diff --git a/ext/README.md b/ext/README.md new file mode 100644 index 0000000000000000000000000000000000000000..933a33d053bf6704be350bc3464a6fd6f02fd2a1 --- /dev/null +++ b/ext/README.md @@ -0,0 +1,8 @@ +## Loadable Extensions + +Various [loadable extensions](https://www.sqlite.org/loadext.html) for +SQLite are found in subfolders. + +Most subfolders are dedicated to a single loadable extension (for +example FTS5, or RTREE). But the misc/ subfolder contains a collection +of smaller single-file extensions. diff --git a/ext/expert/README.md b/ext/expert/README.md new file mode 100644 index 0000000000000000000000000000000000000000..28886fd1f2d75d5074c9287fc3aef0cbfb189db4 --- /dev/null +++ b/ext/expert/README.md @@ -0,0 +1,83 @@ +## SQLite Expert Extension + +This folder contains code for a simple system to propose useful indexes +given a database and a set of SQL queries. It works as follows: + + 1. The user database schema is copied to a temporary database. + + 1. All SQL queries are prepared against the temporary database. + Information regarding the WHERE and ORDER BY clauses, and other query + features that affect index selection are recorded. + + 1. The information gathered in step 2 is used to create candidate + indexes - indexes that the planner might have made use of in the previous + step, had they been available. + + 1. A subset of the data in the user database is used to generate statistics + for all existing indexes and the candidate indexes generated in step 3 + above. + + 1. The SQL queries are prepared a second time. If the planner uses any + of the indexes created in step 3, they are recommended to the user. + +# C API + +The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed +as follows: + + 1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**. + A database handle opened by the user is passed as an argument. + + 1. The sqlite3expert object is configured with one or more SQL statements + by making one or more calls to **sqlite3\_expert\_sql()**. Each call may + specify a single SQL statement, or multiple statements separated by + semi-colons. + + 1. Optionally, the **sqlite3\_expert\_config()** API may be used to + configure the size of the data subset used to generate index statistics. + Using a smaller subset of the data can speed up the analysis. + + 1. **sqlite3\_expert\_analyze()** is called to run the analysis. + + 1. One or more calls are made to **sqlite3\_expert\_report()** to extract + components of the results of the analysis. + + 1. **sqlite3\_expert\_destroy()** is called to free all resources. + +Refer to comments in sqlite3expert.h for further details. + +# sqlite3_expert application + +The file "expert.c" contains the code for a command line application that +uses the API described above. It can be compiled with (for example): + +
+  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+
+ +Assuming the database is named "test.db", it can then be run to analyze a +single query: + +
+  ./sqlite3_expert -sql <sql-query> test.db
+
+ +Or an entire text file worth of queries with: + +
+  ./sqlite3_expert -file <text-file> test.db
+
+ +By default, sqlite3\_expert generates index statistics using all the data in +the user database. For a large database, this may be prohibitively time +consuming. The "-sample" option may be used to configure sqlite3\_expert to +generate statistics based on an integer percentage of the user database as +follows: + +
+  # Generate statistics based on 25% of the user database rows:
+  ./sqlite3_expert -sample 25 -sql <sql-query> test.db
+
+  # Do not generate any statistics at all:
+  ./sqlite3_expert -sample 0 -sql <sql-query> test.db
+
diff --git a/ext/expert/expert.c b/ext/expert/expert.c new file mode 100644 index 0000000000000000000000000000000000000000..051480f896025877c00bfd7c7f569980eb5556d3 --- /dev/null +++ b/ext/expert/expert.c @@ -0,0 +1,156 @@ +/* +** 2017 April 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + + +#include +#include +#include +#include +#include "sqlite3expert.h" + + +static void option_requires_argument(const char *zOpt){ + fprintf(stderr, "Option requires an argument: %s\n", zOpt); + exit(-3); +} + +static int option_integer_arg(const char *zVal){ + return atoi(zVal); +} + +static void usage(char **argv){ + fprintf(stderr, "\n"); + fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "Options are:\n"); + fprintf(stderr, " -sql SQL (analyze SQL statements passed as argument)\n"); + fprintf(stderr, " -file FILE (read SQL statements from file FILE)\n"); + fprintf(stderr, " -verbose LEVEL (integer verbosity level. default 1)\n"); + fprintf(stderr, " -sample PERCENT (percent of db to sample. default 100)\n"); + exit(-1); +} + +static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){ + FILE *in = fopen(zFile, "rb"); + long nIn; + size_t nRead; + char *pBuf; + int rc; + if( in==0 ){ + *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile); + return SQLITE_ERROR; + } + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc64( nIn+1 ); + nRead = fread(pBuf, nIn, 1, in); + fclose(in); + if( nRead!=1 ){ + sqlite3_free(pBuf); + *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile); + return SQLITE_ERROR; + } + pBuf[nIn] = 0; + rc = sqlite3_expert_sql(p, pBuf, pzErr); + sqlite3_free(pBuf); + return rc; +} + +int main(int argc, char **argv){ + const char *zDb; + int rc = 0; + char *zErr = 0; + int i; + int iVerbose = 1; /* -verbose option */ + + sqlite3 *db = 0; + sqlite3expert *p = 0; + + if( argc<2 ) usage(argv); + zDb = argv[argc-1]; + if( zDb[0]=='-' ) usage(argv); + rc = sqlite3_open(zDb, &db); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db)); + exit(-2); + } + + p = sqlite3_expert_new(db, &zErr); + if( p==0 ){ + fprintf(stderr, "Cannot run analysis: %s\n", zErr); + rc = 1; + }else{ + for(i=1; i<(argc-1); i++){ + char *zArg = argv[i]; + int nArg; + if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++; + nArg = (int)strlen(zArg); + if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){ + if( ++i==(argc-1) ) option_requires_argument("-file"); + rc = readSqlFromFile(p, argv[i], &zErr); + } + + else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){ + if( ++i==(argc-1) ) option_requires_argument("-sql"); + rc = sqlite3_expert_sql(p, argv[i], &zErr); + } + + else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){ + int iSample; + if( ++i==(argc-1) ) option_requires_argument("-sample"); + iSample = option_integer_arg(argv[i]); + sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample); + } + + else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){ + if( ++i==(argc-1) ) option_requires_argument("-verbose"); + iVerbose = option_integer_arg(argv[i]); + } + + else{ + usage(argv); + } + } + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_expert_analyze(p, &zErr); + } + + if( rc==SQLITE_OK ){ + int nQuery = sqlite3_expert_count(p); + if( iVerbose>0 ){ + const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); + fprintf(stdout, "-- Candidates -------------------------------\n"); + fprintf(stdout, "%s\n", zCand); + } + for(i=0; i0 ){ + fprintf(stdout, "-- Query %d ----------------------------------\n",i+1); + fprintf(stdout, "%s\n\n", zSql); + } + fprintf(stdout, "%s\n%s\n", zIdx, zEQP); + } + }else{ + fprintf(stderr, "Error: %s\n", zErr ? zErr : "?"); + } + + sqlite3_expert_destroy(p); + sqlite3_free(zErr); + return rc; +} diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test new file mode 100644 index 0000000000000000000000000000000000000000..0c3b512af0ab45a2c4c87ff4518fd614ad7b608b --- /dev/null +++ b/ext/expert/expert1.test @@ -0,0 +1,609 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# TESTRUNNER: shell +# +# The focus of this file is testing the CLI shell tool. Specifically, +# the ".recommend" command. +# +# + +# Test plan: +# +# +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set testprefix expert1 + +if {[info commands sqlite3_expert_new]==""} { + finish_test + return +} + + +set CLI [test_binary_name sqlite3] +set CMD [test_binary_name sqlite3_expert] + +proc squish {txt} { + regsub -all {[[:space:]]+} $txt { } +} + +proc do_setup_rec_test {tn setup sql res} { + reset_db + if {[info exists ::set_main_db_name]} { + dbconfig_maindbname_icecube db + } + db eval $setup + uplevel [list do_rec_test $tn $sql $res] +} + +foreach {tn setup} { + 1 { + if {![file executable $CMD]} { continue } + + proc do_rec_test {tn sql res} { + set res [squish [string trim $res]] + set tst [subst -nocommands { + squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]] + }] + uplevel [list do_test $tn $tst $res] + } + } + 2 { + if {[info commands sqlite3_expert_new]==""} { continue } + + proc do_rec_test {tn sql res} { + set expert [sqlite3_expert_new db] + $expert sql $sql + $expert analyze + + set result [list] + for {set i 0} {$i < [$expert count]} {incr i} { + set idx [string trim [$expert report $i indexes]] + if {$idx==""} {set idx "(no new indexes)"} + lappend result $idx + lappend result [string trim [$expert report $i plan]] + } + + $expert destroy + + set tst [subst -nocommands {set {} [squish [join {$result}]]}] + uplevel [list do_test $tn $tst [string trim [squish $res]]] + } + } + 3 { + if {[info commands sqlite3_expert_new]==""} { continue } + set ::set_main_db_name 1 + } + 4 { + if {![file executable $CLI]} { continue } + + proc do_rec_test {tn sql res} { + set res [squish [string trim $res]] + set tst [subst -nocommands { + squish [string trim [exec $::CLI test.db ".expert" {$sql;}]] + }] + uplevel [list do_test $tn $tst $res] + } + } +} { + + eval $setup + + +do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } { + SELECT * FROM t1 +} { + (no new indexes) + SCAN t1 +} + +do_setup_rec_test $tn.2 { + CREATE TABLE t1(a, b, c); +} { + SELECT * FROM t1 WHERE b>?; +} { + CREATE INDEX t1_idx_00000062 ON t1(b); + SEARCH t1 USING INDEX t1_idx_00000062 (b>?) +} + +do_setup_rec_test $tn.3 { + CREATE TABLE t1(a, b, c); +} { + SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ? +} { + CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE); + SEARCH t1 USING INDEX t1_idx_3e094c27 (b>? AND b? AND b? +} { + (no new indexes) + SEARCH example USING INDEX sqlite_autoindex_example_1 (A=? AND B>?) +} +do_setup_rec_test $tn.17.5 { + CREATE TABLE example (A INTEGER, B INTEGER, C INTEGER, PRIMARY KEY (A,B)); +} { + SELECT * FROM example WHERE a>? AND b=? +} { + CREATE INDEX example_idx_0000cb3f ON example(B, A); + SEARCH example USING INDEX example_idx_0000cb3f (B=? AND A>?) +} + +do_setup_rec_test $tn.18.0 { + CREATE TABLE SomeObject ( + a INTEGER PRIMARY KEY, + x TEXT GENERATED ALWAYS AS(HEX(a)) VIRTUAL + ); +} { + SELECT x FROM SomeObject; +} { + (no new indexes) + SCAN SomeObject +} +do_setup_rec_test $tn.18.1 { + CREATE TABLE SomeObject ( + a INTEGER PRIMARY KEY, + x TEXT GENERATED ALWAYS AS(HEX(a)) VIRTUAL + ); +} { + SELECT * FROM SomeObject WHERE x=?; +} { + CREATE INDEX SomeObject_idx_00000078 ON SomeObject(x); + SEARCH SomeObject USING COVERING INDEX SomeObject_idx_00000078 (x=?) +} + + +do_setup_rec_test $tn.19.0 { + CREATE TABLE t1("index"); +} { + SELECT * FROM t1 ORDER BY "index"; +} { + CREATE INDEX t1_idx_01a7214e ON t1('index'); + SCAN t1 USING COVERING INDEX t1_idx_01a7214e +} + +ifcapable fts5 { + do_setup_rec_test $tn.20.0 { + CREATE VIRTUAL TABLE ft USING fts5(a); + CREATE TABLE t1(x, y); + } { + SELECT * FROM ft, t1 WHERE a=x + } { + CREATE INDEX t1_idx_00000078 ON t1(x); + SCAN ft VIRTUAL TABLE INDEX 0: + SEARCH t1 USING INDEX t1_idx_00000078 (x=?) + } +} + +} + +proc do_candidates_test {tn sql res} { + set res [squish [string trim $res]] + + set expert [sqlite3_expert_new db] + $expert sql $sql + $expert analyze + + set candidates [squish [string trim [$expert report 0 candidates]]] + $expert destroy + + uplevel [list do_test $tn [list set {} $candidates] $res] +} + + +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d); + + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s; + + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s; + + CREATE INDEX i1 ON t1( lower(a) ); +} +do_candidates_test 5.1 { + SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?) +} { + CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 + CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 + CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 + CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5 +} + +do_candidates_test 5.2 { + SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=? +} { + CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17 + CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5 +} + +do_execsql_test 5.3 { + CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 + CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 + CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16 + + CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 + CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5 + CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5 + + ANALYZE; + SELECT * FROM sqlite_stat1 ORDER BY 1, 2; +} { + t1 i1 {100 50} + t1 t1_idx_00000061 {100 50} + t1 t1_idx_00000062 {100 20} + t1 t1_idx_000123a7 {100 50 17} + t2 t2_idx_00000063 {100 20} + t2 t2_idx_00000064 {100 5} + t2 t2_idx_0001295b {100 20 5} +} + +do_catchsql_test 5.4 { + SELECT sqlite_expert_rem(123, 123); +} {1 {no such function: sqlite_expert_rem}} +do_catchsql_test 5.5 { + SELECT sqlite_expert_sample(); +} {1 {no such function: sqlite_expert_sample}} + +if 0 { +do_test expert1-6.0 { + catchcmd :memory: { +.expert +select base64(''); +.expert +select name from pragma_collation_list order by name collate uint; +} +} {0 {(no new indexes) + +SCAN CONSTANT ROW + +(no new indexes) + +SCAN pragma_collation_list VIRTUAL TABLE INDEX 0: +USE TEMP B-TREE FOR ORDER BY +}} +} + +do_execsql_test 6.0 { + CREATE TABLE x1(a, b, c, d); + CREATE INDEX x1ab ON x1(a, lower(b)); + CREATE INDEX x1dcba ON x1(d, b+c, a); +} + +do_candidates_test 6.1 { + SELECT * FROM x1 WHERE b=? ORDER BY a; +} { + CREATE INDEX x1_idx_0001267f ON x1(b, a); + CREATE INDEX x1_idx_00000062 ON x1(b); +} + +#------------------------------------------------------------------------- +ifcapable fts5 { + reset_db + do_execsql_test 7.0 { + CREATE VIRTUAL TABLE ft USING fts5(a); + CREATE TABLE t1(x, y); + } + + do_candidates_test 7.1 { + SELECT * FROM ft, t1 WHERE a=x + } { + CREATE INDEX t1_idx_00000078 ON t1(x); + } + + register_tcl_module db + proc vtab_command {method args} { + global G + + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c);" + } + + xBestIndex { + return [list] + } + + xFilter { + return [list sql "SELECT rowid, * FROM t0"] + } + } + + return {} + } + + do_execsql_test 7.2 { + CREATE TABLE t0(a, b, c); + INSERT INTO t0 VALUES(1, 2, 3), (11, 22, 33); + CREATE VIRTUAL TABLE t2 USING tcl(vtab_command); + } + + do_execsql_test 7.3 { + SELECT * FROM t2 + } { + 1 2 3 + 11 22 33 + } + + do_candidates_test 7.4 { + SELECT * FROM ft, t1 WHERE a=x + } { + CREATE INDEX t1_idx_00000078 ON t1(x); + } + + do_test 7.5 { + set expert [sqlite3_expert_new db] + list [catch { $expert sql "SELECT * FROM ft, t2 WHERE b=1" } msg] $msg + } {1 {no such table: t2}} + $expert destroy + + reset_db + do_execsql_test 7.6 { + BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS 'bfts_idx_data'(id INTEGER PRIMARY KEY, block BLOB); + CREATE TABLE IF NOT EXISTS 'fts_idx_data'(id INTEGER PRIMARY KEY, block BLOB); + INSERT INTO fts_idx_data VALUES(1,X''); + INSERT INTO fts_idx_data VALUES(10,X'00000000ff000001000000'); + CREATE TABLE IF NOT EXISTS 'fts_idx_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID; + CREATE TABLE IF NOT EXISTS 'fts_idx_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER); + CREATE TABLE IF NOT EXISTS 'fts_idx_config'(k PRIMARY KEY, v) WITHOUT ROWID; + INSERT INTO fts_idx_config VALUES('version',4); + PRAGMA writable_schema=ON; + INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)VALUES('table','fts_idx','fts_idx',0,'CREATE VIRTUAL TABLE fts_idx USING fts5(Title, Description, Channel, Tags, content='''', contentless_delete=1)'); + + CREATE TABLE f(x BLOB, y); + COMMIT; + PRAGMA writable_schema = RESET; + } + + do_candidates_test 7.4 { + SELECT * FROM fts_idx, f WHERE x = fts_idx.Channel + } { + CREATE INDEX f_idx_00000078 ON f(x); + } +} + +finish_test diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c new file mode 100644 index 0000000000000000000000000000000000000000..93693cfae99169b92cf924b4d9b8357a09044d61 --- /dev/null +++ b/ext/expert/sqlite3expert.c @@ -0,0 +1,2234 @@ +/* +** 2017 April 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +#include "sqlite3expert.h" +#include +#include +#include + +#if !defined(SQLITE_AMALGAMATION) +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif +#endif /* !defined(SQLITE_AMALGAMATION) */ + + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; + +typedef struct IdxColumn IdxColumn; +typedef struct IdxConstraint IdxConstraint; +typedef struct IdxScan IdxScan; +typedef struct IdxStatement IdxStatement; +typedef struct IdxTable IdxTable; +typedef struct IdxWrite IdxWrite; + +#define STRLEN (int)strlen + +/* +** A temp table name that we assume no user database will actually use. +** If this assumption proves incorrect triggers on the table with the +** conflicting name will be ignored. +*/ +#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776" + +/* +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or +** any other type of single-ended range constraint on a column). +** +** pLink: +** Used to temporarily link IdxConstraint objects into lists while +** creating candidate indexes. +*/ +struct IdxConstraint { + char *zColl; /* Collation sequence */ + int bRange; /* True for range, false for eq */ + int iCol; /* Constrained table column */ + int bFlag; /* Used by idxFindCompatible() */ + int bDesc; /* True if ORDER BY DESC */ + IdxConstraint *pNext; /* Next constraint in pEq or pRange list */ + IdxConstraint *pLink; /* See above */ +}; + +/* +** A single scan of a single table. +*/ +struct IdxScan { + IdxTable *pTab; /* Associated table object */ + int iDb; /* Database containing table zTable */ + i64 covering; /* Mask of columns required for cov. index */ + IdxConstraint *pOrder; /* ORDER BY columns */ + IdxConstraint *pEq; /* List of == constraints */ + IdxConstraint *pRange; /* List of < constraints */ + IdxScan *pNextScan; /* Next IdxScan object for same analysis */ +}; + +/* +** Information regarding a single database table. Extracted from +** "PRAGMA table_info" by function idxGetTableInfo(). +*/ +struct IdxColumn { + char *zName; + char *zColl; + int iPk; +}; +struct IdxTable { + int nCol; + char *zName; /* Table name */ + IdxColumn *aCol; + IdxTable *pNext; /* Next table in linked list of all tables */ +}; + +/* +** An object of the following type is created for each unique table/write-op +** seen. The objects are stored in a singly-linked list beginning at +** sqlite3expert.pWrite. +*/ +struct IdxWrite { + IdxTable *pTab; + int eOp; /* SQLITE_UPDATE, DELETE or INSERT */ + IdxWrite *pNext; +}; + +/* +** Each statement being analyzed is represented by an instance of this +** structure. +*/ +struct IdxStatement { + int iId; /* Statement number */ + char *zSql; /* SQL statement */ + char *zIdx; /* Indexes */ + char *zEQP; /* Plan */ + IdxStatement *pNext; +}; + + +/* +** A hash table for storing strings. With space for a payload string +** with each entry. Methods are: +** +** idxHashInit() +** idxHashClear() +** idxHashAdd() +** idxHashSearch() +*/ +#define IDX_HASH_SIZE 1023 +typedef struct IdxHashEntry IdxHashEntry; +typedef struct IdxHash IdxHash; +struct IdxHashEntry { + char *zKey; /* nul-terminated key */ + char *zVal; /* nul-terminated value string */ + char *zVal2; /* nul-terminated value string 2 */ + IdxHashEntry *pHashNext; /* Next entry in same hash bucket */ + IdxHashEntry *pNext; /* Next entry in hash */ +}; +struct IdxHash { + IdxHashEntry *pFirst; + IdxHashEntry *aHash[IDX_HASH_SIZE]; +}; + +/* +** sqlite3expert object. +*/ +struct sqlite3expert { + int iSample; /* Percentage of tables to sample for stat1 */ + sqlite3 *db; /* User database */ + sqlite3 *dbm; /* In-memory db for this analysis */ + sqlite3 *dbv; /* Vtab schema for this analysis */ + IdxTable *pTable; /* List of all IdxTable objects */ + IdxScan *pScan; /* List of scan objects */ + IdxWrite *pWrite; /* List of write objects */ + IdxStatement *pStatement; /* List of IdxStatement objects */ + int bRun; /* True once analysis has run */ + char **pzErrmsg; + int rc; /* Error code from whereinfo hook */ + IdxHash hIdx; /* Hash containing all candidate indexes */ + char *zCandidates; /* For EXPERT_REPORT_CANDIDATES */ +}; + + +/* +** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). +** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. +*/ +static void *idxMalloc(int *pRc, int nByte){ + void *pRet; + assert( *pRc==SQLITE_OK ); + assert( nByte>0 ); + pRet = sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + *pRc = SQLITE_NOMEM; + } + return pRet; +} + +/* +** Initialize an IdxHash hash table. +*/ +static void idxHashInit(IdxHash *pHash){ + memset(pHash, 0, sizeof(IdxHash)); +} + +/* +** Reset an IdxHash hash table. +*/ +static void idxHashClear(IdxHash *pHash){ + int i; + for(i=0; iaHash[i]; pEntry; pEntry=pNext){ + pNext = pEntry->pHashNext; + sqlite3_free(pEntry->zVal2); + sqlite3_free(pEntry); + } + } + memset(pHash, 0, sizeof(IdxHash)); +} + +/* +** Return the index of the hash bucket that the string specified by the +** arguments to this function belongs. +*/ +static int idxHashString(const char *z, int n){ + unsigned int ret = 0; + int i; + for(i=0; i=0 ); + for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ + if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ + return 1; + } + } + pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); + if( pEntry ){ + pEntry->zKey = (char*)&pEntry[1]; + memcpy(pEntry->zKey, zKey, nKey); + if( zVal ){ + pEntry->zVal = &pEntry->zKey[nKey+1]; + memcpy(pEntry->zVal, zVal, nVal); + } + pEntry->pHashNext = pHash->aHash[iHash]; + pHash->aHash[iHash] = pEntry; + + pEntry->pNext = pHash->pFirst; + pHash->pFirst = pEntry; + } + return 0; +} + +/* +** If zKey/nKey is present in the hash table, return a pointer to the +** hash-entry object. +*/ +static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){ + int iHash; + IdxHashEntry *pEntry; + if( nKey<0 ) nKey = STRLEN(zKey); + iHash = idxHashString(zKey, nKey); + assert( iHash>=0 ); + for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ + if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ + return pEntry; + } + } + return 0; +} + +/* +** If the hash table contains an entry with a key equal to the string +** passed as the final two arguments to this function, return a pointer +** to the payload string. Otherwise, if zKey/nKey is not present in the +** hash table, return NULL. +*/ +static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){ + IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey); + if( pEntry ) return pEntry->zVal; + return 0; +} + +/* +** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl +** variable to point to a copy of nul-terminated string zColl. +*/ +static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ + IdxConstraint *pNew; + int nColl = STRLEN(zColl); + + assert( *pRc==SQLITE_OK ); + pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1); + if( pNew ){ + pNew->zColl = (char*)&pNew[1]; + memcpy(pNew->zColl, zColl, nColl+1); + } + return pNew; +} + +/* +** An error associated with database handle db has just occurred. Pass +** the error message to callback function xOut. +*/ +static void idxDatabaseError( + sqlite3 *db, /* Database handle */ + char **pzErrmsg /* Write error here */ +){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); +} + +/* +** Prepare an SQL statement. +*/ +static int idxPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ + const char *zSql /* SQL statement to compile */ +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *ppStmt = 0; + idxDatabaseError(db, pzErrmsg); + } + return rc; +} + +/* +** Prepare an SQL statement using the results of a printf() formatting. +*/ +static int idxPrintfPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ + const char *zFmt, /* printf() format of SQL statement */ + ... /* Trailing printf() arguments */ +){ + va_list ap; + int rc; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + va_end(ap); + return rc; +} + + +/************************************************************************* +** Beginning of virtual table implementation. +*/ +typedef struct ExpertVtab ExpertVtab; +struct ExpertVtab { + sqlite3_vtab base; + IdxTable *pTab; + sqlite3expert *pExpert; +}; + +typedef struct ExpertCsr ExpertCsr; +struct ExpertCsr { + sqlite3_vtab_cursor base; + sqlite3_stmt *pData; +}; + +static char *expertDequote(const char *zIn){ + int n = STRLEN(zIn); + char *zRet = sqlite3_malloc(n); + + assert( zIn[0]=='\'' ); + assert( zIn[n-1]=='\'' ); + + if( zRet ){ + int iOut = 0; + int iIn = 0; + for(iIn=1; iIn<(n-1); iIn++){ + if( zIn[iIn]=='\'' ){ + assert( zIn[iIn+1]=='\'' ); + iIn++; + } + zRet[iOut++] = zIn[iIn]; + } + zRet[iOut] = '\0'; + } + + return zRet; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int expertConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3expert *pExpert = (sqlite3expert*)pAux; + ExpertVtab *p = 0; + int rc; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("internal error!"); + rc = SQLITE_ERROR; + }else{ + char *zCreateTable = expertDequote(argv[3]); + if( zCreateTable ){ + rc = sqlite3_declare_vtab(db, zCreateTable); + if( rc==SQLITE_OK ){ + p = idxMalloc(&rc, sizeof(ExpertVtab)); + } + if( rc==SQLITE_OK ){ + p->pExpert = pExpert; + p->pTab = pExpert->pTable; + assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 ); + } + sqlite3_free(zCreateTable); + }else{ + rc = SQLITE_NOMEM; + } + } + + *ppVtab = (sqlite3_vtab*)p; + return rc; +} + +static int expertDisconnect(sqlite3_vtab *pVtab){ + ExpertVtab *p = (ExpertVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){ + ExpertVtab *p = (ExpertVtab*)pVtab; + int rc = SQLITE_OK; + int n = 0; + IdxScan *pScan; + const int opmask = + SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT | + SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE | + SQLITE_INDEX_CONSTRAINT_LE; + + pScan = idxMalloc(&rc, sizeof(IdxScan)); + if( pScan ){ + int i; + + /* Link the new scan object into the list */ + pScan->pTab = p->pTab; + pScan->pNextScan = p->pExpert->pScan; + p->pExpert->pScan = pScan; + + /* Add the constraints to the IdxScan object */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + if( pCons->usable + && pCons->iColumn>=0 + && p->pTab->aCol[pCons->iColumn].iPk==0 + && (pCons->op & opmask) + ){ + IdxConstraint *pNew; + const char *zColl = sqlite3_vtab_collation(pIdxInfo, i); + pNew = idxNewConstraint(&rc, zColl); + if( pNew ){ + pNew->iCol = pCons->iColumn; + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pNew->pNext = pScan->pEq; + pScan->pEq = pNew; + }else{ + pNew->bRange = 1; + pNew->pNext = pScan->pRange; + pScan->pRange = pNew; + } + } + n++; + pIdxInfo->aConstraintUsage[i].argvIndex = n; + } + } + + /* Add the ORDER BY to the IdxScan object */ + for(i=pIdxInfo->nOrderBy-1; i>=0; i--){ + int iCol = pIdxInfo->aOrderBy[i].iColumn; + if( iCol>=0 ){ + IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl); + if( pNew ){ + pNew->iCol = iCol; + pNew->bDesc = pIdxInfo->aOrderBy[i].desc; + pNew->pNext = pScan->pOrder; + pNew->pLink = pScan->pOrder; + pScan->pOrder = pNew; + n++; + } + } + } + } + + pIdxInfo->estimatedCost = 1000000.0 / (n+1); + return rc; +} + +static int expertUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **azData, + sqlite_int64 *pRowid +){ + (void)pVtab; + (void)nData; + (void)azData; + (void)pRowid; + return SQLITE_OK; +} + +/* +** Virtual table module xOpen method. +*/ +static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_OK; + ExpertCsr *pCsr; + (void)pVTab; + pCsr = idxMalloc(&rc, sizeof(ExpertCsr)); + *ppCursor = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +/* +** Virtual table module xClose method. +*/ +static int expertClose(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + sqlite3_finalize(pCsr->pData); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Virtual table module xEof method. +** +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int expertEof(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + return pCsr->pData==0; +} + +/* +** Virtual table module xNext method. +*/ +static int expertNext(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + int rc = SQLITE_OK; + + assert( pCsr->pData ); + rc = sqlite3_step(pCsr->pData); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_finalize(pCsr->pData); + pCsr->pData = 0; + }else{ + rc = SQLITE_OK; + } + + return rc; +} + +/* +** Virtual table module xRowid method. +*/ +static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + (void)cur; + *pRowid = 0; + return SQLITE_OK; +} + +/* +** Virtual table module xColumn method. +*/ +static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + sqlite3_value *pVal; + pVal = sqlite3_column_value(pCsr->pData, i); + if( pVal ){ + sqlite3_result_value(ctx, pVal); + } + return SQLITE_OK; +} + +/* +** Virtual table module xFilter method. +*/ +static int expertFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab); + sqlite3expert *pExpert = pVtab->pExpert; + int rc; + + (void)idxNum; + (void)idxStr; + (void)argc; + (void)argv; + rc = sqlite3_finalize(pCsr->pData); + pCsr->pData = 0; + if( rc==SQLITE_OK ){ + rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName + ); + } + + if( rc==SQLITE_OK ){ + rc = expertNext(cur); + } + return rc; +} + +static int idxRegisterVtab(sqlite3expert *p){ + static sqlite3_module expertModule = { + 2, /* iVersion */ + expertConnect, /* xCreate - create a table */ + expertConnect, /* xConnect - connect to an existing table */ + expertBestIndex, /* xBestIndex - Determine search strategy */ + expertDisconnect, /* xDisconnect - Disconnect from a table */ + expertDisconnect, /* xDestroy - Drop a table */ + expertOpen, /* xOpen - open a cursor */ + expertClose, /* xClose - close a cursor */ + expertFilter, /* xFilter - configure scan constraints */ + expertNext, /* xNext - advance a cursor */ + expertEof, /* xEof */ + expertColumn, /* xColumn - read data */ + expertRowid, /* xRowid - read data */ + expertUpdate, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0, /* xIntegrity */ + }; + + return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); +} +/* +** End of virtual table implementation. +*************************************************************************/ +/* +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function +** is called, set it to the return value of sqlite3_finalize() before +** returning. Otherwise, discard the sqlite3_finalize() return value. +*/ +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +/* +** Attempt to allocate an IdxTable structure corresponding to table zTab +** in the main database of connection db. If successful, set (*ppOut) to +** point to the new object and return SQLITE_OK. Otherwise, return an +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be +** set to point to an error string. +** +** It is the responsibility of the caller to eventually free either the +** IdxTable object or error message using sqlite3_free(). +*/ +static int idxGetTableInfo( + sqlite3 *db, /* Database connection to read details from */ + const char *zTab, /* Table name */ + IdxTable **ppOut, /* OUT: New object (if successful) */ + char **pzErrmsg /* OUT: Error message (if not) */ +){ + sqlite3_stmt *p1 = 0; + int nCol = 0; + int nTab; + int nByte; + IdxTable *pNew = 0; + int rc, rc2; + char *pCsr = 0; + int nPk = 0; + + *ppOut = 0; + if( zTab==0 ) return SQLITE_ERROR; + nTab = STRLEN(zTab); + nByte = sizeof(IdxTable) + nTab + 1; + rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_xinfo=%Q", zTab); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = (const char*)sqlite3_column_text(p1, 1); + const char *zColSeq = 0; + if( zCol==0 ){ + rc = SQLITE_ERROR; + break; + } + nByte += 1 + STRLEN(zCol); + rc = sqlite3_table_column_metadata( + db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 + ); + if( zColSeq==0 ) zColSeq = "binary"; + nByte += 1 + STRLEN(zColSeq); + nCol++; + nPk += (sqlite3_column_int(p1, 5)>0); + } + rc2 = sqlite3_reset(p1); + if( rc==SQLITE_OK ) rc = rc2; + + nByte += sizeof(IdxColumn) * nCol; + if( rc==SQLITE_OK ){ + pNew = idxMalloc(&rc, nByte); + } + if( rc==SQLITE_OK ){ + pNew->aCol = (IdxColumn*)&pNew[1]; + pNew->nCol = nCol; + pCsr = (char*)&pNew->aCol[nCol]; + } + + nCol = 0; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = (const char*)sqlite3_column_text(p1, 1); + const char *zColSeq = 0; + int nCopy; + if( zCol==0 ) continue; + nCopy = STRLEN(zCol) + 1; + pNew->aCol[nCol].zName = pCsr; + pNew->aCol[nCol].iPk = (sqlite3_column_int(p1, 5)==1 && nPk==1); + memcpy(pCsr, zCol, nCopy); + pCsr += nCopy; + + rc = sqlite3_table_column_metadata( + db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + if( zColSeq==0 ) zColSeq = "binary"; + nCopy = STRLEN(zColSeq) + 1; + pNew->aCol[nCol].zColl = pCsr; + memcpy(pCsr, zColSeq, nCopy); + pCsr += nCopy; + } + + nCol++; + } + idxFinalize(&rc, p1); + + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + }else if( ALWAYS(pNew!=0) ){ + pNew->zName = pCsr; + if( ALWAYS(pNew->zName!=0) ) memcpy(pNew->zName, zTab, nTab+1); + } + + *ppOut = pNew; + return rc; +} + +/* +** This function is a no-op if *pRc is set to anything other than +** SQLITE_OK when it is called. +** +** If *pRc is initially set to SQLITE_OK, then the text specified by +** the printf() style arguments is appended to zIn and the result returned +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on +** zIn before returning. +*/ +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ + va_list ap; + char *zAppend = 0; + char *zRet = 0; + int nIn = zIn ? STRLEN(zIn) : 0; + int nAppend = 0; + va_start(ap, zFmt); + if( *pRc==SQLITE_OK ){ + zAppend = sqlite3_vmprintf(zFmt, ap); + if( zAppend ){ + nAppend = STRLEN(zAppend); + zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); + } + if( zAppend && zRet ){ + if( nIn ) memcpy(zRet, zIn, nIn); + memcpy(&zRet[nIn], zAppend, nAppend+1); + }else{ + sqlite3_free(zRet); + zRet = 0; + *pRc = SQLITE_NOMEM; + } + sqlite3_free(zAppend); + sqlite3_free(zIn); + } + va_end(ap); + return zRet; +} + +/* +** Return true if zId must be quoted in order to use it as an SQL +** identifier, or false otherwise. +*/ +static int idxIdentifierRequiresQuotes(const char *zId){ + int i; + int nId = STRLEN(zId); + + if( sqlite3_keyword_check(zId, nId) ) return 1; + + for(i=0; zId[i]; i++){ + if( !(zId[i]=='_') + && !(zId[i]>='0' && zId[i]<='9') + && !(zId[i]>='a' && zId[i]<='z') + && !(zId[i]>='A' && zId[i]<='Z') + ){ + return 1; + } + } + return 0; +} + +/* +** This function appends an index column definition suitable for constraint +** pCons to the string passed as zIn and returns the result. +*/ +static char *idxAppendColDefn( + int *pRc, /* IN/OUT: Error code */ + char *zIn, /* Column defn accumulated so far */ + IdxTable *pTab, /* Table index will be created on */ + IdxConstraint *pCons +){ + char *zRet = zIn; + IdxColumn *p = &pTab->aCol[pCons->iCol]; + if( zRet ) zRet = idxAppendText(pRc, zRet, ", "); + + if( idxIdentifierRequiresQuotes(p->zName) ){ + zRet = idxAppendText(pRc, zRet, "%Q", p->zName); + }else{ + zRet = idxAppendText(pRc, zRet, "%s", p->zName); + } + + if( sqlite3_stricmp(p->zColl, pCons->zColl) ){ + if( idxIdentifierRequiresQuotes(pCons->zColl) ){ + zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl); + }else{ + zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl); + } + } + + if( pCons->bDesc ){ + zRet = idxAppendText(pRc, zRet, " DESC"); + } + return zRet; +} + +/* +** Search database dbm for an index compatible with the one idxCreateFromCons() +** would create from arguments pScan, pEq and pTail. If no error occurs and +** such an index is found, return non-zero. Or, if no such index is found, +** return zero. +** +** If an error occurs, set *pRc to an SQLite error code and return zero. +*/ +static int idxFindCompatible( + int *pRc, /* OUT: Error code */ + sqlite3* dbm, /* Database to search */ + IdxScan *pScan, /* Scan for table to search for index on */ + IdxConstraint *pEq, /* List of == constraints */ + IdxConstraint *pTail /* List of range constraints */ +){ + const char *zTbl = pScan->pTab->zName; + sqlite3_stmt *pIdxList = 0; + IdxConstraint *pIter; + int nEq = 0; /* Number of elements in pEq */ + int rc; + + /* Count the elements in list pEq */ + for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++; + + rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl); + while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){ + int bMatch = 1; + IdxConstraint *pT = pTail; + sqlite3_stmt *pInfo = 0; + const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1); + if( zIdx==0 ) continue; + + /* Zero the IdxConstraint.bFlag values in the pEq list */ + for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0; + + rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx); + while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){ + int iIdx = sqlite3_column_int(pInfo, 0); + int iCol = sqlite3_column_int(pInfo, 1); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + + if( iIdxpLink){ + if( pIter->bFlag ) continue; + if( pIter->iCol!=iCol ) continue; + if( sqlite3_stricmp(pIter->zColl, zColl) ) continue; + pIter->bFlag = 1; + break; + } + if( pIter==0 ){ + bMatch = 0; + break; + } + }else{ + if( pT ){ + if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){ + bMatch = 0; + break; + } + pT = pT->pLink; + } + } + } + idxFinalize(&rc, pInfo); + + if( rc==SQLITE_OK && bMatch ){ + sqlite3_finalize(pIdxList); + return 1; + } + } + idxFinalize(&rc, pIdxList); + + *pRc = rc; + return 0; +} + +/* Callback for sqlite3_exec() with query with leading count(*) column. + * The first argument is expected to be an int*, referent to be incremented + * if that leading column is not exactly '0'. + */ +static int countNonzeros(void* pCount, int nc, + char* azResults[], char* azColumns[]){ + (void)azColumns; /* Suppress unused parameter warning */ + if( nc>0 && (azResults[0][0]!='0' || azResults[0][1]!=0) ){ + *((int *)pCount) += 1; + } + return 0; +} + +static int idxCreateFromCons( + sqlite3expert *p, + IdxScan *pScan, + IdxConstraint *pEq, + IdxConstraint *pTail +){ + sqlite3 *dbm = p->dbm; + int rc = SQLITE_OK; + if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){ + IdxTable *pTab = pScan->pTab; + char *zCols = 0; + char *zIdx = 0; + IdxConstraint *pCons; + unsigned int h = 0; + const char *zFmt; + + for(pCons=pEq; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + for(pCons=pTail; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + + if( rc==SQLITE_OK ){ + /* Hash the list of columns to come up with a name for the index */ + const char *zTable = pScan->pTab->zName; + int quoteTable = idxIdentifierRequiresQuotes(zTable); + char *zName = 0; /* Index name */ + int collisions = 0; + do{ + int i; + char *zFind; + for(i=0; zCols[i]; i++){ + h += ((h<<3) + zCols[i]); + } + sqlite3_free(zName); + zName = sqlite3_mprintf("%s_idx_%08x", zTable, h); + if( zName==0 ) break; + /* Is is unique among table, view and index names? */ + zFmt = "SELECT count(*) FROM sqlite_schema WHERE name=%Q" + " AND type in ('index','table','view')"; + zFind = sqlite3_mprintf(zFmt, zName); + i = 0; + rc = sqlite3_exec(dbm, zFind, countNonzeros, &i, 0); + assert(rc==SQLITE_OK); + sqlite3_free(zFind); + if( i==0 ){ + collisions = 0; + break; + } + ++collisions; + }while( collisions<50 && zName!=0 ); + if( collisions ){ + /* This return means "Gave up trying to find a unique index name." */ + rc = SQLITE_BUSY_TIMEOUT; + }else if( zName==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( quoteTable ){ + zFmt = "CREATE INDEX \"%w\" ON \"%w\"(%s)"; + }else{ + zFmt = "CREATE INDEX %s ON %s(%s)"; + } + zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols); + if( !zIdx ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg); + if( rc!=SQLITE_OK ){ + rc = SQLITE_BUSY_TIMEOUT; + }else{ + idxHashAdd(&rc, &p->hIdx, zName, zIdx); + } + } + sqlite3_free(zName); + sqlite3_free(zIdx); + } + } + + sqlite3_free(zCols); + } + return rc; +} + +/* +** Return true if list pList (linked by IdxConstraint.pLink) contains +** a constraint compatible with *p. Otherwise return false. +*/ +static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ + IdxConstraint *pCmp; + for(pCmp=pList; pCmp; pCmp=pCmp->pLink){ + if( p->iCol==pCmp->iCol ) return 1; + } + return 0; +} + +static int idxCreateFromWhere( + sqlite3expert *p, + IdxScan *pScan, /* Create indexes for this scan */ + IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ +){ + IdxConstraint *p1 = 0; + IdxConstraint *pCon; + int rc; + + /* Gather up all the == constraints. */ + for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){ + if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ + pCon->pLink = p1; + p1 = pCon; + } + } + + /* Create an index using the == constraints collected above. And the + ** range constraint/ORDER BY terms passed in by the caller, if any. */ + rc = idxCreateFromCons(p, pScan, p1, pTail); + + /* If no range/ORDER BY passed by the caller, create a version of the + ** index for each range constraint. */ + if( pTail==0 ){ + for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ + assert( pCon->pLink==0 ); + if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ + rc = idxCreateFromCons(p, pScan, p1, pCon); + } + } + } + + return rc; +} + +/* +** Create candidate indexes in database [dbm] based on the data in +** linked-list pScan. +*/ +static int idxCreateCandidates(sqlite3expert *p){ + int rc = SQLITE_OK; + IdxScan *pIter; + + for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + rc = idxCreateFromWhere(p, pIter, 0); + if( rc==SQLITE_OK && pIter->pOrder ){ + rc = idxCreateFromWhere(p, pIter, pIter->pOrder); + } + } + + return rc; +} + +/* +** Free all elements of the linked list starting at pConstraint. +*/ +static void idxConstraintFree(IdxConstraint *pConstraint){ + IdxConstraint *pNext; + IdxConstraint *p; + + for(p=pConstraint; p; p=pNext){ + pNext = p->pNext; + sqlite3_free(p); + } +} + +/* +** Free all elements of the linked list starting from pScan up until pLast +** (pLast is not freed). +*/ +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ + IdxScan *p; + IdxScan *pNext; + for(p=pScan; p!=pLast; p=pNext){ + pNext = p->pNextScan; + idxConstraintFree(p->pOrder); + idxConstraintFree(p->pEq); + idxConstraintFree(p->pRange); + sqlite3_free(p); + } +} + +/* +** Free all elements of the linked list starting from pStatement up +** until pLast (pLast is not freed). +*/ +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ + IdxStatement *p; + IdxStatement *pNext; + for(p=pStatement; p!=pLast; p=pNext){ + pNext = p->pNext; + sqlite3_free(p->zEQP); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Free the linked list of IdxTable objects starting at pTab. +*/ +static void idxTableFree(IdxTable *pTab){ + IdxTable *pIter; + IdxTable *pNext; + for(pIter=pTab; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} + +/* +** Free the linked list of IdxWrite objects starting at pTab. +*/ +static void idxWriteFree(IdxWrite *pTab){ + IdxWrite *pIter; + IdxWrite *pNext; + for(pIter=pTab; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} + + + +/* +** This function is called after candidate indexes have been created. It +** runs all the queries to see which indexes they prefer, and populates +** IdxStatement.zIdx and IdxStatement.zEQP with the results. +*/ +static int idxFindIndexes( + sqlite3expert *p, + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + IdxStatement *pStmt; + sqlite3 *dbm = p->dbm; + int rc = SQLITE_OK; + + IdxHash hIdx; + idxHashInit(&hIdx); + + for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){ + IdxHashEntry *pEntry; + sqlite3_stmt *pExplain = 0; + idxHashClear(&hIdx); + rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr, + "EXPLAIN QUERY PLAN %s", pStmt->zSql + ); + while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ + /* int iId = sqlite3_column_int(pExplain, 0); */ + /* int iParent = sqlite3_column_int(pExplain, 1); */ + /* int iNotUsed = sqlite3_column_int(pExplain, 2); */ + const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); + int nDetail; + int i; + + if( !zDetail ) continue; + nDetail = STRLEN(zDetail); + + for(i=0; ihIdx, zIdx, nIdx); + if( zSql ){ + idxHashAdd(&rc, &hIdx, zSql, 0); + if( rc ) goto find_indexes_out; + } + break; + } + } + + if( zDetail[0]!='-' ){ + pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail); + } + } + + for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ + pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey); + } + + idxFinalize(&rc, pExplain); + } + + find_indexes_out: + idxHashClear(&hIdx); + return rc; +} + +static int idxAuthCallback( + void *pCtx, + int eOp, + const char *z3, + const char *z4, + const char *zDb, + const char *zTrigger +){ + int rc = SQLITE_OK; + (void)z4; + (void)zTrigger; + if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){ + if( sqlite3_stricmp(zDb, "main")==0 ){ + sqlite3expert *p = (sqlite3expert*)pCtx; + IdxTable *pTab; + for(pTab=p->pTable; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_stricmp(z3, pTab->zName) ) break; + } + if( pTab ){ + IdxWrite *pWrite; + for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){ + if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break; + } + if( pWrite==0 ){ + pWrite = idxMalloc(&rc, sizeof(IdxWrite)); + if( rc==SQLITE_OK ){ + pWrite->pTab = pTab; + pWrite->eOp = eOp; + pWrite->pNext = p->pWrite; + p->pWrite = pWrite; + } + } + } + } + } + return rc; +} + +static int idxProcessOneTrigger( + sqlite3expert *p, + IdxWrite *pWrite, + char **pzErr +){ + static const char *zInt = UNIQUE_TABLE_NAME; + static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME; + IdxTable *pTab = pWrite->pTab; + const char *zTab = pTab->zName; + const char *zSql = + "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_schema " + "WHERE tbl_name = %Q AND type IN ('table', 'trigger') " + "ORDER BY type;"; + sqlite3_stmt *pSelect = 0; + int rc = SQLITE_OK; + char *zWrite = 0; + + /* Create the table and its triggers in the temp schema */ + rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0); + if( zCreate==0 ) continue; + rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr); + } + idxFinalize(&rc, pSelect); + + /* Rename the table in the temp schema to zInt */ + if( rc==SQLITE_OK ){ + char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt); + if( z==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr); + sqlite3_free(z); + } + } + + switch( pWrite->eOp ){ + case SQLITE_INSERT: { + int i; + zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt); + for(i=0; inCol; i++){ + zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", "); + } + zWrite = idxAppendText(&rc, zWrite, ")"); + break; + } + case SQLITE_UPDATE: { + int i; + zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt); + for(i=0; inCol; i++){ + zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", + pTab->aCol[i].zName + ); + } + break; + } + default: { + assert( pWrite->eOp==SQLITE_DELETE ); + if( rc==SQLITE_OK ){ + zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt); + if( zWrite==0 ) rc = SQLITE_NOMEM; + } + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pX = 0; + rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0); + idxFinalize(&rc, pX); + if( rc!=SQLITE_OK ){ + idxDatabaseError(p->dbv, pzErr); + } + } + sqlite3_free(zWrite); + + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr); + } + + return rc; +} + +static int idxProcessTriggers(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + IdxWrite *pEnd = 0; + IdxWrite *pFirst = p->pWrite; + + while( rc==SQLITE_OK && pFirst!=pEnd ){ + IdxWrite *pIter; + for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){ + rc = idxProcessOneTrigger(p, pIter, pzErr); + } + pEnd = pFirst; + pFirst = p->pWrite; + } + + return rc; +} + +/* +** This function tests if the schema of the main database of database handle +** db contains an object named zTab. Assuming no error occurs, output parameter +** (*pbContains) is set to true if zTab exists, or false if it does not. +** +** Or, if an error occurs, an SQLite error code is returned. The final value +** of (*pbContains) is undefined in this case. +*/ +static int expertDbContainsObject( + sqlite3 *db, + const char *zTab, + int *pbContains /* OUT: True if object exists */ +){ + const char *zSql = "SELECT 1 FROM sqlite_schema WHERE name = ?"; + sqlite3_stmt *pSql = 0; + int rc = SQLITE_OK; + int ret = 0; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pSql, 1, zTab, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pSql) ){ + ret = 1; + } + rc = sqlite3_finalize(pSql); + } + + *pbContains = ret; + return rc; +} + +/* +** Execute SQL command zSql using database handle db. If no error occurs, +** set (*pzErr) to NULL and return SQLITE_OK. +** +** If an error does occur, return an SQLite error code and set (*pzErr) to +** point to a buffer containing an English language error message. Except, +** if the error message begins with "no such module:", then ignore the +** error and return as if the SQL statement had succeeded. +** +** This is used to copy as much of the database schema as possible while +** ignoring any errors related to missing virtual table modules. +*/ +static int expertSchemaSql(sqlite3 *db, const char *zSql, char **pzErr){ + int rc = SQLITE_OK; + char *zErr = 0; + + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK && zErr ){ + int nErr = STRLEN(zErr); + if( nErr>=15 && memcmp(zErr, "no such module:", 15)==0 ){ + sqlite3_free(zErr); + rc = SQLITE_OK; + zErr = 0; + } + } + + *pzErr = zErr; + return rc; +} + +static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ + int rc = idxRegisterVtab(p); + sqlite3_stmt *pSchema = 0; + + /* For each table in the main db schema: + ** + ** 1) Add an entry to the p->pTable list, and + ** 2) Create the equivalent virtual table in dbv. + */ + rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg, + "SELECT type, name, sql, 1, " + " substr(sql,1,14)=='create virtual' COLLATE nocase " + "FROM sqlite_schema " + "WHERE type IN ('table','view') AND " + " substr(name,1,7)!='sqlite_' COLLATE nocase " + " UNION ALL " + "SELECT type, name, sql, 2, 0 FROM sqlite_schema " + "WHERE type = 'trigger'" + " AND tbl_name IN(SELECT name FROM sqlite_schema WHERE type = 'view') " + "ORDER BY 4, 5 DESC, 1" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){ + const char *zType = (const char*)sqlite3_column_text(pSchema, 0); + const char *zName = (const char*)sqlite3_column_text(pSchema, 1); + const char *zSql = (const char*)sqlite3_column_text(pSchema, 2); + int bVirtual = sqlite3_column_int(pSchema, 4); + int bExists = 0; + + if( zType==0 || zName==0 ) continue; + rc = expertDbContainsObject(p->dbv, zName, &bExists); + if( rc || bExists ) continue; + + if( zType[0]=='v' || zType[1]=='r' || bVirtual ){ + /* A view. Or a trigger on a view. */ + if( zSql ) rc = expertSchemaSql(p->dbv, zSql, pzErrmsg); + }else{ + IdxTable *pTab; + rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg); + if( rc==SQLITE_OK && ALWAYS(pTab!=0) ){ + int i; + char *zInner = 0; + char *zOuter = 0; + pTab->pNext = p->pTable; + p->pTable = pTab; + + /* The statement the vtab will pass to sqlite3_declare_vtab() */ + zInner = idxAppendText(&rc, 0, "CREATE TABLE x("); + for(i=0; inCol; i++){ + zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", + (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl + ); + } + zInner = idxAppendText(&rc, zInner, ")"); + + /* The CVT statement to create the vtab */ + zOuter = idxAppendText(&rc, 0, + "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg); + } + sqlite3_free(zInner); + sqlite3_free(zOuter); + } + } + } + idxFinalize(&rc, pSchema); + return rc; +} + +struct IdxSampleCtx { + int iTarget; + double target; /* Target nRet/nRow value */ + double nRow; /* Number of rows seen */ + double nRet; /* Number of rows returned */ +}; + +static void idxSampleFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx); + int bRet; + + (void)argv; + assert( argc==0 ); + if( p->nRow==0.0 ){ + bRet = 1; + }else{ + bRet = (p->nRet / p->nRow) <= p->target; + if( bRet==0 ){ + unsigned short rnd; + sqlite3_randomness(2, (void*)&rnd); + bRet = ((int)rnd % 100) <= p->iTarget; + } + } + + sqlite3_result_int(pCtx, bRet); + p->nRow += 1.0; + p->nRet += (double)bRet; +} + +struct IdxRemCtx { + int nSlot; + struct IdxRemSlot { + int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ + i64 iVal; /* SQLITE_INTEGER value */ + double rVal; /* SQLITE_FLOAT value */ + int nByte; /* Bytes of space allocated at z */ + int n; /* Size of buffer z */ + char *z; /* SQLITE_TEXT/BLOB value */ + } aSlot[1]; +}; + +/* +** Implementation of scalar function sqlite_expert_rem(). +*/ +static void idxRemFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx); + struct IdxRemSlot *pSlot; + int iSlot; + assert( argc==2 ); + + iSlot = sqlite3_value_int(argv[0]); + assert( iSlotnSlot ); + pSlot = &p->aSlot[iSlot]; + + switch( pSlot->eType ){ + case SQLITE_NULL: + /* no-op */ + break; + + case SQLITE_INTEGER: + sqlite3_result_int64(pCtx, pSlot->iVal); + break; + + case SQLITE_FLOAT: + sqlite3_result_double(pCtx, pSlot->rVal); + break; + + case SQLITE_BLOB: + sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + break; + + case SQLITE_TEXT: + sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + break; + } + + pSlot->eType = sqlite3_value_type(argv[1]); + switch( pSlot->eType ){ + case SQLITE_NULL: + /* no-op */ + break; + + case SQLITE_INTEGER: + pSlot->iVal = sqlite3_value_int64(argv[1]); + break; + + case SQLITE_FLOAT: + pSlot->rVal = sqlite3_value_double(argv[1]); + break; + + case SQLITE_BLOB: + case SQLITE_TEXT: { + int nByte = sqlite3_value_bytes(argv[1]); + const void *pData = 0; + if( nByte>pSlot->nByte ){ + char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); + if( zNew==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + pSlot->nByte = nByte*2; + pSlot->z = zNew; + } + pSlot->n = nByte; + if( pSlot->eType==SQLITE_BLOB ){ + pData = sqlite3_value_blob(argv[1]); + if( pData ) memcpy(pSlot->z, pData, nByte); + }else{ + pData = sqlite3_value_text(argv[1]); + memcpy(pSlot->z, pData, nByte); + } + break; + } + } +} + +static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){ + int rc = SQLITE_OK; + const char *zMax = + "SELECT max(i.seqno) FROM " + " sqlite_schema AS s, " + " pragma_index_list(s.name) AS l, " + " pragma_index_info(l.name) AS i " + "WHERE s.type = 'table'"; + sqlite3_stmt *pMax = 0; + + *pnMax = 0; + rc = idxPrepareStmt(db, &pMax, pzErr, zMax); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + *pnMax = sqlite3_column_int(pMax, 0) + 1; + } + idxFinalize(&rc, pMax); + + return rc; +} + +static int idxPopulateOneStat1( + sqlite3expert *p, + sqlite3_stmt *pIndexXInfo, + sqlite3_stmt *pWriteStat, + const char *zTab, + const char *zIdx, + char **pzErr +){ + char *zCols = 0; + char *zOrder = 0; + char *zQuery = 0; + int nCol = 0; + int i; + sqlite3_stmt *pQuery = 0; + int *aStat = 0; + int rc = SQLITE_OK; + + assert( p->iSample>0 ); + + /* Formulate the query text */ + sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC); + while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){ + const char *zComma = zCols==0 ? "" : ", "; + const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); + const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); + if( zName==0 ){ + /* This index contains an expression. Ignore it. */ + sqlite3_free(zCols); + sqlite3_free(zOrder); + return sqlite3_reset(pIndexXInfo); + } + zCols = idxAppendText(&rc, zCols, + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl + ); + zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); + } + sqlite3_reset(pIndexXInfo); + if( rc==SQLITE_OK ){ + if( p->iSample==100 ){ + zQuery = sqlite3_mprintf( + "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder + ); + }else{ + zQuery = sqlite3_mprintf( + "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder + ); + } + } + sqlite3_free(zCols); + sqlite3_free(zOrder); + + /* Formulate the query text */ + if( rc==SQLITE_OK ){ + sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); + rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery); + } + sqlite3_free(zQuery); + + if( rc==SQLITE_OK ){ + aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); + } + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ + IdxHashEntry *pEntry; + char *zStat = 0; + for(i=0; i<=nCol; i++) aStat[i] = 1; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ + aStat[0]++; + for(i=0; ihIdx, zIdx, STRLEN(zIdx)); + if( pEntry ){ + assert( pEntry->zVal2==0 ); + pEntry->zVal2 = zStat; + }else{ + sqlite3_free(zStat); + } + } + sqlite3_free(aStat); + idxFinalize(&rc, pQuery); + + return rc; +} + +static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){ + int rc; + char *zSql; + + rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); + if( rc!=SQLITE_OK ) return rc; + + zSql = sqlite3_mprintf( + "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab + ); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0); + sqlite3_free(zSql); + + return rc; +} + +/* +** This function is called as part of sqlite3_expert_analyze(). Candidate +** indexes have already been created in database sqlite3expert.dbm, this +** function populates sqlite_stat1 table in the same database. +** +** The stat1 data is generated by querying the +*/ +static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + int nMax =0; + struct IdxRemCtx *pCtx = 0; + struct IdxSampleCtx samplectx; + int i; + i64 iPrev = -100000; + sqlite3_stmt *pAllIndex = 0; + sqlite3_stmt *pIndexXInfo = 0; + sqlite3_stmt *pWrite = 0; + + const char *zAllIndex = + "SELECT s.rowid, s.name, l.name FROM " + " sqlite_schema AS s, " + " pragma_index_list(s.name) AS l " + "WHERE s.type = 'table'"; + const char *zIndexXInfo = + "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key"; + const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)"; + + /* If iSample==0, no sqlite_stat1 data is required. */ + if( p->iSample==0 ) return SQLITE_OK; + + rc = idxLargestIndex(p->dbm, &nMax, pzErr); + if( nMax<=0 || rc!=SQLITE_OK ) return rc; + + rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); + + if( rc==SQLITE_OK ){ + int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); + pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); + } + + if( rc==SQLITE_OK ){ + sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + ); + } + + if( rc==SQLITE_OK ){ + pCtx->nSlot = nMax+1; + rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); + } + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo); + } + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){ + i64 iRowid = sqlite3_column_int64(pAllIndex, 0); + const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1); + const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2); + if( zTab==0 || zIdx==0 ) continue; + if( p->iSample<100 && iPrev!=iRowid ){ + samplectx.target = (double)p->iSample / 100.0; + samplectx.iTarget = p->iSample; + samplectx.nRow = 0.0; + samplectx.nRet = 0.0; + rc = idxBuildSampleTable(p, zTab); + if( rc!=SQLITE_OK ) break; + } + rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr); + iPrev = iRowid; + } + if( rc==SQLITE_OK && p->iSample<100 ){ + rc = sqlite3_exec(p->dbv, + "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0 + ); + } + + idxFinalize(&rc, pAllIndex); + idxFinalize(&rc, pIndexXInfo); + idxFinalize(&rc, pWrite); + + if( pCtx ){ + for(i=0; inSlot; i++){ + sqlite3_free(pCtx->aSlot[i].z); + } + sqlite3_free(pCtx); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); + } + + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); + return rc; +} + +/* +** Define and possibly pretend to use a useless collation sequence. +** This pretense allows expert to accept SQL using custom collations. +*/ +int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){ + (void)up1; + (void)up2; + (void)up3; + (void)up4; + (void)up5; + assert(0); /* VDBE should never be run. */ + return 0; +} +/* And a callback to register above upon actual need */ +void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){ + (void)up1; + sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0); +} + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) +/* +** dummy functions for no-op implementation of UDFs during expert's work +*/ +void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){ + (void)up1; + (void)up2; + (void)up3; + assert(0); /* VDBE should never be run. */ +} +void dummyUDFvalue(sqlite3_context *up1){ + (void)up1; + assert(0); /* VDBE should never be run. */ +} + +/* +** Register UDFs from user database with another. +*/ +int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){ + sqlite3_stmt *pStmt; + int rc = sqlite3_prepare_v2(dbSrc, + "SELECT name,type,enc,narg,flags " + "FROM pragma_function_list() " + "WHERE builtin==0", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + int nargs = sqlite3_column_int(pStmt,3); + int flags = sqlite3_column_int(pStmt,4); + const char *name = (char*)sqlite3_column_text(pStmt,0); + const char *type = (char*)sqlite3_column_text(pStmt,1); + const char *enc = (char*)sqlite3_column_text(pStmt,2); + if( name==0 || type==0 || enc==0 ){ + /* no-op. Only happens on OOM */ + }else{ + int ienc = SQLITE_UTF8; + int rcf = SQLITE_ERROR; + if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE; + else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE; + ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY)); + if( strcmp(type,"w")==0 ){ + rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0, + dummyUDF,dummyUDFvalue,0,0,0); + }else if( strcmp(type,"a")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + 0,dummyUDF,dummyUDFvalue); + }else if( strcmp(type,"s")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + dummyUDF,0,0); + } + if( rcf!=SQLITE_OK ){ + rc = rcf; + break; + } + } + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + return rc; +} +#endif + +/* +** Allocate a new sqlite3expert object. +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ + int rc = SQLITE_OK; + sqlite3expert *pNew; + + pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert)); + + /* Open two in-memory databases to work with. The "vtab database" (dbv) + ** will contain a virtual table corresponding to each real table in + ** the user database schema, and a copy of each view. It is used to + ** collect information regarding the WHERE, ORDER BY and other clauses + ** of the user's query. + */ + if( rc==SQLITE_OK ){ + pNew->db = db; + pNew->iSample = 100; + rc = sqlite3_open(":memory:", &pNew->dbv); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_open(":memory:", &pNew->dbm); + if( rc==SQLITE_OK ){ + sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0); + } + } + + /* Allow custom collations to be dealt with through prepare. */ + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS); + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS); + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + /* Register UDFs from database [db] with [dbm] and [dbv]. */ + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbm); + } + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbv); + } +#endif + + /* Copy the entire schema of database [db] into [dbm]. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSql = 0; + rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, + "SELECT sql, name, substr(sql,1,14)=='create virtual' COLLATE nocase" + " FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' COLLATE nocase" + " ORDER BY 3 DESC, rowid" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + const char *zName = (const char*)sqlite3_column_text(pSql, 1); + int bExists = 0; + rc = expertDbContainsObject(pNew->dbm, zName, &bExists); + if( rc==SQLITE_OK && zSql && bExists==0 ){ + rc = expertSchemaSql(pNew->dbm, zSql, pzErrmsg); + } + } + idxFinalize(&rc, pSql); + } + + /* Create the vtab schema */ + if( rc==SQLITE_OK ){ + rc = idxCreateVtabSchema(pNew, pzErrmsg); + } + + /* Register the auth callback with dbv */ + if( rc==SQLITE_OK ){ + sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew); + } + + /* If an error has occurred, free the new object and reutrn NULL. Otherwise, + ** return the new sqlite3expert handle. */ + if( rc!=SQLITE_OK ){ + sqlite3_expert_destroy(pNew); + pNew = 0; + } + return pNew; +} + +/* +** Configure an sqlite3expert object. +*/ +int sqlite3_expert_config(sqlite3expert *p, int op, ...){ + int rc = SQLITE_OK; + va_list ap; + va_start(ap, op); + switch( op ){ + case EXPERT_CONFIG_SAMPLE: { + int iVal = va_arg(ap, int); + if( iVal<0 ) iVal = 0; + if( iVal>100 ) iVal = 100; + p->iSample = iVal; + break; + } + default: + rc = SQLITE_NOTFOUND; + break; + } + + va_end(ap); + return rc; +} + +/* +** Add an SQL statement to the analysis. +*/ +int sqlite3_expert_sql( + sqlite3expert *p, /* From sqlite3_expert_new() */ + const char *zSql, /* SQL statement to add */ + char **pzErr /* OUT: Error message (if any) */ +){ + IdxScan *pScanOrig = p->pScan; + IdxStatement *pStmtOrig = p->pStatement; + int rc = SQLITE_OK; + const char *zStmt = zSql; + + if( p->bRun ) return SQLITE_MISUSE; + + while( rc==SQLITE_OK && zStmt && zStmt[0] ){ + sqlite3_stmt *pStmt = 0; + /* Ensure that the provided statement compiles against user's DB. */ + rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt); + if( rc!=SQLITE_OK ) break; + sqlite3_finalize(pStmt); + rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt); + if( rc==SQLITE_OK ){ + if( pStmt ){ + IdxStatement *pNew; + const char *z = sqlite3_sql(pStmt); + int n = STRLEN(z); + pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); + if( rc==SQLITE_OK ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, z, n+1); + pNew->pNext = p->pStatement; + if( p->pStatement ) pNew->iId = p->pStatement->iId+1; + p->pStatement = pNew; + } + sqlite3_finalize(pStmt); + } + }else{ + idxDatabaseError(p->dbv, pzErr); + } + } + + if( rc!=SQLITE_OK ){ + idxScanFree(p->pScan, pScanOrig); + idxStatementFree(p->pStatement, pStmtOrig); + p->pScan = pScanOrig; + p->pStatement = pStmtOrig; + } + + return rc; +} + +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ + int rc; + IdxHashEntry *pEntry; + + /* Do trigger processing to collect any extra IdxScan structures */ + rc = idxProcessTriggers(p, pzErr); + + /* Create candidate indexes within the in-memory database file */ + if( rc==SQLITE_OK ){ + rc = idxCreateCandidates(p); + }else if ( rc==SQLITE_BUSY_TIMEOUT ){ + if( pzErr ) + *pzErr = sqlite3_mprintf("Cannot find a unique index name to propose."); + return rc; + } + + /* Generate the stat1 data */ + if( rc==SQLITE_OK ){ + rc = idxPopulateStat1(p, pzErr); + } + + /* Formulate the EXPERT_REPORT_CANDIDATES text */ + for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ + p->zCandidates = idxAppendText(&rc, p->zCandidates, + "%s;%s%s\n", pEntry->zVal, + pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2 + ); + } + + /* Figure out which of the candidate indexes are preferred by the query + ** planner and report the results to the user. */ + if( rc==SQLITE_OK ){ + rc = idxFindIndexes(p, pzErr); + } + + if( rc==SQLITE_OK ){ + p->bRun = 1; + } + return rc; +} + +/* +** Return the total number of statements that have been added to this +** sqlite3expert using sqlite3_expert_sql(). +*/ +int sqlite3_expert_count(sqlite3expert *p){ + int nRet = 0; + if( p->pStatement ) nRet = p->pStatement->iId+1; + return nRet; +} + +/* +** Return a component of the report. +*/ +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){ + const char *zRet = 0; + IdxStatement *pStmt; + + if( p->bRun==0 ) return 0; + for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext); + switch( eReport ){ + case EXPERT_REPORT_SQL: + if( pStmt ) zRet = pStmt->zSql; + break; + case EXPERT_REPORT_INDEXES: + if( pStmt ) zRet = pStmt->zIdx; + break; + case EXPERT_REPORT_PLAN: + if( pStmt ) zRet = pStmt->zEQP; + break; + case EXPERT_REPORT_CANDIDATES: + zRet = p->zCandidates; + break; + } + return zRet; +} + +/* +** Free an sqlite3expert object. +*/ +void sqlite3_expert_destroy(sqlite3expert *p){ + if( p ){ + sqlite3_close(p->dbm); + sqlite3_close(p->dbv); + idxScanFree(p->pScan, 0); + idxStatementFree(p->pStatement, 0); + idxTableFree(p->pTable); + idxWriteFree(p->pWrite); + idxHashClear(&p->hIdx); + sqlite3_free(p->zCandidates); + sqlite3_free(p); + } +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h new file mode 100644 index 0000000000000000000000000000000000000000..6048137237a29a7031132ed6199269f4c76dc9d8 --- /dev/null +++ b/ext/expert/sqlite3expert.h @@ -0,0 +1,168 @@ +/* +** 2017 April 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +#if !defined(SQLITEEXPERT_H) +#define SQLITEEXPERT_H 1 +#include "sqlite3.h" + +typedef struct sqlite3expert sqlite3expert; + +/* +** Create a new sqlite3expert object. +** +** If successful, a pointer to the new object is returned and (*pzErr) set +** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to +** an English-language error message. In this case it is the responsibility +** of the caller to eventually free the error message buffer using +** sqlite3_free(). +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr); + +/* +** Configure an sqlite3expert object. +** +** EXPERT_CONFIG_SAMPLE: +** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for +** each candidate index. This involves scanning and sorting the entire +** contents of each user database table once for each candidate index +** associated with the table. For large databases, this can be +** prohibitively slow. This option allows the sqlite3expert object to +** be configured so that sqlite_stat1 data is instead generated based on a +** subset of each table, or so that no sqlite_stat1 data is used at all. +** +** A single integer argument is passed to this option. If the value is less +** than or equal to zero, then no sqlite_stat1 data is generated or used by +** the analysis - indexes are recommended based on the database schema only. +** Or, if the value is 100 or greater, complete sqlite_stat1 data is +** generated for each candidate index (this is the default). Finally, if the +** value falls between 0 and 100, then it represents the percentage of user +** table rows that should be considered when generating sqlite_stat1 data. +** +** Examples: +** +** // Do not generate any sqlite_stat1 data +** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0); +** +** // Generate sqlite_stat1 data based on 10% of the rows in each table. +** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10); +*/ +int sqlite3_expert_config(sqlite3expert *p, int op, ...); + +#define EXPERT_CONFIG_SAMPLE 1 /* int */ + +/* +** Specify zero or more SQL statements to be included in the analysis. +** +** Buffer zSql must contain zero or more complete SQL statements. This +** function parses all statements contained in the buffer and adds them +** to the internal list of statements to analyze. If successful, SQLITE_OK +** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example +** due to a error in the SQL - an SQLite error code is returned and (*pzErr) +** may be set to point to an English language error message. In this case +** the caller is responsible for eventually freeing the error message buffer +** using sqlite3_free(). +** +** If an error does occur while processing one of the statements in the +** buffer passed as the second argument, none of the statements in the +** buffer are added to the analysis. +** +** This function must be called before sqlite3_expert_analyze(). If a call +** to this function is made on an sqlite3expert object that has already +** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned +** immediately and no statements are added to the analysis. +*/ +int sqlite3_expert_sql( + sqlite3expert *p, /* From a successful sqlite3_expert_new() */ + const char *zSql, /* SQL statement(s) to add */ + char **pzErr /* OUT: Error message (if any) */ +); + + +/* +** This function is called after the sqlite3expert object has been configured +** with all SQL statements using sqlite3_expert_sql() to actually perform +** the analysis. Once this function has been called, it is not possible to +** add further SQL statements to the analysis. +** +** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if +** an error occurs, an SQLite error code is returned and (*pzErr) set to +** point to a buffer containing an English language error message. In this +** case it is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** If an error does occur within this function, the sqlite3expert object +** is no longer useful for any purpose. At that point it is no longer +** possible to add further SQL statements to the object or to re-attempt +** the analysis. The sqlite3expert object must still be freed using a call +** sqlite3_expert_destroy(). +*/ +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr); + +/* +** Return the total number of statements loaded using sqlite3_expert_sql(). +** The total number of SQL statements may be different from the total number +** to calls to sqlite3_expert_sql(). +*/ +int sqlite3_expert_count(sqlite3expert*); + +/* +** Return a component of the report. +** +** This function is called after sqlite3_expert_analyze() to extract the +** results of the analysis. Each call to this function returns either a +** NULL pointer or a pointer to a buffer containing a nul-terminated string. +** The value passed as the third argument must be one of the EXPERT_REPORT_* +** #define constants defined below. +** +** For some EXPERT_REPORT_* parameters, the buffer returned contains +** information relating to a specific SQL statement. In these cases that +** SQL statement is identified by the value passed as the second argument. +** SQL statements are numbered from 0 in the order in which they are parsed. +** If an out-of-range value (less than zero or equal to or greater than the +** value returned by sqlite3_expert_count()) is passed as the second argument +** along with such an EXPERT_REPORT_* parameter, NULL is always returned. +** +** EXPERT_REPORT_SQL: +** Return the text of SQL statement iStmt. +** +** EXPERT_REPORT_INDEXES: +** Return a buffer containing the CREATE INDEX statements for all recommended +** indexes for statement iStmt. If there are no new recommeded indexes, NULL +** is returned. +** +** EXPERT_REPORT_PLAN: +** Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query +** iStmt after the proposed indexes have been added to the database schema. +** +** EXPERT_REPORT_CANDIDATES: +** Return a pointer to a buffer containing the CREATE INDEX statements +** for all indexes that were tested (for all SQL statements). The iStmt +** parameter is ignored for EXPERT_REPORT_CANDIDATES calls. +*/ +const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport); + +/* +** Values for the third argument passed to sqlite3_expert_report(). +*/ +#define EXPERT_REPORT_SQL 1 +#define EXPERT_REPORT_INDEXES 2 +#define EXPERT_REPORT_PLAN 3 +#define EXPERT_REPORT_CANDIDATES 4 + +/* +** Free an (sqlite3expert*) handle and all associated resources. There +** should be one call to this function for each successful call to +** sqlite3-expert_new(). +*/ +void sqlite3_expert_destroy(sqlite3expert*); + +#endif /* !defined(SQLITEEXPERT_H) */ diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c new file mode 100644 index 0000000000000000000000000000000000000000..cae5d0f258f7b9cfa3b5fdfdcd852b3a65ae3070 --- /dev/null +++ b/ext/expert/test_expert.c @@ -0,0 +1,212 @@ +/* +** 2017 April 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#if defined(SQLITE_TEST) + +#include "sqlite3expert.h" +#include +#include +#include "tclsqlite.h" + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Extract an sqlite3* db handle from the object passed as the second +** argument. If successful, set *pDb to point to the db handle and return +** TCL_OK. Otherwise, return TCL_ERROR. +*/ +static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ + Tcl_CmdInfo info; + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + return TCL_ERROR; + } + + *pDb = *(sqlite3 **)info.objClientData; + return TCL_OK; +} + + +/* +** Tclcmd: $expert sql SQL +** $expert analyze +** $expert count +** $expert report STMT EREPORT +** $expert destroy +*/ +static int SQLITE_TCLAPI testExpertCmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3expert *pExpert = (sqlite3expert*)clientData; + struct Subcmd { + const char *zSub; + int nArg; + const char *zMsg; + } aSub[] = { + { "sql", 1, "TABLE", }, /* 0 */ + { "analyze", 0, "", }, /* 1 */ + { "count", 0, "", }, /* 2 */ + { "report", 2, "STMT EREPORT", }, /* 3 */ + { "destroy", 0, "", }, /* 4 */ + { 0 } + }; + int iSub; + int rc = TCL_OK; + char *zErr = 0; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( objc!=2+aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: { /* sql */ + char *zArg = Tcl_GetString(objv[2]); + rc = sqlite3_expert_sql(pExpert, zArg, &zErr); + break; + } + + case 1: { /* analyze */ + rc = sqlite3_expert_analyze(pExpert, &zErr); + break; + } + + case 2: { /* count */ + int n = sqlite3_expert_count(pExpert); + Tcl_SetObjResult(interp, Tcl_NewIntObj(n)); + break; + } + + case 3: { /* report */ + const char *aEnum[] = { + "sql", "indexes", "plan", "candidates", 0 + }; + int iEnum; + int iStmt; + const char *zReport; + + if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) + || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum) + ){ + return TCL_ERROR; + } + + assert( EXPERT_REPORT_SQL==1 ); + assert( EXPERT_REPORT_INDEXES==2 ); + assert( EXPERT_REPORT_PLAN==3 ); + assert( EXPERT_REPORT_CANDIDATES==4 ); + zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1)); + break; + } + + default: /* destroy */ + assert( iSub==4 ); + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + break; + } + + if( rc!=TCL_OK ){ + if( zErr ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + }else{ + extern const char *sqlite3ErrName(int); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + } + sqlite3_free(zErr); + return rc; +} + +static void SQLITE_TCLAPI testExpertDel(void *clientData){ + sqlite3expert *pExpert = (sqlite3expert*)clientData; + sqlite3_expert_destroy(pExpert); +} + +/* +** sqlite3_expert_new DB +*/ +static int SQLITE_TCLAPI test_sqlite3_expert_new( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static int iCmd = 0; + sqlite3 *db; + char *zCmd = 0; + char *zErr = 0; + sqlite3expert *pExpert; + int rc = TCL_OK; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( dbHandleFromObj(interp, objv[1], &db) ){ + return TCL_ERROR; + } + + zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd); + if( zCmd==0 ){ + Tcl_AppendResult(interp, "out of memory", (char*)0); + return TCL_ERROR; + } + + pExpert = sqlite3_expert_new(db, &zErr); + if( pExpert==0 ){ + Tcl_AppendResult(interp, zErr, (char*)0); + rc = TCL_ERROR; + }else{ + void *p = (void*)pExpert; + Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1)); + } + + sqlite3_free(zCmd); + sqlite3_free(zErr); + return rc; +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +int TestExpert_Init(Tcl_Interp *interp){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + struct Cmd { + const char *zCmd; + Tcl_ObjCmdProc *xProc; + } aCmd[] = { + { "sqlite3_expert_new", test_sqlite3_expert_new }, + }; + int i; + + for(i=0; izCmd, p->xProc, 0, 0); + } +#endif + return TCL_OK; +} + +#endif diff --git a/ext/fts3/README.content b/ext/fts3/README.content new file mode 100644 index 0000000000000000000000000000000000000000..b6a75399bec9623fbb1edb6ddb2bb498b8820fd4 --- /dev/null +++ b/ext/fts3/README.content @@ -0,0 +1,176 @@ + +FTS4 CONTENT OPTION + + Normally, in order to create a full-text index on a dataset, the FTS4 + module stores a copy of all indexed documents in a specially created + database table. + + As of SQLite version 3.7.9, FTS4 supports a new option - "content" - + designed to extend FTS4 to support the creation of full-text indexes where: + + * The indexed documents are not stored within the SQLite database + at all (a "contentless" FTS4 table), or + + * The indexed documents are stored in a database table created and + managed by the user (an "external content" FTS4 table). + + Because the indexed documents themselves are usually much larger than + the full-text index, the content option can sometimes be used to achieve + significant space savings. + +CONTENTLESS FTS4 TABLES + + In order to create an FTS4 table that does not store a copy of the indexed + documents at all, the content option should be set to an empty string. + For example, the following SQL creates such an FTS4 table with three + columns - "a", "b", and "c": + + CREATE VIRTUAL TABLE t1 USING fts4(content="", a, b, c); + + Data can be inserted into such an FTS4 table using an INSERT statements. + However, unlike ordinary FTS4 tables, the user must supply an explicit + integer docid value. For example: + + -- This statement is Ok: + INSERT INTO t1(docid, a, b, c) VALUES(1, 'a b c', 'd e f', 'g h i'); + + -- This statement causes an error, as no docid value has been provided: + INSERT INTO t1(a, b, c) VALUES('j k l', 'm n o', 'p q r'); + + It is not possible to UPDATE or DELETE a row stored in a contentless FTS4 + table. Attempting to do so is an error. + + Contentless FTS4 tables also support SELECT statements. However, it is + an error to attempt to retrieve the value of any table column other than + the docid column. The auxiliary function matchinfo() may be used, but + snippet() and offsets() may not. For example: + + -- The following statements are Ok: + SELECT docid FROM t1 WHERE t1 MATCH 'xxx'; + SELECT docid FROM t1 WHERE a MATCH 'xxx'; + SELECT matchinfo(t1) FROM t1 WHERE t1 MATCH 'xxx'; + + -- The following statements all cause errors, as the value of columns + -- other than docid are required to evaluate them. + SELECT * FROM t1; + SELECT a, b FROM t1 WHERE t1 MATCH 'xxx'; + SELECT docid FROM t1 WHERE a LIKE 'xxx%'; + SELECT snippet(t1) FROM t1 WHERE t1 MATCH 'xxx'; + + Errors related to attempting to retrieve column values other than docid + are runtime errors that occur within sqlite3_step(). In some cases, for + example if the MATCH expression in a SELECT query matches zero rows, there + may be no error at all even if a statement does refer to column values + other than docid. + +EXTERNAL CONTENT FTS4 TABLES + + An "external content" FTS4 table is similar to a contentless table, except + that if evaluation of a query requires the value of a column other than + docid, FTS4 attempts to retrieve that value from a table (or view, or + virtual table) nominated by the user (hereafter referred to as the "content + table"). The FTS4 module never writes to the content table, and writing + to the content table does not affect the full-text index. It is the + responsibility of the user to ensure that the content table and the + full-text index are consistent. + + An external content FTS4 table is created by setting the content option + to the name of a table (or view, or virtual table) that may be queried by + FTS4 to retrieve column values when required. If the nominated table does + not exist, then an external content table behaves in the same way as + a contentless table. For example: + + CREATE TABLE t2(id INTEGER PRIMARY KEY, a, b, c); + CREATE VIRTUAL TABLE t3 USING fts4(content="t2", a, c); + + Assuming the nominated table does exist, then its columns must be the same + as or a superset of those defined for the FTS table. + + When a users query on the FTS table requires a column value other than + docid, FTS attempts to read this value from the corresponding column of + the row in the content table with a rowid value equal to the current FTS + docid. Or, if such a row cannot be found in the content table, a NULL + value is used instead. For example: + + CREATE TABLE t2(id INTEGER PRIMARY KEY, a, b, c, d); + CREATE VIRTUAL TABLE t3 USING fts4(content="t2", b, c); + + INSERT INTO t2 VALUES(2, 'a b', 'c d', 'e f'); + INSERT INTO t2 VALUES(3, 'g h', 'i j', 'k l'); + INSERT INTO t3(docid, b, c) SELECT id, b, c FROM t2; + + -- The following query returns a single row with two columns containing + -- the text values "i j" and "k l". + -- + -- The query uses the full-text index to discover that the MATCH + -- term matches the row with docid=3. It then retrieves the values + -- of columns b and c from the row with rowid=3 in the content table + -- to return. + -- + SELECT * FROM t3 WHERE t3 MATCH 'k'; + + -- Following the UPDATE, the query still returns a single row, this + -- time containing the text values "xxx" and "yyy". This is because the + -- full-text index still indicates that the row with docid=3 matches + -- the FTS4 query 'k', even though the documents stored in the content + -- table have been modified. + -- + UPDATE t2 SET b = 'xxx', c = 'yyy' WHERE rowid = 3; + SELECT * FROM t3 WHERE t3 MATCH 'k'; + + -- Following the DELETE below, the query returns one row containing two + -- NULL values. NULL values are returned because FTS is unable to find + -- a row with rowid=3 within the content table. + -- + DELETE FROM t2; + SELECT * FROM t3 WHERE t3 MATCH 'k'; + + When a row is deleted from an external content FTS4 table, FTS4 needs to + retrieve the column values of the row being deleted from the content table. + This is so that FTS4 can update the full-text index entries for each token + that occurs within the deleted row to indicate that that row has been + deleted. If the content table row cannot be found, or if it contains values + inconsistent with the contents of the FTS index, the results can be difficult + to predict. The FTS index may be left containing entries corresponding to the + deleted row, which can lead to seemingly nonsensical results being returned + by subsequent SELECT queries. The same applies when a row is updated, as + internally an UPDATE is the same as a DELETE followed by an INSERT. + + Instead of writing separately to the full-text index and the content table, + some users may wish to use database triggers to keep the full-text index + up to date with respect to the set of documents stored in the content table. + For example, using the tables from earlier examples: + + CREATE TRIGGER t2_bu BEFORE UPDATE ON t2 BEGIN + DELETE FROM t3 WHERE docid=old.rowid; + END; + CREATE TRIGGER t2_bd BEFORE DELETE ON t2 BEGIN + DELETE FROM t3 WHERE docid=old.rowid; + END; + + CREATE TRIGGER t2_bu AFTER UPDATE ON t2 BEGIN + INSERT INTO t3(docid, b, c) VALUES(new.rowid, new.b, new.c); + END; + CREATE TRIGGER t2_bd AFTER INSERT ON t2 BEGIN + INSERT INTO t3(docid, b, c) VALUES(new.rowid, new.b, new.c); + END; + + The DELETE trigger must be fired before the actual delete takes place + on the content table. This is so that FTS4 can still retrieve the original + values in order to update the full-text index. And the INSERT trigger must + be fired after the new row is inserted, so as to handle the case where the + rowid is assigned automatically within the system. The UPDATE trigger must + be split into two parts, one fired before and one after the update of the + content table, for the same reasons. + + FTS4 features a special command similar to the 'optimize' command that + deletes the entire full-text index and rebuilds it based on the current + set of documents in the content table. Assuming again that "t3" is the + name of the external content FTS4 table, the command is: + + INSERT INTO t3(t3) VALUES('rebuild'); + + This command may also be used with ordinary FTS4 tables, although it may + only be useful if the full-text index has somehow become corrupt. It is an + error to attempt to rebuild the full-text index maintained by a contentless + FTS4 table. diff --git a/ext/fts3/README.syntax b/ext/fts3/README.syntax new file mode 100644 index 0000000000000000000000000000000000000000..01bc80c5fbe9fce601884bf9e6dec46dab62c97e --- /dev/null +++ b/ext/fts3/README.syntax @@ -0,0 +1,209 @@ + +1. OVERVIEW + + This README file describes the syntax of the arguments that may be passed to + the FTS3 MATCH operator used for full-text queries. For example, if table + "t1" is an Fts3 virtual table, the following SQL query: + + SELECT * FROM t1 WHERE MATCH + + may be used to retrieve all rows that match a specified for full-text query. + The text "" should be replaced by either the name of the fts3 table + (in this case "t1"), or by the name of one of the columns of the fts3 + table. should be replaced by an SQL expression that + computes to a string containing an Fts3 query. + + If the left-hand-side of the MATCH operator is set to the name of the + fts3 table, then by default the query may be matched against any column + of the table. If it is set to a column name, then by default the query + may only match the specified column. In both cases this may be overriden + as part of the query text (see sections 2 and 3 below). + + As of SQLite version 3.6.8, Fts3 supports two slightly different query + formats; the standard syntax, which is used by default, and the enhanced + query syntax which can be selected by compiling with the pre-processor + symbol SQLITE_ENABLE_FTS3_PARENTHESIS defined. + + -DSQLITE_ENABLE_FTS3_PARENTHESIS + +2. STANDARD QUERY SYNTAX + + When using the standard Fts3 query syntax, a query usually consists of a + list of terms (words) separated by white-space characters. To match a + query, a row (or column) of an Fts3 table must contain each of the specified + terms. For example, the following query: + + MATCH 'hello world' + + matches rows (or columns, if is the name of a column name) that + contain at least one instance of the token "hello", and at least one + instance of the token "world". Tokens may be grouped into phrases using + quotation marks. In this case, a matching row or column must contain each + of the tokens in the phrase in the order specified, with no intervening + tokens. For example, the query: + + MATCH '"hello world" joe" + + matches the first of the following two documents, but not the second or + third: + + "'Hello world', said Joe." + "One should always greet the world with a cheery hello, thought Joe." + "How many hello world programs could their be?" + + As well as grouping tokens together by phrase, the binary NEAR operator + may be used to search for rows that contain two or more specified tokens + or phrases within a specified proximity of each other. The NEAR operator + must always be specified in upper case. The word "near" in lower or mixed + case is treated as an ordinary token. For example, the following query: + + MATCH 'engineering NEAR consultancy' + + matches rows that contain both the "engineering" and "consultancy" tokens + in the same column with not more than 10 other words between them. It does + not matter which of the two terms occurs first in the document, only that + they be seperated by only 10 tokens or less. The user may also specify + a different required proximity by adding "/N" immediately after the NEAR + operator, where N is an integer. For example: + + MATCH 'engineering NEAR/5 consultancy' + + searches for a row containing an instance of each specified token seperated + by not more than 5 other tokens. More than one NEAR operator can be used + in as sequence. For example this query: + + MATCH 'reliable NEAR/2 engineering NEAR/5 consultancy' + + searches for a row that contains an instance of the token "reliable" + seperated by not more than two tokens from an instance of "engineering", + which is in turn separated by not more than 5 other tokens from an + instance of the term "consultancy". Phrases enclosed in quotes may + also be used as arguments to the NEAR operator. + + Similar to the NEAR operator, one or more tokens or phrases may be + separated by OR operators. In this case, only one of the specified tokens + or phrases must appear in the document. For example, the query: + + MATCH 'hello OR world' + + matches rows that contain either the term "hello", or the term "world", + or both. Note that unlike in many programming languages, the OR operator + has a higher precedence than the AND operators implied between white-space + separated tokens. The following query matches documents that contain the + term 'sqlite' and at least one of the terms 'fantastic' or 'impressive', + not those that contain both 'sqlite' and 'fantastic' or 'impressive': + + MATCH 'sqlite fantastic OR impressive' + + Any token that is part of an Fts3 query expression, whether or not it is + part of a phrase enclosed in quotes, may have a '*' character appended to + it. In this case, the token matches all terms that begin with the characters + of the token, not just those that exactly match it. For example, the + following query: + + MATCH 'sql*' + + matches all rows that contain the term "SQLite", as well as those that + contain "SQL". + + A token that is not part of a quoted phrase may be preceded by a '-' + character, which indicates that matching rows must not contain the + specified term. For example, the following: + + MATCH '"database engine" -sqlite' + + matches rows that contain the phrase "database engine" but do not contain + the term "sqlite". If the '-' character occurs inside a quoted phrase, + it is ignored. It is possible to use both the '-' prefix and the '*' postfix + on a single term. At this time, all Fts3 queries must contain at least + one term or phrase that is not preceded by the '-' prefix. + + Regardless of whether or not a table name or column name is used on the + left hand side of the MATCH operator, a specific column of the fts3 table + may be associated with each token in a query by preceding a token with + a column name followed by a ':' character. For example, regardless of what + is specified for , the following query requires that column "col1" + of the table contains the term "hello", and that column "col2" of the + table contains the term "world". If the table does not contain columns + named "col1" and "col2", then an error is returned and the query is + not run. + + MATCH 'col1:hello col2:world' + + It is not possible to associate a specific table column with a quoted + phrase or a term preceded by a '-' operator. A '*' character may be + appended to a term associated with a specific column for prefix matching. + +3. ENHANCED QUERY SYNTAX + + The enhanced query syntax is quite similar to the standard query syntax, + with the following four differences: + + 1) Parenthesis are supported. When using the enhanced query syntax, + parenthesis may be used to overcome the built-in precedence of the + supplied binary operators. For example, the following query: + + MATCH '(hello world) OR (simple example)' + + matches documents that contain both "hello" and "world", and documents + that contain both "simple" and "example". It is not possible to forumlate + such a query using the standard syntax. + + 2) Instead of separating tokens and phrases by whitespace, an AND operator + may be explicitly specified. This does not change query processing at + all, but may be used to improve readability. For example, the following + query is handled identically to the one above: + + MATCH '(hello AND world) OR (simple AND example)' + + As with the OR and NEAR operators, the AND operator must be specified + in upper case. The word "and" specified in lower or mixed case is + handled as a regular token. + + 3) The '-' token prefix is not supported. Instead, a new binary operator, + NOT, is included. The NOT operator requires that the query specified + as its left-hand operator matches, but that the query specified as the + right-hand operator does not. For example, to query for all rows that + contain the term "example" but not the term "simple", the following + query could be used: + + MATCH 'example NOT simple' + + As for all other operators, the NOT operator must be specified in + upper case. Otherwise it will be treated as a regular token. + + 4) Unlike in the standard syntax, where the OR operator has a higher + precedence than the implicit AND operator, when using the enhanced + syntax implicit and explict AND operators have a higher precedence + than OR operators. Using the enhanced syntax, the following two + queries are equivalent: + + MATCH 'sqlite fantastic OR impressive' + MATCH '(sqlite AND fantastic) OR impressive' + + however, when using the standard syntax, the query: + + MATCH 'sqlite fantastic OR impressive' + + is equivalent to the enhanced syntax query: + + MATCH 'sqlite AND (fantastic OR impressive)' + + The precedence of all enhanced syntax operators, in order from highest + to lowest, is: + + NEAR (highest precedence, tightest grouping) + NOT + AND + OR (lowest precedence, loosest grouping) + + Using the advanced syntax, it is possible to specify expressions enclosed + in parenthesis as operands to the NOT, AND and OR operators. However both + the left and right hand side operands of NEAR operators must be either + tokens or phrases. Attempting the following query will return an error: + + MATCH 'sqlite NEAR (fantastic OR impressive)' + + Queries of this form must be re-written as: + + MATCH 'sqlite NEAR fantastic OR sqlite NEAR impressive' diff --git a/ext/fts3/README.tokenizers b/ext/fts3/README.tokenizers new file mode 100644 index 0000000000000000000000000000000000000000..70bdceff061d02a6a5ebb144aca876421d4782a0 --- /dev/null +++ b/ext/fts3/README.tokenizers @@ -0,0 +1,135 @@ + +1. FTS3 Tokenizers + + When creating a new full-text table, FTS3 allows the user to select + the text tokenizer implementation to be used when indexing text + by specifying a "tokenize" clause as part of the CREATE VIRTUAL TABLE + statement: + + CREATE VIRTUAL TABLE USING fts3( + [, tokenize []] + ); + + The built-in tokenizers (valid values to pass as ) are + "simple", "porter" and "unicode". + + should consist of zero or more white-space separated + arguments to pass to the selected tokenizer implementation. The + interpretation of the arguments, if any, depends on the individual + tokenizer. + +2. Custom Tokenizers + + FTS3 allows users to provide custom tokenizer implementations. The + interface used to create a new tokenizer is defined and described in + the fts3_tokenizer.h source file. + + Registering a new FTS3 tokenizer is similar to registering a new + virtual table module with SQLite. The user passes a pointer to a + structure containing pointers to various callback functions that + make up the implementation of the new tokenizer type. For tokenizers, + the structure (defined in fts3_tokenizer.h) is called + "sqlite3_tokenizer_module". + + FTS3 does not expose a C-function that users call to register new + tokenizer types with a database handle. Instead, the pointer must + be encoded as an SQL blob value and passed to FTS3 through the SQL + engine by evaluating a special scalar function, "fts3_tokenizer()". + The fts3_tokenizer() function may be called with one or two arguments, + as follows: + + SELECT fts3_tokenizer(); + SELECT fts3_tokenizer(, ); + + Where is a string identifying the tokenizer and + is a pointer to an sqlite3_tokenizer_module + structure encoded as an SQL blob. If the second argument is present, + it is registered as tokenizer and a copy of it + returned. If only one argument is passed, a pointer to the tokenizer + implementation currently registered as is returned, + encoded as a blob. Or, if no such tokenizer exists, an SQL exception + (error) is raised. + + SECURITY: If the fts3 extension is used in an environment where potentially + malicious users may execute arbitrary SQL (i.e. gears), they should be + prevented from invoking the fts3_tokenizer() function. The + fts3_tokenizer() function is disabled by default. It is only enabled + by SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER. Do not enable it in + security sensitive environments. + + See "Sample code" below for an example of calling the fts3_tokenizer() + function from C code. + +3. ICU Library Tokenizers + + If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor + symbol defined, then there exists a built-in tokenizer named "icu" + implemented using the ICU library. The first argument passed to the + xCreate() method (see fts3_tokenizer.h) of this tokenizer may be + an ICU locale identifier. For example "tr_TR" for Turkish as used + in Turkey, or "en_AU" for English as used in Australia. For example: + + "CREATE VIRTUAL TABLE thai_text USING fts3(text, tokenizer icu th_TH)" + + The ICU tokenizer implementation is very simple. It splits the input + text according to the ICU rules for finding word boundaries and discards + any tokens that consist entirely of white-space. This may be suitable + for some applications in some locales, but not all. If more complex + processing is required, for example to implement stemming or + discard punctuation, this can be done by creating a tokenizer + implementation that uses the ICU tokenizer as part of its implementation. + + When using the ICU tokenizer this way, it is safe to overwrite the + contents of the strings returned by the xNext() method (see + fts3_tokenizer.h). + +4. Sample code. + + The following two code samples illustrate the way C code should invoke + the fts3_tokenizer() scalar function: + + int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p + ){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); + } + + int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp + ){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); + } diff --git a/ext/fts3/README.txt b/ext/fts3/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..517a2a043466cac040cb45686e397c9c650bb93e --- /dev/null +++ b/ext/fts3/README.txt @@ -0,0 +1,4 @@ +This folder contains source code to the second full-text search +extension for SQLite. While the API is the same, this version uses a +substantially different storage schema from fts1, so tables will need +to be rebuilt. diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c new file mode 100644 index 0000000000000000000000000000000000000000..e58f256a4887267f80f672fcfa81c592aad9eed0 --- /dev/null +++ b/ext/fts3/fts3.c @@ -0,0 +1,6194 @@ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ + +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +#include +#include +#include +#include +#include +#include + +#include "fts3.h" +#ifndef SQLITE_CORE +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif + +typedef struct Fts3HashWrapper Fts3HashWrapper; +struct Fts3HashWrapper { + Fts3Hash hash; /* Hash table */ + int nRef; /* Number of pointers to this object */ +}; + +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts3 code are activated - conditions that are +** only true if it is guaranteed that the fts3 database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +int sqlite3_fts3_may_be_corrupt = 1; +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + +int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0] and +** not extending past pEnd[-1]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarintBounded( + const char *pBuf, + const char *pEnd, + sqlite_int64 *v +){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + const unsigned char *pX = (const unsigned char*)pEnd; + u64 b = 0; + int shift; + for(shift=0; shift<=63; shift+=7){ + u64 c = p=0 ); + return 5; +} + +/* +** Return the number of bytes required to encode v as a varint +*/ +int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** +*/ +void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } +} + +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Write an error message into *pzErr +*/ +void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ + + /* Drop the shadow tables */ + fts3DbExec(&rc, db, + "DROP TABLE IF EXISTS %Q.'%q_segments';" + "DROP TABLE IF EXISTS %Q.'%q_segdir';" + "DROP TABLE IF EXISTS %Q.'%q_docsize';" + "DROP TABLE IF EXISTS %Q.'%q_stat';" + "%s DROP TABLE IF EXISTS %Q.'%q_content';", + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + (p->zContentTbl ? "--" : ""), zDb,p->zName + ); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } +} + +/* +** Create the %_stat table if it does not already exist. +*/ +void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; +} + +/* +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. +*/ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + sqlite3_int64 nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc64(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} + +/* +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. +*/ +int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} + +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + int nInt = 0; /* Output value */ + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; + } + *pnOut = nInt; + *pp += nByte; + return SQLITE_OK; +} + +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } + + *pnIndex = nIndex; + return SQLITE_OK; +} + +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + sqlite3_int64 nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = &((Fts3HashWrapper*)pAux)->hash; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3_int64 nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc64(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc64(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; + + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + + default: + assert( iOpt==SizeofArray(aFts4Opt) ); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + break; + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iCol0 ){ + memcpy(zCsr, z, n); + } + zCsr[n] = '\0'; + sqlite3Fts3Dequote(zCsr); + p->azColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } + + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + +#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST) + p->nMergeCount = FTS3_MERGE_COUNT; +#endif + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + if( p->bLock ){ + return SQLITE_ERROR; + } + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + +/* +** Free all resources currently held by the cursor passed as the only +** argument. +*/ +static void fts3ClearCursor(Fts3Cursor *pCsr){ + fts3CursorFinalizeStmt(pCsr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + fts3ClearCursor(pCsr); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + p->bLock++; + rc = sqlite3_prepare_v3( + p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; + } + return rc; +} + +/* +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. +*/ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; + pTab->bLock++; + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + pTab->bLock--; + return SQLITE_OK; + }else{ + pTab->bLock--; + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} + +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + i64 nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + u64 iChild; /* Block id of child node to descend to */ + int nBuffer = 0; /* Total term size */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrnBuffer ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + } + isFirstTerm = 0; + zCsr += fts3GetVarint32(zCsr, &nSuffix); + + assert( nPrefix>=0 && nSuffix>=0 ); + if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( (i64)nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = ((i64)nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = (i64)iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = (i64)iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = (i64)iChild; + if( piLast ) *piLast = (i64)iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + int iNewHeight = 0; + fts3GetVarint32(zBlob, &iNewHeight); + if( iNewHeight>=iHeight ){ + rc = FTS_CORRUPT_VTAB; + }else{ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + } + sqlite3_free(zBlob); + } + + return rc; +} + +/* +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert_fts3_nc( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an position-list. This must be +** as large or larger than any value that might appear on the +** position-list, even a position list that has been corrupted. +*/ +#define POSITION_LIST_END LARGEST_INT64 + +/* +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + int iVal; + *pp += fts3GetVarint32((*pp), &iVal); + *pi += iVal; + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static int fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ + + if( *p1==POS_COLUMN ){ + fts3GetVarint32(&p1[1], &iCol1); + if( iCol1==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p1==POS_END ) iCol1 = 0x7fffffff; + else iCol1 = 0; + + if( *p2==POS_COLUMN ){ + fts3GetVarint32(&p2[1], &iCol2); + if( iCol2==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p2==POS_END ) iCol2 = 0x7fffffff; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + if( i1<2 || i2<2 ){ + break; + } + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. +*/ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert_fts3_nc( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + /* iCol1==0 indicates corruption. Column 0 does not have a POS_COLUMN + ** entry, so this is actually end-of-doclist. */ + if( iCol1==0 ) return 0; + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + /* As above, iCol2==0 indicates corruption. */ + if( iCol2==0 ) return 0; + } + + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; + + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } + + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + if( iPos1<0 || iPos2<0 ) break; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1=pEnd ){ + *pp = 0; + }else{ + u64 iVal; + *pp += sqlite3Fts3GetVarintU(*pp, &iVal); + if( bDescIdx ){ + *pVal = (i64)((u64)*pVal - iVal); + }else{ + *pVal = (i64)((u64)*pVal + iVal); + } + } +} + +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_uint64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev ); + iWrite = (u64)iVal - (u64)*piPrev; + }else{ + assert_fts3_nc( *piPrev>=iVal ); + iWrite = (u64)*piPrev - (u64)iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert_fts3_nc( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + + +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1))) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + int rc = SQLITE_OK; + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + rc = fts3PoslistMerge(&p, &p1, &p2); + if( rc ) break; + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + + assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) ); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + p = aOut = 0; + }else{ + assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 ); + memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING); + } + *paOut = aOut; + *pnOut = (int)(p-aOut); + return rc; +} + +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ +static int fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *aRight = *paRight; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut; + + assert( nDist>0 ); + if( bDescDoclist ){ + aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } + p = aOut; + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} + +/* +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. +*/ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + } + return SQLITE_OK; +} + +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ + + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm && zRoot ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; +} + +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelbase.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; + } + } + + *ppSegcsr = pSegcsr; + return rc; +} + +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} + +/* +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + } + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + Fts3Table *pTab = (Fts3Table*)pCursor->pVtab; + pTab->bLock++; + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } + pTab->bLock--; + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. +** +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_OK; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + if( p->bLock ){ + return SQLITE_ERROR; + } + + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); + + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + + /* In case the cursor has been used before, clear it now. */ + fts3ClearCursor(pCsr); + + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; + + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); + + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; + } + + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + if( pDocidGe || pDocidLe ){ + zSql = sqlite3_mprintf( + "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", + p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } + if( zSql ){ + p->bLock++; + rc = sqlite3_prepare_v3( + p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } + } + if( rc!=SQLITE_OK ) return rc; + + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; + if( pCsr->isEof ){ + fts3ClearCursor(pCsr); + pCsr->isEof = 1; + } + return pCsr->isEof; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); + + switch( iCol-p->nColumn ){ + case 0: + /* The special 'table-name' column */ + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); + break; + + case 1: + /* The docid column */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + break; + + case 2: + if( pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + break; + }else if( p->zLanguageid==0 ){ + sqlite3_result_int(pCtx, 0); + break; + }else{ + iCol = p->nColumn; + /* no break */ deliberate_fall_through + } + + default: + /* A user column. Or, if this is a full-table scan, possibly the + ** language-id column. Seek the cursor. */ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + break; + } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + char *zTbl = sqlite3_mprintf("%s_stat", p->zName); + if( zTbl ){ + int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); + sqlite3_free(zTbl); + p->bHasStat = (res==SQLITE_OK); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + int rc; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + p->nLeafAdd = 0; + rc = fts3SetHasStat(p); +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK ){ + p->inTransaction = 1; + p->mxSavepoint = -1; + } +#endif + return rc; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ + while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + int rc; + *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); + if( (*ppCsr)!=0 ){ + rc = SQLITE_OK; + }else{ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + /* no break */ deliberate_fall_through + case 5: iCol = sqlite3_value_int(apVal[4]); + /* no break */ deliberate_fall_through + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + /* no break */ deliberate_fall_through + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + /* no break */ deliberate_fall_through + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} + +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} + +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); + } +} + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; idb; /* Database connection */ + int rc; /* Return Code */ + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + p->bIgnoreSavepoint = 1; + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + + p->bIgnoreSavepoint = 0; + return rc; +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint<=iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + + if( pTab->bIgnoreSavepoint==0 ){ + if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){ + char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", + pTab->zDb, pTab->zName, pTab->zName + ); + if( zSql ){ + pTab->bIgnoreSavepoint = 1; + rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); + pTab->bIgnoreSavepoint = 0; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint >= iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint-1 ); + pTab->iSavepoint = iSavepoint; + return SQLITE_OK; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *pTab = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( pTab->inTransaction ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + sqlite3Fts3PendingTermsClear(pTab); + } + return SQLITE_OK; +} + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts3ShadowName(const char *zName){ + static const char *azName[] = { + "content", "docsize", "segdir", "segments", "stat", + }; + unsigned int i; + for(i=0; ibFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", + p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + + + +static const sqlite3_module fts3Module = { + /* iVersion */ 4, + /* xCreate */ fts3CreateMethod, + /* xConnect */ fts3ConnectMethod, + /* xBestIndex */ fts3BestIndexMethod, + /* xDisconnect */ fts3DisconnectMethod, + /* xDestroy */ fts3DestroyMethod, + /* xOpen */ fts3OpenMethod, + /* xClose */ fts3CloseMethod, + /* xFilter */ fts3FilterMethod, + /* xNext */ fts3NextMethod, + /* xEof */ fts3EofMethod, + /* xColumn */ fts3ColumnMethod, + /* xRowid */ fts3RowidMethod, + /* xUpdate */ fts3UpdateMethod, + /* xBegin */ fts3BeginMethod, + /* xSync */ fts3SyncMethod, + /* xCommit */ fts3CommitMethod, + /* xRollback */ fts3RollbackMethod, + /* xFindFunction */ fts3FindFunctionMethod, + /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, + /* xShadowName */ fts3ShadowName, + /* xIntegrity */ fts3IntegrityMethod, +}; + +/* +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. +*/ +static void hashDestroy(void *p){ + Fts3HashWrapper *pHash = (Fts3HashWrapper *)p; + pHash->nRef--; + if( pHash->nRef<=0 ){ + sqlite3Fts3HashClear(&pHash->hash); + sqlite3_free(pHash); + } +} + +/* +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. +** +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. +*/ +void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif +#ifdef SQLITE_ENABLE_ICU +void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif + +/* +** Initialize the fts3 extension. If this extension is built as part +** of the sqlite library, then this function is called directly by +** SQLite. If fts3 is built as a dynamically loadable extension, this +** function is called by the sqlite3_extension_init() entry point. +*/ +int sqlite3Fts3Init(sqlite3 *db){ + int rc = SQLITE_OK; + Fts3HashWrapper *pHash = 0; + const sqlite3_tokenizer_module *pSimple = 0; + const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + const sqlite3_tokenizer_module *pUnicode = 0; +#endif + +#ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif + +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; + + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); + + /* Allocate and initialize the hash-table used to store tokenizers. */ + pHash = sqlite3_malloc(sizeof(Fts3HashWrapper)); + if( !pHash ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Fts3HashInit(&pHash->hash, FTS3_HASH_STRING, 1); + pHash->nRef = 0; + } + + /* Load the built-in tokenizers into the hash table */ + if( rc==SQLITE_OK ){ + if( sqlite3Fts3HashInsert(&pHash->hash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(&pHash->hash, "porter", 7, (void *)pPorter) + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + || sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode) +#endif +#ifdef SQLITE_ENABLE_ICU + || (pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu)) +#endif + ){ + rc = SQLITE_NOMEM; + } + } + +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ExprInitTestInterface(db, &pHash->hash); + } +#endif + + /* Create the virtual table wrapper around the hash-table and overload + ** the four scalar functions. If this is successful, register the + ** module with sqlite. + */ + if( SQLITE_OK==rc + && SQLITE_OK==(rc=sqlite3Fts3InitHashTable(db,&pHash->hash,"fts3_tokenizer")) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) + ){ + pHash->nRef++; + rc = sqlite3_create_module_v2( + db, "fts3", &fts3Module, (void *)pHash, hashDestroy + ); + if( rc==SQLITE_OK ){ + pHash->nRef++; + rc = sqlite3_create_module_v2( + db, "fts4", &fts3Module, (void *)pHash, hashDestroy + ); + } + if( rc==SQLITE_OK ){ + pHash->nRef++; + rc = sqlite3Fts3InitTok(db, (void *)pHash, hashDestroy); + } + return rc; + } + + + /* An error has occurred. Delete the hash table and return the error code. */ + assert( rc!=SQLITE_OK ); + if( pHash ){ + sqlite3Fts3HashClear(&pHash->hash); + sqlite3_free(pHash); + } + return rc; +} + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +*/ +static int fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + int rc = SQLITE_OK; + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; +} + +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ + char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0); + + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + sqlite3_free(aFree); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + sqlite3_free(aFree); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + assert( p1 && p2 ); + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree); + return SQLITE_OK; +} +#endif /* SQLITE_DISABLE_FTS4_DEFERRED */ + +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; + + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. +*/ +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert_fts3_nc( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid0 ); + assert( *pbEof==0 ); + assert_fts3_nc( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + assert( pDL->aAll!=0 || pIter==0 ); + }else{ + pIter = pDL->aAll; + } + + if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); + } + + return rc; +} + +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int nToken = pExpr->pPhrase->nToken; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + } + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; + +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + int rc = SQLITE_OK; + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + testcase( a==0 ); /* If %_stat.value set to X'' */ + if( a ){ + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + } + + *pnPage = pCsr->nRowAvg; + return rc; +} + +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; + } + } + assert( pTC ); + + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } + if( rc==SQLITE_OK ){ + int nCount; + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; + } + + return rc; +} + +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + aTC = (Fts3TokenAndCost *)sqlite3_malloc64( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + Fts3Expr **apOr = (Fts3Expr **)&aTC[nToken]; + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + return rc; +} + +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert_fts3_nc( nNew<=pPhrase->doclist.nList && nNew>0 ); + if( nNew>=0 && nNew<=pPhrase->doclist.nList ){ + assert( pPhrase->doclist.pList[nNew]=='\0' ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + } + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK && pExpr->bEof==0 ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + assert( pRight->eType==FTSQUERY_PHRASE ); + if( pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + pRight->bEof = pLeft->bEof = 1; + } + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert_fts3_nc( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert_fts3_nc( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || iCmp>0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + sqlite3_int64 nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + assert( p->pRight->pPhrase->doclist.nList>0 ); + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc64(nTmp*2 + FTS3_VARINT_MAX); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + +/* +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. +*/ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred && (pExpr->bDeferred || ( + pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList + ))){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = ( + pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId + && pExpr->pPhrase->doclist.nList>0 + ); + } + break; + } + } + } + return bHit; +} + +/* +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + do{ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + }while( iColpLeft, nCol); + fts3EvalUpdateCounts(pExpr->pRight, nCol); + } +} + +/* +** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array +** has not yet been allocated, allocate and zero it. Otherwise, just zero +** it. +*/ +static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){ + Fts3Table *pTab = (Fts3Table*)pCtx; + UNUSED_PARAMETER(iPhrase); + if( pExpr->aMI==0 ){ + pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( pExpr->aMI==0 ) return SQLITE_NOMEM; + } + memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + return SQLITE_OK; +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent + && (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred) + ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab); + if( rc!=SQLITE_OK ) return rc; + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot, pTab->nColumn); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + if( pRoot->bEof ) rc = FTS_CORRUPT_VTAB; + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; + + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + Fts3Expr *pRun; /* Closest non-deferred ancestor of pNear */ + int bMatch; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + pRun = pNear; + while( pRun->bDeferred ){ + assert( pRun->pParent ); + pRun = pRun->pParent; + } + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int bEofSave = pRun->bEof; + fts3EvalRestart(pCsr, pRun, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); + if( bEofSave==0 && pRun->iDocid==iDocid ) break; + } + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( bTreeEof ){ + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); + } + } + if( rc!=SQLITE_OK ) return rc; + + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; + + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; + if( pCsr->bDesc==bDescDoclist ){ + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; + } + + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } + } + if( pIter==0 ) return SQLITE_OK; + + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !defined(SQLITE_CORE) +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + +#endif diff --git a/ext/fts3/fts3.h b/ext/fts3/fts3.h new file mode 100644 index 0000000000000000000000000000000000000000..c1aa8caf0927920d6bb78d578d9c111d4161a34a --- /dev/null +++ b/ext/fts3/fts3.h @@ -0,0 +1,26 @@ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +int sqlite3Fts3Init(sqlite3 *db); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h new file mode 100644 index 0000000000000000000000000000000000000000..3b236faf495e758417116d7c7fb1201e8d7d2fcb --- /dev/null +++ b/ext/fts3/fts3Int.h @@ -0,0 +1,659 @@ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* FTS3/FTS4 require virtual tables */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT3 +#endif + +#include "sqlite3.h" +#include "fts3_tokenizer.h" +#include "fts3_hash.h" + +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +#define FTS3_BUFFER_PADDING 8 + +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) +#endif + +/* +** Terminator values for position-lists and column-lists. +*/ +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** The assert_fts3_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +extern int sqlite3_fts3_may_be_corrupt; +# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x)) +#else +# define assert_fts3_nc(x) assert(x) +#endif + +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +/* +** Internal types used by SQLite. +*/ +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#define deliberate_fall_through + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +typedef struct MatchinfoBuffer MatchinfoBuffer; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + int bLock; /* Used to prevent recursive content= tbls */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + int iSavepoint; + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; + + /* Number of segments in a level */ + int nMergeCount; +#endif +}; + +/* Macro to find the number of segments to merge */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +# define MergeCount(P) ((P)->nMergeCount) +#else +# define MergeCount(P) FTS3_MERGE_COUNT +#endif + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_write.c */ +int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +int sqlite3Fts3PendingTermsFlush(Fts3Table *); +void sqlite3Fts3PendingTermsClear(Fts3Table *); +int sqlite3Fts3Optimize(Fts3Table *); +int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +void sqlite3Fts3SegReaderFree(Fts3SegReader *); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + +void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +void sqlite3Fts3ErrMsg(char**,const char*,...); +int sqlite3Fts3PutVarint(char *, sqlite3_int64); +int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); +int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); +int sqlite3Fts3GetVarint32(const char *, int *); +int sqlite3Fts3VarintLen(sqlite3_uint64); +void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +int sqlite3Fts3ReadInt(const char *z, int *pnOut); + +/* fts3_tokenizer.c */ +const char *sqlite3Fts3NextToken(const char *, int *); +int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); + +/* fts3_expr.c */ +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); +int sqlite3Fts3InitTerm(sqlite3 *db); +#endif +void *sqlite3Fts3MallocZero(i64 nByte); + +int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + +/* fts3_aux.c */ +int sqlite3Fts3InitAux(sqlite3 *db); + +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +/* fts3_tokenize_vtab.c */ +int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); + +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +int sqlite3FtsUnicodeFold(int, int); +int sqlite3FtsUnicodeIsalnum(int); +int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); + +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c new file mode 100644 index 0000000000000000000000000000000000000000..439d57936620734f2e741738ed26968dfff09ba3 --- /dev/null +++ b/ext/fts3/fts3_aux.c @@ -0,0 +1,557 @@ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + sqlite3_int64 nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ + + UNUSED_PARAMETER(pVTab); + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } + + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + rc = SQLITE_OK; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( iCol<1 ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); + + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr->zStop); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm); + } + } + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + pCsr->nStop = (int)strlen(pCsr->zStop); + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c new file mode 100644 index 0000000000000000000000000000000000000000..9e201b168415c81ababc35826ba583e81325ba70 --- /dev/null +++ b/ext/fts3/fts3_expr.c @@ -0,0 +1,1292 @@ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +#ifdef SQLITE_TEST +int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +#include +#include + +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +void *sqlite3Fts3MallocZero(sqlite3_int64 nByte){ + void *pRet = sqlite3_malloc64(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + sqlite3_int64 nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + + } + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pModule->xClose(pCursor); + } + + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){ + void *pRet = sqlite3_realloc64(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp || !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; + } + } + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + assert( nTemp==0 || zTemp ); + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + } + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } + rc = SQLITE_OK; + } + + getnextstring_out: + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + p = 0; + } + *ppExpr = p; + return rc; +} + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + pParse->isNot = 0; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)sqlite3Fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; +#if !defined(SQLITE_MAX_EXPR_DEPTH) + if( pParse->nNest>1000 ) return SQLITE_ERROR; +#elif SQLITE_MAX_EXPR_DEPTH>0 + if( pParse->nNest>SQLITE_MAX_EXPR_DEPTH ) return SQLITE_ERROR; +#endif + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = sqlite3Fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = sqlite3Fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + pPrev = p; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; + } + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + sqlite3Fts3ErrMsg(pzErr, + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + } + } + + return rc; +} + +/* +** Free a single node of an expression tree. +*/ +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +#include + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); + } + return zBuf; + } + + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; + } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTestCommon( + int bRebalance, + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context); + const char *zTokenizer = 0; + char *zErr = 0; + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + zTokenizer = (const char*)sqlite3_value_text(argv[0]); + rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_error(context, zErr, -1); + } + sqlite3_free(zErr); + return; + } + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iipModule->xDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(0, context, argc, argv); +} +static void fts3ExprTestRebalance( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(1, context, argc, argv); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0 + ); + } + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c new file mode 100644 index 0000000000000000000000000000000000000000..63e55b3dc9ac3d152e104af29b95b9e179e1ce48 --- /dev/null +++ b/ext/fts3/fts3_hash.c @@ -0,0 +1,383 @@ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include +#include + +#include "fts3_hash.h" + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(sqlite3_int64 n){ + void *p = sqlite3_malloc64(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. +*/ +void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: +** +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. +*/ +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} + +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } + pEntry->count++; + pEntry->chain = pNew; +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; + } + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); + } +} + +Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ +){ + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_hash.h b/ext/fts3/fts3_hash.h new file mode 100644 index 0000000000000000000000000000000000000000..dc3fcf8334fb26962bc887fb1fc30e21c543185c --- /dev/null +++ b/ext/fts3/fts3_hash.h @@ -0,0 +1,112 @@ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ + +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +void sqlite3Fts3HashClear(Fts3Hash*); +Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) + +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ diff --git a/ext/fts3/fts3_icu.c b/ext/fts3/fts3_icu.c new file mode 100644 index 0000000000000000000000000000000000000000..0848a5aabaeb642279d24d8d8dbeb5a38fda131f --- /dev/null +++ b/ext/fts3/fts3_icu.c @@ -0,0 +1,262 @@ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_ENABLE_ICU + +#include +#include +#include "fts3_tokenizer.h" + +#include +#include +#include +#include + +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; + +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; +}; + +struct IcuCursor { + sqlite3_tokenizer_cursor base; + + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ + + int nBuffer; + char *zBuffer; + + int iToken; +}; + +/* +** Create a new tokenizer instance. +*/ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +){ + IcuTokenizer *p; + int n = 0; + + if( argc>0 ){ + n = strlen(argv[0])+1; + } + p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); + + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); + } + + *ppTokenizer = (sqlite3_tokenizer *)p; + + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; + + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; + + UChar32 c; + int iInput = 0; + int iOut = 0; + + *ppCursor = 0; + + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); + } + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc64( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; + + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->aOffset[iOut] = iInput; + + if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->nChar = iOut; + + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to icuOpen(). +*/ +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. +*/ +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + + int iStart = 0; + int iEnd = 0; + int nByte = 0; + + while( iStart==iEnd ){ + UChar32 c; + + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; + } + + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; + }else{ + break; + } + } + assert(iStart<=iEnd); + } + + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ + return SQLITE_NOMEM; + } + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; + } + + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); + + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; + + return SQLITE_OK; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ +}; + +/* +** Set *ppModule to point at the implementation of the ICU tokenizer. +*/ +void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &icuTokenizerModule; +} + +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c new file mode 100644 index 0000000000000000000000000000000000000000..fbe7913020a4e975ce997b02a619ca589dc1dc1d --- /dev/null +++ b/ext/fts3/fts3_porter.c @@ -0,0 +1,662 @@ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include +#include +#include + +#include "fts3_tokenizer.h" + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlite3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; + + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/* +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. +*/ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/* +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. +*/ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; + + while( c->iOffsetnInput ){ + int iStartOffset, ch; + + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc64(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c new file mode 100644 index 0000000000000000000000000000000000000000..80f62eb3bbfcfe55968e0394d00ec3be5cf9ee37 --- /dev/null +++ b/ext/fts3/fts3_snippet.c @@ -0,0 +1,1756 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include + +#ifndef SQLITE_AMALGAMATION +typedef sqlite3_int64 i64; +#endif + +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ + +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" + + +/* +** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. +*/ +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; + +/* +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. +*/ +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; + +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; + +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + i64 iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + i64 iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; + +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; + +/* +** This type is used as an sqlite3Fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. +*/ +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; + +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; + + +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. +*/ +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; + + +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); + + pRet = sqlite3Fts3MallocZero(nByte + nStr+1); + if( pRet ){ + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } + + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } + + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } +} + +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; + + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } + } +} + +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + + +/* +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: +** +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. +*/ +static void fts3GetDeltaPosition(char **pp, i64 *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for sqlite3Fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; + } + return rc; +} + +/* +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. +*/ +int sqlite3Fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +} + +/* +** This is an sqlite3Fts3ExprIterate() callback used while loading the +** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pPhrase->nToken; + + return rc; +} + +/* +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. +** +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. +*/ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ +){ + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + pExpr->iPhrase = iPhrase; + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} + +/* +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. +*/ +static void fts3SnippetAdvance(char **ppIter, i64 *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + i64 iIter = *piIter; + + while( iIteriCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; + + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } + } + if( iEnd==0x7FFFFFFF ){ + return 1; + } + + assert( pIter->nSnippet>=0 ); + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + } + } + + return 0; +} + +/* +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ +){ + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + i64 iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){ + int j; + u64 mPhrase = (u64)1 << (i%64); + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart && (iCsr - iStart)<=64 ); + assert( i>=0 ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken && jnSnippet; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } + } + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; +} + +/* +** This function is an sqlite3Fts3ExprIterate() callback used by +** fts3BestSnippet(). Each invocation populates an element of the +** SnippetIter.aPhrase[] array. +*/ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; + + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + i64 iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + if( iFirst<0 ){ + rc = FTS_CORRUPT_VTAB; + }else{ + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + } + }else{ + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); + } + + return rc; +} + +/* +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurrence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. +*/ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ +){ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + sqlite3_int64 nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ + + memset(&sIter, 0, sizeof(sIter)); + + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3Fts3MallocZero(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; + } + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter + ); + if( rc==SQLITE_OK ){ + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + *piScore = iBestScore; + } + sqlite3_free(sIter.aPhrase); + return rc; +} + + +/* +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). +*/ +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } + + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100; + char *zNew = sqlite3_realloc64(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; + } + pStr->z = zNew; + pStr->nAlloc = nAlloc; + } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); + + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; + + return SQLITE_OK; +} + +/* +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: +** +** ........X.....X +** +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: +** +** ....X.....X.... +** +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. +*/ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ + + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ + + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 ); + nDesired = (nLeft-nRight)/2; + + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; + } + } + } + return SQLITE_OK; +} + +/* +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. +*/ +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + return SQLITE_OK; + } + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; + } + + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; + } + + pMod->xClose(pC); + return rc; +} + + +/* +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). +** +** The number of elements in the column-list is returned. +*/ +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; + + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; + } + + *ppCollist = pEnd; + return nEntry; +} + +/* +** This function gathers 'y' or 'b' data for a single phrase. +*/ +static int fts3ExprLHits( + Fts3Expr *pExpr, /* Phrase expression node */ + MatchInfo *p /* Matchinfo context */ +){ + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; + + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } + + if( pIter ) while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); + if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static int fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + int rc = SQLITE_OK; + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + rc = fts3ExprLHitGather(pExpr->pLeft, p); + if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p); + }else{ + rc = fts3ExprLHits(pExpr, p); + } + } + return rc; +} + +/* +** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo +** stats for a single query. +** +** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); +} + +/* +** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. +*/ +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; + + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; + } + } + + return rc; +} + +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) + ){ + return SQLITE_OK; + } + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } + + return nVal; +} + +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen, + const char **ppEnd +){ + sqlite3_stmt *pStmt; + const char *a; + const char *pEnd; + sqlite3_int64 nDoc; + int n; + + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); + + n = sqlite3_column_bytes(pStmt, 0); + a = sqlite3_column_blob(pStmt, 0); + if( a==0 ){ + return FTS_CORRUPT_VTAB; + } + pEnd = a + n; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + if( nDoc<=0 || a>pEnd ){ + return FTS_CORRUPT_VTAB; + } + *pnDoc = nDoc; + + if( paLen ) *paLen = a; + if( ppEnd ) *ppEnd = pEnd; + return SQLITE_OK; +} + +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ +}; + +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; + +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} + +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead; + sqlite3_int64 iRead; + int rc = 0; + + if( NEVER(pIter==0) ) return 1; + pRead = pIter->pRead; + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); + } + + pIter->pRead = pRead; + return rc; +} + +/* +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. +** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. +*/ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + int rc = SQLITE_OK; + + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } + + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) goto matchinfo_lcs_out; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(pIt); + if( pIt->pRead==0 ){ + rc = FTS_CORRUPT_VTAB; + goto matchinfo_lcs_out; + } + nLive++; + } + } + + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + matchinfo_lcs_out: + sqlite3_free(aIter); + return rc; +} + +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; + + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + pInfo->flag = zArg[i]; + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; + + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + const char *pEnd; /* First byte past end of length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; + } + } + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } + + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); + if( rc!=SQLITE_OK ) break; + } + rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); + if( rc!=SQLITE_OK ) break; + } + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } + + sqlite3_reset(pSelect); + return rc; +} + + +/* +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +*/ +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ + + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; + + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; + + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; + } + + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + if( pCsr->pMIBuffer==0 ){ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ + int i; /* Used to iterate through zArg */ + + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; + + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } + + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; + + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; + } + + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } +} + +/* +** Implementation of snippet() function. +*/ +void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; + + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. + */ + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + /* Limit the snippet length to 64 tokens. */ + if( nToken<-64 ) nToken = -64; + if( nToken>+64 ) nToken = +64; + + for(nSnippet=1; 1; nSnippet++){ + + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ + + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } + + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS = 0; + if( iCol>=0 && iRead!=iCol ) continue; + + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } + + mCovered |= pFragment->covered; + } + + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + } + + assert( nFToken>0 ); + + for(i=0; ipCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert_fts3_nc( iPos>=0 ); + } + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } + + return rc; +} + +/* +** Implementation of offsets() function. +*/ +void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); + + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; + + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3Fts3MallocZero(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; + + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; + + /* Initialize the contents of sCtx.aTerm[] for column iCol. This + ** operation may fail if the database contains corrupt records. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx + ); + if( rc!=SQLITE_OK ) goto offsets_out; + + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } + + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; + + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ + + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } + + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; + }else{ + assert_fts3_nc( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; + } + + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + } + return; +} + +/* +** Implementation of matchinfo() function. +*/ +void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + const char *zFormat; + + if( zArg ){ + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } + + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + }else{ + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + } +} + +#endif diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c new file mode 100644 index 0000000000000000000000000000000000000000..655dd9f35a82a0804cb69f54e32e9bce7e7a091c --- /dev/null +++ b/ext/fts3/fts3_term.c @@ -0,0 +1,383 @@ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is not part of the production FTS code. It is only used for +** testing. It contains a virtual table implementation that provides direct +** access to the full-text index of an FTS table. +*/ + +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_TEST + +#include +#include +#include + +typedef struct Fts3termTable Fts3termTable; +typedef struct Fts3termCursor Fts3termCursor; + +struct Fts3termTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + int iIndex; /* Index for Fts3Table.aIndex[] */ + Fts3Table *pFts3Tab; +}; + +struct Fts3termCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + + int isEof; /* True if cursor is at EOF */ + char *pNext; + + sqlite3_int64 iRowid; /* Current 'rowid' value */ + sqlite3_int64 iDocid; /* Current 'docid' value */ + int iCol; /* Current 'col' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, docid, col, pos)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3termConnectMethod( + sqlite3 *db, /* Database connection */ + void *pCtx, /* Non-zero for an fts4prefix table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + sqlite3_int64 nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3termTable *p; /* Virtual table object to return */ + int iIndex = 0; + + UNUSED_PARAMETER(pCtx); + if( argc==5 ){ + iIndex = atoi(argv[4]); + argc--; + } + + *ppVtab = 0; + + /* The user should specify a single argument - the name of an fts3 table. */ + if( argc!=4 ){ + sqlite3Fts3ErrMsg(pzErr, + "wrong number of arguments to fts4term constructor" + ); + return SQLITE_ERROR; + } + + zDb = argv[1]; + nDb = (int)strlen(zDb); + zFts3 = argv[3]; + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3termTable); + p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte); + if( !p ) return SQLITE_NOMEM; + + p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero( + sizeof(Fts3Table) + nDb + nFts3 + 2 + ); + if( p->pFts3Tab==0 ){ + sqlite3_free(p); + return SQLITE_NOMEM; + } + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = iIndex+1; + p->iIndex = iIndex; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3termTable *p = (Fts3termTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(pFts3); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3termBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + UNUSED_PARAMETER(pVTab); + + /* This vtab naturally does "ORDER BY term, docid, col, pos". */ + if( pInfo->nOrderBy ){ + int i; + for(i=0; inOrderBy; i++){ + if( pInfo->aOrderBy[i].iColumn!=i || pInfo->aOrderBy[i].desc ) break; + } + if( i==pInfo->nOrderBy ){ + pInfo->orderByConsumed = 1; + } + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3termOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3termCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3termCursor *)sqlite3_malloc(sizeof(Fts3termCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3termCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3termCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + int rc; + sqlite3_int64 v; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + /* Advance to the next term in the full-text index. */ + if( pCsr->csr.aDoclist==0 + || pCsr->pNext>=&pCsr->csr.aDoclist[pCsr->csr.nDoclist-1] + ){ + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc!=SQLITE_ROW ){ + pCsr->isEof = 1; + return rc; + } + + pCsr->iCol = 0; + pCsr->iPos = 0; + pCsr->iDocid = 0; + pCsr->pNext = pCsr->csr.aDoclist; + + /* Read docid */ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &pCsr->iDocid); + } + + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + if( v==0 ){ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iDocid += v; + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iCol = 0; + pCsr->iPos = 0; + } + + if( v==1 ){ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iCol += (int)v; + pCsr->iPos = 0; + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + } + + pCsr->iPos += (int)(v - 2); + + return SQLITE_OK; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3termFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + Fts3termTable *p = (Fts3termTable *)pCursor->pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int rc; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(apVal); + + assert( idxStr==0 && idxNum==0 ); + + /* In case this cursor is being reused, close and zero it. */ + testcase(pCsr->filter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, p->iIndex, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + if( rc==SQLITE_OK ){ + rc = fts3termNextMethod(pCursor); + } + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3termEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3termColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3termCursor *p = (Fts3termCursor *)pCursor; + + assert( iCol>=0 && iCol<=3 ); + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(pCtx, p->iDocid); + break; + case 2: + sqlite3_result_int64(pCtx, p->iCol); + break; + default: + sqlite3_result_int64(pCtx, p->iPos); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3termRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3term module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTerm(sqlite3 *db){ + static const sqlite3_module fts3term_module = { + 0, /* iVersion */ + fts3termConnectMethod, /* xCreate */ + fts3termConnectMethod, /* xConnect */ + fts3termBestIndexMethod, /* xBestIndex */ + fts3termDisconnectMethod, /* xDisconnect */ + fts3termDisconnectMethod, /* xDestroy */ + fts3termOpenMethod, /* xOpen */ + fts3termCloseMethod, /* xClose */ + fts3termFilterMethod, /* xFilter */ + fts3termNextMethod, /* xNext */ + fts3termEofMethod, /* xEof */ + fts3termColumnMethod, /* xColumn */ + fts3termRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c new file mode 100644 index 0000000000000000000000000000000000000000..3c42a7bf02963c8430d99eae92ad988f073c4b69 --- /dev/null +++ b/ext/fts3/fts3_test.c @@ -0,0 +1,620 @@ +/* +** 2011 Jun 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is not part of the production FTS code. It is only used for +** testing. It contains a Tcl command that can be used to test if a document +** matches an FTS NEAR expression. +** +** As of March 2012, it also contains a version 1 tokenizer used for testing +** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. +*/ + +#include "tclsqlite.h" +#include +#include + +#if defined(SQLITE_TEST) +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +/* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ +#include "fts3Int.h" + +#define NM_MAX_TOKEN 12 + +typedef struct NearPhrase NearPhrase; +typedef struct NearDocument NearDocument; +typedef struct NearToken NearToken; + +struct NearDocument { + int nToken; /* Length of token in bytes */ + NearToken *aToken; /* Token array */ +}; + +struct NearToken { + int n; /* Length of token in bytes */ + const char *z; /* Pointer to token string */ +}; + +struct NearPhrase { + int nNear; /* Preceding NEAR value */ + int nToken; /* Number of tokens in this phrase */ + NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */ +}; + +static int nm_phrase_match( + NearPhrase *p, + NearToken *aToken +){ + int ii; + + for(ii=0; iinToken; ii++){ + NearToken *pToken = &p->aToken[ii]; + if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){ + if( aToken[ii].n<(pToken->n-1) ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0; + }else{ + if( aToken[ii].n!=pToken->n ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0; + } + } + + return 1; +} + +static int nm_near_chain( + int iDir, /* Direction to iterate through aPhrase[] */ + NearDocument *pDoc, /* Document to match against */ + int iPos, /* Position at which iPhrase was found */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase found */ +){ + int iStart; + int iStop; + int ii; + int nNear; + int iPhrase2; + NearPhrase *p; + NearPhrase *pPrev; + + assert( iDir==1 || iDir==-1 ); + + if( iDir==1 ){ + if( (iPhrase+1)==nPhrase ) return 1; + nNear = aPhrase[iPhrase+1].nNear; + }else{ + if( iPhrase==0 ) return 1; + nNear = aPhrase[iPhrase].nNear; + } + pPrev = &aPhrase[iPhrase]; + iPhrase2 = iPhrase+iDir; + p = &aPhrase[iPhrase2]; + + iStart = iPos - nNear - p->nToken; + iStop = iPos + nNear + pPrev->nToken; + + if( iStart<0 ) iStart = 0; + if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; + + for(ii=iStart; ii<=iStop; ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1; + } + } + + return 0; +} + +static int nm_match_count( + NearDocument *pDoc, /* Document to match against */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase to count matches for */ +){ + int nOcc = 0; + int ii; + NearPhrase *p = &aPhrase[iPhrase]; + + for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + /* Test forward NEAR chain (i>iPhrase) */ + if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; + + /* Test reverse NEAR chain (iNM_MAX_TOKEN ){ + Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + rc = TCL_ERROR; + goto near_match_out; + } + for(jj=0; jj<(int)nToken; jj++){ + NearToken *pT = &aPhrase[ii].aToken[jj]; + pT->z = Tcl_GetStringFromObj(apToken[jj], &nn); + pT->n = (int)nn; + } + aPhrase[ii].nToken = (int)nToken; + } + for(ii=1; ii0)); + + near_match_out: + ckfree((char *)aPhrase); + ckfree((char *)doc.aToken); + return rc; +} + +/* +** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD? +** +** Normally, FTS uses hard-coded values to determine the minimum doclist +** size eligible for incremental loading, and the size of the chunks loaded +** when a doclist is incrementally loaded. This command allows the built-in +** values to be overridden for testing purposes. +** +** If present, the first argument is the chunksize in bytes to load doclists +** in. The second argument is the minimum doclist size in bytes to use +** incremental loading with. +** +** Whether or not the arguments are present, this command returns a list of +** two integers - the initial chunksize and threshold when the command is +** invoked. This can be used to restore the default behavior after running +** tests. For example: +** +** # Override incr-load settings for testing: +** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] +** +** .... run tests .... +** +** # Restore initial incr-load settings: +** eval fts3_configure_incr_load $cfg +*/ +static int SQLITE_TCLAPI fts3_configure_incr_load_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + extern int test_fts3_node_chunksize; + extern int test_fts3_node_chunk_threshold; + Tcl_Obj *pRet; + + if( objc!=1 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?"); + return TCL_ERROR; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize)); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold)); + + if( objc==3 ){ + int iArg1; + int iArg2; + if( Tcl_GetIntFromObj(interp, objv[1], &iArg1) + || Tcl_GetIntFromObj(interp, objv[2], &iArg2) + ){ + Tcl_DecrRefCount(pRet); + return TCL_ERROR; + } + test_fts3_node_chunksize = iArg1; + test_fts3_node_chunk_threshold = iArg2; + } + + Tcl_SetObjResult(interp, pRet); + Tcl_DecrRefCount(pRet); +#endif + UNUSED_PARAMETER(clientData); + return TCL_OK; +} + +#ifdef SQLITE_ENABLE_FTS3 +/************************************************************************** +** Beginning of test tokenizer code. +** +** For language 0, this tokenizer is similar to the default 'simple' +** tokenizer. For other languages L, the following: +** +** * Odd numbered languages are case-sensitive. Even numbered +** languages are not. +** +** * Language ids 100 or greater are considered an error. +** +** The implementation assumes that the input contains only ASCII characters +** (i.e. those that may be encoded in UTF-8 using a single byte). +*/ +typedef struct test_tokenizer { + sqlite3_tokenizer base; +} test_tokenizer; + +typedef struct test_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *aInput; /* Input being tokenized */ + int nInput; /* Size of the input in bytes */ + int iInput; /* Current offset in aInput */ + int iToken; /* Index of next token to be returned */ + char *aBuffer; /* Buffer containing current token */ + int nBuffer; /* Number of bytes allocated at pToken */ + int iLangid; /* Configured language id */ +} test_tokenizer_cursor; + +static int testTokenizerCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + test_tokenizer *pNew; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + pNew = sqlite3_malloc(sizeof(test_tokenizer)); + if( !pNew ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(test_tokenizer)); + + *ppTokenizer = (sqlite3_tokenizer *)pNew; + return SQLITE_OK; +} + +static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){ + test_tokenizer *p = (test_tokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +static int testTokenizerOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + int rc = SQLITE_OK; /* Return code */ + test_tokenizer_cursor *pCsr; /* New cursor object */ + + UNUSED_PARAMETER(pTokenizer); + + pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor)); + if( pCsr==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(test_tokenizer_cursor)); + pCsr->aInput = pInput; + if( nBytes<0 ){ + pCsr->nInput = (int)strlen(pInput); + }else{ + pCsr->nInput = nBytes; + } + } + + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return rc; +} + +static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){ + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + sqlite3_free(pCsr->aBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int testIsTokenChar(char c){ + return (c>='a' && c<='z') || (c>='A' && c<='Z'); +} +static int testTolower(char c){ + char ret = c; + if( ret>='A' && ret<='Z') ret = ret - ('A'-'a'); + return ret; +} + +static int testTokenizerNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + int rc = SQLITE_OK; + const char *p; + const char *pEnd; + + p = &pCsr->aInput[pCsr->iInput]; + pEnd = &pCsr->aInput[pCsr->nInput]; + + /* Skip past any white-space */ + assert( p<=pEnd ); + while( ppCsr->nBuffer ){ + sqlite3_free(pCsr->aBuffer); + pCsr->aBuffer = sqlite3_malloc64(nToken); + } + if( pCsr->aBuffer==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + + if( pCsr->iLangid & 0x00000001 ){ + for(i=0; iaBuffer[i] = pToken[i]; + }else{ + for(i=0; iaBuffer[i] = (char)testTolower(pToken[i]); + } + pCsr->iToken++; + pCsr->iInput = (int)(p - pCsr->aInput); + + *ppToken = pCsr->aBuffer; + *pnBytes = (int)nToken; + *piStartOffset = (int)(pToken - pCsr->aInput); + *piEndOffset = (int)(p - pCsr->aInput); + *piPosition = pCsr->iToken; + } + } + + return rc; +} + +static int testTokenizerLanguage( + sqlite3_tokenizer_cursor *pCursor, + int iLangid +){ + int rc = SQLITE_OK; + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + pCsr->iLangid = iLangid; + if( pCsr->iLangid>=100 ){ + rc = SQLITE_ERROR; + } + return rc; +} +#endif + +static int SQLITE_TCLAPI fts3_test_tokenizer_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + static const sqlite3_tokenizer_module testTokenizerModule = { + 1, + testTokenizerCreate, + testTokenizerDestroy, + testTokenizerOpen, + testTokenizerClose, + testTokenizerNext, + testTokenizerLanguage + }; + const sqlite3_tokenizer_module *pPtr = &testTokenizerModule; + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj( + (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) + )); +#endif + UNUSED_PARAMETER(clientData); + return TCL_OK; +} + +static int SQLITE_TCLAPI fts3_test_varint_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + char aBuf[24]; + int rc; + Tcl_WideInt w; + sqlite3_int64 w2; + int nByte, nByte2; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "INTEGER"); + return TCL_ERROR; + } + + rc = Tcl_GetWideIntFromObj(interp, objv[1], &w); + if( rc!=TCL_OK ) return rc; + + nByte = sqlite3Fts3PutVarint(aBuf, w); + nByte2 = sqlite3Fts3GetVarint(aBuf, &w2); + if( w!=w2 || nByte!=nByte2 ){ + char *zErr = sqlite3_mprintf("error testing %lld", w); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErr, 0); + return TCL_ERROR; + } + + if( w<=2147483647 && w>=0 ){ + int i; + nByte2 = fts3GetVarint32(aBuf, &i); + if( (int)w!=i || nByte!=nByte2 ){ + char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErr, 0); + return TCL_ERROR; + } + } + +#endif + UNUSED_PARAMETER(clientData); + return TCL_OK; +} + +/* +** End of tokenizer code. +**************************************************************************/ + +/* +** sqlite3_fts3_may_be_corrupt BOOLEAN +** +** Set or clear the global "may-be-corrupt" flag. Return the old value. +*/ +static int SQLITE_TCLAPI fts3_may_be_corrupt( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_DEBUG + int bOld = sqlite3_fts3_may_be_corrupt; + + if( objc!=2 && objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?"); + return TCL_ERROR; + } + if( objc==2 ){ + int bNew; + if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR; + sqlite3_fts3_may_be_corrupt = bNew; + } + + Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld)); +#endif + return TCL_OK; +} + +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + Tcl_CreateObjCommand(interp, + "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "sqlite3_fts3_may_be_corrupt", fts3_may_be_corrupt, 0, 0 + ); + return TCL_OK; +} +#endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */ +#endif /* ifdef SQLITE_TEST */ diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c new file mode 100644 index 0000000000000000000000000000000000000000..7e8d09bd4834e73cb13f8bdf808b1d55b98ee808 --- /dev/null +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -0,0 +1,459 @@ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; i1 ) azArg = (const char * const *)&azDequote[1]; + rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module_v2( + db, "fts3tokenize", &fts3tok_module, (void*)pHash, xDestroy + ); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c new file mode 100644 index 0000000000000000000000000000000000000000..24c237a89d5a79142fb6cf12d00bc7260d0a28c7 --- /dev/null +++ b/ext/fts3/fts3_tokenizer.c @@ -0,0 +1,516 @@ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include + +/* +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +*/ +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} + +/* +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: +** +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). +*/ +static void fts3TokenizerFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); + return; + } + }else{ + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){ + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + } +} + +int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + +#ifdef SQLITE_TEST + +#include "tclsqlite.h" +#include + +/* +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: +** +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** +*/ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; + + const char *zErr = 0; + + const char *zName; + int nName; + const char *zInput; + int nInput; + + const char *azArg[64]; + + const char *zToken; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; + + Tcl_Obj *pRet; + + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } + + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + + if( !p ){ + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); + return; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; + } + + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } + + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; + } + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); +} + +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); +} + + +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB + && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp) + ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); + +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); + + /* Test the storage function */ + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + } + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); +} + +#endif + +/* +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: +** +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. +*/ +int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName +){ + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_UTF8|SQLITE_DIRECTONLY; + +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif + + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } +#endif + +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif + + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_tokenizer.h b/ext/fts3/fts3_tokenizer.h new file mode 100644 index 0000000000000000000000000000000000000000..4a40b2b38503833bf17ded9ee2e2742e86267fc6 --- /dev/null +++ b/ext/fts3/fts3_tokenizer.h @@ -0,0 +1,161 @@ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +#include "sqlite3.h" + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c new file mode 100644 index 0000000000000000000000000000000000000000..78e5889da5250184db89879b059a66eb5637aa93 --- /dev/null +++ b/ext/fts3/fts3_tokenizer1.c @@ -0,0 +1,234 @@ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include +#include +#include + +#include "fts3_tokenizer.h" + +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; + +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. + */ + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; + } + } + + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + simple_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); + }else{ + c->nBytes = nBytes; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; + + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; + } + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c new file mode 100644 index 0000000000000000000000000000000000000000..2fd4b39fb7721a8df03477546875b23a0e6ba43f --- /dev/null +++ b/ext/fts3/fts3_unicode.c @@ -0,0 +1,397 @@ +/* +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include +#include +#include + +#include "fts3_tokenizer.h" + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; + +struct unicode_tokenizer { + sqlite3_tokenizer base; + int eRemoveDiacritic; + int nException; + int *aiException; +}; + +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; + + +/* +** Destroy a tokenizer allocated by unicodeCreate(). +*/ +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; +} + +/* +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). +** +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. +** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. +*/ +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + unsigned int iCode; + int nEntry = 0; + + assert( bAlnum==0 || bAlnum==1 ); + + while( zaiException,(p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; + + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = (int)iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } + + return SQLITE_OK; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). +*/ +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +} + +/* +** Create a new tokenizer instance. +*/ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; + + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->eRemoveDiacritic = 1; + + for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; + } + } + + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; + return rc; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; + + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); + + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + pCsr->aInput = (const unsigned char*)""; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; + } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ +){ + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + unsigned int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; + + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; + } + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) + ); + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; +} + +/* +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. +*/ +void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c new file mode 100644 index 0000000000000000000000000000000000000000..c510162496dd95c8d21c0b588b8f247114503aca --- /dev/null +++ b/ext/fts3/fts3_unicode2.c @@ -0,0 +1,383 @@ +/* +** 2012-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +#include + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int remove_diacritic(int c, int bComplex){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, + }; +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c new file mode 100644 index 0000000000000000000000000000000000000000..5a449dec1efe76d54644a4e7a07cfc68b06ab14a --- /dev/null +++ b/ext/fts3/fts3_write.c @@ -0,0 +1,5834 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include +#include +#include + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + +/* +** The values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +** +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 + +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" + " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM (SELECT * FROM %Q.'%q_segdir' " + " WHERE level = ? ORDER BY idx ASC LIMIT ?" + " )", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + f &= ~SQLITE_PREPARE_NO_VTAB; + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + + return rc; +} + +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert_fts3_nc( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root +*/ +int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; +} + + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +** +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: +** +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc64(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + i64 nNew = p->nSpace * 2; + p = sqlite3_realloc64(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = (int)nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; + + char const *zToken; + int nToken = 0; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } + + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); + + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; + return SQLITE_OK; +} + +/* +** Discard the contents of the pending-terms hash tables. +*/ +void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; +} + +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=MergeCount(p) ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + }else if( rc==SQLITE_ERROR ){ + rc = FTS_CORRUPT_VTAB; + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} + +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; + PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; + + int nTerm = fts3HashKeysize(pElem); + if( (nTerm+1)>pReader->nTermAlloc ){ + sqlite3_free(pReader->zTerm); + pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2); + if( !pReader->zTerm ) return SQLITE_NOMEM; + pReader->nTermAlloc = (nTerm+1)*2; + } + memcpy(pReader->zTerm, fts3HashKey(pElem), nTerm); + pReader->zTerm[nTerm] = '\0'; + pReader->nTerm = nTerm; + + aCopy = (char*)sqlite3_malloc64(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } + + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ +#ifdef CORRUPT_DB + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB ); +#endif + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } + pNext = pReader->aNode; + } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nSuffix<=0 + || (&pReader->aNode[pReader->nNode] - pNext)pReader->nTerm + ){ + return FTS_CORRUPT_VTAB; + } + + /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are + ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer + ** overflow - hence the (i64) casts. */ + if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){ + i64 nNew = ((i64)nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc64(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 + ){ + return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +){ + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; + + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + u64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta); + }else{ + pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta); + } + } + } + } + + return rc; +} + + +int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + sqlite3_free(pReader->zTerm); + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + sqlite3_blob_close(pReader->pBlob); + } + sqlite3_free(pReader); +} + +/* +** Allocate a new SegReader object. +*/ +int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + assert( zRoot!=0 || nRoot==0 ); +#ifdef CORRUPT_DB + assert( zRoot!=0 || CORRUPT_DB ); +#endif + + if( iStartLeaf==0 ){ + if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB; + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} + +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; + + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc64( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + } + } + + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert_fts3_nc( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + } + return rc; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc64(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, (i64)nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + i64 nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc64(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + assert( nSuffix>0 ); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + assert( nDoclist>0 ); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + assert( nTerm>0 ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); + } + p->nLeafAdd++; + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + (((u64)iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + +/* +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. +*/ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ + + for(i=0; rc==SQLITE_OK && i=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. +*/ +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( p0){ + memset(&pList[nList], 0, pEnd - &pList[nList]); + } + *ppList = pList; + *pnList = nList; +} + +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + i64 nList +){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ + char *pNew; + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; + } + + assert( nList>0 ); + memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); + return SQLITE_OK; +} + +int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); + } + + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } + } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + +int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){ + if( nReq>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = nReq*2; + aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } + return SQLITE_OK; +} + + +int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; + + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } + + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } + + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, + (i64)pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iPrev - (u64)iDocid); + }else{ + if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iDocid - (u64)iPrev); + } + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + + rc = fts3GrowSegReaderBuffer(pCsr, + (i64)nByte+nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } + + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; + memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); + + return rc; +} + + +void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); + + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } +} + +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + u64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = (i64)iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = ((i64)iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + +/* +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. +** +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. +*/ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ + + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* This call is to merge all segments in the database to a single + ** segment. The level of the new segment is equal to the numerically + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ + rc = SQLITE_DONE; + goto finished; + } + iNewLevel = iMaxLevel; + bIgnoreEmpty = 1; + + }else{ + /* This call is to merge all segments at level iLevel. find the next + ** available segment index at level iLevel+1. The call to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** a single iLevel+2 segment if necessary. */ + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); + } + if( rc!=SQLITE_OK ) goto finished; + + assert( csr.nSegment>0 ); + assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert_fts3_nc( + iNewLevelnLeafData); + } + } + } + + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); + return rc; +} + + +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } + rc = sqlite3_reset(pStmt); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } + return rc; +} + +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); +} + +/* +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** +*/ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + const int nStat = p->nColumn+2; + + if( *pRC ) return; + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); + }else{ + memset(a, 0, sizeof(u32)*(nStat) ); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + sqlite3_free(a); +} + +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + } + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + sqlite3Fts3SegmentsClose(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){ + return FTS_CORRUPT_VTAB; + } + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK && ALWAYS(p->term.a!=0) ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + if( (p->nNode-p->iOff)nDoclist ){ + return FTS_CORRUPT_VTAB; + } + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert_fts3_nc( p->iOff<=p->nNode ); + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( aNode && aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return aNode ? nodeReaderNext(p) : SQLITE_OK; +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + assert( nPrefix+nSuffix<=nTerm ); + assert( nPrefix>=0 ); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + assert( pPrev!=0 ); + assert( pPrev->a!=0 ); + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize bytes, + ** and if there is still room for another leaf page, write this block out to + ** the database. */ + if( pLeaf->block.n>0 + && (pLeaf->block.n + nSpace)>p->nNodeSize + && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst) + ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + if( nCmp && ALWAYS(zLhs) && ALWAYS(zRhs) ){ + res = memcmp(zLhs, zRhs, nCmp); + }else{ + res = 0; + } + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + if( aRoot==0 ){ + sqlite3_reset(pSelect); + return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB; + } + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + if( nHeight<1 || nHeight>=FTS_MAX_APPENDABLE_HEIGHT ){ + sqlite3_reset(pSelect); + return FTS_CORRUPT_VTAB; + } + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, + MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING); + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + memset(&reader, 0, sizeof(reader)); + pNode = &pWriter->aNodeWriter[i]; + + if( pNode->block.a){ + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + assert_fts3_nc( reader.term.n>0 || reader.aNode==0 ); + if( reader.term.n>0 ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + } + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); + blobGrowBuffer(&pNode->block, + MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING); + } + sqlite3_free(aBlock); + } + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf; /* True for a leaf node */ + + if( nNode<1 ) return FTS_CORRUPT_VTAB; + bLeaf = aNode[0]=='\0'; + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + if( ALWAYS(pHint->a!=0) ) memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-1; + if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + if( i==0 ) return FTS_CORRUPT_VTAB; + i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + assert( i<=nHint ); + if( i!=nHint ) return FTS_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. +*/ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + /* Based on the scan in the block above, it is known that there + ** are no levels with a relative level smaller than that of + ** iAbsLevel with more than nSeg segments, or if nSeg is -1, + ** no levels with more than nMin segments. Use this to limit the + ** value of nHintSeg to avoid a large memory allocation in case the + ** merge-hint is corrupt*/ + iAbsLevel = iHintAbsLevel; + nSeg = MIN(MAX(nMin,nSeg), nHintSeg); + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<=0 ) break; + + assert( nMod<=0x7FFFFFFF ); + if( iAbsLevel<0 || iAbsLevel>(nMod<<32) ){ + rc = FTS_CORRUPT_VTAB; + break; + } + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + ){ + int bEmpty = 0; + rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( rc==SQLITE_OK ){ + bEmpty = 1; + }else if( rc!=SQLITE_ROW ){ + sqlite3Fts3SegReaderFinish(pCsr); + break; + } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + if( bEmpty==0 ){ + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (MergeCount(p) / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){ + p->nAutoincrmerge = 8; + } + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + if( *pRc ) return 0; + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + u64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrbDescIdx ){ + iDocid = (i64)((u64)iDocid - iVal); + }else{ + iDocid = (i64)((u64)iDocid + iVal); + } + } + }else{ + iPos += (iVal - 2); + cksum = cksum ^ fts3ChecksumEntry( + csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, + (int)iCol, (int)iPos + ); + } + } + } + } + } + sqlite3Fts3SegReaderFinish(&csr); + + *pRc = rc; + return cksum; +} + +/* +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. +** +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. +*/ +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ + + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + } + + sqlite3_finalize(pStmt); + } + + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = sqlite3Fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; + return rc; +} + +/* +** Handle a 'special' INSERT of the form: +** +** "INSERT INTO tbl(tbl) VALUES()" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc = SQLITE_ERROR; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); + + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); + }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + else{ + int v; + if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + v = atoi(&zVal[9]); + if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + v = atoi(&zVal[11]); + if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){ + v = atoi(&zVal[11]); + if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; + rc = SQLITE_OK; + } + } +#endif + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; + } +} + +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} + +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } + } + + return rc; +} + +int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc64(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc64(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} +#endif + +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel +){ + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + } + + return rc; +} + +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: +** +** CREATE TABLE ( +** , +**
HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** +*/ +int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); + + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; + + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } + } + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( rc==SQLITE_OK ){ + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; + } + + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); + } + + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + } + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +#endif diff --git a/ext/fts3/fts3speed.tcl b/ext/fts3/fts3speed.tcl new file mode 100644 index 0000000000000000000000000000000000000000..377cb196065699557b0a035a1e9d74808321187a --- /dev/null +++ b/ext/fts3/fts3speed.tcl @@ -0,0 +1,122 @@ + + +#-------------------------------------------------------------------------- +# This script contains several sub-programs used to test FTS3/FTS4 +# performance. It does not run the queries directly, but generates SQL +# scripts that can be run using the shell tool. +# +# The following cases are tested: +# +# 1. Inserting documents into an FTS3 table. +# 2. Optimizing an FTS3 table (i.e. "INSERT INTO t1 VALUES('optimize')"). +# 3. Deleting documents from an FTS3 table. +# 4. Querying FTS3 tables. +# + +# Number of tokens in vocabulary. And number of tokens in each document. +# +set VOCAB_SIZE 2000 +set DOC_SIZE 100 + +set NUM_INSERTS 100000 +set NUM_SELECTS 1000 + +# Force everything in this script to be deterministic. +# +expr {srand(0)} + +proc usage {} { + puts stderr "Usage: $::argv0 " + exit -1 +} + +proc sql {sql} { + puts $::fd $sql +} + + +# Return a list of $nWord randomly generated tokens each between 2 and 10 +# characters in length. +# +proc build_vocab {nWord} { + set ret [list] + set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + for {set i 0} {$i<$nWord} {incr i} { + set len [expr {int((rand()*9.0)+2)}] + set term "" + for {set j 0} {$j<$len} {incr j} { + append term [lindex $chars [expr {int(rand()*[llength $chars])}]] + } + lappend ret $term + } + set ret +} + +proc select_term {} { + set n [llength $::vocab] + set t [expr int(rand()*$n*3)] + if {$t>=2*$n} { set t [expr {($t-2*$n)/100}] } + if {$t>=$n} { set t [expr {($t-$n)/10}] } + lindex $::vocab $t +} + +proc select_doc {nTerm} { + set ret [list] + for {set i 0} {$i<$nTerm} {incr i} { + lappend ret [select_term] + } + set ret +} + +proc test_1 {nInsert} { + sql "PRAGMA synchronous = OFF;" + sql "DROP TABLE IF EXISTS t1;" + sql "CREATE VIRTUAL TABLE t1 USING fts4;" + for {set i 0} {$i < $nInsert} {incr i} { + set doc [select_doc $::DOC_SIZE] + sql "INSERT INTO t1 VALUES('$doc');" + } +} + +proc test_2 {} { + sql "INSERT INTO t1(t1) VALUES('optimize');" +} + +proc test_3 {nSelect} { + for {set i 0} {$i < $nSelect} {incr i} { + sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term]';" + } +} + +proc test_4 {nSelect} { + for {set i 0} {$i < $nSelect} {incr i} { + sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term] [select_term]';" + } +} + +if {[llength $argv]!=0} usage + +set ::vocab [build_vocab $::VOCAB_SIZE] + +set ::fd [open fts3speed_insert.sql w] +test_1 $NUM_INSERTS +close $::fd + +set ::fd [open fts3speed_select.sql w] +test_3 $NUM_SELECTS +close $::fd + +set ::fd [open fts3speed_select2.sql w] +test_4 $NUM_SELECTS +close $::fd + +set ::fd [open fts3speed_optimize.sql w] +test_2 +close $::fd + +puts "Success. Created files:" +puts " fts3speed_insert.sql" +puts " fts3speed_select.sql" +puts " fts3speed_select2.sql" +puts " fts3speed_optimize.sql" + diff --git a/ext/fts3/tool/fts3cov.sh b/ext/fts3/tool/fts3cov.sh new file mode 100644 index 0000000000000000000000000000000000000000..b1f34dce76bb46c8ac59044856861083a0be4050 --- /dev/null +++ b/ext/fts3/tool/fts3cov.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +srcdir=`dirname $(dirname $(dirname $(dirname $0)))` +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt + +echo "" + +for f in `ls $srcdir/ext/fts3/*.c` +do + f=`basename $f` + echo -ne "$f: " + gcov -b $f | grep Taken | sed 's/Taken at least once://' +done + diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c new file mode 100644 index 0000000000000000000000000000000000000000..9558cde0d06af937217c0b4811f92a480603b6f3 --- /dev/null +++ b/ext/fts3/tool/fts3view.c @@ -0,0 +1,875 @@ +/* +** This program is a debugging and analysis utility that displays +** information about an FTS3 or FTS4 index. +** +** Link this program against the SQLite3 amalgamation with the +** SQLITE_ENABLE_FTS4 compile-time option. Then run it as: +** +** fts3view DATABASE +** +** to get a list of all FTS3/4 tables in DATABASE, or do +** +** fts3view DATABASE TABLE COMMAND .... +** +** to see various aspects of the TABLE table. Type fts3view with no +** arguments for a list of available COMMANDs. +*/ +#include +#include +#include +#include +#include +#include "sqlite3.h" + +/* +** Extra command-line arguments: +*/ +int nExtra; +char **azExtra; + +/* +** Look for a command-line argument. +*/ +const char *findOption(const char *zName, int hasArg, const char *zDefault){ + int i; + const char *zResult = zDefault; + for(i=0; i=2000 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/1000); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 0.1%% or less of docs...... %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + if( nDoc>=200 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/100); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 1%% or less of docs........ %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + nTop = atoi(findOption("top", 1, "25")); + printf("The %d most common tokens:\n", nTop); + pStmt = prepare(db, + "SELECT term, documents FROM %s" + " WHERE col='*'" + " ORDER BY documents DESC, term" + " LIMIT %d", zAux, nTop); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i++; + n = sqlite3_column_int(pStmt, 1); + printf(" %2d. %-30s %9d docs %5.2f%%\n", i, + sqlite3_column_text(pStmt, 0), n, n*100.0/nDoc); + } + sqlite3_finalize(pStmt); + +end_vocab: + runSql(db, "ROLLBACK"); + sqlite3_free(zAux); +} + +/* +** Report on the number and sizes of segments +*/ +static void showSegmentStats(sqlite3 *db, const char *zTab){ + sqlite3_stmt *pStmt; + int nSeg = 0; + sqlite3_int64 szSeg = 0, mxSeg = 0; + int nIdx = 0; + sqlite3_int64 szIdx = 0, mxIdx = 0; + int nRoot = 0; + sqlite3_int64 szRoot = 0, mxRoot = 0; + sqlite3_int64 mx; + int nLeaf; + int n; + int pgsz; + int mxLevel; + int i; + + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nSeg = sqlite3_column_int(pStmt, 0); + szSeg = sqlite3_column_int64(pStmt, 1); + mxSeg = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE a.blockid BETWEEN b.leaves_end_block+1 AND b.end_block", + zTab, zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nIdx = sqlite3_column_int(pStmt, 0); + szIdx = sqlite3_column_int64(pStmt, 1); + mxIdx = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(root)), max(length(root))" + " FROM '%q_segdir'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nRoot = sqlite3_column_int(pStmt, 0); + szRoot = sqlite3_column_int64(pStmt, 1); + mxRoot = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + + printf("Number of segments....................... %9d\n", nSeg+nRoot); + printf("Number of leaf segments.................. %9d\n", nSeg-nIdx); + printf("Number of index segments................. %9d\n", nIdx); + printf("Number of root segments.................. %9d\n", nRoot); + printf("Total size of all segments............... %9lld\n", szSeg+szRoot); + printf("Total size of all leaf segments.......... %9lld\n", szSeg-szIdx); + printf("Total size of all index segments......... %9lld\n", szIdx); + printf("Total size of all root segments.......... %9lld\n", szRoot); + if( nSeg>0 ){ + printf("Average size of all segments............. %11.1f\n", + (double)(szSeg+szRoot)/(double)(nSeg+nRoot)); + printf("Average size of leaf segments............ %11.1f\n", + (double)(szSeg-szIdx)/(double)(nSeg-nIdx)); + } + if( nIdx>0 ){ + printf("Average size of index segments........... %11.1f\n", + (double)szIdx/(double)nIdx); + } + if( nRoot>0 ){ + printf("Average size of root segments............ %11.1f\n", + (double)szRoot/(double)nRoot); + } + mx = mxSeg; + if( mx%d", + zTab, zTab, pgsz-45); + n = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + nLeaf = nSeg - nIdx; + printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", + pgsz-45, n, nLeaf>0 ? n*100.0/nLeaf : 0.0); + + pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); + mxLevel = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + + for(i=0; i<=mxLevel; i++){ + pStmt = prepare(db, + "SELECT count(*), sum(len), avg(len), max(len), sum(len>%d)," + " count(distinct idx)" + " FROM (SELECT length(a.block) AS len, idx" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE (a.blockid BETWEEN b.start_block" + " AND b.leaves_end_block)" + " AND (b.level%%1024)==%d)", + pgsz-45, zTab, zTab, i); + if( sqlite3_step(pStmt)==SQLITE_ROW + && (nLeaf = sqlite3_column_int(pStmt, 0))>0 + ){ + sqlite3_int64 sz; + nIdx = sqlite3_column_int(pStmt, 5); + printf("For level %d:\n", i); + printf(" Number of indexes...................... %9d\n", nIdx); + printf(" Number of leaf segments................ %9d\n", nLeaf); + if( nIdx>1 ){ + printf(" Average leaf segments per index........ %11.1f\n", + (double)nLeaf/(double)nIdx); + } + printf(" Total size of all leaf segments........ %9lld\n", + (sz = sqlite3_column_int64(pStmt, 1))); + printf(" Average size of leaf segments.......... %11.1f\n", + sqlite3_column_double(pStmt, 2)); + if( nIdx>1 ){ + printf(" Average leaf segment size per index.... %11.1f\n", + (double)sz/(double)nIdx); + } + printf(" Maximum leaf segment size.............. %9lld\n", + sqlite3_column_int64(pStmt, 3)); + n = sqlite3_column_int(pStmt, 4); + printf(" Leaf segments larger than %5d bytes.. %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + } + sqlite3_finalize(pStmt); + } +} + +/* +** Print a single "tree" line of the segdir map output. +*/ +static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){ + printf(" tree %9lld", iLower); + if( iUpper>iLower ){ + printf(" thru %9lld (%lld blocks)", iUpper, iUpper-iLower+1); + } + printf("\n"); +} + +/* +** Check to see if the block of a %_segments entry is NULL. +*/ +static int isNullSegment(sqlite3 *db, const char *zTab, sqlite3_int64 iBlockId){ + sqlite3_stmt *pStmt; + int rc = 1; + + pStmt = prepare(db, "SELECT block IS NULL FROM '%q_segments'" + " WHERE blockid=%lld", zTab, iBlockId); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + rc = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** Show a map of segments derived from the %_segdir table. +*/ +static void showSegdirMap(sqlite3 *db, const char *zTab){ + int mxIndex, iIndex; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pStmt2 = 0; + int prevLevel; + + pStmt = prepare(db, "SELECT max(level/1024) FROM '%q_segdir'", zTab); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxIndex = sqlite3_column_int(pStmt, 0); + }else{ + mxIndex = 0; + } + sqlite3_finalize(pStmt); + + printf("Number of inverted indices............... %3d\n", mxIndex+1); + pStmt = prepare(db, + "SELECT level, idx, start_block, leaves_end_block, end_block, rowid" + " FROM '%q_segdir'" + " WHERE level/1024==?" + " ORDER BY level DESC, idx", + zTab); + pStmt2 = prepare(db, + "SELECT blockid FROM '%q_segments'" + " WHERE blockid BETWEEN ? AND ? ORDER BY blockid", + zTab); + for(iIndex=0; iIndex<=mxIndex; iIndex++){ + if( mxIndex>0 ){ + printf("**************************** Index %d " + "****************************\n", iIndex); + } + sqlite3_bind_int(pStmt, 1, iIndex); + prevLevel = -1; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int iLevel = sqlite3_column_int(pStmt, 0)%1024; + int iIdx = sqlite3_column_int(pStmt, 1); + sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3); + sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4); + char rtag[20]; + if( iLevel!=prevLevel ){ + printf("level %2d idx %2d", iLevel, iIdx); + prevLevel = iLevel; + }else{ + printf(" idx %2d", iIdx); + } + sqlite3_snprintf(sizeof(rtag), rtag, "r%lld", + sqlite3_column_int64(pStmt,5)); + printf(" root %9s\n", rtag); + if( iLEnd>iStart ){ + sqlite3_int64 iLower, iPrev = 0, iX; + if( iLEnd+1<=iEnd ){ + sqlite3_bind_int64(pStmt2, 1, iLEnd+1); + sqlite3_bind_int64(pStmt2, 2, iEnd); + iLower = -1; + while( sqlite3_step(pStmt2)==SQLITE_ROW ){ + iX = sqlite3_column_int64(pStmt2, 0); + if( iLower<0 ){ + iLower = iPrev = iX; + }else if( iX==iPrev+1 ){ + iPrev = iX; + }else{ + printTreeLine(iLower, iPrev); + iLower = iPrev = iX; + } + } + sqlite3_reset(pStmt2); + if( iLower>=0 ){ + if( iLower==iPrev && iLower==iEnd + && isNullSegment(db,zTab,iLower) + ){ + printf(" null %9lld\n", iLower); + }else{ + printTreeLine(iLower, iPrev); + } + } + } + printf(" leaves %9lld thru %9lld (%lld blocks)\n", + iStart, iLEnd, iLEnd - iStart + 1); + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + sqlite3_finalize(pStmt2); +} + +/* +** Decode a single segment block and display the results on stdout. +*/ +static void decodeSegment( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iChild = 0; + sqlite3_int64 iPrefix; + sqlite3_int64 nTerm; + sqlite3_int64 n; + sqlite3_int64 iDocsz; + int iHeight; + sqlite3_int64 i = 0; + int cnt = 0; + char zTerm[1000]; + + i += getVarint(aData, &n); + iHeight = (int)n; + printf("height: %d\n", iHeight); + if( iHeight>0 ){ + i += getVarint(aData+i, &iChild); + printf("left-child: %lld\n", iChild); + } + while( i0 ){ + i += getVarint(aData+i, &iPrefix); + }else{ + iPrefix = 0; + } + i += getVarint(aData+i, &nTerm); + if( iPrefix+nTerm+1 >= sizeof(zTerm) ){ + fprintf(stderr, "term to long\n"); + exit(1); + } + memcpy(zTerm+iPrefix, aData+i, (size_t)nTerm); + zTerm[iPrefix+nTerm] = 0; + i += nTerm; + if( iHeight==0 ){ + i += getVarint(aData+i, &iDocsz); + printf("term: %-25s doclist %7lld bytes offset %lld\n", zTerm, iDocsz, i); + i += iDocsz; + }else{ + printf("term: %-25s child %lld\n", zTerm, ++iChild); + } + } +} + + +/* +** Print a a blob as hex and ascii. +*/ +static void printBlob( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + int i, j; + const char *zOfstFmt; + const int perLine = 16; + + if( (nData&~0xfff)==0 ){ + zOfstFmt = " %03x: "; + }else if( (nData&~0xffff)==0 ){ + zOfstFmt = " %04x: "; + }else if( (nData&~0xfffff)==0 ){ + zOfstFmt = " %05x: "; + }else if( (nData&~0xffffff)==0 ){ + zOfstFmt = " %06x: "; + }else{ + zOfstFmt = " %08x: "; + } + + for(i=0; inData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%02x ", aData[i+j]); + } + } + for(j=0; jnData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + } + } + fprintf(stdout,"\n"); + } +} + +/* +** Convert text to a 64-bit integer +*/ +static sqlite3_int64 atoi64(const char *z){ + sqlite3_int64 v = 0; + while( z[0]>='0' && z[0]<='9' ){ + v = v*10 + z[0] - '0'; + z++; + } + return v; +} + +/* +** Return a prepared statement which, when stepped, will return in its +** first column the blob associated with segment zId. If zId begins with +** 'r' then it is a rowid of a %_segdir entry. Otherwise it is a +** %_segment entry. +*/ +static sqlite3_stmt *prepareToGetSegment( + sqlite3 *db, /* The database */ + const char *zTab, /* The FTS3/4 table name */ + const char *zId /* ID of the segment to open */ +){ + sqlite3_stmt *pStmt; + if( zId[0]=='r' ){ + pStmt = prepare(db, "SELECT root FROM '%q_segdir' WHERE rowid=%lld", + zTab, atoi64(zId+1)); + }else{ + pStmt = prepare(db, "SELECT block FROM '%q_segments' WHERE blockid=%lld", + zTab, atoi64(zId)); + } + return pStmt; +} + +/* +** Print the content of a segment or of the root of a segdir. The segment +** or root is identified by azExtra[0]. If the first character of azExtra[0] +** is 'r' then the remainder is the integer rowid of the %_segdir entry. +** If the first character of azExtra[0] is not 'r' then, then all of +** azExtra[0] is an integer which is the block number. +** +** If the --raw option is present in azExtra, then a hex dump is provided. +** Otherwise a decoding is shown. +*/ +static void showSegment(sqlite3 *db, const char *zTab){ + const unsigned char *aData; + int nData; + sqlite3_stmt *pStmt; + + pStmt = prepareToGetSegment(db, zTab, azExtra[0]); + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + sqlite3_finalize(pStmt); + return; + } + nData = sqlite3_column_bytes(pStmt, 0); + aData = sqlite3_column_blob(pStmt, 0); + printf("Segment %s of size %d bytes:\n", azExtra[0], nData); + if( findOption("raw", 0, 0)!=0 ){ + printBlob(aData, nData); + }else{ + decodeSegment(aData, nData); + } + sqlite3_finalize(pStmt); +} + +/* +** Decode a single doclist and display the results on stdout. +*/ +static void decodeDoclist( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iPrevDocid = 0; + sqlite3_int64 iDocid; + sqlite3_int64 iPos; + sqlite3_int64 iPrevPos = 0; + sqlite3_int64 iCol; + int i = 0; + + while( i; ; ; # +# +# The status field is: +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# Note that the Turkic mappings do not maintain canonical equivalence without additional processing. +# See the discussions of case mapping in the Unicode Standard for more information. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) +# +# ================================================================= + +# Property: Case_Folding + +# All code points not explicitly listed for Case_Folding +# have the value C for the status field, and the code point itself for the mapping field. + +# @missing: 0000..10FFFF; C; + +# ================================================================= +0041; C; 0061; # LATIN CAPITAL LETTER A +0042; C; 0062; # LATIN CAPITAL LETTER B +0043; C; 0063; # LATIN CAPITAL LETTER C +0044; C; 0064; # LATIN CAPITAL LETTER D +0045; C; 0065; # LATIN CAPITAL LETTER E +0046; C; 0066; # LATIN CAPITAL LETTER F +0047; C; 0067; # LATIN CAPITAL LETTER G +0048; C; 0068; # LATIN CAPITAL LETTER H +0049; C; 0069; # LATIN CAPITAL LETTER I +0049; T; 0131; # LATIN CAPITAL LETTER I +004A; C; 006A; # LATIN CAPITAL LETTER J +004B; C; 006B; # LATIN CAPITAL LETTER K +004C; C; 006C; # LATIN CAPITAL LETTER L +004D; C; 006D; # LATIN CAPITAL LETTER M +004E; C; 006E; # LATIN CAPITAL LETTER N +004F; C; 006F; # LATIN CAPITAL LETTER O +0050; C; 0070; # LATIN CAPITAL LETTER P +0051; C; 0071; # LATIN CAPITAL LETTER Q +0052; C; 0072; # LATIN CAPITAL LETTER R +0053; C; 0073; # LATIN CAPITAL LETTER S +0054; C; 0074; # LATIN CAPITAL LETTER T +0055; C; 0075; # LATIN CAPITAL LETTER U +0056; C; 0076; # LATIN CAPITAL LETTER V +0057; C; 0077; # LATIN CAPITAL LETTER W +0058; C; 0078; # LATIN CAPITAL LETTER X +0059; C; 0079; # LATIN CAPITAL LETTER Y +005A; C; 007A; # LATIN CAPITAL LETTER Z +00B5; C; 03BC; # MICRO SIGN +00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE +00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE +00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE +00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS +00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE +00C6; C; 00E6; # LATIN CAPITAL LETTER AE +00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA +00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE +00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE +00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS +00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE +00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS +00D0; C; 00F0; # LATIN CAPITAL LETTER ETH +00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE +00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE +00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE +00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE +00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS +00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE +00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE +00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE +00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS +00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE +00DE; C; 00FE; # LATIN CAPITAL LETTER THORN +00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S +0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON +0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE +0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK +0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE +0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE +010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON +010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON +0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE +0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON +0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE +0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE +0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK +011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON +011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE +0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE +0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA +0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE +0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE +012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON +012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE +012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0132; C; 0133; # LATIN CAPITAL LIGATURE IJ +0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA +0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE +013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA +013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON +013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE +0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE +0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA +0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON +0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014A; C; 014B; # LATIN CAPITAL LETTER ENG +014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON +014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE +0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152; C; 0153; # LATIN CAPITAL LIGATURE OE +0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE +0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA +0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON +015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE +015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA +0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON +0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA +0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON +0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE +0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE +016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON +016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE +016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE +0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK +0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS +0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE +017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON +017F; C; 0073; # LATIN SMALL LETTER LONG S +0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK +0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR +0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX +0186; C; 0254; # LATIN CAPITAL LETTER OPEN O +0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK +0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D +018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK +018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR +018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E +018F; C; 0259; # LATIN CAPITAL LETTER SCHWA +0190; C; 025B; # LATIN CAPITAL LETTER OPEN E +0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK +0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK +0194; C; 0263; # LATIN CAPITAL LETTER GAMMA +0196; C; 0269; # LATIN CAPITAL LETTER IOTA +0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE +0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK +019C; C; 026F; # LATIN CAPITAL LETTER TURNED M +019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK +019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN +01A2; C; 01A3; # LATIN CAPITAL LETTER OI +01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK +01A6; C; 0280; # LATIN LETTER YR +01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO +01A9; C; 0283; # LATIN CAPITAL LETTER ESH +01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK +01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN +01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON +01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK +01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK +01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE +01B7; C; 0292; # LATIN CAPITAL LETTER EZH +01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED +01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE +01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON +01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7; C; 01C9; # LATIN CAPITAL LETTER LJ +01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA; C; 01CC; # LATIN CAPITAL LETTER NJ +01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON +01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON +01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON +01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON +01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON +01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE +01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON +01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON +01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK +01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON +01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON +01F1; C; 01F3; # LATIN CAPITAL LETTER DZ +01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE +01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR +01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN +01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE +01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE +01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW +021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW +021C; C; 021D; # LATIN CAPITAL LETTER YOGH +021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON +0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222; C; 0223; # LATIN CAPITAL LETTER OU +0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK +0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE +0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA +022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE +0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON +023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE +023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE +023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR +023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP +0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE +0244; C; 0289; # LATIN CAPITAL LETTER U BAR +0245; C; 028C; # LATIN CAPITAL LETTER TURNED V +0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE +0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE +024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE +024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE +0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI +0370; C; 0371; # GREEK CAPITAL LETTER HETA +0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI +0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS +0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS +0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS +038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS +038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS +038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS +038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS +0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA +0392; C; 03B2; # GREEK CAPITAL LETTER BETA +0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA +0394; C; 03B4; # GREEK CAPITAL LETTER DELTA +0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON +0396; C; 03B6; # GREEK CAPITAL LETTER ZETA +0397; C; 03B7; # GREEK CAPITAL LETTER ETA +0398; C; 03B8; # GREEK CAPITAL LETTER THETA +0399; C; 03B9; # GREEK CAPITAL LETTER IOTA +039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA +039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA +039C; C; 03BC; # GREEK CAPITAL LETTER MU +039D; C; 03BD; # GREEK CAPITAL LETTER NU +039E; C; 03BE; # GREEK CAPITAL LETTER XI +039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON +03A0; C; 03C0; # GREEK CAPITAL LETTER PI +03A1; C; 03C1; # GREEK CAPITAL LETTER RHO +03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA +03A4; C; 03C4; # GREEK CAPITAL LETTER TAU +03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON +03A6; C; 03C6; # GREEK CAPITAL LETTER PHI +03A7; C; 03C7; # GREEK CAPITAL LETTER CHI +03A8; C; 03C8; # GREEK CAPITAL LETTER PSI +03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA +03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA +03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL +03D0; C; 03B2; # GREEK BETA SYMBOL +03D1; C; 03B8; # GREEK THETA SYMBOL +03D5; C; 03C6; # GREEK PHI SYMBOL +03D6; C; 03C0; # GREEK PI SYMBOL +03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA +03DA; C; 03DB; # GREEK LETTER STIGMA +03DC; C; 03DD; # GREEK LETTER DIGAMMA +03DE; C; 03DF; # GREEK LETTER KOPPA +03E0; C; 03E1; # GREEK LETTER SAMPI +03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI +03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI +03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI +03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI +03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA +03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA +03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI +03F0; C; 03BA; # GREEK KAPPA SYMBOL +03F1; C; 03C1; # GREEK RHO SYMBOL +03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL +03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL +03F7; C; 03F8; # GREEK CAPITAL LETTER SHO +03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL +03FA; C; 03FB; # GREEK CAPITAL LETTER SAN +03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL +03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL +03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401; C; 0451; # CYRILLIC CAPITAL LETTER IO +0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE +0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE +0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE +0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE +0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +0407; C; 0457; # CYRILLIC CAPITAL LETTER YI +0408; C; 0458; # CYRILLIC CAPITAL LETTER JE +0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE +040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE +040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE +040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE +040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE +040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U +040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE +0410; C; 0430; # CYRILLIC CAPITAL LETTER A +0411; C; 0431; # CYRILLIC CAPITAL LETTER BE +0412; C; 0432; # CYRILLIC CAPITAL LETTER VE +0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE +0414; C; 0434; # CYRILLIC CAPITAL LETTER DE +0415; C; 0435; # CYRILLIC CAPITAL LETTER IE +0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE +0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE +0418; C; 0438; # CYRILLIC CAPITAL LETTER I +0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I +041A; C; 043A; # CYRILLIC CAPITAL LETTER KA +041B; C; 043B; # CYRILLIC CAPITAL LETTER EL +041C; C; 043C; # CYRILLIC CAPITAL LETTER EM +041D; C; 043D; # CYRILLIC CAPITAL LETTER EN +041E; C; 043E; # CYRILLIC CAPITAL LETTER O +041F; C; 043F; # CYRILLIC CAPITAL LETTER PE +0420; C; 0440; # CYRILLIC CAPITAL LETTER ER +0421; C; 0441; # CYRILLIC CAPITAL LETTER ES +0422; C; 0442; # CYRILLIC CAPITAL LETTER TE +0423; C; 0443; # CYRILLIC CAPITAL LETTER U +0424; C; 0444; # CYRILLIC CAPITAL LETTER EF +0425; C; 0445; # CYRILLIC CAPITAL LETTER HA +0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE +0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE +0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA +0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA +042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN +042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU +042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN +042D; C; 044D; # CYRILLIC CAPITAL LETTER E +042E; C; 044E; # CYRILLIC CAPITAL LETTER YU +042F; C; 044F; # CYRILLIC CAPITAL LETTER YA +0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA +0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT +0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E +0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS +0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS +046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI +0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI +0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA +0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA +0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478; C; 0479; # CYRILLIC CAPITAL LETTER UK +047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA +047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E; C; 047F; # CYRILLIC CAPITAL LETTER OT +0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA +048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK +0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA +04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE +04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U +04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE +04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA +04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA +04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE +04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE +04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA +04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON +04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O +04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON +04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE +0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE +0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE +0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE +0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE +0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE +050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE +050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE +050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE +0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE +0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK +0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA +0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA +0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE +051A; C; 051B; # CYRILLIC CAPITAL LETTER QA +051C; C; 051D; # CYRILLIC CAPITAL LETTER WE +051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA +0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER +0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB +0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN +0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM +0534; C; 0564; # ARMENIAN CAPITAL LETTER DA +0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH +0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA +0537; C; 0567; # ARMENIAN CAPITAL LETTER EH +0538; C; 0568; # ARMENIAN CAPITAL LETTER ET +0539; C; 0569; # ARMENIAN CAPITAL LETTER TO +053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE +053B; C; 056B; # ARMENIAN CAPITAL LETTER INI +053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN +053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH +053E; C; 056E; # ARMENIAN CAPITAL LETTER CA +053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN +0540; C; 0570; # ARMENIAN CAPITAL LETTER HO +0541; C; 0571; # ARMENIAN CAPITAL LETTER JA +0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD +0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH +0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN +0545; C; 0575; # ARMENIAN CAPITAL LETTER YI +0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW +0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA +0548; C; 0578; # ARMENIAN CAPITAL LETTER VO +0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA +054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH +054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH +054C; C; 057C; # ARMENIAN CAPITAL LETTER RA +054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH +054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW +054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN +0550; C; 0580; # ARMENIAN CAPITAL LETTER REH +0551; C; 0581; # ARMENIAN CAPITAL LETTER CO +0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN +0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR +0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH +0555; C; 0585; # ARMENIAN CAPITAL LETTER OH +0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH +0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN +10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN +10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN +10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN +10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON +10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN +10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN +10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN +10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN +10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN +10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN +10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS +10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN +10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR +10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON +10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR +10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR +10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE +10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN +10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR +10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN +10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR +10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR +10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN +10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR +10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN +10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN +10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN +10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL +10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL +10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR +10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN +10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN +10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE +10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE +10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE +10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE +10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR +10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE +10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN +10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN +1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW +1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW +1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW +1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW +1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA +1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON +1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW +1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS +1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA +1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE +1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW +1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW +1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW +1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE +1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW +1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW +1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE +1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW +1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW +1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW +1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE +1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW +1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE +1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE +1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS +1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW +1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S +1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S +1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE +1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE +1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE +1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP +1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI +1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA +1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI +1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE; C; 03B9; # GREEK PROSGEGRAMMENI +1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA +1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA +1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA +1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA +1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126; C; 03C9; # OHM SIGN +212A; C; 006B; # KELVIN SIGN +212B; C; 00E5; # ANGSTROM SIGN +2132; C; 214E; # TURNED CAPITAL F +2160; C; 2170; # ROMAN NUMERAL ONE +2161; C; 2171; # ROMAN NUMERAL TWO +2162; C; 2172; # ROMAN NUMERAL THREE +2163; C; 2173; # ROMAN NUMERAL FOUR +2164; C; 2174; # ROMAN NUMERAL FIVE +2165; C; 2175; # ROMAN NUMERAL SIX +2166; C; 2176; # ROMAN NUMERAL SEVEN +2167; C; 2177; # ROMAN NUMERAL EIGHT +2168; C; 2178; # ROMAN NUMERAL NINE +2169; C; 2179; # ROMAN NUMERAL TEN +216A; C; 217A; # ROMAN NUMERAL ELEVEN +216B; C; 217B; # ROMAN NUMERAL TWELVE +216C; C; 217C; # ROMAN NUMERAL FIFTY +216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED +216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED +216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND +2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED +24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A +24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B +24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C +24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D +24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E +24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F +24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G +24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H +24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I +24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J +24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K +24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L +24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M +24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N +24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O +24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P +24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q +24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R +24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S +24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T +24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U +24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V +24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W +24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X +24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y +24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z +2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU +2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY +2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE +2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI +2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO +2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU +2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE +2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO +2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA +2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE +2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE +2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I +2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI +2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO +2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE +2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE +2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI +2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU +2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI +2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI +2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO +2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO +2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU +2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU +2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU +2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU +2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE +2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA +2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI +2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI +2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA +2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU +2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI +2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI +2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA +2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU +2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS +2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL +2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO +2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS +2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS +2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS +2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA +2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA +2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC +2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A +2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE +2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE +2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL +2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER +2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER +2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA +2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK +2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A +2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA +2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK +2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H +2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL +2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA +2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA +2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA +2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA +2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE +2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU +2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA +2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE +2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE +2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA +2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA +2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA +2C98; C; 2C99; # COPTIC CAPITAL LETTER MI +2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI +2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI +2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O +2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI +2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO +2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA +2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU +2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA +2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI +2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI +2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI +2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU +2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI +2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI +2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI +2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA +2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI +A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA +A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO +A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE +A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA +A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV +A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER +A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU +A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A +A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS +A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN +A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE +A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE +A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL +A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM +A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O +A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O +A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680; C; A681; # CYRILLIC CAPITAL LETTER DWE +A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE +A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE +A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE +A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE +A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE +A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE +A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE +A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE +A694; C; A695; # CYRILLIC CAPITAL LETTER HWE +A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE +A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726; C; A727; # LATIN CAPITAL LETTER HENG +A728; C; A729; # LATIN CAPITAL LETTER TZ +A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO +A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO +A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732; C; A733; # LATIN CAPITAL LETTER AA +A734; C; A735; # LATIN CAPITAL LETTER AO +A736; C; A737; # LATIN CAPITAL LETTER AU +A738; C; A739; # LATIN CAPITAL LETTER AV +A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C; C; A73D; # LATIN CAPITAL LETTER AY +A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT +A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE +A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746; C; A747; # LATIN CAPITAL LETTER BROKEN L +A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP +A74E; C; A74F; # LATIN CAPITAL LETTER OO +A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH +A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA +A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA +A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760; C; A761; # LATIN CAPITAL LETTER VY +A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z +A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE +A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768; C; A769; # LATIN CAPITAL LETTER VEND +A76A; C; A76B; # LATIN CAPITAL LETTER ET +A76C; C; A76D; # LATIN CAPITAL LETTER IS +A76E; C; A76F; # LATIN CAPITAL LETTER CON +A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D +A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F +A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G +A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G +A780; C; A781; # LATIN CAPITAL LETTER TURNED L +A782; C; A783; # LATIN CAPITAL LETTER INSULAR R +A784; C; A785; # LATIN CAPITAL LETTER INSULAR S +A786; C; A787; # LATIN CAPITAL LETTER INSULAR T +A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO +A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H +A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER +A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR +A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE +A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE +A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE +A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE +A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE +A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK +FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF +FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI +FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL +FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI +FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL +FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST +FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH +FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A +FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B +FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C +FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D +FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E +FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F +FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G +FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H +FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I +FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J +FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K +FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L +FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M +FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N +FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O +FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P +FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q +FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R +FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S +FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T +FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U +FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V +FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W +FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X +FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y +FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z +10400; C; 10428; # DESERET CAPITAL LETTER LONG I +10401; C; 10429; # DESERET CAPITAL LETTER LONG E +10402; C; 1042A; # DESERET CAPITAL LETTER LONG A +10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH +10404; C; 1042C; # DESERET CAPITAL LETTER LONG O +10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO +10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I +10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E +10408; C; 10430; # DESERET CAPITAL LETTER SHORT A +10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH +1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O +1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO +1040C; C; 10434; # DESERET CAPITAL LETTER AY +1040D; C; 10435; # DESERET CAPITAL LETTER OW +1040E; C; 10436; # DESERET CAPITAL LETTER WU +1040F; C; 10437; # DESERET CAPITAL LETTER YEE +10410; C; 10438; # DESERET CAPITAL LETTER H +10411; C; 10439; # DESERET CAPITAL LETTER PEE +10412; C; 1043A; # DESERET CAPITAL LETTER BEE +10413; C; 1043B; # DESERET CAPITAL LETTER TEE +10414; C; 1043C; # DESERET CAPITAL LETTER DEE +10415; C; 1043D; # DESERET CAPITAL LETTER CHEE +10416; C; 1043E; # DESERET CAPITAL LETTER JEE +10417; C; 1043F; # DESERET CAPITAL LETTER KAY +10418; C; 10440; # DESERET CAPITAL LETTER GAY +10419; C; 10441; # DESERET CAPITAL LETTER EF +1041A; C; 10442; # DESERET CAPITAL LETTER VEE +1041B; C; 10443; # DESERET CAPITAL LETTER ETH +1041C; C; 10444; # DESERET CAPITAL LETTER THEE +1041D; C; 10445; # DESERET CAPITAL LETTER ES +1041E; C; 10446; # DESERET CAPITAL LETTER ZEE +1041F; C; 10447; # DESERET CAPITAL LETTER ESH +10420; C; 10448; # DESERET CAPITAL LETTER ZHEE +10421; C; 10449; # DESERET CAPITAL LETTER ER +10422; C; 1044A; # DESERET CAPITAL LETTER EL +10423; C; 1044B; # DESERET CAPITAL LETTER EM +10424; C; 1044C; # DESERET CAPITAL LETTER EN +10425; C; 1044D; # DESERET CAPITAL LETTER ENG +10426; C; 1044E; # DESERET CAPITAL LETTER OI +10427; C; 1044F; # DESERET CAPITAL LETTER EW diff --git a/ext/fts3/unicode/UnicodeData.txt b/ext/fts3/unicode/UnicodeData.txt new file mode 100644 index 0000000000000000000000000000000000000000..9f204050c6bb2e587cb1c3154372b8131f9e2f28 --- /dev/null +++ b/ext/fts3/unicode/UnicodeData.txt @@ -0,0 +1,24428 @@ +0000;;Cc;0;BN;;;;;N;NULL;;;; +0001;;Cc;0;BN;;;;;N;START OF HEADING;;;; +0002;;Cc;0;BN;;;;;N;START OF TEXT;;;; +0003;;Cc;0;BN;;;;;N;END OF TEXT;;;; +0004;;Cc;0;BN;;;;;N;END OF TRANSMISSION;;;; +0005;;Cc;0;BN;;;;;N;ENQUIRY;;;; +0006;;Cc;0;BN;;;;;N;ACKNOWLEDGE;;;; +0007;;Cc;0;BN;;;;;N;BELL;;;; +0008;;Cc;0;BN;;;;;N;BACKSPACE;;;; +0009;;Cc;0;S;;;;;N;CHARACTER TABULATION;;;; +000A;;Cc;0;B;;;;;N;LINE FEED (LF);;;; +000B;;Cc;0;S;;;;;N;LINE TABULATION;;;; +000C;;Cc;0;WS;;;;;N;FORM FEED (FF);;;; +000D;;Cc;0;B;;;;;N;CARRIAGE RETURN (CR);;;; +000E;;Cc;0;BN;;;;;N;SHIFT OUT;;;; +000F;;Cc;0;BN;;;;;N;SHIFT IN;;;; +0010;;Cc;0;BN;;;;;N;DATA LINK ESCAPE;;;; +0011;;Cc;0;BN;;;;;N;DEVICE CONTROL ONE;;;; +0012;;Cc;0;BN;;;;;N;DEVICE CONTROL TWO;;;; +0013;;Cc;0;BN;;;;;N;DEVICE CONTROL THREE;;;; +0014;;Cc;0;BN;;;;;N;DEVICE CONTROL FOUR;;;; +0015;;Cc;0;BN;;;;;N;NEGATIVE ACKNOWLEDGE;;;; +0016;;Cc;0;BN;;;;;N;SYNCHRONOUS IDLE;;;; +0017;;Cc;0;BN;;;;;N;END OF TRANSMISSION BLOCK;;;; +0018;;Cc;0;BN;;;;;N;CANCEL;;;; +0019;;Cc;0;BN;;;;;N;END OF MEDIUM;;;; +001A;;Cc;0;BN;;;;;N;SUBSTITUTE;;;; +001B;;Cc;0;BN;;;;;N;ESCAPE;;;; +001C;;Cc;0;B;;;;;N;INFORMATION SEPARATOR FOUR;;;; +001D;;Cc;0;B;;;;;N;INFORMATION SEPARATOR THREE;;;; +001E;;Cc;0;B;;;;;N;INFORMATION SEPARATOR TWO;;;; +001F;;Cc;0;S;;;;;N;INFORMATION SEPARATOR ONE;;;; +0020;SPACE;Zs;0;WS;;;;;N;;;;; +0021;EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; +0022;QUOTATION MARK;Po;0;ON;;;;;N;;;;; +0023;NUMBER SIGN;Po;0;ET;;;;;N;;;;; +0024;DOLLAR SIGN;Sc;0;ET;;;;;N;;;;; +0025;PERCENT SIGN;Po;0;ET;;;;;N;;;;; +0026;AMPERSAND;Po;0;ON;;;;;N;;;;; +0027;APOSTROPHE;Po;0;ON;;;;;N;APOSTROPHE-QUOTE;;;; +0028;LEFT PARENTHESIS;Ps;0;ON;;;;;Y;OPENING PARENTHESIS;;;; +0029;RIGHT PARENTHESIS;Pe;0;ON;;;;;Y;CLOSING PARENTHESIS;;;; +002A;ASTERISK;Po;0;ON;;;;;N;;;;; +002B;PLUS SIGN;Sm;0;ES;;;;;N;;;;; +002C;COMMA;Po;0;CS;;;;;N;;;;; +002D;HYPHEN-MINUS;Pd;0;ES;;;;;N;;;;; +002E;FULL STOP;Po;0;CS;;;;;N;PERIOD;;;; +002F;SOLIDUS;Po;0;CS;;;;;N;SLASH;;;; +0030;DIGIT ZERO;Nd;0;EN;;0;0;0;N;;;;; +0031;DIGIT ONE;Nd;0;EN;;1;1;1;N;;;;; +0032;DIGIT TWO;Nd;0;EN;;2;2;2;N;;;;; +0033;DIGIT THREE;Nd;0;EN;;3;3;3;N;;;;; +0034;DIGIT FOUR;Nd;0;EN;;4;4;4;N;;;;; +0035;DIGIT FIVE;Nd;0;EN;;5;5;5;N;;;;; +0036;DIGIT SIX;Nd;0;EN;;6;6;6;N;;;;; +0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;; +0038;DIGIT EIGHT;Nd;0;EN;;8;8;8;N;;;;; +0039;DIGIT NINE;Nd;0;EN;;9;9;9;N;;;;; +003A;COLON;Po;0;CS;;;;;N;;;;; +003B;SEMICOLON;Po;0;ON;;;;;N;;;;; +003C;LESS-THAN SIGN;Sm;0;ON;;;;;Y;;;;; +003D;EQUALS SIGN;Sm;0;ON;;;;;N;;;;; +003E;GREATER-THAN SIGN;Sm;0;ON;;;;;Y;;;;; +003F;QUESTION MARK;Po;0;ON;;;;;N;;;;; +0040;COMMERCIAL AT;Po;0;ON;;;;;N;;;;; +0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061; +0042;LATIN CAPITAL LETTER B;Lu;0;L;;;;;N;;;;0062; +0043;LATIN CAPITAL LETTER C;Lu;0;L;;;;;N;;;;0063; +0044;LATIN CAPITAL LETTER D;Lu;0;L;;;;;N;;;;0064; +0045;LATIN CAPITAL LETTER E;Lu;0;L;;;;;N;;;;0065; +0046;LATIN CAPITAL LETTER F;Lu;0;L;;;;;N;;;;0066; +0047;LATIN CAPITAL LETTER G;Lu;0;L;;;;;N;;;;0067; +0048;LATIN CAPITAL LETTER H;Lu;0;L;;;;;N;;;;0068; +0049;LATIN CAPITAL LETTER I;Lu;0;L;;;;;N;;;;0069; +004A;LATIN CAPITAL LETTER J;Lu;0;L;;;;;N;;;;006A; +004B;LATIN CAPITAL LETTER K;Lu;0;L;;;;;N;;;;006B; +004C;LATIN CAPITAL LETTER L;Lu;0;L;;;;;N;;;;006C; +004D;LATIN CAPITAL LETTER M;Lu;0;L;;;;;N;;;;006D; +004E;LATIN CAPITAL LETTER N;Lu;0;L;;;;;N;;;;006E; +004F;LATIN CAPITAL LETTER O;Lu;0;L;;;;;N;;;;006F; +0050;LATIN CAPITAL LETTER P;Lu;0;L;;;;;N;;;;0070; +0051;LATIN CAPITAL LETTER Q;Lu;0;L;;;;;N;;;;0071; +0052;LATIN CAPITAL LETTER R;Lu;0;L;;;;;N;;;;0072; +0053;LATIN CAPITAL LETTER S;Lu;0;L;;;;;N;;;;0073; +0054;LATIN CAPITAL LETTER T;Lu;0;L;;;;;N;;;;0074; +0055;LATIN CAPITAL LETTER U;Lu;0;L;;;;;N;;;;0075; +0056;LATIN CAPITAL LETTER V;Lu;0;L;;;;;N;;;;0076; +0057;LATIN CAPITAL LETTER W;Lu;0;L;;;;;N;;;;0077; +0058;LATIN CAPITAL LETTER X;Lu;0;L;;;;;N;;;;0078; +0059;LATIN CAPITAL LETTER Y;Lu;0;L;;;;;N;;;;0079; +005A;LATIN CAPITAL LETTER Z;Lu;0;L;;;;;N;;;;007A; +005B;LEFT SQUARE BRACKET;Ps;0;ON;;;;;Y;OPENING SQUARE BRACKET;;;; +005C;REVERSE SOLIDUS;Po;0;ON;;;;;N;BACKSLASH;;;; +005D;RIGHT SQUARE BRACKET;Pe;0;ON;;;;;Y;CLOSING SQUARE BRACKET;;;; +005E;CIRCUMFLEX ACCENT;Sk;0;ON;;;;;N;SPACING CIRCUMFLEX;;;; +005F;LOW LINE;Pc;0;ON;;;;;N;SPACING UNDERSCORE;;;; +0060;GRAVE ACCENT;Sk;0;ON;;;;;N;SPACING GRAVE;;;; +0061;LATIN SMALL LETTER A;Ll;0;L;;;;;N;;;0041;;0041 +0062;LATIN SMALL LETTER B;Ll;0;L;;;;;N;;;0042;;0042 +0063;LATIN SMALL LETTER C;Ll;0;L;;;;;N;;;0043;;0043 +0064;LATIN SMALL LETTER D;Ll;0;L;;;;;N;;;0044;;0044 +0065;LATIN SMALL LETTER E;Ll;0;L;;;;;N;;;0045;;0045 +0066;LATIN SMALL LETTER F;Ll;0;L;;;;;N;;;0046;;0046 +0067;LATIN SMALL LETTER G;Ll;0;L;;;;;N;;;0047;;0047 +0068;LATIN SMALL LETTER H;Ll;0;L;;;;;N;;;0048;;0048 +0069;LATIN SMALL LETTER I;Ll;0;L;;;;;N;;;0049;;0049 +006A;LATIN SMALL LETTER J;Ll;0;L;;;;;N;;;004A;;004A +006B;LATIN SMALL LETTER K;Ll;0;L;;;;;N;;;004B;;004B +006C;LATIN SMALL LETTER L;Ll;0;L;;;;;N;;;004C;;004C +006D;LATIN SMALL LETTER M;Ll;0;L;;;;;N;;;004D;;004D +006E;LATIN SMALL LETTER N;Ll;0;L;;;;;N;;;004E;;004E +006F;LATIN SMALL LETTER O;Ll;0;L;;;;;N;;;004F;;004F +0070;LATIN SMALL LETTER P;Ll;0;L;;;;;N;;;0050;;0050 +0071;LATIN SMALL LETTER Q;Ll;0;L;;;;;N;;;0051;;0051 +0072;LATIN SMALL LETTER R;Ll;0;L;;;;;N;;;0052;;0052 +0073;LATIN SMALL LETTER S;Ll;0;L;;;;;N;;;0053;;0053 +0074;LATIN SMALL LETTER T;Ll;0;L;;;;;N;;;0054;;0054 +0075;LATIN SMALL LETTER U;Ll;0;L;;;;;N;;;0055;;0055 +0076;LATIN SMALL LETTER V;Ll;0;L;;;;;N;;;0056;;0056 +0077;LATIN SMALL LETTER W;Ll;0;L;;;;;N;;;0057;;0057 +0078;LATIN SMALL LETTER X;Ll;0;L;;;;;N;;;0058;;0058 +0079;LATIN SMALL LETTER Y;Ll;0;L;;;;;N;;;0059;;0059 +007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A +007B;LEFT CURLY BRACKET;Ps;0;ON;;;;;Y;OPENING CURLY BRACKET;;;; +007C;VERTICAL LINE;Sm;0;ON;;;;;N;VERTICAL BAR;;;; +007D;RIGHT CURLY BRACKET;Pe;0;ON;;;;;Y;CLOSING CURLY BRACKET;;;; +007E;TILDE;Sm;0;ON;;;;;N;;;;; +007F;;Cc;0;BN;;;;;N;DELETE;;;; +0080;;Cc;0;BN;;;;;N;;;;; +0081;;Cc;0;BN;;;;;N;;;;; +0082;;Cc;0;BN;;;;;N;BREAK PERMITTED HERE;;;; +0083;;Cc;0;BN;;;;;N;NO BREAK HERE;;;; +0084;;Cc;0;BN;;;;;N;;;;; +0085;;Cc;0;B;;;;;N;NEXT LINE (NEL);;;; +0086;;Cc;0;BN;;;;;N;START OF SELECTED AREA;;;; +0087;;Cc;0;BN;;;;;N;END OF SELECTED AREA;;;; +0088;;Cc;0;BN;;;;;N;CHARACTER TABULATION SET;;;; +0089;;Cc;0;BN;;;;;N;CHARACTER TABULATION WITH JUSTIFICATION;;;; +008A;;Cc;0;BN;;;;;N;LINE TABULATION SET;;;; +008B;;Cc;0;BN;;;;;N;PARTIAL LINE FORWARD;;;; +008C;;Cc;0;BN;;;;;N;PARTIAL LINE BACKWARD;;;; +008D;;Cc;0;BN;;;;;N;REVERSE LINE FEED;;;; +008E;;Cc;0;BN;;;;;N;SINGLE SHIFT TWO;;;; +008F;;Cc;0;BN;;;;;N;SINGLE SHIFT THREE;;;; +0090;;Cc;0;BN;;;;;N;DEVICE CONTROL STRING;;;; +0091;;Cc;0;BN;;;;;N;PRIVATE USE ONE;;;; +0092;;Cc;0;BN;;;;;N;PRIVATE USE TWO;;;; +0093;;Cc;0;BN;;;;;N;SET TRANSMIT STATE;;;; +0094;;Cc;0;BN;;;;;N;CANCEL CHARACTER;;;; +0095;;Cc;0;BN;;;;;N;MESSAGE WAITING;;;; +0096;;Cc;0;BN;;;;;N;START OF GUARDED AREA;;;; +0097;;Cc;0;BN;;;;;N;END OF GUARDED AREA;;;; +0098;;Cc;0;BN;;;;;N;START OF STRING;;;; +0099;;Cc;0;BN;;;;;N;;;;; +009A;;Cc;0;BN;;;;;N;SINGLE CHARACTER INTRODUCER;;;; +009B;;Cc;0;BN;;;;;N;CONTROL SEQUENCE INTRODUCER;;;; +009C;;Cc;0;BN;;;;;N;STRING TERMINATOR;;;; +009D;;Cc;0;BN;;;;;N;OPERATING SYSTEM COMMAND;;;; +009E;;Cc;0;BN;;;;;N;PRIVACY MESSAGE;;;; +009F;;Cc;0;BN;;;;;N;APPLICATION PROGRAM COMMAND;;;; +00A0;NO-BREAK SPACE;Zs;0;CS; 0020;;;;N;NON-BREAKING SPACE;;;; +00A1;INVERTED EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; +00A2;CENT SIGN;Sc;0;ET;;;;;N;;;;; +00A3;POUND SIGN;Sc;0;ET;;;;;N;;;;; +00A4;CURRENCY SIGN;Sc;0;ET;;;;;N;;;;; +00A5;YEN SIGN;Sc;0;ET;;;;;N;;;;; +00A6;BROKEN BAR;So;0;ON;;;;;N;BROKEN VERTICAL BAR;;;; +00A7;SECTION SIGN;Po;0;ON;;;;;N;;;;; +00A8;DIAERESIS;Sk;0;ON; 0020 0308;;;;N;SPACING DIAERESIS;;;; +00A9;COPYRIGHT SIGN;So;0;ON;;;;;N;;;;; +00AA;FEMININE ORDINAL INDICATOR;Lo;0;L; 0061;;;;N;;;;; +00AB;LEFT-POINTING DOUBLE ANGLE QUOTATION MARK;Pi;0;ON;;;;;Y;LEFT POINTING GUILLEMET;;;; +00AC;NOT SIGN;Sm;0;ON;;;;;N;;;;; +00AD;SOFT HYPHEN;Cf;0;BN;;;;;N;;;;; +00AE;REGISTERED SIGN;So;0;ON;;;;;N;REGISTERED TRADE MARK SIGN;;;; +00AF;MACRON;Sk;0;ON; 0020 0304;;;;N;SPACING MACRON;;;; +00B0;DEGREE SIGN;So;0;ET;;;;;N;;;;; +00B1;PLUS-MINUS SIGN;Sm;0;ET;;;;;N;PLUS-OR-MINUS SIGN;;;; +00B2;SUPERSCRIPT TWO;No;0;EN; 0032;;2;2;N;SUPERSCRIPT DIGIT TWO;;;; +00B3;SUPERSCRIPT THREE;No;0;EN; 0033;;3;3;N;SUPERSCRIPT DIGIT THREE;;;; +00B4;ACUTE ACCENT;Sk;0;ON; 0020 0301;;;;N;SPACING ACUTE;;;; +00B5;MICRO SIGN;Ll;0;L; 03BC;;;;N;;;039C;;039C +00B6;PILCROW SIGN;Po;0;ON;;;;;N;PARAGRAPH SIGN;;;; +00B7;MIDDLE DOT;Po;0;ON;;;;;N;;;;; +00B8;CEDILLA;Sk;0;ON; 0020 0327;;;;N;SPACING CEDILLA;;;; +00B9;SUPERSCRIPT ONE;No;0;EN; 0031;;1;1;N;SUPERSCRIPT DIGIT ONE;;;; +00BA;MASCULINE ORDINAL INDICATOR;Lo;0;L; 006F;;;;N;;;;; +00BB;RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK;Pf;0;ON;;;;;Y;RIGHT POINTING GUILLEMET;;;; +00BC;VULGAR FRACTION ONE QUARTER;No;0;ON; 0031 2044 0034;;;1/4;N;FRACTION ONE QUARTER;;;; +00BD;VULGAR FRACTION ONE HALF;No;0;ON; 0031 2044 0032;;;1/2;N;FRACTION ONE HALF;;;; +00BE;VULGAR FRACTION THREE QUARTERS;No;0;ON; 0033 2044 0034;;;3/4;N;FRACTION THREE QUARTERS;;;; +00BF;INVERTED QUESTION MARK;Po;0;ON;;;;;N;;;;; +00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;0;L;0041 0300;;;;N;LATIN CAPITAL LETTER A GRAVE;;;00E0; +00C1;LATIN CAPITAL LETTER A WITH ACUTE;Lu;0;L;0041 0301;;;;N;LATIN CAPITAL LETTER A ACUTE;;;00E1; +00C2;LATIN CAPITAL LETTER A WITH CIRCUMFLEX;Lu;0;L;0041 0302;;;;N;LATIN CAPITAL LETTER A CIRCUMFLEX;;;00E2; +00C3;LATIN CAPITAL LETTER A WITH TILDE;Lu;0;L;0041 0303;;;;N;LATIN CAPITAL LETTER A TILDE;;;00E3; +00C4;LATIN CAPITAL LETTER A WITH DIAERESIS;Lu;0;L;0041 0308;;;;N;LATIN CAPITAL LETTER A DIAERESIS;;;00E4; +00C5;LATIN CAPITAL LETTER A WITH RING ABOVE;Lu;0;L;0041 030A;;;;N;LATIN CAPITAL LETTER A RING;;;00E5; +00C6;LATIN CAPITAL LETTER AE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER A E;;;00E6; +00C7;LATIN CAPITAL LETTER C WITH CEDILLA;Lu;0;L;0043 0327;;;;N;LATIN CAPITAL LETTER C CEDILLA;;;00E7; +00C8;LATIN CAPITAL LETTER E WITH GRAVE;Lu;0;L;0045 0300;;;;N;LATIN CAPITAL LETTER E GRAVE;;;00E8; +00C9;LATIN CAPITAL LETTER E WITH ACUTE;Lu;0;L;0045 0301;;;;N;LATIN CAPITAL LETTER E ACUTE;;;00E9; +00CA;LATIN CAPITAL LETTER E WITH CIRCUMFLEX;Lu;0;L;0045 0302;;;;N;LATIN CAPITAL LETTER E CIRCUMFLEX;;;00EA; +00CB;LATIN CAPITAL LETTER E WITH DIAERESIS;Lu;0;L;0045 0308;;;;N;LATIN CAPITAL LETTER E DIAERESIS;;;00EB; +00CC;LATIN CAPITAL LETTER I WITH GRAVE;Lu;0;L;0049 0300;;;;N;LATIN CAPITAL LETTER I GRAVE;;;00EC; +00CD;LATIN CAPITAL LETTER I WITH ACUTE;Lu;0;L;0049 0301;;;;N;LATIN CAPITAL LETTER I ACUTE;;;00ED; +00CE;LATIN CAPITAL LETTER I WITH CIRCUMFLEX;Lu;0;L;0049 0302;;;;N;LATIN CAPITAL LETTER I CIRCUMFLEX;;;00EE; +00CF;LATIN CAPITAL LETTER I WITH DIAERESIS;Lu;0;L;0049 0308;;;;N;LATIN CAPITAL LETTER I DIAERESIS;;;00EF; +00D0;LATIN CAPITAL LETTER ETH;Lu;0;L;;;;;N;;;;00F0; +00D1;LATIN CAPITAL LETTER N WITH TILDE;Lu;0;L;004E 0303;;;;N;LATIN CAPITAL LETTER N TILDE;;;00F1; +00D2;LATIN CAPITAL LETTER O WITH GRAVE;Lu;0;L;004F 0300;;;;N;LATIN CAPITAL LETTER O GRAVE;;;00F2; +00D3;LATIN CAPITAL LETTER O WITH ACUTE;Lu;0;L;004F 0301;;;;N;LATIN CAPITAL LETTER O ACUTE;;;00F3; +00D4;LATIN CAPITAL LETTER O WITH CIRCUMFLEX;Lu;0;L;004F 0302;;;;N;LATIN CAPITAL LETTER O CIRCUMFLEX;;;00F4; +00D5;LATIN CAPITAL LETTER O WITH TILDE;Lu;0;L;004F 0303;;;;N;LATIN CAPITAL LETTER O TILDE;;;00F5; +00D6;LATIN CAPITAL LETTER O WITH DIAERESIS;Lu;0;L;004F 0308;;;;N;LATIN CAPITAL LETTER O DIAERESIS;;;00F6; +00D7;MULTIPLICATION SIGN;Sm;0;ON;;;;;N;;;;; +00D8;LATIN CAPITAL LETTER O WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER O SLASH;;;00F8; +00D9;LATIN CAPITAL LETTER U WITH GRAVE;Lu;0;L;0055 0300;;;;N;LATIN CAPITAL LETTER U GRAVE;;;00F9; +00DA;LATIN CAPITAL LETTER U WITH ACUTE;Lu;0;L;0055 0301;;;;N;LATIN CAPITAL LETTER U ACUTE;;;00FA; +00DB;LATIN CAPITAL LETTER U WITH CIRCUMFLEX;Lu;0;L;0055 0302;;;;N;LATIN CAPITAL LETTER U CIRCUMFLEX;;;00FB; +00DC;LATIN CAPITAL LETTER U WITH DIAERESIS;Lu;0;L;0055 0308;;;;N;LATIN CAPITAL LETTER U DIAERESIS;;;00FC; +00DD;LATIN CAPITAL LETTER Y WITH ACUTE;Lu;0;L;0059 0301;;;;N;LATIN CAPITAL LETTER Y ACUTE;;;00FD; +00DE;LATIN CAPITAL LETTER THORN;Lu;0;L;;;;;N;;;;00FE; +00DF;LATIN SMALL LETTER SHARP S;Ll;0;L;;;;;N;;;;; +00E0;LATIN SMALL LETTER A WITH GRAVE;Ll;0;L;0061 0300;;;;N;LATIN SMALL LETTER A GRAVE;;00C0;;00C0 +00E1;LATIN SMALL LETTER A WITH ACUTE;Ll;0;L;0061 0301;;;;N;LATIN SMALL LETTER A ACUTE;;00C1;;00C1 +00E2;LATIN SMALL LETTER A WITH CIRCUMFLEX;Ll;0;L;0061 0302;;;;N;LATIN SMALL LETTER A CIRCUMFLEX;;00C2;;00C2 +00E3;LATIN SMALL LETTER A WITH TILDE;Ll;0;L;0061 0303;;;;N;LATIN SMALL LETTER A TILDE;;00C3;;00C3 +00E4;LATIN SMALL LETTER A WITH DIAERESIS;Ll;0;L;0061 0308;;;;N;LATIN SMALL LETTER A DIAERESIS;;00C4;;00C4 +00E5;LATIN SMALL LETTER A WITH RING ABOVE;Ll;0;L;0061 030A;;;;N;LATIN SMALL LETTER A RING;;00C5;;00C5 +00E6;LATIN SMALL LETTER AE;Ll;0;L;;;;;N;LATIN SMALL LETTER A E;;00C6;;00C6 +00E7;LATIN SMALL LETTER C WITH CEDILLA;Ll;0;L;0063 0327;;;;N;LATIN SMALL LETTER C CEDILLA;;00C7;;00C7 +00E8;LATIN SMALL LETTER E WITH GRAVE;Ll;0;L;0065 0300;;;;N;LATIN SMALL LETTER E GRAVE;;00C8;;00C8 +00E9;LATIN SMALL LETTER E WITH ACUTE;Ll;0;L;0065 0301;;;;N;LATIN SMALL LETTER E ACUTE;;00C9;;00C9 +00EA;LATIN SMALL LETTER E WITH CIRCUMFLEX;Ll;0;L;0065 0302;;;;N;LATIN SMALL LETTER E CIRCUMFLEX;;00CA;;00CA +00EB;LATIN SMALL LETTER E WITH DIAERESIS;Ll;0;L;0065 0308;;;;N;LATIN SMALL LETTER E DIAERESIS;;00CB;;00CB +00EC;LATIN SMALL LETTER I WITH GRAVE;Ll;0;L;0069 0300;;;;N;LATIN SMALL LETTER I GRAVE;;00CC;;00CC +00ED;LATIN SMALL LETTER I WITH ACUTE;Ll;0;L;0069 0301;;;;N;LATIN SMALL LETTER I ACUTE;;00CD;;00CD +00EE;LATIN SMALL LETTER I WITH CIRCUMFLEX;Ll;0;L;0069 0302;;;;N;LATIN SMALL LETTER I CIRCUMFLEX;;00CE;;00CE +00EF;LATIN SMALL LETTER I WITH DIAERESIS;Ll;0;L;0069 0308;;;;N;LATIN SMALL LETTER I DIAERESIS;;00CF;;00CF +00F0;LATIN SMALL LETTER ETH;Ll;0;L;;;;;N;;;00D0;;00D0 +00F1;LATIN SMALL LETTER N WITH TILDE;Ll;0;L;006E 0303;;;;N;LATIN SMALL LETTER N TILDE;;00D1;;00D1 +00F2;LATIN SMALL LETTER O WITH GRAVE;Ll;0;L;006F 0300;;;;N;LATIN SMALL LETTER O GRAVE;;00D2;;00D2 +00F3;LATIN SMALL LETTER O WITH ACUTE;Ll;0;L;006F 0301;;;;N;LATIN SMALL LETTER O ACUTE;;00D3;;00D3 +00F4;LATIN SMALL LETTER O WITH CIRCUMFLEX;Ll;0;L;006F 0302;;;;N;LATIN SMALL LETTER O CIRCUMFLEX;;00D4;;00D4 +00F5;LATIN SMALL LETTER O WITH TILDE;Ll;0;L;006F 0303;;;;N;LATIN SMALL LETTER O TILDE;;00D5;;00D5 +00F6;LATIN SMALL LETTER O WITH DIAERESIS;Ll;0;L;006F 0308;;;;N;LATIN SMALL LETTER O DIAERESIS;;00D6;;00D6 +00F7;DIVISION SIGN;Sm;0;ON;;;;;N;;;;; +00F8;LATIN SMALL LETTER O WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER O SLASH;;00D8;;00D8 +00F9;LATIN SMALL LETTER U WITH GRAVE;Ll;0;L;0075 0300;;;;N;LATIN SMALL LETTER U GRAVE;;00D9;;00D9 +00FA;LATIN SMALL LETTER U WITH ACUTE;Ll;0;L;0075 0301;;;;N;LATIN SMALL LETTER U ACUTE;;00DA;;00DA +00FB;LATIN SMALL LETTER U WITH CIRCUMFLEX;Ll;0;L;0075 0302;;;;N;LATIN SMALL LETTER U CIRCUMFLEX;;00DB;;00DB +00FC;LATIN SMALL LETTER U WITH DIAERESIS;Ll;0;L;0075 0308;;;;N;LATIN SMALL LETTER U DIAERESIS;;00DC;;00DC +00FD;LATIN SMALL LETTER Y WITH ACUTE;Ll;0;L;0079 0301;;;;N;LATIN SMALL LETTER Y ACUTE;;00DD;;00DD +00FE;LATIN SMALL LETTER THORN;Ll;0;L;;;;;N;;;00DE;;00DE +00FF;LATIN SMALL LETTER Y WITH DIAERESIS;Ll;0;L;0079 0308;;;;N;LATIN SMALL LETTER Y DIAERESIS;;0178;;0178 +0100;LATIN CAPITAL LETTER A WITH MACRON;Lu;0;L;0041 0304;;;;N;LATIN CAPITAL LETTER A MACRON;;;0101; +0101;LATIN SMALL LETTER A WITH MACRON;Ll;0;L;0061 0304;;;;N;LATIN SMALL LETTER A MACRON;;0100;;0100 +0102;LATIN CAPITAL LETTER A WITH BREVE;Lu;0;L;0041 0306;;;;N;LATIN CAPITAL LETTER A BREVE;;;0103; +0103;LATIN SMALL LETTER A WITH BREVE;Ll;0;L;0061 0306;;;;N;LATIN SMALL LETTER A BREVE;;0102;;0102 +0104;LATIN CAPITAL LETTER A WITH OGONEK;Lu;0;L;0041 0328;;;;N;LATIN CAPITAL LETTER A OGONEK;;;0105; +0105;LATIN SMALL LETTER A WITH OGONEK;Ll;0;L;0061 0328;;;;N;LATIN SMALL LETTER A OGONEK;;0104;;0104 +0106;LATIN CAPITAL LETTER C WITH ACUTE;Lu;0;L;0043 0301;;;;N;LATIN CAPITAL LETTER C ACUTE;;;0107; +0107;LATIN SMALL LETTER C WITH ACUTE;Ll;0;L;0063 0301;;;;N;LATIN SMALL LETTER C ACUTE;;0106;;0106 +0108;LATIN CAPITAL LETTER C WITH CIRCUMFLEX;Lu;0;L;0043 0302;;;;N;LATIN CAPITAL LETTER C CIRCUMFLEX;;;0109; +0109;LATIN SMALL LETTER C WITH CIRCUMFLEX;Ll;0;L;0063 0302;;;;N;LATIN SMALL LETTER C CIRCUMFLEX;;0108;;0108 +010A;LATIN CAPITAL LETTER C WITH DOT ABOVE;Lu;0;L;0043 0307;;;;N;LATIN CAPITAL LETTER C DOT;;;010B; +010B;LATIN SMALL LETTER C WITH DOT ABOVE;Ll;0;L;0063 0307;;;;N;LATIN SMALL LETTER C DOT;;010A;;010A +010C;LATIN CAPITAL LETTER C WITH CARON;Lu;0;L;0043 030C;;;;N;LATIN CAPITAL LETTER C HACEK;;;010D; +010D;LATIN SMALL LETTER C WITH CARON;Ll;0;L;0063 030C;;;;N;LATIN SMALL LETTER C HACEK;;010C;;010C +010E;LATIN CAPITAL LETTER D WITH CARON;Lu;0;L;0044 030C;;;;N;LATIN CAPITAL LETTER D HACEK;;;010F; +010F;LATIN SMALL LETTER D WITH CARON;Ll;0;L;0064 030C;;;;N;LATIN SMALL LETTER D HACEK;;010E;;010E +0110;LATIN CAPITAL LETTER D WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER D BAR;;;0111; +0111;LATIN SMALL LETTER D WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER D BAR;;0110;;0110 +0112;LATIN CAPITAL LETTER E WITH MACRON;Lu;0;L;0045 0304;;;;N;LATIN CAPITAL LETTER E MACRON;;;0113; +0113;LATIN SMALL LETTER E WITH MACRON;Ll;0;L;0065 0304;;;;N;LATIN SMALL LETTER E MACRON;;0112;;0112 +0114;LATIN CAPITAL LETTER E WITH BREVE;Lu;0;L;0045 0306;;;;N;LATIN CAPITAL LETTER E BREVE;;;0115; +0115;LATIN SMALL LETTER E WITH BREVE;Ll;0;L;0065 0306;;;;N;LATIN SMALL LETTER E BREVE;;0114;;0114 +0116;LATIN CAPITAL LETTER E WITH DOT ABOVE;Lu;0;L;0045 0307;;;;N;LATIN CAPITAL LETTER E DOT;;;0117; +0117;LATIN SMALL LETTER E WITH DOT ABOVE;Ll;0;L;0065 0307;;;;N;LATIN SMALL LETTER E DOT;;0116;;0116 +0118;LATIN CAPITAL LETTER E WITH OGONEK;Lu;0;L;0045 0328;;;;N;LATIN CAPITAL LETTER E OGONEK;;;0119; +0119;LATIN SMALL LETTER E WITH OGONEK;Ll;0;L;0065 0328;;;;N;LATIN SMALL LETTER E OGONEK;;0118;;0118 +011A;LATIN CAPITAL LETTER E WITH CARON;Lu;0;L;0045 030C;;;;N;LATIN CAPITAL LETTER E HACEK;;;011B; +011B;LATIN SMALL LETTER E WITH CARON;Ll;0;L;0065 030C;;;;N;LATIN SMALL LETTER E HACEK;;011A;;011A +011C;LATIN CAPITAL LETTER G WITH CIRCUMFLEX;Lu;0;L;0047 0302;;;;N;LATIN CAPITAL LETTER G CIRCUMFLEX;;;011D; +011D;LATIN SMALL LETTER G WITH CIRCUMFLEX;Ll;0;L;0067 0302;;;;N;LATIN SMALL LETTER G CIRCUMFLEX;;011C;;011C +011E;LATIN CAPITAL LETTER G WITH BREVE;Lu;0;L;0047 0306;;;;N;LATIN CAPITAL LETTER G BREVE;;;011F; +011F;LATIN SMALL LETTER G WITH BREVE;Ll;0;L;0067 0306;;;;N;LATIN SMALL LETTER G BREVE;;011E;;011E +0120;LATIN CAPITAL LETTER G WITH DOT ABOVE;Lu;0;L;0047 0307;;;;N;LATIN CAPITAL LETTER G DOT;;;0121; +0121;LATIN SMALL LETTER G WITH DOT ABOVE;Ll;0;L;0067 0307;;;;N;LATIN SMALL LETTER G DOT;;0120;;0120 +0122;LATIN CAPITAL LETTER G WITH CEDILLA;Lu;0;L;0047 0327;;;;N;LATIN CAPITAL LETTER G CEDILLA;;;0123; +0123;LATIN SMALL LETTER G WITH CEDILLA;Ll;0;L;0067 0327;;;;N;LATIN SMALL LETTER G CEDILLA;;0122;;0122 +0124;LATIN CAPITAL LETTER H WITH CIRCUMFLEX;Lu;0;L;0048 0302;;;;N;LATIN CAPITAL LETTER H CIRCUMFLEX;;;0125; +0125;LATIN SMALL LETTER H WITH CIRCUMFLEX;Ll;0;L;0068 0302;;;;N;LATIN SMALL LETTER H CIRCUMFLEX;;0124;;0124 +0126;LATIN CAPITAL LETTER H WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER H BAR;;;0127; +0127;LATIN SMALL LETTER H WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER H BAR;;0126;;0126 +0128;LATIN CAPITAL LETTER I WITH TILDE;Lu;0;L;0049 0303;;;;N;LATIN CAPITAL LETTER I TILDE;;;0129; +0129;LATIN SMALL LETTER I WITH TILDE;Ll;0;L;0069 0303;;;;N;LATIN SMALL LETTER I TILDE;;0128;;0128 +012A;LATIN CAPITAL LETTER I WITH MACRON;Lu;0;L;0049 0304;;;;N;LATIN CAPITAL LETTER I MACRON;;;012B; +012B;LATIN SMALL LETTER I WITH MACRON;Ll;0;L;0069 0304;;;;N;LATIN SMALL LETTER I MACRON;;012A;;012A +012C;LATIN CAPITAL LETTER I WITH BREVE;Lu;0;L;0049 0306;;;;N;LATIN CAPITAL LETTER I BREVE;;;012D; +012D;LATIN SMALL LETTER I WITH BREVE;Ll;0;L;0069 0306;;;;N;LATIN SMALL LETTER I BREVE;;012C;;012C +012E;LATIN CAPITAL LETTER I WITH OGONEK;Lu;0;L;0049 0328;;;;N;LATIN CAPITAL LETTER I OGONEK;;;012F; +012F;LATIN SMALL LETTER I WITH OGONEK;Ll;0;L;0069 0328;;;;N;LATIN SMALL LETTER I OGONEK;;012E;;012E +0130;LATIN CAPITAL LETTER I WITH DOT ABOVE;Lu;0;L;0049 0307;;;;N;LATIN CAPITAL LETTER I DOT;;;0069; +0131;LATIN SMALL LETTER DOTLESS I;Ll;0;L;;;;;N;;;0049;;0049 +0132;LATIN CAPITAL LIGATURE IJ;Lu;0;L; 0049 004A;;;;N;LATIN CAPITAL LETTER I J;;;0133; +0133;LATIN SMALL LIGATURE IJ;Ll;0;L; 0069 006A;;;;N;LATIN SMALL LETTER I J;;0132;;0132 +0134;LATIN CAPITAL LETTER J WITH CIRCUMFLEX;Lu;0;L;004A 0302;;;;N;LATIN CAPITAL LETTER J CIRCUMFLEX;;;0135; +0135;LATIN SMALL LETTER J WITH CIRCUMFLEX;Ll;0;L;006A 0302;;;;N;LATIN SMALL LETTER J CIRCUMFLEX;;0134;;0134 +0136;LATIN CAPITAL LETTER K WITH CEDILLA;Lu;0;L;004B 0327;;;;N;LATIN CAPITAL LETTER K CEDILLA;;;0137; +0137;LATIN SMALL LETTER K WITH CEDILLA;Ll;0;L;006B 0327;;;;N;LATIN SMALL LETTER K CEDILLA;;0136;;0136 +0138;LATIN SMALL LETTER KRA;Ll;0;L;;;;;N;;;;; +0139;LATIN CAPITAL LETTER L WITH ACUTE;Lu;0;L;004C 0301;;;;N;LATIN CAPITAL LETTER L ACUTE;;;013A; +013A;LATIN SMALL LETTER L WITH ACUTE;Ll;0;L;006C 0301;;;;N;LATIN SMALL LETTER L ACUTE;;0139;;0139 +013B;LATIN CAPITAL LETTER L WITH CEDILLA;Lu;0;L;004C 0327;;;;N;LATIN CAPITAL LETTER L CEDILLA;;;013C; +013C;LATIN SMALL LETTER L WITH CEDILLA;Ll;0;L;006C 0327;;;;N;LATIN SMALL LETTER L CEDILLA;;013B;;013B +013D;LATIN CAPITAL LETTER L WITH CARON;Lu;0;L;004C 030C;;;;N;LATIN CAPITAL LETTER L HACEK;;;013E; +013E;LATIN SMALL LETTER L WITH CARON;Ll;0;L;006C 030C;;;;N;LATIN SMALL LETTER L HACEK;;013D;;013D +013F;LATIN CAPITAL LETTER L WITH MIDDLE DOT;Lu;0;L; 004C 00B7;;;;N;;;;0140; +0140;LATIN SMALL LETTER L WITH MIDDLE DOT;Ll;0;L; 006C 00B7;;;;N;;;013F;;013F +0141;LATIN CAPITAL LETTER L WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER L SLASH;;;0142; +0142;LATIN SMALL LETTER L WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER L SLASH;;0141;;0141 +0143;LATIN CAPITAL LETTER N WITH ACUTE;Lu;0;L;004E 0301;;;;N;LATIN CAPITAL LETTER N ACUTE;;;0144; +0144;LATIN SMALL LETTER N WITH ACUTE;Ll;0;L;006E 0301;;;;N;LATIN SMALL LETTER N ACUTE;;0143;;0143 +0145;LATIN CAPITAL LETTER N WITH CEDILLA;Lu;0;L;004E 0327;;;;N;LATIN CAPITAL LETTER N CEDILLA;;;0146; +0146;LATIN SMALL LETTER N WITH CEDILLA;Ll;0;L;006E 0327;;;;N;LATIN SMALL LETTER N CEDILLA;;0145;;0145 +0147;LATIN CAPITAL LETTER N WITH CARON;Lu;0;L;004E 030C;;;;N;LATIN CAPITAL LETTER N HACEK;;;0148; +0148;LATIN SMALL LETTER N WITH CARON;Ll;0;L;006E 030C;;;;N;LATIN SMALL LETTER N HACEK;;0147;;0147 +0149;LATIN SMALL LETTER N PRECEDED BY APOSTROPHE;Ll;0;L; 02BC 006E;;;;N;LATIN SMALL LETTER APOSTROPHE N;;;; +014A;LATIN CAPITAL LETTER ENG;Lu;0;L;;;;;N;;;;014B; +014B;LATIN SMALL LETTER ENG;Ll;0;L;;;;;N;;;014A;;014A +014C;LATIN CAPITAL LETTER O WITH MACRON;Lu;0;L;004F 0304;;;;N;LATIN CAPITAL LETTER O MACRON;;;014D; +014D;LATIN SMALL LETTER O WITH MACRON;Ll;0;L;006F 0304;;;;N;LATIN SMALL LETTER O MACRON;;014C;;014C +014E;LATIN CAPITAL LETTER O WITH BREVE;Lu;0;L;004F 0306;;;;N;LATIN CAPITAL LETTER O BREVE;;;014F; +014F;LATIN SMALL LETTER O WITH BREVE;Ll;0;L;006F 0306;;;;N;LATIN SMALL LETTER O BREVE;;014E;;014E +0150;LATIN CAPITAL LETTER O WITH DOUBLE ACUTE;Lu;0;L;004F 030B;;;;N;LATIN CAPITAL LETTER O DOUBLE ACUTE;;;0151; +0151;LATIN SMALL LETTER O WITH DOUBLE ACUTE;Ll;0;L;006F 030B;;;;N;LATIN SMALL LETTER O DOUBLE ACUTE;;0150;;0150 +0152;LATIN CAPITAL LIGATURE OE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER O E;;;0153; +0153;LATIN SMALL LIGATURE OE;Ll;0;L;;;;;N;LATIN SMALL LETTER O E;;0152;;0152 +0154;LATIN CAPITAL LETTER R WITH ACUTE;Lu;0;L;0052 0301;;;;N;LATIN CAPITAL LETTER R ACUTE;;;0155; +0155;LATIN SMALL LETTER R WITH ACUTE;Ll;0;L;0072 0301;;;;N;LATIN SMALL LETTER R ACUTE;;0154;;0154 +0156;LATIN CAPITAL LETTER R WITH CEDILLA;Lu;0;L;0052 0327;;;;N;LATIN CAPITAL LETTER R CEDILLA;;;0157; +0157;LATIN SMALL LETTER R WITH CEDILLA;Ll;0;L;0072 0327;;;;N;LATIN SMALL LETTER R CEDILLA;;0156;;0156 +0158;LATIN CAPITAL LETTER R WITH CARON;Lu;0;L;0052 030C;;;;N;LATIN CAPITAL LETTER R HACEK;;;0159; +0159;LATIN SMALL LETTER R WITH CARON;Ll;0;L;0072 030C;;;;N;LATIN SMALL LETTER R HACEK;;0158;;0158 +015A;LATIN CAPITAL LETTER S WITH ACUTE;Lu;0;L;0053 0301;;;;N;LATIN CAPITAL LETTER S ACUTE;;;015B; +015B;LATIN SMALL LETTER S WITH ACUTE;Ll;0;L;0073 0301;;;;N;LATIN SMALL LETTER S ACUTE;;015A;;015A +015C;LATIN CAPITAL LETTER S WITH CIRCUMFLEX;Lu;0;L;0053 0302;;;;N;LATIN CAPITAL LETTER S CIRCUMFLEX;;;015D; +015D;LATIN SMALL LETTER S WITH CIRCUMFLEX;Ll;0;L;0073 0302;;;;N;LATIN SMALL LETTER S CIRCUMFLEX;;015C;;015C +015E;LATIN CAPITAL LETTER S WITH CEDILLA;Lu;0;L;0053 0327;;;;N;LATIN CAPITAL LETTER S CEDILLA;;;015F; +015F;LATIN SMALL LETTER S WITH CEDILLA;Ll;0;L;0073 0327;;;;N;LATIN SMALL LETTER S CEDILLA;;015E;;015E +0160;LATIN CAPITAL LETTER S WITH CARON;Lu;0;L;0053 030C;;;;N;LATIN CAPITAL LETTER S HACEK;;;0161; +0161;LATIN SMALL LETTER S WITH CARON;Ll;0;L;0073 030C;;;;N;LATIN SMALL LETTER S HACEK;;0160;;0160 +0162;LATIN CAPITAL LETTER T WITH CEDILLA;Lu;0;L;0054 0327;;;;N;LATIN CAPITAL LETTER T CEDILLA;;;0163; +0163;LATIN SMALL LETTER T WITH CEDILLA;Ll;0;L;0074 0327;;;;N;LATIN SMALL LETTER T CEDILLA;;0162;;0162 +0164;LATIN CAPITAL LETTER T WITH CARON;Lu;0;L;0054 030C;;;;N;LATIN CAPITAL LETTER T HACEK;;;0165; +0165;LATIN SMALL LETTER T WITH CARON;Ll;0;L;0074 030C;;;;N;LATIN SMALL LETTER T HACEK;;0164;;0164 +0166;LATIN CAPITAL LETTER T WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER T BAR;;;0167; +0167;LATIN SMALL LETTER T WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER T BAR;;0166;;0166 +0168;LATIN CAPITAL LETTER U WITH TILDE;Lu;0;L;0055 0303;;;;N;LATIN CAPITAL LETTER U TILDE;;;0169; +0169;LATIN SMALL LETTER U WITH TILDE;Ll;0;L;0075 0303;;;;N;LATIN SMALL LETTER U TILDE;;0168;;0168 +016A;LATIN CAPITAL LETTER U WITH MACRON;Lu;0;L;0055 0304;;;;N;LATIN CAPITAL LETTER U MACRON;;;016B; +016B;LATIN SMALL LETTER U WITH MACRON;Ll;0;L;0075 0304;;;;N;LATIN SMALL LETTER U MACRON;;016A;;016A +016C;LATIN CAPITAL LETTER U WITH BREVE;Lu;0;L;0055 0306;;;;N;LATIN CAPITAL LETTER U BREVE;;;016D; +016D;LATIN SMALL LETTER U WITH BREVE;Ll;0;L;0075 0306;;;;N;LATIN SMALL LETTER U BREVE;;016C;;016C +016E;LATIN CAPITAL LETTER U WITH RING ABOVE;Lu;0;L;0055 030A;;;;N;LATIN CAPITAL LETTER U RING;;;016F; +016F;LATIN SMALL LETTER U WITH RING ABOVE;Ll;0;L;0075 030A;;;;N;LATIN SMALL LETTER U RING;;016E;;016E +0170;LATIN CAPITAL LETTER U WITH DOUBLE ACUTE;Lu;0;L;0055 030B;;;;N;LATIN CAPITAL LETTER U DOUBLE ACUTE;;;0171; +0171;LATIN SMALL LETTER U WITH DOUBLE ACUTE;Ll;0;L;0075 030B;;;;N;LATIN SMALL LETTER U DOUBLE ACUTE;;0170;;0170 +0172;LATIN CAPITAL LETTER U WITH OGONEK;Lu;0;L;0055 0328;;;;N;LATIN CAPITAL LETTER U OGONEK;;;0173; +0173;LATIN SMALL LETTER U WITH OGONEK;Ll;0;L;0075 0328;;;;N;LATIN SMALL LETTER U OGONEK;;0172;;0172 +0174;LATIN CAPITAL LETTER W WITH CIRCUMFLEX;Lu;0;L;0057 0302;;;;N;LATIN CAPITAL LETTER W CIRCUMFLEX;;;0175; +0175;LATIN SMALL LETTER W WITH CIRCUMFLEX;Ll;0;L;0077 0302;;;;N;LATIN SMALL LETTER W CIRCUMFLEX;;0174;;0174 +0176;LATIN CAPITAL LETTER Y WITH CIRCUMFLEX;Lu;0;L;0059 0302;;;;N;LATIN CAPITAL LETTER Y CIRCUMFLEX;;;0177; +0177;LATIN SMALL LETTER Y WITH CIRCUMFLEX;Ll;0;L;0079 0302;;;;N;LATIN SMALL LETTER Y CIRCUMFLEX;;0176;;0176 +0178;LATIN CAPITAL LETTER Y WITH DIAERESIS;Lu;0;L;0059 0308;;;;N;LATIN CAPITAL LETTER Y DIAERESIS;;;00FF; +0179;LATIN CAPITAL LETTER Z WITH ACUTE;Lu;0;L;005A 0301;;;;N;LATIN CAPITAL LETTER Z ACUTE;;;017A; +017A;LATIN SMALL LETTER Z WITH ACUTE;Ll;0;L;007A 0301;;;;N;LATIN SMALL LETTER Z ACUTE;;0179;;0179 +017B;LATIN CAPITAL LETTER Z WITH DOT ABOVE;Lu;0;L;005A 0307;;;;N;LATIN CAPITAL LETTER Z DOT;;;017C; +017C;LATIN SMALL LETTER Z WITH DOT ABOVE;Ll;0;L;007A 0307;;;;N;LATIN SMALL LETTER Z DOT;;017B;;017B +017D;LATIN CAPITAL LETTER Z WITH CARON;Lu;0;L;005A 030C;;;;N;LATIN CAPITAL LETTER Z HACEK;;;017E; +017E;LATIN SMALL LETTER Z WITH CARON;Ll;0;L;007A 030C;;;;N;LATIN SMALL LETTER Z HACEK;;017D;;017D +017F;LATIN SMALL LETTER LONG S;Ll;0;L; 0073;;;;N;;;0053;;0053 +0180;LATIN SMALL LETTER B WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER B BAR;;0243;;0243 +0181;LATIN CAPITAL LETTER B WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER B HOOK;;;0253; +0182;LATIN CAPITAL LETTER B WITH TOPBAR;Lu;0;L;;;;;N;LATIN CAPITAL LETTER B TOPBAR;;;0183; +0183;LATIN SMALL LETTER B WITH TOPBAR;Ll;0;L;;;;;N;LATIN SMALL LETTER B TOPBAR;;0182;;0182 +0184;LATIN CAPITAL LETTER TONE SIX;Lu;0;L;;;;;N;;;;0185; +0185;LATIN SMALL LETTER TONE SIX;Ll;0;L;;;;;N;;;0184;;0184 +0186;LATIN CAPITAL LETTER OPEN O;Lu;0;L;;;;;N;;;;0254; +0187;LATIN CAPITAL LETTER C WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER C HOOK;;;0188; +0188;LATIN SMALL LETTER C WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER C HOOK;;0187;;0187 +0189;LATIN CAPITAL LETTER AFRICAN D;Lu;0;L;;;;;N;;;;0256; +018A;LATIN CAPITAL LETTER D WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER D HOOK;;;0257; +018B;LATIN CAPITAL LETTER D WITH TOPBAR;Lu;0;L;;;;;N;LATIN CAPITAL LETTER D TOPBAR;;;018C; +018C;LATIN SMALL LETTER D WITH TOPBAR;Ll;0;L;;;;;N;LATIN SMALL LETTER D TOPBAR;;018B;;018B +018D;LATIN SMALL LETTER TURNED DELTA;Ll;0;L;;;;;N;;;;; +018E;LATIN CAPITAL LETTER REVERSED E;Lu;0;L;;;;;N;LATIN CAPITAL LETTER TURNED E;;;01DD; +018F;LATIN CAPITAL LETTER SCHWA;Lu;0;L;;;;;N;;;;0259; +0190;LATIN CAPITAL LETTER OPEN E;Lu;0;L;;;;;N;LATIN CAPITAL LETTER EPSILON;;;025B; +0191;LATIN CAPITAL LETTER F WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER F HOOK;;;0192; +0192;LATIN SMALL LETTER F WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER SCRIPT F;;0191;;0191 +0193;LATIN CAPITAL LETTER G WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER G HOOK;;;0260; +0194;LATIN CAPITAL LETTER GAMMA;Lu;0;L;;;;;N;;;;0263; +0195;LATIN SMALL LETTER HV;Ll;0;L;;;;;N;LATIN SMALL LETTER H V;;01F6;;01F6 +0196;LATIN CAPITAL LETTER IOTA;Lu;0;L;;;;;N;;;;0269; +0197;LATIN CAPITAL LETTER I WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER BARRED I;;;0268; +0198;LATIN CAPITAL LETTER K WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER K HOOK;;;0199; +0199;LATIN SMALL LETTER K WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER K HOOK;;0198;;0198 +019A;LATIN SMALL LETTER L WITH BAR;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED L;;023D;;023D +019B;LATIN SMALL LETTER LAMBDA WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED LAMBDA;;;; +019C;LATIN CAPITAL LETTER TURNED M;Lu;0;L;;;;;N;;;;026F; +019D;LATIN CAPITAL LETTER N WITH LEFT HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER N HOOK;;;0272; +019E;LATIN SMALL LETTER N WITH LONG RIGHT LEG;Ll;0;L;;;;;N;;;0220;;0220 +019F;LATIN CAPITAL LETTER O WITH MIDDLE TILDE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER BARRED O;;;0275; +01A0;LATIN CAPITAL LETTER O WITH HORN;Lu;0;L;004F 031B;;;;N;LATIN CAPITAL LETTER O HORN;;;01A1; +01A1;LATIN SMALL LETTER O WITH HORN;Ll;0;L;006F 031B;;;;N;LATIN SMALL LETTER O HORN;;01A0;;01A0 +01A2;LATIN CAPITAL LETTER OI;Lu;0;L;;;;;N;LATIN CAPITAL LETTER O I;;;01A3; +01A3;LATIN SMALL LETTER OI;Ll;0;L;;;;;N;LATIN SMALL LETTER O I;;01A2;;01A2 +01A4;LATIN CAPITAL LETTER P WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER P HOOK;;;01A5; +01A5;LATIN SMALL LETTER P WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER P HOOK;;01A4;;01A4 +01A6;LATIN LETTER YR;Lu;0;L;;;;;N;LATIN LETTER Y R;;;0280; +01A7;LATIN CAPITAL LETTER TONE TWO;Lu;0;L;;;;;N;;;;01A8; +01A8;LATIN SMALL LETTER TONE TWO;Ll;0;L;;;;;N;;;01A7;;01A7 +01A9;LATIN CAPITAL LETTER ESH;Lu;0;L;;;;;N;;;;0283; +01AA;LATIN LETTER REVERSED ESH LOOP;Ll;0;L;;;;;N;;;;; +01AB;LATIN SMALL LETTER T WITH PALATAL HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER T PALATAL HOOK;;;; +01AC;LATIN CAPITAL LETTER T WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER T HOOK;;;01AD; +01AD;LATIN SMALL LETTER T WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER T HOOK;;01AC;;01AC +01AE;LATIN CAPITAL LETTER T WITH RETROFLEX HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER T RETROFLEX HOOK;;;0288; +01AF;LATIN CAPITAL LETTER U WITH HORN;Lu;0;L;0055 031B;;;;N;LATIN CAPITAL LETTER U HORN;;;01B0; +01B0;LATIN SMALL LETTER U WITH HORN;Ll;0;L;0075 031B;;;;N;LATIN SMALL LETTER U HORN;;01AF;;01AF +01B1;LATIN CAPITAL LETTER UPSILON;Lu;0;L;;;;;N;;;;028A; +01B2;LATIN CAPITAL LETTER V WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER SCRIPT V;;;028B; +01B3;LATIN CAPITAL LETTER Y WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER Y HOOK;;;01B4; +01B4;LATIN SMALL LETTER Y WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER Y HOOK;;01B3;;01B3 +01B5;LATIN CAPITAL LETTER Z WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER Z BAR;;;01B6; +01B6;LATIN SMALL LETTER Z WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER Z BAR;;01B5;;01B5 +01B7;LATIN CAPITAL LETTER EZH;Lu;0;L;;;;;N;LATIN CAPITAL LETTER YOGH;;;0292; +01B8;LATIN CAPITAL LETTER EZH REVERSED;Lu;0;L;;;;;N;LATIN CAPITAL LETTER REVERSED YOGH;;;01B9; +01B9;LATIN SMALL LETTER EZH REVERSED;Ll;0;L;;;;;N;LATIN SMALL LETTER REVERSED YOGH;;01B8;;01B8 +01BA;LATIN SMALL LETTER EZH WITH TAIL;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH WITH TAIL;;;; +01BB;LATIN LETTER TWO WITH STROKE;Lo;0;L;;;;;N;LATIN LETTER TWO BAR;;;; +01BC;LATIN CAPITAL LETTER TONE FIVE;Lu;0;L;;;;;N;;;;01BD; +01BD;LATIN SMALL LETTER TONE FIVE;Ll;0;L;;;;;N;;;01BC;;01BC +01BE;LATIN LETTER INVERTED GLOTTAL STOP WITH STROKE;Ll;0;L;;;;;N;LATIN LETTER INVERTED GLOTTAL STOP BAR;;;; +01BF;LATIN LETTER WYNN;Ll;0;L;;;;;N;;;01F7;;01F7 +01C0;LATIN LETTER DENTAL CLICK;Lo;0;L;;;;;N;LATIN LETTER PIPE;;;; +01C1;LATIN LETTER LATERAL CLICK;Lo;0;L;;;;;N;LATIN LETTER DOUBLE PIPE;;;; +01C2;LATIN LETTER ALVEOLAR CLICK;Lo;0;L;;;;;N;LATIN LETTER PIPE DOUBLE BAR;;;; +01C3;LATIN LETTER RETROFLEX CLICK;Lo;0;L;;;;;N;LATIN LETTER EXCLAMATION MARK;;;; +01C4;LATIN CAPITAL LETTER DZ WITH CARON;Lu;0;L; 0044 017D;;;;N;LATIN CAPITAL LETTER D Z HACEK;;;01C6;01C5 +01C5;LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON;Lt;0;L; 0044 017E;;;;N;LATIN LETTER CAPITAL D SMALL Z HACEK;;01C4;01C6;01C5 +01C6;LATIN SMALL LETTER DZ WITH CARON;Ll;0;L; 0064 017E;;;;N;LATIN SMALL LETTER D Z HACEK;;01C4;;01C5 +01C7;LATIN CAPITAL LETTER LJ;Lu;0;L; 004C 004A;;;;N;LATIN CAPITAL LETTER L J;;;01C9;01C8 +01C8;LATIN CAPITAL LETTER L WITH SMALL LETTER J;Lt;0;L; 004C 006A;;;;N;LATIN LETTER CAPITAL L SMALL J;;01C7;01C9;01C8 +01C9;LATIN SMALL LETTER LJ;Ll;0;L; 006C 006A;;;;N;LATIN SMALL LETTER L J;;01C7;;01C8 +01CA;LATIN CAPITAL LETTER NJ;Lu;0;L; 004E 004A;;;;N;LATIN CAPITAL LETTER N J;;;01CC;01CB +01CB;LATIN CAPITAL LETTER N WITH SMALL LETTER J;Lt;0;L; 004E 006A;;;;N;LATIN LETTER CAPITAL N SMALL J;;01CA;01CC;01CB +01CC;LATIN SMALL LETTER NJ;Ll;0;L; 006E 006A;;;;N;LATIN SMALL LETTER N J;;01CA;;01CB +01CD;LATIN CAPITAL LETTER A WITH CARON;Lu;0;L;0041 030C;;;;N;LATIN CAPITAL LETTER A HACEK;;;01CE; +01CE;LATIN SMALL LETTER A WITH CARON;Ll;0;L;0061 030C;;;;N;LATIN SMALL LETTER A HACEK;;01CD;;01CD +01CF;LATIN CAPITAL LETTER I WITH CARON;Lu;0;L;0049 030C;;;;N;LATIN CAPITAL LETTER I HACEK;;;01D0; +01D0;LATIN SMALL LETTER I WITH CARON;Ll;0;L;0069 030C;;;;N;LATIN SMALL LETTER I HACEK;;01CF;;01CF +01D1;LATIN CAPITAL LETTER O WITH CARON;Lu;0;L;004F 030C;;;;N;LATIN CAPITAL LETTER O HACEK;;;01D2; +01D2;LATIN SMALL LETTER O WITH CARON;Ll;0;L;006F 030C;;;;N;LATIN SMALL LETTER O HACEK;;01D1;;01D1 +01D3;LATIN CAPITAL LETTER U WITH CARON;Lu;0;L;0055 030C;;;;N;LATIN CAPITAL LETTER U HACEK;;;01D4; +01D4;LATIN SMALL LETTER U WITH CARON;Ll;0;L;0075 030C;;;;N;LATIN SMALL LETTER U HACEK;;01D3;;01D3 +01D5;LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON;Lu;0;L;00DC 0304;;;;N;LATIN CAPITAL LETTER U DIAERESIS MACRON;;;01D6; +01D6;LATIN SMALL LETTER U WITH DIAERESIS AND MACRON;Ll;0;L;00FC 0304;;;;N;LATIN SMALL LETTER U DIAERESIS MACRON;;01D5;;01D5 +01D7;LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE;Lu;0;L;00DC 0301;;;;N;LATIN CAPITAL LETTER U DIAERESIS ACUTE;;;01D8; +01D8;LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE;Ll;0;L;00FC 0301;;;;N;LATIN SMALL LETTER U DIAERESIS ACUTE;;01D7;;01D7 +01D9;LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON;Lu;0;L;00DC 030C;;;;N;LATIN CAPITAL LETTER U DIAERESIS HACEK;;;01DA; +01DA;LATIN SMALL LETTER U WITH DIAERESIS AND CARON;Ll;0;L;00FC 030C;;;;N;LATIN SMALL LETTER U DIAERESIS HACEK;;01D9;;01D9 +01DB;LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE;Lu;0;L;00DC 0300;;;;N;LATIN CAPITAL LETTER U DIAERESIS GRAVE;;;01DC; +01DC;LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE;Ll;0;L;00FC 0300;;;;N;LATIN SMALL LETTER U DIAERESIS GRAVE;;01DB;;01DB +01DD;LATIN SMALL LETTER TURNED E;Ll;0;L;;;;;N;;;018E;;018E +01DE;LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON;Lu;0;L;00C4 0304;;;;N;LATIN CAPITAL LETTER A DIAERESIS MACRON;;;01DF; +01DF;LATIN SMALL LETTER A WITH DIAERESIS AND MACRON;Ll;0;L;00E4 0304;;;;N;LATIN SMALL LETTER A DIAERESIS MACRON;;01DE;;01DE +01E0;LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON;Lu;0;L;0226 0304;;;;N;LATIN CAPITAL LETTER A DOT MACRON;;;01E1; +01E1;LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON;Ll;0;L;0227 0304;;;;N;LATIN SMALL LETTER A DOT MACRON;;01E0;;01E0 +01E2;LATIN CAPITAL LETTER AE WITH MACRON;Lu;0;L;00C6 0304;;;;N;LATIN CAPITAL LETTER A E MACRON;;;01E3; +01E3;LATIN SMALL LETTER AE WITH MACRON;Ll;0;L;00E6 0304;;;;N;LATIN SMALL LETTER A E MACRON;;01E2;;01E2 +01E4;LATIN CAPITAL LETTER G WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER G BAR;;;01E5; +01E5;LATIN SMALL LETTER G WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER G BAR;;01E4;;01E4 +01E6;LATIN CAPITAL LETTER G WITH CARON;Lu;0;L;0047 030C;;;;N;LATIN CAPITAL LETTER G HACEK;;;01E7; +01E7;LATIN SMALL LETTER G WITH CARON;Ll;0;L;0067 030C;;;;N;LATIN SMALL LETTER G HACEK;;01E6;;01E6 +01E8;LATIN CAPITAL LETTER K WITH CARON;Lu;0;L;004B 030C;;;;N;LATIN CAPITAL LETTER K HACEK;;;01E9; +01E9;LATIN SMALL LETTER K WITH CARON;Ll;0;L;006B 030C;;;;N;LATIN SMALL LETTER K HACEK;;01E8;;01E8 +01EA;LATIN CAPITAL LETTER O WITH OGONEK;Lu;0;L;004F 0328;;;;N;LATIN CAPITAL LETTER O OGONEK;;;01EB; +01EB;LATIN SMALL LETTER O WITH OGONEK;Ll;0;L;006F 0328;;;;N;LATIN SMALL LETTER O OGONEK;;01EA;;01EA +01EC;LATIN CAPITAL LETTER O WITH OGONEK AND MACRON;Lu;0;L;01EA 0304;;;;N;LATIN CAPITAL LETTER O OGONEK MACRON;;;01ED; +01ED;LATIN SMALL LETTER O WITH OGONEK AND MACRON;Ll;0;L;01EB 0304;;;;N;LATIN SMALL LETTER O OGONEK MACRON;;01EC;;01EC +01EE;LATIN CAPITAL LETTER EZH WITH CARON;Lu;0;L;01B7 030C;;;;N;LATIN CAPITAL LETTER YOGH HACEK;;;01EF; +01EF;LATIN SMALL LETTER EZH WITH CARON;Ll;0;L;0292 030C;;;;N;LATIN SMALL LETTER YOGH HACEK;;01EE;;01EE +01F0;LATIN SMALL LETTER J WITH CARON;Ll;0;L;006A 030C;;;;N;LATIN SMALL LETTER J HACEK;;;; +01F1;LATIN CAPITAL LETTER DZ;Lu;0;L; 0044 005A;;;;N;;;;01F3;01F2 +01F2;LATIN CAPITAL LETTER D WITH SMALL LETTER Z;Lt;0;L; 0044 007A;;;;N;;;01F1;01F3;01F2 +01F3;LATIN SMALL LETTER DZ;Ll;0;L; 0064 007A;;;;N;;;01F1;;01F2 +01F4;LATIN CAPITAL LETTER G WITH ACUTE;Lu;0;L;0047 0301;;;;N;;;;01F5; +01F5;LATIN SMALL LETTER G WITH ACUTE;Ll;0;L;0067 0301;;;;N;;;01F4;;01F4 +01F6;LATIN CAPITAL LETTER HWAIR;Lu;0;L;;;;;N;;;;0195; +01F7;LATIN CAPITAL LETTER WYNN;Lu;0;L;;;;;N;;;;01BF; +01F8;LATIN CAPITAL LETTER N WITH GRAVE;Lu;0;L;004E 0300;;;;N;;;;01F9; +01F9;LATIN SMALL LETTER N WITH GRAVE;Ll;0;L;006E 0300;;;;N;;;01F8;;01F8 +01FA;LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE;Lu;0;L;00C5 0301;;;;N;;;;01FB; +01FB;LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE;Ll;0;L;00E5 0301;;;;N;;;01FA;;01FA +01FC;LATIN CAPITAL LETTER AE WITH ACUTE;Lu;0;L;00C6 0301;;;;N;;;;01FD; +01FD;LATIN SMALL LETTER AE WITH ACUTE;Ll;0;L;00E6 0301;;;;N;;;01FC;;01FC +01FE;LATIN CAPITAL LETTER O WITH STROKE AND ACUTE;Lu;0;L;00D8 0301;;;;N;;;;01FF; +01FF;LATIN SMALL LETTER O WITH STROKE AND ACUTE;Ll;0;L;00F8 0301;;;;N;;;01FE;;01FE +0200;LATIN CAPITAL LETTER A WITH DOUBLE GRAVE;Lu;0;L;0041 030F;;;;N;;;;0201; +0201;LATIN SMALL LETTER A WITH DOUBLE GRAVE;Ll;0;L;0061 030F;;;;N;;;0200;;0200 +0202;LATIN CAPITAL LETTER A WITH INVERTED BREVE;Lu;0;L;0041 0311;;;;N;;;;0203; +0203;LATIN SMALL LETTER A WITH INVERTED BREVE;Ll;0;L;0061 0311;;;;N;;;0202;;0202 +0204;LATIN CAPITAL LETTER E WITH DOUBLE GRAVE;Lu;0;L;0045 030F;;;;N;;;;0205; +0205;LATIN SMALL LETTER E WITH DOUBLE GRAVE;Ll;0;L;0065 030F;;;;N;;;0204;;0204 +0206;LATIN CAPITAL LETTER E WITH INVERTED BREVE;Lu;0;L;0045 0311;;;;N;;;;0207; +0207;LATIN SMALL LETTER E WITH INVERTED BREVE;Ll;0;L;0065 0311;;;;N;;;0206;;0206 +0208;LATIN CAPITAL LETTER I WITH DOUBLE GRAVE;Lu;0;L;0049 030F;;;;N;;;;0209; +0209;LATIN SMALL LETTER I WITH DOUBLE GRAVE;Ll;0;L;0069 030F;;;;N;;;0208;;0208 +020A;LATIN CAPITAL LETTER I WITH INVERTED BREVE;Lu;0;L;0049 0311;;;;N;;;;020B; +020B;LATIN SMALL LETTER I WITH INVERTED BREVE;Ll;0;L;0069 0311;;;;N;;;020A;;020A +020C;LATIN CAPITAL LETTER O WITH DOUBLE GRAVE;Lu;0;L;004F 030F;;;;N;;;;020D; +020D;LATIN SMALL LETTER O WITH DOUBLE GRAVE;Ll;0;L;006F 030F;;;;N;;;020C;;020C +020E;LATIN CAPITAL LETTER O WITH INVERTED BREVE;Lu;0;L;004F 0311;;;;N;;;;020F; +020F;LATIN SMALL LETTER O WITH INVERTED BREVE;Ll;0;L;006F 0311;;;;N;;;020E;;020E +0210;LATIN CAPITAL LETTER R WITH DOUBLE GRAVE;Lu;0;L;0052 030F;;;;N;;;;0211; +0211;LATIN SMALL LETTER R WITH DOUBLE GRAVE;Ll;0;L;0072 030F;;;;N;;;0210;;0210 +0212;LATIN CAPITAL LETTER R WITH INVERTED BREVE;Lu;0;L;0052 0311;;;;N;;;;0213; +0213;LATIN SMALL LETTER R WITH INVERTED BREVE;Ll;0;L;0072 0311;;;;N;;;0212;;0212 +0214;LATIN CAPITAL LETTER U WITH DOUBLE GRAVE;Lu;0;L;0055 030F;;;;N;;;;0215; +0215;LATIN SMALL LETTER U WITH DOUBLE GRAVE;Ll;0;L;0075 030F;;;;N;;;0214;;0214 +0216;LATIN CAPITAL LETTER U WITH INVERTED BREVE;Lu;0;L;0055 0311;;;;N;;;;0217; +0217;LATIN SMALL LETTER U WITH INVERTED BREVE;Ll;0;L;0075 0311;;;;N;;;0216;;0216 +0218;LATIN CAPITAL LETTER S WITH COMMA BELOW;Lu;0;L;0053 0326;;;;N;;;;0219; +0219;LATIN SMALL LETTER S WITH COMMA BELOW;Ll;0;L;0073 0326;;;;N;;;0218;;0218 +021A;LATIN CAPITAL LETTER T WITH COMMA BELOW;Lu;0;L;0054 0326;;;;N;;;;021B; +021B;LATIN SMALL LETTER T WITH COMMA BELOW;Ll;0;L;0074 0326;;;;N;;;021A;;021A +021C;LATIN CAPITAL LETTER YOGH;Lu;0;L;;;;;N;;;;021D; +021D;LATIN SMALL LETTER YOGH;Ll;0;L;;;;;N;;;021C;;021C +021E;LATIN CAPITAL LETTER H WITH CARON;Lu;0;L;0048 030C;;;;N;;;;021F; +021F;LATIN SMALL LETTER H WITH CARON;Ll;0;L;0068 030C;;;;N;;;021E;;021E +0220;LATIN CAPITAL LETTER N WITH LONG RIGHT LEG;Lu;0;L;;;;;N;;;;019E; +0221;LATIN SMALL LETTER D WITH CURL;Ll;0;L;;;;;N;;;;; +0222;LATIN CAPITAL LETTER OU;Lu;0;L;;;;;N;;;;0223; +0223;LATIN SMALL LETTER OU;Ll;0;L;;;;;N;;;0222;;0222 +0224;LATIN CAPITAL LETTER Z WITH HOOK;Lu;0;L;;;;;N;;;;0225; +0225;LATIN SMALL LETTER Z WITH HOOK;Ll;0;L;;;;;N;;;0224;;0224 +0226;LATIN CAPITAL LETTER A WITH DOT ABOVE;Lu;0;L;0041 0307;;;;N;;;;0227; +0227;LATIN SMALL LETTER A WITH DOT ABOVE;Ll;0;L;0061 0307;;;;N;;;0226;;0226 +0228;LATIN CAPITAL LETTER E WITH CEDILLA;Lu;0;L;0045 0327;;;;N;;;;0229; +0229;LATIN SMALL LETTER E WITH CEDILLA;Ll;0;L;0065 0327;;;;N;;;0228;;0228 +022A;LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON;Lu;0;L;00D6 0304;;;;N;;;;022B; +022B;LATIN SMALL LETTER O WITH DIAERESIS AND MACRON;Ll;0;L;00F6 0304;;;;N;;;022A;;022A +022C;LATIN CAPITAL LETTER O WITH TILDE AND MACRON;Lu;0;L;00D5 0304;;;;N;;;;022D; +022D;LATIN SMALL LETTER O WITH TILDE AND MACRON;Ll;0;L;00F5 0304;;;;N;;;022C;;022C +022E;LATIN CAPITAL LETTER O WITH DOT ABOVE;Lu;0;L;004F 0307;;;;N;;;;022F; +022F;LATIN SMALL LETTER O WITH DOT ABOVE;Ll;0;L;006F 0307;;;;N;;;022E;;022E +0230;LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON;Lu;0;L;022E 0304;;;;N;;;;0231; +0231;LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON;Ll;0;L;022F 0304;;;;N;;;0230;;0230 +0232;LATIN CAPITAL LETTER Y WITH MACRON;Lu;0;L;0059 0304;;;;N;;;;0233; +0233;LATIN SMALL LETTER Y WITH MACRON;Ll;0;L;0079 0304;;;;N;;;0232;;0232 +0234;LATIN SMALL LETTER L WITH CURL;Ll;0;L;;;;;N;;;;; +0235;LATIN SMALL LETTER N WITH CURL;Ll;0;L;;;;;N;;;;; +0236;LATIN SMALL LETTER T WITH CURL;Ll;0;L;;;;;N;;;;; +0237;LATIN SMALL LETTER DOTLESS J;Ll;0;L;;;;;N;;;;; +0238;LATIN SMALL LETTER DB DIGRAPH;Ll;0;L;;;;;N;;;;; +0239;LATIN SMALL LETTER QP DIGRAPH;Ll;0;L;;;;;N;;;;; +023A;LATIN CAPITAL LETTER A WITH STROKE;Lu;0;L;;;;;N;;;;2C65; +023B;LATIN CAPITAL LETTER C WITH STROKE;Lu;0;L;;;;;N;;;;023C; +023C;LATIN SMALL LETTER C WITH STROKE;Ll;0;L;;;;;N;;;023B;;023B +023D;LATIN CAPITAL LETTER L WITH BAR;Lu;0;L;;;;;N;;;;019A; +023E;LATIN CAPITAL LETTER T WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;2C66; +023F;LATIN SMALL LETTER S WITH SWASH TAIL;Ll;0;L;;;;;N;;;2C7E;;2C7E +0240;LATIN SMALL LETTER Z WITH SWASH TAIL;Ll;0;L;;;;;N;;;2C7F;;2C7F +0241;LATIN CAPITAL LETTER GLOTTAL STOP;Lu;0;L;;;;;N;;;;0242; +0242;LATIN SMALL LETTER GLOTTAL STOP;Ll;0;L;;;;;N;;;0241;;0241 +0243;LATIN CAPITAL LETTER B WITH STROKE;Lu;0;L;;;;;N;;;;0180; +0244;LATIN CAPITAL LETTER U BAR;Lu;0;L;;;;;N;;;;0289; +0245;LATIN CAPITAL LETTER TURNED V;Lu;0;L;;;;;N;;;;028C; +0246;LATIN CAPITAL LETTER E WITH STROKE;Lu;0;L;;;;;N;;;;0247; +0247;LATIN SMALL LETTER E WITH STROKE;Ll;0;L;;;;;N;;;0246;;0246 +0248;LATIN CAPITAL LETTER J WITH STROKE;Lu;0;L;;;;;N;;;;0249; +0249;LATIN SMALL LETTER J WITH STROKE;Ll;0;L;;;;;N;;;0248;;0248 +024A;LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL;Lu;0;L;;;;;N;;;;024B; +024B;LATIN SMALL LETTER Q WITH HOOK TAIL;Ll;0;L;;;;;N;;;024A;;024A +024C;LATIN CAPITAL LETTER R WITH STROKE;Lu;0;L;;;;;N;;;;024D; +024D;LATIN SMALL LETTER R WITH STROKE;Ll;0;L;;;;;N;;;024C;;024C +024E;LATIN CAPITAL LETTER Y WITH STROKE;Lu;0;L;;;;;N;;;;024F; +024F;LATIN SMALL LETTER Y WITH STROKE;Ll;0;L;;;;;N;;;024E;;024E +0250;LATIN SMALL LETTER TURNED A;Ll;0;L;;;;;N;;;2C6F;;2C6F +0251;LATIN SMALL LETTER ALPHA;Ll;0;L;;;;;N;LATIN SMALL LETTER SCRIPT A;;2C6D;;2C6D +0252;LATIN SMALL LETTER TURNED ALPHA;Ll;0;L;;;;;N;LATIN SMALL LETTER TURNED SCRIPT A;;2C70;;2C70 +0253;LATIN SMALL LETTER B WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER B HOOK;;0181;;0181 +0254;LATIN SMALL LETTER OPEN O;Ll;0;L;;;;;N;;;0186;;0186 +0255;LATIN SMALL LETTER C WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER C CURL;;;; +0256;LATIN SMALL LETTER D WITH TAIL;Ll;0;L;;;;;N;LATIN SMALL LETTER D RETROFLEX HOOK;;0189;;0189 +0257;LATIN SMALL LETTER D WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER D HOOK;;018A;;018A +0258;LATIN SMALL LETTER REVERSED E;Ll;0;L;;;;;N;;;;; +0259;LATIN SMALL LETTER SCHWA;Ll;0;L;;;;;N;;;018F;;018F +025A;LATIN SMALL LETTER SCHWA WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER SCHWA HOOK;;;; +025B;LATIN SMALL LETTER OPEN E;Ll;0;L;;;;;N;LATIN SMALL LETTER EPSILON;;0190;;0190 +025C;LATIN SMALL LETTER REVERSED OPEN E;Ll;0;L;;;;;N;LATIN SMALL LETTER REVERSED EPSILON;;;; +025D;LATIN SMALL LETTER REVERSED OPEN E WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER REVERSED EPSILON HOOK;;;; +025E;LATIN SMALL LETTER CLOSED REVERSED OPEN E;Ll;0;L;;;;;N;LATIN SMALL LETTER CLOSED REVERSED EPSILON;;;; +025F;LATIN SMALL LETTER DOTLESS J WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER DOTLESS J BAR;;;; +0260;LATIN SMALL LETTER G WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER G HOOK;;0193;;0193 +0261;LATIN SMALL LETTER SCRIPT G;Ll;0;L;;;;;N;;;;; +0262;LATIN LETTER SMALL CAPITAL G;Ll;0;L;;;;;N;;;;; +0263;LATIN SMALL LETTER GAMMA;Ll;0;L;;;;;N;;;0194;;0194 +0264;LATIN SMALL LETTER RAMS HORN;Ll;0;L;;;;;N;LATIN SMALL LETTER BABY GAMMA;;;; +0265;LATIN SMALL LETTER TURNED H;Ll;0;L;;;;;N;;;A78D;;A78D +0266;LATIN SMALL LETTER H WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER H HOOK;;A7AA;;A7AA +0267;LATIN SMALL LETTER HENG WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER HENG HOOK;;;; +0268;LATIN SMALL LETTER I WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED I;;0197;;0197 +0269;LATIN SMALL LETTER IOTA;Ll;0;L;;;;;N;;;0196;;0196 +026A;LATIN LETTER SMALL CAPITAL I;Ll;0;L;;;;;N;;;;; +026B;LATIN SMALL LETTER L WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;2C62;;2C62 +026C;LATIN SMALL LETTER L WITH BELT;Ll;0;L;;;;;N;LATIN SMALL LETTER L BELT;;;; +026D;LATIN SMALL LETTER L WITH RETROFLEX HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER L RETROFLEX HOOK;;;; +026E;LATIN SMALL LETTER LEZH;Ll;0;L;;;;;N;LATIN SMALL LETTER L YOGH;;;; +026F;LATIN SMALL LETTER TURNED M;Ll;0;L;;;;;N;;;019C;;019C +0270;LATIN SMALL LETTER TURNED M WITH LONG LEG;Ll;0;L;;;;;N;;;;; +0271;LATIN SMALL LETTER M WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER M HOOK;;2C6E;;2C6E +0272;LATIN SMALL LETTER N WITH LEFT HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER N HOOK;;019D;;019D +0273;LATIN SMALL LETTER N WITH RETROFLEX HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER N RETROFLEX HOOK;;;; +0274;LATIN LETTER SMALL CAPITAL N;Ll;0;L;;;;;N;;;;; +0275;LATIN SMALL LETTER BARRED O;Ll;0;L;;;;;N;;;019F;;019F +0276;LATIN LETTER SMALL CAPITAL OE;Ll;0;L;;;;;N;LATIN LETTER SMALL CAPITAL O E;;;; +0277;LATIN SMALL LETTER CLOSED OMEGA;Ll;0;L;;;;;N;;;;; +0278;LATIN SMALL LETTER PHI;Ll;0;L;;;;;N;;;;; +0279;LATIN SMALL LETTER TURNED R;Ll;0;L;;;;;N;;;;; +027A;LATIN SMALL LETTER TURNED R WITH LONG LEG;Ll;0;L;;;;;N;;;;; +027B;LATIN SMALL LETTER TURNED R WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER TURNED R HOOK;;;; +027C;LATIN SMALL LETTER R WITH LONG LEG;Ll;0;L;;;;;N;;;;; +027D;LATIN SMALL LETTER R WITH TAIL;Ll;0;L;;;;;N;LATIN SMALL LETTER R HOOK;;2C64;;2C64 +027E;LATIN SMALL LETTER R WITH FISHHOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER FISHHOOK R;;;; +027F;LATIN SMALL LETTER REVERSED R WITH FISHHOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER REVERSED FISHHOOK R;;;; +0280;LATIN LETTER SMALL CAPITAL R;Ll;0;L;;;;;N;;;01A6;;01A6 +0281;LATIN LETTER SMALL CAPITAL INVERTED R;Ll;0;L;;;;;N;;;;; +0282;LATIN SMALL LETTER S WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER S HOOK;;;; +0283;LATIN SMALL LETTER ESH;Ll;0;L;;;;;N;;;01A9;;01A9 +0284;LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER DOTLESS J BAR HOOK;;;; +0285;LATIN SMALL LETTER SQUAT REVERSED ESH;Ll;0;L;;;;;N;;;;; +0286;LATIN SMALL LETTER ESH WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER ESH CURL;;;; +0287;LATIN SMALL LETTER TURNED T;Ll;0;L;;;;;N;;;;; +0288;LATIN SMALL LETTER T WITH RETROFLEX HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER T RETROFLEX HOOK;;01AE;;01AE +0289;LATIN SMALL LETTER U BAR;Ll;0;L;;;;;N;;;0244;;0244 +028A;LATIN SMALL LETTER UPSILON;Ll;0;L;;;;;N;;;01B1;;01B1 +028B;LATIN SMALL LETTER V WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER SCRIPT V;;01B2;;01B2 +028C;LATIN SMALL LETTER TURNED V;Ll;0;L;;;;;N;;;0245;;0245 +028D;LATIN SMALL LETTER TURNED W;Ll;0;L;;;;;N;;;;; +028E;LATIN SMALL LETTER TURNED Y;Ll;0;L;;;;;N;;;;; +028F;LATIN LETTER SMALL CAPITAL Y;Ll;0;L;;;;;N;;;;; +0290;LATIN SMALL LETTER Z WITH RETROFLEX HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER Z RETROFLEX HOOK;;;; +0291;LATIN SMALL LETTER Z WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER Z CURL;;;; +0292;LATIN SMALL LETTER EZH;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH;;01B7;;01B7 +0293;LATIN SMALL LETTER EZH WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH CURL;;;; +0294;LATIN LETTER GLOTTAL STOP;Lo;0;L;;;;;N;;;;; +0295;LATIN LETTER PHARYNGEAL VOICED FRICATIVE;Ll;0;L;;;;;N;LATIN LETTER REVERSED GLOTTAL STOP;;;; +0296;LATIN LETTER INVERTED GLOTTAL STOP;Ll;0;L;;;;;N;;;;; +0297;LATIN LETTER STRETCHED C;Ll;0;L;;;;;N;;;;; +0298;LATIN LETTER BILABIAL CLICK;Ll;0;L;;;;;N;LATIN LETTER BULLSEYE;;;; +0299;LATIN LETTER SMALL CAPITAL B;Ll;0;L;;;;;N;;;;; +029A;LATIN SMALL LETTER CLOSED OPEN E;Ll;0;L;;;;;N;LATIN SMALL LETTER CLOSED EPSILON;;;; +029B;LATIN LETTER SMALL CAPITAL G WITH HOOK;Ll;0;L;;;;;N;LATIN LETTER SMALL CAPITAL G HOOK;;;; +029C;LATIN LETTER SMALL CAPITAL H;Ll;0;L;;;;;N;;;;; +029D;LATIN SMALL LETTER J WITH CROSSED-TAIL;Ll;0;L;;;;;N;LATIN SMALL LETTER CROSSED-TAIL J;;;; +029E;LATIN SMALL LETTER TURNED K;Ll;0;L;;;;;N;;;;; +029F;LATIN LETTER SMALL CAPITAL L;Ll;0;L;;;;;N;;;;; +02A0;LATIN SMALL LETTER Q WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER Q HOOK;;;; +02A1;LATIN LETTER GLOTTAL STOP WITH STROKE;Ll;0;L;;;;;N;LATIN LETTER GLOTTAL STOP BAR;;;; +02A2;LATIN LETTER REVERSED GLOTTAL STOP WITH STROKE;Ll;0;L;;;;;N;LATIN LETTER REVERSED GLOTTAL STOP BAR;;;; +02A3;LATIN SMALL LETTER DZ DIGRAPH;Ll;0;L;;;;;N;LATIN SMALL LETTER D Z;;;; +02A4;LATIN SMALL LETTER DEZH DIGRAPH;Ll;0;L;;;;;N;LATIN SMALL LETTER D YOGH;;;; +02A5;LATIN SMALL LETTER DZ DIGRAPH WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER D Z CURL;;;; +02A6;LATIN SMALL LETTER TS DIGRAPH;Ll;0;L;;;;;N;LATIN SMALL LETTER T S;;;; +02A7;LATIN SMALL LETTER TESH DIGRAPH;Ll;0;L;;;;;N;LATIN SMALL LETTER T ESH;;;; +02A8;LATIN SMALL LETTER TC DIGRAPH WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER T C CURL;;;; +02A9;LATIN SMALL LETTER FENG DIGRAPH;Ll;0;L;;;;;N;;;;; +02AA;LATIN SMALL LETTER LS DIGRAPH;Ll;0;L;;;;;N;;;;; +02AB;LATIN SMALL LETTER LZ DIGRAPH;Ll;0;L;;;;;N;;;;; +02AC;LATIN LETTER BILABIAL PERCUSSIVE;Ll;0;L;;;;;N;;;;; +02AD;LATIN LETTER BIDENTAL PERCUSSIVE;Ll;0;L;;;;;N;;;;; +02AE;LATIN SMALL LETTER TURNED H WITH FISHHOOK;Ll;0;L;;;;;N;;;;; +02AF;LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL;Ll;0;L;;;;;N;;;;; +02B0;MODIFIER LETTER SMALL H;Lm;0;L; 0068;;;;N;;;;; +02B1;MODIFIER LETTER SMALL H WITH HOOK;Lm;0;L; 0266;;;;N;MODIFIER LETTER SMALL H HOOK;;;; +02B2;MODIFIER LETTER SMALL J;Lm;0;L; 006A;;;;N;;;;; +02B3;MODIFIER LETTER SMALL R;Lm;0;L; 0072;;;;N;;;;; +02B4;MODIFIER LETTER SMALL TURNED R;Lm;0;L; 0279;;;;N;;;;; +02B5;MODIFIER LETTER SMALL TURNED R WITH HOOK;Lm;0;L; 027B;;;;N;MODIFIER LETTER SMALL TURNED R HOOK;;;; +02B6;MODIFIER LETTER SMALL CAPITAL INVERTED R;Lm;0;L; 0281;;;;N;;;;; +02B7;MODIFIER LETTER SMALL W;Lm;0;L; 0077;;;;N;;;;; +02B8;MODIFIER LETTER SMALL Y;Lm;0;L; 0079;;;;N;;;;; +02B9;MODIFIER LETTER PRIME;Lm;0;ON;;;;;N;;;;; +02BA;MODIFIER LETTER DOUBLE PRIME;Lm;0;ON;;;;;N;;;;; +02BB;MODIFIER LETTER TURNED COMMA;Lm;0;L;;;;;N;;;;; +02BC;MODIFIER LETTER APOSTROPHE;Lm;0;L;;;;;N;;;;; +02BD;MODIFIER LETTER REVERSED COMMA;Lm;0;L;;;;;N;;;;; +02BE;MODIFIER LETTER RIGHT HALF RING;Lm;0;L;;;;;N;;;;; +02BF;MODIFIER LETTER LEFT HALF RING;Lm;0;L;;;;;N;;;;; +02C0;MODIFIER LETTER GLOTTAL STOP;Lm;0;L;;;;;N;;;;; +02C1;MODIFIER LETTER REVERSED GLOTTAL STOP;Lm;0;L;;;;;N;;;;; +02C2;MODIFIER LETTER LEFT ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02C3;MODIFIER LETTER RIGHT ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02C4;MODIFIER LETTER UP ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02C5;MODIFIER LETTER DOWN ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02C6;MODIFIER LETTER CIRCUMFLEX ACCENT;Lm;0;ON;;;;;N;MODIFIER LETTER CIRCUMFLEX;;;; +02C7;CARON;Lm;0;ON;;;;;N;MODIFIER LETTER HACEK;;;; +02C8;MODIFIER LETTER VERTICAL LINE;Lm;0;ON;;;;;N;;;;; +02C9;MODIFIER LETTER MACRON;Lm;0;ON;;;;;N;;;;; +02CA;MODIFIER LETTER ACUTE ACCENT;Lm;0;ON;;;;;N;MODIFIER LETTER ACUTE;;;; +02CB;MODIFIER LETTER GRAVE ACCENT;Lm;0;ON;;;;;N;MODIFIER LETTER GRAVE;;;; +02CC;MODIFIER LETTER LOW VERTICAL LINE;Lm;0;ON;;;;;N;;;;; +02CD;MODIFIER LETTER LOW MACRON;Lm;0;ON;;;;;N;;;;; +02CE;MODIFIER LETTER LOW GRAVE ACCENT;Lm;0;ON;;;;;N;MODIFIER LETTER LOW GRAVE;;;; +02CF;MODIFIER LETTER LOW ACUTE ACCENT;Lm;0;ON;;;;;N;MODIFIER LETTER LOW ACUTE;;;; +02D0;MODIFIER LETTER TRIANGULAR COLON;Lm;0;L;;;;;N;;;;; +02D1;MODIFIER LETTER HALF TRIANGULAR COLON;Lm;0;L;;;;;N;;;;; +02D2;MODIFIER LETTER CENTRED RIGHT HALF RING;Sk;0;ON;;;;;N;MODIFIER LETTER CENTERED RIGHT HALF RING;;;; +02D3;MODIFIER LETTER CENTRED LEFT HALF RING;Sk;0;ON;;;;;N;MODIFIER LETTER CENTERED LEFT HALF RING;;;; +02D4;MODIFIER LETTER UP TACK;Sk;0;ON;;;;;N;;;;; +02D5;MODIFIER LETTER DOWN TACK;Sk;0;ON;;;;;N;;;;; +02D6;MODIFIER LETTER PLUS SIGN;Sk;0;ON;;;;;N;;;;; +02D7;MODIFIER LETTER MINUS SIGN;Sk;0;ON;;;;;N;;;;; +02D8;BREVE;Sk;0;ON; 0020 0306;;;;N;SPACING BREVE;;;; +02D9;DOT ABOVE;Sk;0;ON; 0020 0307;;;;N;SPACING DOT ABOVE;;;; +02DA;RING ABOVE;Sk;0;ON; 0020 030A;;;;N;SPACING RING ABOVE;;;; +02DB;OGONEK;Sk;0;ON; 0020 0328;;;;N;SPACING OGONEK;;;; +02DC;SMALL TILDE;Sk;0;ON; 0020 0303;;;;N;SPACING TILDE;;;; +02DD;DOUBLE ACUTE ACCENT;Sk;0;ON; 0020 030B;;;;N;SPACING DOUBLE ACUTE;;;; +02DE;MODIFIER LETTER RHOTIC HOOK;Sk;0;ON;;;;;N;;;;; +02DF;MODIFIER LETTER CROSS ACCENT;Sk;0;ON;;;;;N;;;;; +02E0;MODIFIER LETTER SMALL GAMMA;Lm;0;L; 0263;;;;N;;;;; +02E1;MODIFIER LETTER SMALL L;Lm;0;L; 006C;;;;N;;;;; +02E2;MODIFIER LETTER SMALL S;Lm;0;L; 0073;;;;N;;;;; +02E3;MODIFIER LETTER SMALL X;Lm;0;L; 0078;;;;N;;;;; +02E4;MODIFIER LETTER SMALL REVERSED GLOTTAL STOP;Lm;0;L; 0295;;;;N;;;;; +02E5;MODIFIER LETTER EXTRA-HIGH TONE BAR;Sk;0;ON;;;;;N;;;;; +02E6;MODIFIER LETTER HIGH TONE BAR;Sk;0;ON;;;;;N;;;;; +02E7;MODIFIER LETTER MID TONE BAR;Sk;0;ON;;;;;N;;;;; +02E8;MODIFIER LETTER LOW TONE BAR;Sk;0;ON;;;;;N;;;;; +02E9;MODIFIER LETTER EXTRA-LOW TONE BAR;Sk;0;ON;;;;;N;;;;; +02EA;MODIFIER LETTER YIN DEPARTING TONE MARK;Sk;0;ON;;;;;N;;;;; +02EB;MODIFIER LETTER YANG DEPARTING TONE MARK;Sk;0;ON;;;;;N;;;;; +02EC;MODIFIER LETTER VOICING;Lm;0;ON;;;;;N;;;;; +02ED;MODIFIER LETTER UNASPIRATED;Sk;0;ON;;;;;N;;;;; +02EE;MODIFIER LETTER DOUBLE APOSTROPHE;Lm;0;L;;;;;N;;;;; +02EF;MODIFIER LETTER LOW DOWN ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02F0;MODIFIER LETTER LOW UP ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02F1;MODIFIER LETTER LOW LEFT ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02F2;MODIFIER LETTER LOW RIGHT ARROWHEAD;Sk;0;ON;;;;;N;;;;; +02F3;MODIFIER LETTER LOW RING;Sk;0;ON;;;;;N;;;;; +02F4;MODIFIER LETTER MIDDLE GRAVE ACCENT;Sk;0;ON;;;;;N;;;;; +02F5;MODIFIER LETTER MIDDLE DOUBLE GRAVE ACCENT;Sk;0;ON;;;;;N;;;;; +02F6;MODIFIER LETTER MIDDLE DOUBLE ACUTE ACCENT;Sk;0;ON;;;;;N;;;;; +02F7;MODIFIER LETTER LOW TILDE;Sk;0;ON;;;;;N;;;;; +02F8;MODIFIER LETTER RAISED COLON;Sk;0;ON;;;;;N;;;;; +02F9;MODIFIER LETTER BEGIN HIGH TONE;Sk;0;ON;;;;;N;;;;; +02FA;MODIFIER LETTER END HIGH TONE;Sk;0;ON;;;;;N;;;;; +02FB;MODIFIER LETTER BEGIN LOW TONE;Sk;0;ON;;;;;N;;;;; +02FC;MODIFIER LETTER END LOW TONE;Sk;0;ON;;;;;N;;;;; +02FD;MODIFIER LETTER SHELF;Sk;0;ON;;;;;N;;;;; +02FE;MODIFIER LETTER OPEN SHELF;Sk;0;ON;;;;;N;;;;; +02FF;MODIFIER LETTER LOW LEFT ARROW;Sk;0;ON;;;;;N;;;;; +0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;NON-SPACING GRAVE;;;; +0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;NON-SPACING ACUTE;;;; +0302;COMBINING CIRCUMFLEX ACCENT;Mn;230;NSM;;;;;N;NON-SPACING CIRCUMFLEX;;;; +0303;COMBINING TILDE;Mn;230;NSM;;;;;N;NON-SPACING TILDE;;;; +0304;COMBINING MACRON;Mn;230;NSM;;;;;N;NON-SPACING MACRON;;;; +0305;COMBINING OVERLINE;Mn;230;NSM;;;;;N;NON-SPACING OVERSCORE;;;; +0306;COMBINING BREVE;Mn;230;NSM;;;;;N;NON-SPACING BREVE;;;; +0307;COMBINING DOT ABOVE;Mn;230;NSM;;;;;N;NON-SPACING DOT ABOVE;;;; +0308;COMBINING DIAERESIS;Mn;230;NSM;;;;;N;NON-SPACING DIAERESIS;;;; +0309;COMBINING HOOK ABOVE;Mn;230;NSM;;;;;N;NON-SPACING HOOK ABOVE;;;; +030A;COMBINING RING ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RING ABOVE;;;; +030B;COMBINING DOUBLE ACUTE ACCENT;Mn;230;NSM;;;;;N;NON-SPACING DOUBLE ACUTE;;;; +030C;COMBINING CARON;Mn;230;NSM;;;;;N;NON-SPACING HACEK;;;; +030D;COMBINING VERTICAL LINE ABOVE;Mn;230;NSM;;;;;N;NON-SPACING VERTICAL LINE ABOVE;;;; +030E;COMBINING DOUBLE VERTICAL LINE ABOVE;Mn;230;NSM;;;;;N;NON-SPACING DOUBLE VERTICAL LINE ABOVE;;;; +030F;COMBINING DOUBLE GRAVE ACCENT;Mn;230;NSM;;;;;N;NON-SPACING DOUBLE GRAVE;;;; +0310;COMBINING CANDRABINDU;Mn;230;NSM;;;;;N;NON-SPACING CANDRABINDU;;;; +0311;COMBINING INVERTED BREVE;Mn;230;NSM;;;;;N;NON-SPACING INVERTED BREVE;;;; +0312;COMBINING TURNED COMMA ABOVE;Mn;230;NSM;;;;;N;NON-SPACING TURNED COMMA ABOVE;;;; +0313;COMBINING COMMA ABOVE;Mn;230;NSM;;;;;N;NON-SPACING COMMA ABOVE;;;; +0314;COMBINING REVERSED COMMA ABOVE;Mn;230;NSM;;;;;N;NON-SPACING REVERSED COMMA ABOVE;;;; +0315;COMBINING COMMA ABOVE RIGHT;Mn;232;NSM;;;;;N;NON-SPACING COMMA ABOVE RIGHT;;;; +0316;COMBINING GRAVE ACCENT BELOW;Mn;220;NSM;;;;;N;NON-SPACING GRAVE BELOW;;;; +0317;COMBINING ACUTE ACCENT BELOW;Mn;220;NSM;;;;;N;NON-SPACING ACUTE BELOW;;;; +0318;COMBINING LEFT TACK BELOW;Mn;220;NSM;;;;;N;NON-SPACING LEFT TACK BELOW;;;; +0319;COMBINING RIGHT TACK BELOW;Mn;220;NSM;;;;;N;NON-SPACING RIGHT TACK BELOW;;;; +031A;COMBINING LEFT ANGLE ABOVE;Mn;232;NSM;;;;;N;NON-SPACING LEFT ANGLE ABOVE;;;; +031B;COMBINING HORN;Mn;216;NSM;;;;;N;NON-SPACING HORN;;;; +031C;COMBINING LEFT HALF RING BELOW;Mn;220;NSM;;;;;N;NON-SPACING LEFT HALF RING BELOW;;;; +031D;COMBINING UP TACK BELOW;Mn;220;NSM;;;;;N;NON-SPACING UP TACK BELOW;;;; +031E;COMBINING DOWN TACK BELOW;Mn;220;NSM;;;;;N;NON-SPACING DOWN TACK BELOW;;;; +031F;COMBINING PLUS SIGN BELOW;Mn;220;NSM;;;;;N;NON-SPACING PLUS SIGN BELOW;;;; +0320;COMBINING MINUS SIGN BELOW;Mn;220;NSM;;;;;N;NON-SPACING MINUS SIGN BELOW;;;; +0321;COMBINING PALATALIZED HOOK BELOW;Mn;202;NSM;;;;;N;NON-SPACING PALATALIZED HOOK BELOW;;;; +0322;COMBINING RETROFLEX HOOK BELOW;Mn;202;NSM;;;;;N;NON-SPACING RETROFLEX HOOK BELOW;;;; +0323;COMBINING DOT BELOW;Mn;220;NSM;;;;;N;NON-SPACING DOT BELOW;;;; +0324;COMBINING DIAERESIS BELOW;Mn;220;NSM;;;;;N;NON-SPACING DOUBLE DOT BELOW;;;; +0325;COMBINING RING BELOW;Mn;220;NSM;;;;;N;NON-SPACING RING BELOW;;;; +0326;COMBINING COMMA BELOW;Mn;220;NSM;;;;;N;NON-SPACING COMMA BELOW;;;; +0327;COMBINING CEDILLA;Mn;202;NSM;;;;;N;NON-SPACING CEDILLA;;;; +0328;COMBINING OGONEK;Mn;202;NSM;;;;;N;NON-SPACING OGONEK;;;; +0329;COMBINING VERTICAL LINE BELOW;Mn;220;NSM;;;;;N;NON-SPACING VERTICAL LINE BELOW;;;; +032A;COMBINING BRIDGE BELOW;Mn;220;NSM;;;;;N;NON-SPACING BRIDGE BELOW;;;; +032B;COMBINING INVERTED DOUBLE ARCH BELOW;Mn;220;NSM;;;;;N;NON-SPACING INVERTED DOUBLE ARCH BELOW;;;; +032C;COMBINING CARON BELOW;Mn;220;NSM;;;;;N;NON-SPACING HACEK BELOW;;;; +032D;COMBINING CIRCUMFLEX ACCENT BELOW;Mn;220;NSM;;;;;N;NON-SPACING CIRCUMFLEX BELOW;;;; +032E;COMBINING BREVE BELOW;Mn;220;NSM;;;;;N;NON-SPACING BREVE BELOW;;;; +032F;COMBINING INVERTED BREVE BELOW;Mn;220;NSM;;;;;N;NON-SPACING INVERTED BREVE BELOW;;;; +0330;COMBINING TILDE BELOW;Mn;220;NSM;;;;;N;NON-SPACING TILDE BELOW;;;; +0331;COMBINING MACRON BELOW;Mn;220;NSM;;;;;N;NON-SPACING MACRON BELOW;;;; +0332;COMBINING LOW LINE;Mn;220;NSM;;;;;N;NON-SPACING UNDERSCORE;;;; +0333;COMBINING DOUBLE LOW LINE;Mn;220;NSM;;;;;N;NON-SPACING DOUBLE UNDERSCORE;;;; +0334;COMBINING TILDE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING TILDE OVERLAY;;;; +0335;COMBINING SHORT STROKE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING SHORT BAR OVERLAY;;;; +0336;COMBINING LONG STROKE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG BAR OVERLAY;;;; +0337;COMBINING SHORT SOLIDUS OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING SHORT SLASH OVERLAY;;;; +0338;COMBINING LONG SOLIDUS OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG SLASH OVERLAY;;;; +0339;COMBINING RIGHT HALF RING BELOW;Mn;220;NSM;;;;;N;NON-SPACING RIGHT HALF RING BELOW;;;; +033A;COMBINING INVERTED BRIDGE BELOW;Mn;220;NSM;;;;;N;NON-SPACING INVERTED BRIDGE BELOW;;;; +033B;COMBINING SQUARE BELOW;Mn;220;NSM;;;;;N;NON-SPACING SQUARE BELOW;;;; +033C;COMBINING SEAGULL BELOW;Mn;220;NSM;;;;;N;NON-SPACING SEAGULL BELOW;;;; +033D;COMBINING X ABOVE;Mn;230;NSM;;;;;N;NON-SPACING X ABOVE;;;; +033E;COMBINING VERTICAL TILDE;Mn;230;NSM;;;;;N;NON-SPACING VERTICAL TILDE;;;; +033F;COMBINING DOUBLE OVERLINE;Mn;230;NSM;;;;;N;NON-SPACING DOUBLE OVERSCORE;;;; +0340;COMBINING GRAVE TONE MARK;Mn;230;NSM;0300;;;;N;NON-SPACING GRAVE TONE MARK;;;; +0341;COMBINING ACUTE TONE MARK;Mn;230;NSM;0301;;;;N;NON-SPACING ACUTE TONE MARK;;;; +0342;COMBINING GREEK PERISPOMENI;Mn;230;NSM;;;;;N;;;;; +0343;COMBINING GREEK KORONIS;Mn;230;NSM;0313;;;;N;;;;; +0344;COMBINING GREEK DIALYTIKA TONOS;Mn;230;NSM;0308 0301;;;;N;GREEK NON-SPACING DIAERESIS TONOS;;;; +0345;COMBINING GREEK YPOGEGRAMMENI;Mn;240;NSM;;;;;N;GREEK NON-SPACING IOTA BELOW;;0399;;0399 +0346;COMBINING BRIDGE ABOVE;Mn;230;NSM;;;;;N;;;;; +0347;COMBINING EQUALS SIGN BELOW;Mn;220;NSM;;;;;N;;;;; +0348;COMBINING DOUBLE VERTICAL LINE BELOW;Mn;220;NSM;;;;;N;;;;; +0349;COMBINING LEFT ANGLE BELOW;Mn;220;NSM;;;;;N;;;;; +034A;COMBINING NOT TILDE ABOVE;Mn;230;NSM;;;;;N;;;;; +034B;COMBINING HOMOTHETIC ABOVE;Mn;230;NSM;;;;;N;;;;; +034C;COMBINING ALMOST EQUAL TO ABOVE;Mn;230;NSM;;;;;N;;;;; +034D;COMBINING LEFT RIGHT ARROW BELOW;Mn;220;NSM;;;;;N;;;;; +034E;COMBINING UPWARDS ARROW BELOW;Mn;220;NSM;;;;;N;;;;; +034F;COMBINING GRAPHEME JOINER;Mn;0;NSM;;;;;N;;;;; +0350;COMBINING RIGHT ARROWHEAD ABOVE;Mn;230;NSM;;;;;N;;;;; +0351;COMBINING LEFT HALF RING ABOVE;Mn;230;NSM;;;;;N;;;;; +0352;COMBINING FERMATA;Mn;230;NSM;;;;;N;;;;; +0353;COMBINING X BELOW;Mn;220;NSM;;;;;N;;;;; +0354;COMBINING LEFT ARROWHEAD BELOW;Mn;220;NSM;;;;;N;;;;; +0355;COMBINING RIGHT ARROWHEAD BELOW;Mn;220;NSM;;;;;N;;;;; +0356;COMBINING RIGHT ARROWHEAD AND UP ARROWHEAD BELOW;Mn;220;NSM;;;;;N;;;;; +0357;COMBINING RIGHT HALF RING ABOVE;Mn;230;NSM;;;;;N;;;;; +0358;COMBINING DOT ABOVE RIGHT;Mn;232;NSM;;;;;N;;;;; +0359;COMBINING ASTERISK BELOW;Mn;220;NSM;;;;;N;;;;; +035A;COMBINING DOUBLE RING BELOW;Mn;220;NSM;;;;;N;;;;; +035B;COMBINING ZIGZAG ABOVE;Mn;230;NSM;;;;;N;;;;; +035C;COMBINING DOUBLE BREVE BELOW;Mn;233;NSM;;;;;N;;;;; +035D;COMBINING DOUBLE BREVE;Mn;234;NSM;;;;;N;;;;; +035E;COMBINING DOUBLE MACRON;Mn;234;NSM;;;;;N;;;;; +035F;COMBINING DOUBLE MACRON BELOW;Mn;233;NSM;;;;;N;;;;; +0360;COMBINING DOUBLE TILDE;Mn;234;NSM;;;;;N;;;;; +0361;COMBINING DOUBLE INVERTED BREVE;Mn;234;NSM;;;;;N;;;;; +0362;COMBINING DOUBLE RIGHTWARDS ARROW BELOW;Mn;233;NSM;;;;;N;;;;; +0363;COMBINING LATIN SMALL LETTER A;Mn;230;NSM;;;;;N;;;;; +0364;COMBINING LATIN SMALL LETTER E;Mn;230;NSM;;;;;N;;;;; +0365;COMBINING LATIN SMALL LETTER I;Mn;230;NSM;;;;;N;;;;; +0366;COMBINING LATIN SMALL LETTER O;Mn;230;NSM;;;;;N;;;;; +0367;COMBINING LATIN SMALL LETTER U;Mn;230;NSM;;;;;N;;;;; +0368;COMBINING LATIN SMALL LETTER C;Mn;230;NSM;;;;;N;;;;; +0369;COMBINING LATIN SMALL LETTER D;Mn;230;NSM;;;;;N;;;;; +036A;COMBINING LATIN SMALL LETTER H;Mn;230;NSM;;;;;N;;;;; +036B;COMBINING LATIN SMALL LETTER M;Mn;230;NSM;;;;;N;;;;; +036C;COMBINING LATIN SMALL LETTER R;Mn;230;NSM;;;;;N;;;;; +036D;COMBINING LATIN SMALL LETTER T;Mn;230;NSM;;;;;N;;;;; +036E;COMBINING LATIN SMALL LETTER V;Mn;230;NSM;;;;;N;;;;; +036F;COMBINING LATIN SMALL LETTER X;Mn;230;NSM;;;;;N;;;;; +0370;GREEK CAPITAL LETTER HETA;Lu;0;L;;;;;N;;;;0371; +0371;GREEK SMALL LETTER HETA;Ll;0;L;;;;;N;;;0370;;0370 +0372;GREEK CAPITAL LETTER ARCHAIC SAMPI;Lu;0;L;;;;;N;;;;0373; +0373;GREEK SMALL LETTER ARCHAIC SAMPI;Ll;0;L;;;;;N;;;0372;;0372 +0374;GREEK NUMERAL SIGN;Lm;0;ON;02B9;;;;N;GREEK UPPER NUMERAL SIGN;;;; +0375;GREEK LOWER NUMERAL SIGN;Sk;0;ON;;;;;N;;;;; +0376;GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA;Lu;0;L;;;;;N;;;;0377; +0377;GREEK SMALL LETTER PAMPHYLIAN DIGAMMA;Ll;0;L;;;;;N;;;0376;;0376 +037A;GREEK YPOGEGRAMMENI;Lm;0;L; 0020 0345;;;;N;GREEK SPACING IOTA BELOW;;;; +037B;GREEK SMALL REVERSED LUNATE SIGMA SYMBOL;Ll;0;L;;;;;N;;;03FD;;03FD +037C;GREEK SMALL DOTTED LUNATE SIGMA SYMBOL;Ll;0;L;;;;;N;;;03FE;;03FE +037D;GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL;Ll;0;L;;;;;N;;;03FF;;03FF +037E;GREEK QUESTION MARK;Po;0;ON;003B;;;;N;;;;; +0384;GREEK TONOS;Sk;0;ON; 0020 0301;;;;N;GREEK SPACING TONOS;;;; +0385;GREEK DIALYTIKA TONOS;Sk;0;ON;00A8 0301;;;;N;GREEK SPACING DIAERESIS TONOS;;;; +0386;GREEK CAPITAL LETTER ALPHA WITH TONOS;Lu;0;L;0391 0301;;;;N;GREEK CAPITAL LETTER ALPHA TONOS;;;03AC; +0387;GREEK ANO TELEIA;Po;0;ON;00B7;;;;N;;;;; +0388;GREEK CAPITAL LETTER EPSILON WITH TONOS;Lu;0;L;0395 0301;;;;N;GREEK CAPITAL LETTER EPSILON TONOS;;;03AD; +0389;GREEK CAPITAL LETTER ETA WITH TONOS;Lu;0;L;0397 0301;;;;N;GREEK CAPITAL LETTER ETA TONOS;;;03AE; +038A;GREEK CAPITAL LETTER IOTA WITH TONOS;Lu;0;L;0399 0301;;;;N;GREEK CAPITAL LETTER IOTA TONOS;;;03AF; +038C;GREEK CAPITAL LETTER OMICRON WITH TONOS;Lu;0;L;039F 0301;;;;N;GREEK CAPITAL LETTER OMICRON TONOS;;;03CC; +038E;GREEK CAPITAL LETTER UPSILON WITH TONOS;Lu;0;L;03A5 0301;;;;N;GREEK CAPITAL LETTER UPSILON TONOS;;;03CD; +038F;GREEK CAPITAL LETTER OMEGA WITH TONOS;Lu;0;L;03A9 0301;;;;N;GREEK CAPITAL LETTER OMEGA TONOS;;;03CE; +0390;GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS;Ll;0;L;03CA 0301;;;;N;GREEK SMALL LETTER IOTA DIAERESIS TONOS;;;; +0391;GREEK CAPITAL LETTER ALPHA;Lu;0;L;;;;;N;;;;03B1; +0392;GREEK CAPITAL LETTER BETA;Lu;0;L;;;;;N;;;;03B2; +0393;GREEK CAPITAL LETTER GAMMA;Lu;0;L;;;;;N;;;;03B3; +0394;GREEK CAPITAL LETTER DELTA;Lu;0;L;;;;;N;;;;03B4; +0395;GREEK CAPITAL LETTER EPSILON;Lu;0;L;;;;;N;;;;03B5; +0396;GREEK CAPITAL LETTER ZETA;Lu;0;L;;;;;N;;;;03B6; +0397;GREEK CAPITAL LETTER ETA;Lu;0;L;;;;;N;;;;03B7; +0398;GREEK CAPITAL LETTER THETA;Lu;0;L;;;;;N;;;;03B8; +0399;GREEK CAPITAL LETTER IOTA;Lu;0;L;;;;;N;;;;03B9; +039A;GREEK CAPITAL LETTER KAPPA;Lu;0;L;;;;;N;;;;03BA; +039B;GREEK CAPITAL LETTER LAMDA;Lu;0;L;;;;;N;GREEK CAPITAL LETTER LAMBDA;;;03BB; +039C;GREEK CAPITAL LETTER MU;Lu;0;L;;;;;N;;;;03BC; +039D;GREEK CAPITAL LETTER NU;Lu;0;L;;;;;N;;;;03BD; +039E;GREEK CAPITAL LETTER XI;Lu;0;L;;;;;N;;;;03BE; +039F;GREEK CAPITAL LETTER OMICRON;Lu;0;L;;;;;N;;;;03BF; +03A0;GREEK CAPITAL LETTER PI;Lu;0;L;;;;;N;;;;03C0; +03A1;GREEK CAPITAL LETTER RHO;Lu;0;L;;;;;N;;;;03C1; +03A3;GREEK CAPITAL LETTER SIGMA;Lu;0;L;;;;;N;;;;03C3; +03A4;GREEK CAPITAL LETTER TAU;Lu;0;L;;;;;N;;;;03C4; +03A5;GREEK CAPITAL LETTER UPSILON;Lu;0;L;;;;;N;;;;03C5; +03A6;GREEK CAPITAL LETTER PHI;Lu;0;L;;;;;N;;;;03C6; +03A7;GREEK CAPITAL LETTER CHI;Lu;0;L;;;;;N;;;;03C7; +03A8;GREEK CAPITAL LETTER PSI;Lu;0;L;;;;;N;;;;03C8; +03A9;GREEK CAPITAL LETTER OMEGA;Lu;0;L;;;;;N;;;;03C9; +03AA;GREEK CAPITAL LETTER IOTA WITH DIALYTIKA;Lu;0;L;0399 0308;;;;N;GREEK CAPITAL LETTER IOTA DIAERESIS;;;03CA; +03AB;GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA;Lu;0;L;03A5 0308;;;;N;GREEK CAPITAL LETTER UPSILON DIAERESIS;;;03CB; +03AC;GREEK SMALL LETTER ALPHA WITH TONOS;Ll;0;L;03B1 0301;;;;N;GREEK SMALL LETTER ALPHA TONOS;;0386;;0386 +03AD;GREEK SMALL LETTER EPSILON WITH TONOS;Ll;0;L;03B5 0301;;;;N;GREEK SMALL LETTER EPSILON TONOS;;0388;;0388 +03AE;GREEK SMALL LETTER ETA WITH TONOS;Ll;0;L;03B7 0301;;;;N;GREEK SMALL LETTER ETA TONOS;;0389;;0389 +03AF;GREEK SMALL LETTER IOTA WITH TONOS;Ll;0;L;03B9 0301;;;;N;GREEK SMALL LETTER IOTA TONOS;;038A;;038A +03B0;GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS;Ll;0;L;03CB 0301;;;;N;GREEK SMALL LETTER UPSILON DIAERESIS TONOS;;;; +03B1;GREEK SMALL LETTER ALPHA;Ll;0;L;;;;;N;;;0391;;0391 +03B2;GREEK SMALL LETTER BETA;Ll;0;L;;;;;N;;;0392;;0392 +03B3;GREEK SMALL LETTER GAMMA;Ll;0;L;;;;;N;;;0393;;0393 +03B4;GREEK SMALL LETTER DELTA;Ll;0;L;;;;;N;;;0394;;0394 +03B5;GREEK SMALL LETTER EPSILON;Ll;0;L;;;;;N;;;0395;;0395 +03B6;GREEK SMALL LETTER ZETA;Ll;0;L;;;;;N;;;0396;;0396 +03B7;GREEK SMALL LETTER ETA;Ll;0;L;;;;;N;;;0397;;0397 +03B8;GREEK SMALL LETTER THETA;Ll;0;L;;;;;N;;;0398;;0398 +03B9;GREEK SMALL LETTER IOTA;Ll;0;L;;;;;N;;;0399;;0399 +03BA;GREEK SMALL LETTER KAPPA;Ll;0;L;;;;;N;;;039A;;039A +03BB;GREEK SMALL LETTER LAMDA;Ll;0;L;;;;;N;GREEK SMALL LETTER LAMBDA;;039B;;039B +03BC;GREEK SMALL LETTER MU;Ll;0;L;;;;;N;;;039C;;039C +03BD;GREEK SMALL LETTER NU;Ll;0;L;;;;;N;;;039D;;039D +03BE;GREEK SMALL LETTER XI;Ll;0;L;;;;;N;;;039E;;039E +03BF;GREEK SMALL LETTER OMICRON;Ll;0;L;;;;;N;;;039F;;039F +03C0;GREEK SMALL LETTER PI;Ll;0;L;;;;;N;;;03A0;;03A0 +03C1;GREEK SMALL LETTER RHO;Ll;0;L;;;;;N;;;03A1;;03A1 +03C2;GREEK SMALL LETTER FINAL SIGMA;Ll;0;L;;;;;N;;;03A3;;03A3 +03C3;GREEK SMALL LETTER SIGMA;Ll;0;L;;;;;N;;;03A3;;03A3 +03C4;GREEK SMALL LETTER TAU;Ll;0;L;;;;;N;;;03A4;;03A4 +03C5;GREEK SMALL LETTER UPSILON;Ll;0;L;;;;;N;;;03A5;;03A5 +03C6;GREEK SMALL LETTER PHI;Ll;0;L;;;;;N;;;03A6;;03A6 +03C7;GREEK SMALL LETTER CHI;Ll;0;L;;;;;N;;;03A7;;03A7 +03C8;GREEK SMALL LETTER PSI;Ll;0;L;;;;;N;;;03A8;;03A8 +03C9;GREEK SMALL LETTER OMEGA;Ll;0;L;;;;;N;;;03A9;;03A9 +03CA;GREEK SMALL LETTER IOTA WITH DIALYTIKA;Ll;0;L;03B9 0308;;;;N;GREEK SMALL LETTER IOTA DIAERESIS;;03AA;;03AA +03CB;GREEK SMALL LETTER UPSILON WITH DIALYTIKA;Ll;0;L;03C5 0308;;;;N;GREEK SMALL LETTER UPSILON DIAERESIS;;03AB;;03AB +03CC;GREEK SMALL LETTER OMICRON WITH TONOS;Ll;0;L;03BF 0301;;;;N;GREEK SMALL LETTER OMICRON TONOS;;038C;;038C +03CD;GREEK SMALL LETTER UPSILON WITH TONOS;Ll;0;L;03C5 0301;;;;N;GREEK SMALL LETTER UPSILON TONOS;;038E;;038E +03CE;GREEK SMALL LETTER OMEGA WITH TONOS;Ll;0;L;03C9 0301;;;;N;GREEK SMALL LETTER OMEGA TONOS;;038F;;038F +03CF;GREEK CAPITAL KAI SYMBOL;Lu;0;L;;;;;N;;;;03D7; +03D0;GREEK BETA SYMBOL;Ll;0;L; 03B2;;;;N;GREEK SMALL LETTER CURLED BETA;;0392;;0392 +03D1;GREEK THETA SYMBOL;Ll;0;L; 03B8;;;;N;GREEK SMALL LETTER SCRIPT THETA;;0398;;0398 +03D2;GREEK UPSILON WITH HOOK SYMBOL;Lu;0;L; 03A5;;;;N;GREEK CAPITAL LETTER UPSILON HOOK;;;; +03D3;GREEK UPSILON WITH ACUTE AND HOOK SYMBOL;Lu;0;L;03D2 0301;;;;N;GREEK CAPITAL LETTER UPSILON HOOK TONOS;;;; +03D4;GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL;Lu;0;L;03D2 0308;;;;N;GREEK CAPITAL LETTER UPSILON HOOK DIAERESIS;;;; +03D5;GREEK PHI SYMBOL;Ll;0;L; 03C6;;;;N;GREEK SMALL LETTER SCRIPT PHI;;03A6;;03A6 +03D6;GREEK PI SYMBOL;Ll;0;L; 03C0;;;;N;GREEK SMALL LETTER OMEGA PI;;03A0;;03A0 +03D7;GREEK KAI SYMBOL;Ll;0;L;;;;;N;;;03CF;;03CF +03D8;GREEK LETTER ARCHAIC KOPPA;Lu;0;L;;;;;N;;;;03D9; +03D9;GREEK SMALL LETTER ARCHAIC KOPPA;Ll;0;L;;;;;N;;;03D8;;03D8 +03DA;GREEK LETTER STIGMA;Lu;0;L;;;;;N;GREEK CAPITAL LETTER STIGMA;;;03DB; +03DB;GREEK SMALL LETTER STIGMA;Ll;0;L;;;;;N;;;03DA;;03DA +03DC;GREEK LETTER DIGAMMA;Lu;0;L;;;;;N;GREEK CAPITAL LETTER DIGAMMA;;;03DD; +03DD;GREEK SMALL LETTER DIGAMMA;Ll;0;L;;;;;N;;;03DC;;03DC +03DE;GREEK LETTER KOPPA;Lu;0;L;;;;;N;GREEK CAPITAL LETTER KOPPA;;;03DF; +03DF;GREEK SMALL LETTER KOPPA;Ll;0;L;;;;;N;;;03DE;;03DE +03E0;GREEK LETTER SAMPI;Lu;0;L;;;;;N;GREEK CAPITAL LETTER SAMPI;;;03E1; +03E1;GREEK SMALL LETTER SAMPI;Ll;0;L;;;;;N;;;03E0;;03E0 +03E2;COPTIC CAPITAL LETTER SHEI;Lu;0;L;;;;;N;GREEK CAPITAL LETTER SHEI;;;03E3; +03E3;COPTIC SMALL LETTER SHEI;Ll;0;L;;;;;N;GREEK SMALL LETTER SHEI;;03E2;;03E2 +03E4;COPTIC CAPITAL LETTER FEI;Lu;0;L;;;;;N;GREEK CAPITAL LETTER FEI;;;03E5; +03E5;COPTIC SMALL LETTER FEI;Ll;0;L;;;;;N;GREEK SMALL LETTER FEI;;03E4;;03E4 +03E6;COPTIC CAPITAL LETTER KHEI;Lu;0;L;;;;;N;GREEK CAPITAL LETTER KHEI;;;03E7; +03E7;COPTIC SMALL LETTER KHEI;Ll;0;L;;;;;N;GREEK SMALL LETTER KHEI;;03E6;;03E6 +03E8;COPTIC CAPITAL LETTER HORI;Lu;0;L;;;;;N;GREEK CAPITAL LETTER HORI;;;03E9; +03E9;COPTIC SMALL LETTER HORI;Ll;0;L;;;;;N;GREEK SMALL LETTER HORI;;03E8;;03E8 +03EA;COPTIC CAPITAL LETTER GANGIA;Lu;0;L;;;;;N;GREEK CAPITAL LETTER GANGIA;;;03EB; +03EB;COPTIC SMALL LETTER GANGIA;Ll;0;L;;;;;N;GREEK SMALL LETTER GANGIA;;03EA;;03EA +03EC;COPTIC CAPITAL LETTER SHIMA;Lu;0;L;;;;;N;GREEK CAPITAL LETTER SHIMA;;;03ED; +03ED;COPTIC SMALL LETTER SHIMA;Ll;0;L;;;;;N;GREEK SMALL LETTER SHIMA;;03EC;;03EC +03EE;COPTIC CAPITAL LETTER DEI;Lu;0;L;;;;;N;GREEK CAPITAL LETTER DEI;;;03EF; +03EF;COPTIC SMALL LETTER DEI;Ll;0;L;;;;;N;GREEK SMALL LETTER DEI;;03EE;;03EE +03F0;GREEK KAPPA SYMBOL;Ll;0;L; 03BA;;;;N;GREEK SMALL LETTER SCRIPT KAPPA;;039A;;039A +03F1;GREEK RHO SYMBOL;Ll;0;L; 03C1;;;;N;GREEK SMALL LETTER TAILED RHO;;03A1;;03A1 +03F2;GREEK LUNATE SIGMA SYMBOL;Ll;0;L; 03C2;;;;N;GREEK SMALL LETTER LUNATE SIGMA;;03F9;;03F9 +03F3;GREEK LETTER YOT;Ll;0;L;;;;;N;;;;; +03F4;GREEK CAPITAL THETA SYMBOL;Lu;0;L; 0398;;;;N;;;;03B8; +03F5;GREEK LUNATE EPSILON SYMBOL;Ll;0;L; 03B5;;;;N;;;0395;;0395 +03F6;GREEK REVERSED LUNATE EPSILON SYMBOL;Sm;0;ON;;;;;N;;;;; +03F7;GREEK CAPITAL LETTER SHO;Lu;0;L;;;;;N;;;;03F8; +03F8;GREEK SMALL LETTER SHO;Ll;0;L;;;;;N;;;03F7;;03F7 +03F9;GREEK CAPITAL LUNATE SIGMA SYMBOL;Lu;0;L; 03A3;;;;N;;;;03F2; +03FA;GREEK CAPITAL LETTER SAN;Lu;0;L;;;;;N;;;;03FB; +03FB;GREEK SMALL LETTER SAN;Ll;0;L;;;;;N;;;03FA;;03FA +03FC;GREEK RHO WITH STROKE SYMBOL;Ll;0;L;;;;;N;;;;; +03FD;GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL;Lu;0;L;;;;;N;;;;037B; +03FE;GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL;Lu;0;L;;;;;N;;;;037C; +03FF;GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL;Lu;0;L;;;;;N;;;;037D; +0400;CYRILLIC CAPITAL LETTER IE WITH GRAVE;Lu;0;L;0415 0300;;;;N;;;;0450; +0401;CYRILLIC CAPITAL LETTER IO;Lu;0;L;0415 0308;;;;N;;;;0451; +0402;CYRILLIC CAPITAL LETTER DJE;Lu;0;L;;;;;N;;;;0452; +0403;CYRILLIC CAPITAL LETTER GJE;Lu;0;L;0413 0301;;;;N;;;;0453; +0404;CYRILLIC CAPITAL LETTER UKRAINIAN IE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER E;;;0454; +0405;CYRILLIC CAPITAL LETTER DZE;Lu;0;L;;;;;N;;;;0455; +0406;CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER I;;;0456; +0407;CYRILLIC CAPITAL LETTER YI;Lu;0;L;0406 0308;;;;N;;;;0457; +0408;CYRILLIC CAPITAL LETTER JE;Lu;0;L;;;;;N;;;;0458; +0409;CYRILLIC CAPITAL LETTER LJE;Lu;0;L;;;;;N;;;;0459; +040A;CYRILLIC CAPITAL LETTER NJE;Lu;0;L;;;;;N;;;;045A; +040B;CYRILLIC CAPITAL LETTER TSHE;Lu;0;L;;;;;N;;;;045B; +040C;CYRILLIC CAPITAL LETTER KJE;Lu;0;L;041A 0301;;;;N;;;;045C; +040D;CYRILLIC CAPITAL LETTER I WITH GRAVE;Lu;0;L;0418 0300;;;;N;;;;045D; +040E;CYRILLIC CAPITAL LETTER SHORT U;Lu;0;L;0423 0306;;;;N;;;;045E; +040F;CYRILLIC CAPITAL LETTER DZHE;Lu;0;L;;;;;N;;;;045F; +0410;CYRILLIC CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0430; +0411;CYRILLIC CAPITAL LETTER BE;Lu;0;L;;;;;N;;;;0431; +0412;CYRILLIC CAPITAL LETTER VE;Lu;0;L;;;;;N;;;;0432; +0413;CYRILLIC CAPITAL LETTER GHE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER GE;;;0433; +0414;CYRILLIC CAPITAL LETTER DE;Lu;0;L;;;;;N;;;;0434; +0415;CYRILLIC CAPITAL LETTER IE;Lu;0;L;;;;;N;;;;0435; +0416;CYRILLIC CAPITAL LETTER ZHE;Lu;0;L;;;;;N;;;;0436; +0417;CYRILLIC CAPITAL LETTER ZE;Lu;0;L;;;;;N;;;;0437; +0418;CYRILLIC CAPITAL LETTER I;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER II;;;0438; +0419;CYRILLIC CAPITAL LETTER SHORT I;Lu;0;L;0418 0306;;;;N;CYRILLIC CAPITAL LETTER SHORT II;;;0439; +041A;CYRILLIC CAPITAL LETTER KA;Lu;0;L;;;;;N;;;;043A; +041B;CYRILLIC CAPITAL LETTER EL;Lu;0;L;;;;;N;;;;043B; +041C;CYRILLIC CAPITAL LETTER EM;Lu;0;L;;;;;N;;;;043C; +041D;CYRILLIC CAPITAL LETTER EN;Lu;0;L;;;;;N;;;;043D; +041E;CYRILLIC CAPITAL LETTER O;Lu;0;L;;;;;N;;;;043E; +041F;CYRILLIC CAPITAL LETTER PE;Lu;0;L;;;;;N;;;;043F; +0420;CYRILLIC CAPITAL LETTER ER;Lu;0;L;;;;;N;;;;0440; +0421;CYRILLIC CAPITAL LETTER ES;Lu;0;L;;;;;N;;;;0441; +0422;CYRILLIC CAPITAL LETTER TE;Lu;0;L;;;;;N;;;;0442; +0423;CYRILLIC CAPITAL LETTER U;Lu;0;L;;;;;N;;;;0443; +0424;CYRILLIC CAPITAL LETTER EF;Lu;0;L;;;;;N;;;;0444; +0425;CYRILLIC CAPITAL LETTER HA;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER KHA;;;0445; +0426;CYRILLIC CAPITAL LETTER TSE;Lu;0;L;;;;;N;;;;0446; +0427;CYRILLIC CAPITAL LETTER CHE;Lu;0;L;;;;;N;;;;0447; +0428;CYRILLIC CAPITAL LETTER SHA;Lu;0;L;;;;;N;;;;0448; +0429;CYRILLIC CAPITAL LETTER SHCHA;Lu;0;L;;;;;N;;;;0449; +042A;CYRILLIC CAPITAL LETTER HARD SIGN;Lu;0;L;;;;;N;;;;044A; +042B;CYRILLIC CAPITAL LETTER YERU;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER YERI;;;044B; +042C;CYRILLIC CAPITAL LETTER SOFT SIGN;Lu;0;L;;;;;N;;;;044C; +042D;CYRILLIC CAPITAL LETTER E;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER REVERSED E;;;044D; +042E;CYRILLIC CAPITAL LETTER YU;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER IU;;;044E; +042F;CYRILLIC CAPITAL LETTER YA;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER IA;;;044F; +0430;CYRILLIC SMALL LETTER A;Ll;0;L;;;;;N;;;0410;;0410 +0431;CYRILLIC SMALL LETTER BE;Ll;0;L;;;;;N;;;0411;;0411 +0432;CYRILLIC SMALL LETTER VE;Ll;0;L;;;;;N;;;0412;;0412 +0433;CYRILLIC SMALL LETTER GHE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER GE;;0413;;0413 +0434;CYRILLIC SMALL LETTER DE;Ll;0;L;;;;;N;;;0414;;0414 +0435;CYRILLIC SMALL LETTER IE;Ll;0;L;;;;;N;;;0415;;0415 +0436;CYRILLIC SMALL LETTER ZHE;Ll;0;L;;;;;N;;;0416;;0416 +0437;CYRILLIC SMALL LETTER ZE;Ll;0;L;;;;;N;;;0417;;0417 +0438;CYRILLIC SMALL LETTER I;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER II;;0418;;0418 +0439;CYRILLIC SMALL LETTER SHORT I;Ll;0;L;0438 0306;;;;N;CYRILLIC SMALL LETTER SHORT II;;0419;;0419 +043A;CYRILLIC SMALL LETTER KA;Ll;0;L;;;;;N;;;041A;;041A +043B;CYRILLIC SMALL LETTER EL;Ll;0;L;;;;;N;;;041B;;041B +043C;CYRILLIC SMALL LETTER EM;Ll;0;L;;;;;N;;;041C;;041C +043D;CYRILLIC SMALL LETTER EN;Ll;0;L;;;;;N;;;041D;;041D +043E;CYRILLIC SMALL LETTER O;Ll;0;L;;;;;N;;;041E;;041E +043F;CYRILLIC SMALL LETTER PE;Ll;0;L;;;;;N;;;041F;;041F +0440;CYRILLIC SMALL LETTER ER;Ll;0;L;;;;;N;;;0420;;0420 +0441;CYRILLIC SMALL LETTER ES;Ll;0;L;;;;;N;;;0421;;0421 +0442;CYRILLIC SMALL LETTER TE;Ll;0;L;;;;;N;;;0422;;0422 +0443;CYRILLIC SMALL LETTER U;Ll;0;L;;;;;N;;;0423;;0423 +0444;CYRILLIC SMALL LETTER EF;Ll;0;L;;;;;N;;;0424;;0424 +0445;CYRILLIC SMALL LETTER HA;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER KHA;;0425;;0425 +0446;CYRILLIC SMALL LETTER TSE;Ll;0;L;;;;;N;;;0426;;0426 +0447;CYRILLIC SMALL LETTER CHE;Ll;0;L;;;;;N;;;0427;;0427 +0448;CYRILLIC SMALL LETTER SHA;Ll;0;L;;;;;N;;;0428;;0428 +0449;CYRILLIC SMALL LETTER SHCHA;Ll;0;L;;;;;N;;;0429;;0429 +044A;CYRILLIC SMALL LETTER HARD SIGN;Ll;0;L;;;;;N;;;042A;;042A +044B;CYRILLIC SMALL LETTER YERU;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER YERI;;042B;;042B +044C;CYRILLIC SMALL LETTER SOFT SIGN;Ll;0;L;;;;;N;;;042C;;042C +044D;CYRILLIC SMALL LETTER E;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER REVERSED E;;042D;;042D +044E;CYRILLIC SMALL LETTER YU;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER IU;;042E;;042E +044F;CYRILLIC SMALL LETTER YA;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER IA;;042F;;042F +0450;CYRILLIC SMALL LETTER IE WITH GRAVE;Ll;0;L;0435 0300;;;;N;;;0400;;0400 +0451;CYRILLIC SMALL LETTER IO;Ll;0;L;0435 0308;;;;N;;;0401;;0401 +0452;CYRILLIC SMALL LETTER DJE;Ll;0;L;;;;;N;;;0402;;0402 +0453;CYRILLIC SMALL LETTER GJE;Ll;0;L;0433 0301;;;;N;;;0403;;0403 +0454;CYRILLIC SMALL LETTER UKRAINIAN IE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER E;;0404;;0404 +0455;CYRILLIC SMALL LETTER DZE;Ll;0;L;;;;;N;;;0405;;0405 +0456;CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER I;;0406;;0406 +0457;CYRILLIC SMALL LETTER YI;Ll;0;L;0456 0308;;;;N;;;0407;;0407 +0458;CYRILLIC SMALL LETTER JE;Ll;0;L;;;;;N;;;0408;;0408 +0459;CYRILLIC SMALL LETTER LJE;Ll;0;L;;;;;N;;;0409;;0409 +045A;CYRILLIC SMALL LETTER NJE;Ll;0;L;;;;;N;;;040A;;040A +045B;CYRILLIC SMALL LETTER TSHE;Ll;0;L;;;;;N;;;040B;;040B +045C;CYRILLIC SMALL LETTER KJE;Ll;0;L;043A 0301;;;;N;;;040C;;040C +045D;CYRILLIC SMALL LETTER I WITH GRAVE;Ll;0;L;0438 0300;;;;N;;;040D;;040D +045E;CYRILLIC SMALL LETTER SHORT U;Ll;0;L;0443 0306;;;;N;;;040E;;040E +045F;CYRILLIC SMALL LETTER DZHE;Ll;0;L;;;;;N;;;040F;;040F +0460;CYRILLIC CAPITAL LETTER OMEGA;Lu;0;L;;;;;N;;;;0461; +0461;CYRILLIC SMALL LETTER OMEGA;Ll;0;L;;;;;N;;;0460;;0460 +0462;CYRILLIC CAPITAL LETTER YAT;Lu;0;L;;;;;N;;;;0463; +0463;CYRILLIC SMALL LETTER YAT;Ll;0;L;;;;;N;;;0462;;0462 +0464;CYRILLIC CAPITAL LETTER IOTIFIED E;Lu;0;L;;;;;N;;;;0465; +0465;CYRILLIC SMALL LETTER IOTIFIED E;Ll;0;L;;;;;N;;;0464;;0464 +0466;CYRILLIC CAPITAL LETTER LITTLE YUS;Lu;0;L;;;;;N;;;;0467; +0467;CYRILLIC SMALL LETTER LITTLE YUS;Ll;0;L;;;;;N;;;0466;;0466 +0468;CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS;Lu;0;L;;;;;N;;;;0469; +0469;CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS;Ll;0;L;;;;;N;;;0468;;0468 +046A;CYRILLIC CAPITAL LETTER BIG YUS;Lu;0;L;;;;;N;;;;046B; +046B;CYRILLIC SMALL LETTER BIG YUS;Ll;0;L;;;;;N;;;046A;;046A +046C;CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS;Lu;0;L;;;;;N;;;;046D; +046D;CYRILLIC SMALL LETTER IOTIFIED BIG YUS;Ll;0;L;;;;;N;;;046C;;046C +046E;CYRILLIC CAPITAL LETTER KSI;Lu;0;L;;;;;N;;;;046F; +046F;CYRILLIC SMALL LETTER KSI;Ll;0;L;;;;;N;;;046E;;046E +0470;CYRILLIC CAPITAL LETTER PSI;Lu;0;L;;;;;N;;;;0471; +0471;CYRILLIC SMALL LETTER PSI;Ll;0;L;;;;;N;;;0470;;0470 +0472;CYRILLIC CAPITAL LETTER FITA;Lu;0;L;;;;;N;;;;0473; +0473;CYRILLIC SMALL LETTER FITA;Ll;0;L;;;;;N;;;0472;;0472 +0474;CYRILLIC CAPITAL LETTER IZHITSA;Lu;0;L;;;;;N;;;;0475; +0475;CYRILLIC SMALL LETTER IZHITSA;Ll;0;L;;;;;N;;;0474;;0474 +0476;CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT;Lu;0;L;0474 030F;;;;N;CYRILLIC CAPITAL LETTER IZHITSA DOUBLE GRAVE;;;0477; +0477;CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT;Ll;0;L;0475 030F;;;;N;CYRILLIC SMALL LETTER IZHITSA DOUBLE GRAVE;;0476;;0476 +0478;CYRILLIC CAPITAL LETTER UK;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER UK DIGRAPH;;;0479; +0479;CYRILLIC SMALL LETTER UK;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER UK DIGRAPH;;0478;;0478 +047A;CYRILLIC CAPITAL LETTER ROUND OMEGA;Lu;0;L;;;;;N;;;;047B; +047B;CYRILLIC SMALL LETTER ROUND OMEGA;Ll;0;L;;;;;N;;;047A;;047A +047C;CYRILLIC CAPITAL LETTER OMEGA WITH TITLO;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER OMEGA TITLO;;;047D; +047D;CYRILLIC SMALL LETTER OMEGA WITH TITLO;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER OMEGA TITLO;;047C;;047C +047E;CYRILLIC CAPITAL LETTER OT;Lu;0;L;;;;;N;;;;047F; +047F;CYRILLIC SMALL LETTER OT;Ll;0;L;;;;;N;;;047E;;047E +0480;CYRILLIC CAPITAL LETTER KOPPA;Lu;0;L;;;;;N;;;;0481; +0481;CYRILLIC SMALL LETTER KOPPA;Ll;0;L;;;;;N;;;0480;;0480 +0482;CYRILLIC THOUSANDS SIGN;So;0;L;;;;;N;;;;; +0483;COMBINING CYRILLIC TITLO;Mn;230;NSM;;;;;N;CYRILLIC NON-SPACING TITLO;;;; +0484;COMBINING CYRILLIC PALATALIZATION;Mn;230;NSM;;;;;N;CYRILLIC NON-SPACING PALATALIZATION;;;; +0485;COMBINING CYRILLIC DASIA PNEUMATA;Mn;230;NSM;;;;;N;CYRILLIC NON-SPACING DASIA PNEUMATA;;;; +0486;COMBINING CYRILLIC PSILI PNEUMATA;Mn;230;NSM;;;;;N;CYRILLIC NON-SPACING PSILI PNEUMATA;;;; +0487;COMBINING CYRILLIC POKRYTIE;Mn;230;NSM;;;;;N;;;;; +0488;COMBINING CYRILLIC HUNDRED THOUSANDS SIGN;Me;0;NSM;;;;;N;;;;; +0489;COMBINING CYRILLIC MILLIONS SIGN;Me;0;NSM;;;;;N;;;;; +048A;CYRILLIC CAPITAL LETTER SHORT I WITH TAIL;Lu;0;L;;;;;N;;;;048B; +048B;CYRILLIC SMALL LETTER SHORT I WITH TAIL;Ll;0;L;;;;;N;;;048A;;048A +048C;CYRILLIC CAPITAL LETTER SEMISOFT SIGN;Lu;0;L;;;;;N;;;;048D; +048D;CYRILLIC SMALL LETTER SEMISOFT SIGN;Ll;0;L;;;;;N;;;048C;;048C +048E;CYRILLIC CAPITAL LETTER ER WITH TICK;Lu;0;L;;;;;N;;;;048F; +048F;CYRILLIC SMALL LETTER ER WITH TICK;Ll;0;L;;;;;N;;;048E;;048E +0490;CYRILLIC CAPITAL LETTER GHE WITH UPTURN;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER GE WITH UPTURN;;;0491; +0491;CYRILLIC SMALL LETTER GHE WITH UPTURN;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER GE WITH UPTURN;;0490;;0490 +0492;CYRILLIC CAPITAL LETTER GHE WITH STROKE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER GE BAR;;;0493; +0493;CYRILLIC SMALL LETTER GHE WITH STROKE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER GE BAR;;0492;;0492 +0494;CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER GE HOOK;;;0495; +0495;CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER GE HOOK;;0494;;0494 +0496;CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER ZHE WITH RIGHT DESCENDER;;;0497; +0497;CYRILLIC SMALL LETTER ZHE WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER ZHE WITH RIGHT DESCENDER;;0496;;0496 +0498;CYRILLIC CAPITAL LETTER ZE WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER ZE CEDILLA;;;0499; +0499;CYRILLIC SMALL LETTER ZE WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER ZE CEDILLA;;0498;;0498 +049A;CYRILLIC CAPITAL LETTER KA WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER KA WITH RIGHT DESCENDER;;;049B; +049B;CYRILLIC SMALL LETTER KA WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER KA WITH RIGHT DESCENDER;;049A;;049A +049C;CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER KA VERTICAL BAR;;;049D; +049D;CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER KA VERTICAL BAR;;049C;;049C +049E;CYRILLIC CAPITAL LETTER KA WITH STROKE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER KA BAR;;;049F; +049F;CYRILLIC SMALL LETTER KA WITH STROKE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER KA BAR;;049E;;049E +04A0;CYRILLIC CAPITAL LETTER BASHKIR KA;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER REVERSED GE KA;;;04A1; +04A1;CYRILLIC SMALL LETTER BASHKIR KA;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER REVERSED GE KA;;04A0;;04A0 +04A2;CYRILLIC CAPITAL LETTER EN WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER EN WITH RIGHT DESCENDER;;;04A3; +04A3;CYRILLIC SMALL LETTER EN WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER EN WITH RIGHT DESCENDER;;04A2;;04A2 +04A4;CYRILLIC CAPITAL LIGATURE EN GHE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER EN GE;;;04A5; +04A5;CYRILLIC SMALL LIGATURE EN GHE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER EN GE;;04A4;;04A4 +04A6;CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER PE HOOK;;;04A7; +04A7;CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER PE HOOK;;04A6;;04A6 +04A8;CYRILLIC CAPITAL LETTER ABKHASIAN HA;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER O HOOK;;;04A9; +04A9;CYRILLIC SMALL LETTER ABKHASIAN HA;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER O HOOK;;04A8;;04A8 +04AA;CYRILLIC CAPITAL LETTER ES WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER ES CEDILLA;;;04AB; +04AB;CYRILLIC SMALL LETTER ES WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER ES CEDILLA;;04AA;;04AA +04AC;CYRILLIC CAPITAL LETTER TE WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER TE WITH RIGHT DESCENDER;;;04AD; +04AD;CYRILLIC SMALL LETTER TE WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER TE WITH RIGHT DESCENDER;;04AC;;04AC +04AE;CYRILLIC CAPITAL LETTER STRAIGHT U;Lu;0;L;;;;;N;;;;04AF; +04AF;CYRILLIC SMALL LETTER STRAIGHT U;Ll;0;L;;;;;N;;;04AE;;04AE +04B0;CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER STRAIGHT U BAR;;;04B1; +04B1;CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER STRAIGHT U BAR;;04B0;;04B0 +04B2;CYRILLIC CAPITAL LETTER HA WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER KHA WITH RIGHT DESCENDER;;;04B3; +04B3;CYRILLIC SMALL LETTER HA WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER KHA WITH RIGHT DESCENDER;;04B2;;04B2 +04B4;CYRILLIC CAPITAL LIGATURE TE TSE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER TE TSE;;;04B5; +04B5;CYRILLIC SMALL LIGATURE TE TSE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER TE TSE;;04B4;;04B4 +04B6;CYRILLIC CAPITAL LETTER CHE WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER CHE WITH RIGHT DESCENDER;;;04B7; +04B7;CYRILLIC SMALL LETTER CHE WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER CHE WITH RIGHT DESCENDER;;04B6;;04B6 +04B8;CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER CHE VERTICAL BAR;;;04B9; +04B9;CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER CHE VERTICAL BAR;;04B8;;04B8 +04BA;CYRILLIC CAPITAL LETTER SHHA;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER H;;;04BB; +04BB;CYRILLIC SMALL LETTER SHHA;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER H;;04BA;;04BA +04BC;CYRILLIC CAPITAL LETTER ABKHASIAN CHE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER IE HOOK;;;04BD; +04BD;CYRILLIC SMALL LETTER ABKHASIAN CHE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER IE HOOK;;04BC;;04BC +04BE;CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER IE HOOK OGONEK;;;04BF; +04BF;CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER IE HOOK OGONEK;;04BE;;04BE +04C0;CYRILLIC LETTER PALOCHKA;Lu;0;L;;;;;N;CYRILLIC LETTER I;;;04CF; +04C1;CYRILLIC CAPITAL LETTER ZHE WITH BREVE;Lu;0;L;0416 0306;;;;N;CYRILLIC CAPITAL LETTER SHORT ZHE;;;04C2; +04C2;CYRILLIC SMALL LETTER ZHE WITH BREVE;Ll;0;L;0436 0306;;;;N;CYRILLIC SMALL LETTER SHORT ZHE;;04C1;;04C1 +04C3;CYRILLIC CAPITAL LETTER KA WITH HOOK;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER KA HOOK;;;04C4; +04C4;CYRILLIC SMALL LETTER KA WITH HOOK;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER KA HOOK;;04C3;;04C3 +04C5;CYRILLIC CAPITAL LETTER EL WITH TAIL;Lu;0;L;;;;;N;;;;04C6; +04C6;CYRILLIC SMALL LETTER EL WITH TAIL;Ll;0;L;;;;;N;;;04C5;;04C5 +04C7;CYRILLIC CAPITAL LETTER EN WITH HOOK;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER EN HOOK;;;04C8; +04C8;CYRILLIC SMALL LETTER EN WITH HOOK;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER EN HOOK;;04C7;;04C7 +04C9;CYRILLIC CAPITAL LETTER EN WITH TAIL;Lu;0;L;;;;;N;;;;04CA; +04CA;CYRILLIC SMALL LETTER EN WITH TAIL;Ll;0;L;;;;;N;;;04C9;;04C9 +04CB;CYRILLIC CAPITAL LETTER KHAKASSIAN CHE;Lu;0;L;;;;;N;CYRILLIC CAPITAL LETTER CHE WITH LEFT DESCENDER;;;04CC; +04CC;CYRILLIC SMALL LETTER KHAKASSIAN CHE;Ll;0;L;;;;;N;CYRILLIC SMALL LETTER CHE WITH LEFT DESCENDER;;04CB;;04CB +04CD;CYRILLIC CAPITAL LETTER EM WITH TAIL;Lu;0;L;;;;;N;;;;04CE; +04CE;CYRILLIC SMALL LETTER EM WITH TAIL;Ll;0;L;;;;;N;;;04CD;;04CD +04CF;CYRILLIC SMALL LETTER PALOCHKA;Ll;0;L;;;;;N;;;04C0;;04C0 +04D0;CYRILLIC CAPITAL LETTER A WITH BREVE;Lu;0;L;0410 0306;;;;N;;;;04D1; +04D1;CYRILLIC SMALL LETTER A WITH BREVE;Ll;0;L;0430 0306;;;;N;;;04D0;;04D0 +04D2;CYRILLIC CAPITAL LETTER A WITH DIAERESIS;Lu;0;L;0410 0308;;;;N;;;;04D3; +04D3;CYRILLIC SMALL LETTER A WITH DIAERESIS;Ll;0;L;0430 0308;;;;N;;;04D2;;04D2 +04D4;CYRILLIC CAPITAL LIGATURE A IE;Lu;0;L;;;;;N;;;;04D5; +04D5;CYRILLIC SMALL LIGATURE A IE;Ll;0;L;;;;;N;;;04D4;;04D4 +04D6;CYRILLIC CAPITAL LETTER IE WITH BREVE;Lu;0;L;0415 0306;;;;N;;;;04D7; +04D7;CYRILLIC SMALL LETTER IE WITH BREVE;Ll;0;L;0435 0306;;;;N;;;04D6;;04D6 +04D8;CYRILLIC CAPITAL LETTER SCHWA;Lu;0;L;;;;;N;;;;04D9; +04D9;CYRILLIC SMALL LETTER SCHWA;Ll;0;L;;;;;N;;;04D8;;04D8 +04DA;CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS;Lu;0;L;04D8 0308;;;;N;;;;04DB; +04DB;CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS;Ll;0;L;04D9 0308;;;;N;;;04DA;;04DA +04DC;CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS;Lu;0;L;0416 0308;;;;N;;;;04DD; +04DD;CYRILLIC SMALL LETTER ZHE WITH DIAERESIS;Ll;0;L;0436 0308;;;;N;;;04DC;;04DC +04DE;CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS;Lu;0;L;0417 0308;;;;N;;;;04DF; +04DF;CYRILLIC SMALL LETTER ZE WITH DIAERESIS;Ll;0;L;0437 0308;;;;N;;;04DE;;04DE +04E0;CYRILLIC CAPITAL LETTER ABKHASIAN DZE;Lu;0;L;;;;;N;;;;04E1; +04E1;CYRILLIC SMALL LETTER ABKHASIAN DZE;Ll;0;L;;;;;N;;;04E0;;04E0 +04E2;CYRILLIC CAPITAL LETTER I WITH MACRON;Lu;0;L;0418 0304;;;;N;;;;04E3; +04E3;CYRILLIC SMALL LETTER I WITH MACRON;Ll;0;L;0438 0304;;;;N;;;04E2;;04E2 +04E4;CYRILLIC CAPITAL LETTER I WITH DIAERESIS;Lu;0;L;0418 0308;;;;N;;;;04E5; +04E5;CYRILLIC SMALL LETTER I WITH DIAERESIS;Ll;0;L;0438 0308;;;;N;;;04E4;;04E4 +04E6;CYRILLIC CAPITAL LETTER O WITH DIAERESIS;Lu;0;L;041E 0308;;;;N;;;;04E7; +04E7;CYRILLIC SMALL LETTER O WITH DIAERESIS;Ll;0;L;043E 0308;;;;N;;;04E6;;04E6 +04E8;CYRILLIC CAPITAL LETTER BARRED O;Lu;0;L;;;;;N;;;;04E9; +04E9;CYRILLIC SMALL LETTER BARRED O;Ll;0;L;;;;;N;;;04E8;;04E8 +04EA;CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS;Lu;0;L;04E8 0308;;;;N;;;;04EB; +04EB;CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS;Ll;0;L;04E9 0308;;;;N;;;04EA;;04EA +04EC;CYRILLIC CAPITAL LETTER E WITH DIAERESIS;Lu;0;L;042D 0308;;;;N;;;;04ED; +04ED;CYRILLIC SMALL LETTER E WITH DIAERESIS;Ll;0;L;044D 0308;;;;N;;;04EC;;04EC +04EE;CYRILLIC CAPITAL LETTER U WITH MACRON;Lu;0;L;0423 0304;;;;N;;;;04EF; +04EF;CYRILLIC SMALL LETTER U WITH MACRON;Ll;0;L;0443 0304;;;;N;;;04EE;;04EE +04F0;CYRILLIC CAPITAL LETTER U WITH DIAERESIS;Lu;0;L;0423 0308;;;;N;;;;04F1; +04F1;CYRILLIC SMALL LETTER U WITH DIAERESIS;Ll;0;L;0443 0308;;;;N;;;04F0;;04F0 +04F2;CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE;Lu;0;L;0423 030B;;;;N;;;;04F3; +04F3;CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE;Ll;0;L;0443 030B;;;;N;;;04F2;;04F2 +04F4;CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS;Lu;0;L;0427 0308;;;;N;;;;04F5; +04F5;CYRILLIC SMALL LETTER CHE WITH DIAERESIS;Ll;0;L;0447 0308;;;;N;;;04F4;;04F4 +04F6;CYRILLIC CAPITAL LETTER GHE WITH DESCENDER;Lu;0;L;;;;;N;;;;04F7; +04F7;CYRILLIC SMALL LETTER GHE WITH DESCENDER;Ll;0;L;;;;;N;;;04F6;;04F6 +04F8;CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS;Lu;0;L;042B 0308;;;;N;;;;04F9; +04F9;CYRILLIC SMALL LETTER YERU WITH DIAERESIS;Ll;0;L;044B 0308;;;;N;;;04F8;;04F8 +04FA;CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK;Lu;0;L;;;;;N;;;;04FB; +04FB;CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK;Ll;0;L;;;;;N;;;04FA;;04FA +04FC;CYRILLIC CAPITAL LETTER HA WITH HOOK;Lu;0;L;;;;;N;;;;04FD; +04FD;CYRILLIC SMALL LETTER HA WITH HOOK;Ll;0;L;;;;;N;;;04FC;;04FC +04FE;CYRILLIC CAPITAL LETTER HA WITH STROKE;Lu;0;L;;;;;N;;;;04FF; +04FF;CYRILLIC SMALL LETTER HA WITH STROKE;Ll;0;L;;;;;N;;;04FE;;04FE +0500;CYRILLIC CAPITAL LETTER KOMI DE;Lu;0;L;;;;;N;;;;0501; +0501;CYRILLIC SMALL LETTER KOMI DE;Ll;0;L;;;;;N;;;0500;;0500 +0502;CYRILLIC CAPITAL LETTER KOMI DJE;Lu;0;L;;;;;N;;;;0503; +0503;CYRILLIC SMALL LETTER KOMI DJE;Ll;0;L;;;;;N;;;0502;;0502 +0504;CYRILLIC CAPITAL LETTER KOMI ZJE;Lu;0;L;;;;;N;;;;0505; +0505;CYRILLIC SMALL LETTER KOMI ZJE;Ll;0;L;;;;;N;;;0504;;0504 +0506;CYRILLIC CAPITAL LETTER KOMI DZJE;Lu;0;L;;;;;N;;;;0507; +0507;CYRILLIC SMALL LETTER KOMI DZJE;Ll;0;L;;;;;N;;;0506;;0506 +0508;CYRILLIC CAPITAL LETTER KOMI LJE;Lu;0;L;;;;;N;;;;0509; +0509;CYRILLIC SMALL LETTER KOMI LJE;Ll;0;L;;;;;N;;;0508;;0508 +050A;CYRILLIC CAPITAL LETTER KOMI NJE;Lu;0;L;;;;;N;;;;050B; +050B;CYRILLIC SMALL LETTER KOMI NJE;Ll;0;L;;;;;N;;;050A;;050A +050C;CYRILLIC CAPITAL LETTER KOMI SJE;Lu;0;L;;;;;N;;;;050D; +050D;CYRILLIC SMALL LETTER KOMI SJE;Ll;0;L;;;;;N;;;050C;;050C +050E;CYRILLIC CAPITAL LETTER KOMI TJE;Lu;0;L;;;;;N;;;;050F; +050F;CYRILLIC SMALL LETTER KOMI TJE;Ll;0;L;;;;;N;;;050E;;050E +0510;CYRILLIC CAPITAL LETTER REVERSED ZE;Lu;0;L;;;;;N;;;;0511; +0511;CYRILLIC SMALL LETTER REVERSED ZE;Ll;0;L;;;;;N;;;0510;;0510 +0512;CYRILLIC CAPITAL LETTER EL WITH HOOK;Lu;0;L;;;;;N;;;;0513; +0513;CYRILLIC SMALL LETTER EL WITH HOOK;Ll;0;L;;;;;N;;;0512;;0512 +0514;CYRILLIC CAPITAL LETTER LHA;Lu;0;L;;;;;N;;;;0515; +0515;CYRILLIC SMALL LETTER LHA;Ll;0;L;;;;;N;;;0514;;0514 +0516;CYRILLIC CAPITAL LETTER RHA;Lu;0;L;;;;;N;;;;0517; +0517;CYRILLIC SMALL LETTER RHA;Ll;0;L;;;;;N;;;0516;;0516 +0518;CYRILLIC CAPITAL LETTER YAE;Lu;0;L;;;;;N;;;;0519; +0519;CYRILLIC SMALL LETTER YAE;Ll;0;L;;;;;N;;;0518;;0518 +051A;CYRILLIC CAPITAL LETTER QA;Lu;0;L;;;;;N;;;;051B; +051B;CYRILLIC SMALL LETTER QA;Ll;0;L;;;;;N;;;051A;;051A +051C;CYRILLIC CAPITAL LETTER WE;Lu;0;L;;;;;N;;;;051D; +051D;CYRILLIC SMALL LETTER WE;Ll;0;L;;;;;N;;;051C;;051C +051E;CYRILLIC CAPITAL LETTER ALEUT KA;Lu;0;L;;;;;N;;;;051F; +051F;CYRILLIC SMALL LETTER ALEUT KA;Ll;0;L;;;;;N;;;051E;;051E +0520;CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK;Lu;0;L;;;;;N;;;;0521; +0521;CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK;Ll;0;L;;;;;N;;;0520;;0520 +0522;CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK;Lu;0;L;;;;;N;;;;0523; +0523;CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK;Ll;0;L;;;;;N;;;0522;;0522 +0524;CYRILLIC CAPITAL LETTER PE WITH DESCENDER;Lu;0;L;;;;;N;;;;0525; +0525;CYRILLIC SMALL LETTER PE WITH DESCENDER;Ll;0;L;;;;;N;;;0524;;0524 +0526;CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER;Lu;0;L;;;;;N;;;;0527; +0527;CYRILLIC SMALL LETTER SHHA WITH DESCENDER;Ll;0;L;;;;;N;;;0526;;0526 +0531;ARMENIAN CAPITAL LETTER AYB;Lu;0;L;;;;;N;;;;0561; +0532;ARMENIAN CAPITAL LETTER BEN;Lu;0;L;;;;;N;;;;0562; +0533;ARMENIAN CAPITAL LETTER GIM;Lu;0;L;;;;;N;;;;0563; +0534;ARMENIAN CAPITAL LETTER DA;Lu;0;L;;;;;N;;;;0564; +0535;ARMENIAN CAPITAL LETTER ECH;Lu;0;L;;;;;N;;;;0565; +0536;ARMENIAN CAPITAL LETTER ZA;Lu;0;L;;;;;N;;;;0566; +0537;ARMENIAN CAPITAL LETTER EH;Lu;0;L;;;;;N;;;;0567; +0538;ARMENIAN CAPITAL LETTER ET;Lu;0;L;;;;;N;;;;0568; +0539;ARMENIAN CAPITAL LETTER TO;Lu;0;L;;;;;N;;;;0569; +053A;ARMENIAN CAPITAL LETTER ZHE;Lu;0;L;;;;;N;;;;056A; +053B;ARMENIAN CAPITAL LETTER INI;Lu;0;L;;;;;N;;;;056B; +053C;ARMENIAN CAPITAL LETTER LIWN;Lu;0;L;;;;;N;;;;056C; +053D;ARMENIAN CAPITAL LETTER XEH;Lu;0;L;;;;;N;;;;056D; +053E;ARMENIAN CAPITAL LETTER CA;Lu;0;L;;;;;N;;;;056E; +053F;ARMENIAN CAPITAL LETTER KEN;Lu;0;L;;;;;N;;;;056F; +0540;ARMENIAN CAPITAL LETTER HO;Lu;0;L;;;;;N;;;;0570; +0541;ARMENIAN CAPITAL LETTER JA;Lu;0;L;;;;;N;;;;0571; +0542;ARMENIAN CAPITAL LETTER GHAD;Lu;0;L;;;;;N;ARMENIAN CAPITAL LETTER LAD;;;0572; +0543;ARMENIAN CAPITAL LETTER CHEH;Lu;0;L;;;;;N;;;;0573; +0544;ARMENIAN CAPITAL LETTER MEN;Lu;0;L;;;;;N;;;;0574; +0545;ARMENIAN CAPITAL LETTER YI;Lu;0;L;;;;;N;;;;0575; +0546;ARMENIAN CAPITAL LETTER NOW;Lu;0;L;;;;;N;;;;0576; +0547;ARMENIAN CAPITAL LETTER SHA;Lu;0;L;;;;;N;;;;0577; +0548;ARMENIAN CAPITAL LETTER VO;Lu;0;L;;;;;N;;;;0578; +0549;ARMENIAN CAPITAL LETTER CHA;Lu;0;L;;;;;N;;;;0579; +054A;ARMENIAN CAPITAL LETTER PEH;Lu;0;L;;;;;N;;;;057A; +054B;ARMENIAN CAPITAL LETTER JHEH;Lu;0;L;;;;;N;;;;057B; +054C;ARMENIAN CAPITAL LETTER RA;Lu;0;L;;;;;N;;;;057C; +054D;ARMENIAN CAPITAL LETTER SEH;Lu;0;L;;;;;N;;;;057D; +054E;ARMENIAN CAPITAL LETTER VEW;Lu;0;L;;;;;N;;;;057E; +054F;ARMENIAN CAPITAL LETTER TIWN;Lu;0;L;;;;;N;;;;057F; +0550;ARMENIAN CAPITAL LETTER REH;Lu;0;L;;;;;N;;;;0580; +0551;ARMENIAN CAPITAL LETTER CO;Lu;0;L;;;;;N;;;;0581; +0552;ARMENIAN CAPITAL LETTER YIWN;Lu;0;L;;;;;N;;;;0582; +0553;ARMENIAN CAPITAL LETTER PIWR;Lu;0;L;;;;;N;;;;0583; +0554;ARMENIAN CAPITAL LETTER KEH;Lu;0;L;;;;;N;;;;0584; +0555;ARMENIAN CAPITAL LETTER OH;Lu;0;L;;;;;N;;;;0585; +0556;ARMENIAN CAPITAL LETTER FEH;Lu;0;L;;;;;N;;;;0586; +0559;ARMENIAN MODIFIER LETTER LEFT HALF RING;Lm;0;L;;;;;N;;;;; +055A;ARMENIAN APOSTROPHE;Po;0;L;;;;;N;ARMENIAN MODIFIER LETTER RIGHT HALF RING;;;; +055B;ARMENIAN EMPHASIS MARK;Po;0;L;;;;;N;;;;; +055C;ARMENIAN EXCLAMATION MARK;Po;0;L;;;;;N;;;;; +055D;ARMENIAN COMMA;Po;0;L;;;;;N;;;;; +055E;ARMENIAN QUESTION MARK;Po;0;L;;;;;N;;;;; +055F;ARMENIAN ABBREVIATION MARK;Po;0;L;;;;;N;;;;; +0561;ARMENIAN SMALL LETTER AYB;Ll;0;L;;;;;N;;;0531;;0531 +0562;ARMENIAN SMALL LETTER BEN;Ll;0;L;;;;;N;;;0532;;0532 +0563;ARMENIAN SMALL LETTER GIM;Ll;0;L;;;;;N;;;0533;;0533 +0564;ARMENIAN SMALL LETTER DA;Ll;0;L;;;;;N;;;0534;;0534 +0565;ARMENIAN SMALL LETTER ECH;Ll;0;L;;;;;N;;;0535;;0535 +0566;ARMENIAN SMALL LETTER ZA;Ll;0;L;;;;;N;;;0536;;0536 +0567;ARMENIAN SMALL LETTER EH;Ll;0;L;;;;;N;;;0537;;0537 +0568;ARMENIAN SMALL LETTER ET;Ll;0;L;;;;;N;;;0538;;0538 +0569;ARMENIAN SMALL LETTER TO;Ll;0;L;;;;;N;;;0539;;0539 +056A;ARMENIAN SMALL LETTER ZHE;Ll;0;L;;;;;N;;;053A;;053A +056B;ARMENIAN SMALL LETTER INI;Ll;0;L;;;;;N;;;053B;;053B +056C;ARMENIAN SMALL LETTER LIWN;Ll;0;L;;;;;N;;;053C;;053C +056D;ARMENIAN SMALL LETTER XEH;Ll;0;L;;;;;N;;;053D;;053D +056E;ARMENIAN SMALL LETTER CA;Ll;0;L;;;;;N;;;053E;;053E +056F;ARMENIAN SMALL LETTER KEN;Ll;0;L;;;;;N;;;053F;;053F +0570;ARMENIAN SMALL LETTER HO;Ll;0;L;;;;;N;;;0540;;0540 +0571;ARMENIAN SMALL LETTER JA;Ll;0;L;;;;;N;;;0541;;0541 +0572;ARMENIAN SMALL LETTER GHAD;Ll;0;L;;;;;N;ARMENIAN SMALL LETTER LAD;;0542;;0542 +0573;ARMENIAN SMALL LETTER CHEH;Ll;0;L;;;;;N;;;0543;;0543 +0574;ARMENIAN SMALL LETTER MEN;Ll;0;L;;;;;N;;;0544;;0544 +0575;ARMENIAN SMALL LETTER YI;Ll;0;L;;;;;N;;;0545;;0545 +0576;ARMENIAN SMALL LETTER NOW;Ll;0;L;;;;;N;;;0546;;0546 +0577;ARMENIAN SMALL LETTER SHA;Ll;0;L;;;;;N;;;0547;;0547 +0578;ARMENIAN SMALL LETTER VO;Ll;0;L;;;;;N;;;0548;;0548 +0579;ARMENIAN SMALL LETTER CHA;Ll;0;L;;;;;N;;;0549;;0549 +057A;ARMENIAN SMALL LETTER PEH;Ll;0;L;;;;;N;;;054A;;054A +057B;ARMENIAN SMALL LETTER JHEH;Ll;0;L;;;;;N;;;054B;;054B +057C;ARMENIAN SMALL LETTER RA;Ll;0;L;;;;;N;;;054C;;054C +057D;ARMENIAN SMALL LETTER SEH;Ll;0;L;;;;;N;;;054D;;054D +057E;ARMENIAN SMALL LETTER VEW;Ll;0;L;;;;;N;;;054E;;054E +057F;ARMENIAN SMALL LETTER TIWN;Ll;0;L;;;;;N;;;054F;;054F +0580;ARMENIAN SMALL LETTER REH;Ll;0;L;;;;;N;;;0550;;0550 +0581;ARMENIAN SMALL LETTER CO;Ll;0;L;;;;;N;;;0551;;0551 +0582;ARMENIAN SMALL LETTER YIWN;Ll;0;L;;;;;N;;;0552;;0552 +0583;ARMENIAN SMALL LETTER PIWR;Ll;0;L;;;;;N;;;0553;;0553 +0584;ARMENIAN SMALL LETTER KEH;Ll;0;L;;;;;N;;;0554;;0554 +0585;ARMENIAN SMALL LETTER OH;Ll;0;L;;;;;N;;;0555;;0555 +0586;ARMENIAN SMALL LETTER FEH;Ll;0;L;;;;;N;;;0556;;0556 +0587;ARMENIAN SMALL LIGATURE ECH YIWN;Ll;0;L; 0565 0582;;;;N;;;;; +0589;ARMENIAN FULL STOP;Po;0;L;;;;;N;ARMENIAN PERIOD;;;; +058A;ARMENIAN HYPHEN;Pd;0;ON;;;;;N;;;;; +058F;ARMENIAN DRAM SIGN;Sc;0;ET;;;;;N;;;;; +0591;HEBREW ACCENT ETNAHTA;Mn;220;NSM;;;;;N;;;;; +0592;HEBREW ACCENT SEGOL;Mn;230;NSM;;;;;N;;;;; +0593;HEBREW ACCENT SHALSHELET;Mn;230;NSM;;;;;N;;;;; +0594;HEBREW ACCENT ZAQEF QATAN;Mn;230;NSM;;;;;N;;;;; +0595;HEBREW ACCENT ZAQEF GADOL;Mn;230;NSM;;;;;N;;;;; +0596;HEBREW ACCENT TIPEHA;Mn;220;NSM;;;;;N;;;;; +0597;HEBREW ACCENT REVIA;Mn;230;NSM;;;;;N;;;;; +0598;HEBREW ACCENT ZARQA;Mn;230;NSM;;;;;N;;;;; +0599;HEBREW ACCENT PASHTA;Mn;230;NSM;;;;;N;;;;; +059A;HEBREW ACCENT YETIV;Mn;222;NSM;;;;;N;;;;; +059B;HEBREW ACCENT TEVIR;Mn;220;NSM;;;;;N;;;;; +059C;HEBREW ACCENT GERESH;Mn;230;NSM;;;;;N;;;;; +059D;HEBREW ACCENT GERESH MUQDAM;Mn;230;NSM;;;;;N;;;;; +059E;HEBREW ACCENT GERSHAYIM;Mn;230;NSM;;;;;N;;;;; +059F;HEBREW ACCENT QARNEY PARA;Mn;230;NSM;;;;;N;;;;; +05A0;HEBREW ACCENT TELISHA GEDOLA;Mn;230;NSM;;;;;N;;;;; +05A1;HEBREW ACCENT PAZER;Mn;230;NSM;;;;;N;;;;; +05A2;HEBREW ACCENT ATNAH HAFUKH;Mn;220;NSM;;;;;N;;;;; +05A3;HEBREW ACCENT MUNAH;Mn;220;NSM;;;;;N;;;;; +05A4;HEBREW ACCENT MAHAPAKH;Mn;220;NSM;;;;;N;;;;; +05A5;HEBREW ACCENT MERKHA;Mn;220;NSM;;;;;N;;;;; +05A6;HEBREW ACCENT MERKHA KEFULA;Mn;220;NSM;;;;;N;;;;; +05A7;HEBREW ACCENT DARGA;Mn;220;NSM;;;;;N;;;;; +05A8;HEBREW ACCENT QADMA;Mn;230;NSM;;;;;N;;;;; +05A9;HEBREW ACCENT TELISHA QETANA;Mn;230;NSM;;;;;N;;;;; +05AA;HEBREW ACCENT YERAH BEN YOMO;Mn;220;NSM;;;;;N;;;;; +05AB;HEBREW ACCENT OLE;Mn;230;NSM;;;;;N;;;;; +05AC;HEBREW ACCENT ILUY;Mn;230;NSM;;;;;N;;;;; +05AD;HEBREW ACCENT DEHI;Mn;222;NSM;;;;;N;;;;; +05AE;HEBREW ACCENT ZINOR;Mn;228;NSM;;;;;N;;;;; +05AF;HEBREW MARK MASORA CIRCLE;Mn;230;NSM;;;;;N;;;;; +05B0;HEBREW POINT SHEVA;Mn;10;NSM;;;;;N;;;;; +05B1;HEBREW POINT HATAF SEGOL;Mn;11;NSM;;;;;N;;;;; +05B2;HEBREW POINT HATAF PATAH;Mn;12;NSM;;;;;N;;;;; +05B3;HEBREW POINT HATAF QAMATS;Mn;13;NSM;;;;;N;;;;; +05B4;HEBREW POINT HIRIQ;Mn;14;NSM;;;;;N;;;;; +05B5;HEBREW POINT TSERE;Mn;15;NSM;;;;;N;;;;; +05B6;HEBREW POINT SEGOL;Mn;16;NSM;;;;;N;;;;; +05B7;HEBREW POINT PATAH;Mn;17;NSM;;;;;N;;;;; +05B8;HEBREW POINT QAMATS;Mn;18;NSM;;;;;N;;;;; +05B9;HEBREW POINT HOLAM;Mn;19;NSM;;;;;N;;;;; +05BA;HEBREW POINT HOLAM HASER FOR VAV;Mn;19;NSM;;;;;N;;;;; +05BB;HEBREW POINT QUBUTS;Mn;20;NSM;;;;;N;;;;; +05BC;HEBREW POINT DAGESH OR MAPIQ;Mn;21;NSM;;;;;N;HEBREW POINT DAGESH;;;; +05BD;HEBREW POINT METEG;Mn;22;NSM;;;;;N;;;;; +05BE;HEBREW PUNCTUATION MAQAF;Pd;0;R;;;;;N;;;;; +05BF;HEBREW POINT RAFE;Mn;23;NSM;;;;;N;;;;; +05C0;HEBREW PUNCTUATION PASEQ;Po;0;R;;;;;N;HEBREW POINT PASEQ;;;; +05C1;HEBREW POINT SHIN DOT;Mn;24;NSM;;;;;N;;;;; +05C2;HEBREW POINT SIN DOT;Mn;25;NSM;;;;;N;;;;; +05C3;HEBREW PUNCTUATION SOF PASUQ;Po;0;R;;;;;N;;;;; +05C4;HEBREW MARK UPPER DOT;Mn;230;NSM;;;;;N;;;;; +05C5;HEBREW MARK LOWER DOT;Mn;220;NSM;;;;;N;;;;; +05C6;HEBREW PUNCTUATION NUN HAFUKHA;Po;0;R;;;;;N;;;;; +05C7;HEBREW POINT QAMATS QATAN;Mn;18;NSM;;;;;N;;;;; +05D0;HEBREW LETTER ALEF;Lo;0;R;;;;;N;;;;; +05D1;HEBREW LETTER BET;Lo;0;R;;;;;N;;;;; +05D2;HEBREW LETTER GIMEL;Lo;0;R;;;;;N;;;;; +05D3;HEBREW LETTER DALET;Lo;0;R;;;;;N;;;;; +05D4;HEBREW LETTER HE;Lo;0;R;;;;;N;;;;; +05D5;HEBREW LETTER VAV;Lo;0;R;;;;;N;;;;; +05D6;HEBREW LETTER ZAYIN;Lo;0;R;;;;;N;;;;; +05D7;HEBREW LETTER HET;Lo;0;R;;;;;N;;;;; +05D8;HEBREW LETTER TET;Lo;0;R;;;;;N;;;;; +05D9;HEBREW LETTER YOD;Lo;0;R;;;;;N;;;;; +05DA;HEBREW LETTER FINAL KAF;Lo;0;R;;;;;N;;;;; +05DB;HEBREW LETTER KAF;Lo;0;R;;;;;N;;;;; +05DC;HEBREW LETTER LAMED;Lo;0;R;;;;;N;;;;; +05DD;HEBREW LETTER FINAL MEM;Lo;0;R;;;;;N;;;;; +05DE;HEBREW LETTER MEM;Lo;0;R;;;;;N;;;;; +05DF;HEBREW LETTER FINAL NUN;Lo;0;R;;;;;N;;;;; +05E0;HEBREW LETTER NUN;Lo;0;R;;;;;N;;;;; +05E1;HEBREW LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +05E2;HEBREW LETTER AYIN;Lo;0;R;;;;;N;;;;; +05E3;HEBREW LETTER FINAL PE;Lo;0;R;;;;;N;;;;; +05E4;HEBREW LETTER PE;Lo;0;R;;;;;N;;;;; +05E5;HEBREW LETTER FINAL TSADI;Lo;0;R;;;;;N;;;;; +05E6;HEBREW LETTER TSADI;Lo;0;R;;;;;N;;;;; +05E7;HEBREW LETTER QOF;Lo;0;R;;;;;N;;;;; +05E8;HEBREW LETTER RESH;Lo;0;R;;;;;N;;;;; +05E9;HEBREW LETTER SHIN;Lo;0;R;;;;;N;;;;; +05EA;HEBREW LETTER TAV;Lo;0;R;;;;;N;;;;; +05F0;HEBREW LIGATURE YIDDISH DOUBLE VAV;Lo;0;R;;;;;N;HEBREW LETTER DOUBLE VAV;;;; +05F1;HEBREW LIGATURE YIDDISH VAV YOD;Lo;0;R;;;;;N;HEBREW LETTER VAV YOD;;;; +05F2;HEBREW LIGATURE YIDDISH DOUBLE YOD;Lo;0;R;;;;;N;HEBREW LETTER DOUBLE YOD;;;; +05F3;HEBREW PUNCTUATION GERESH;Po;0;R;;;;;N;;;;; +05F4;HEBREW PUNCTUATION GERSHAYIM;Po;0;R;;;;;N;;;;; +0600;ARABIC NUMBER SIGN;Cf;0;AN;;;;;N;;;;; +0601;ARABIC SIGN SANAH;Cf;0;AN;;;;;N;;;;; +0602;ARABIC FOOTNOTE MARKER;Cf;0;AN;;;;;N;;;;; +0603;ARABIC SIGN SAFHA;Cf;0;AN;;;;;N;;;;; +0604;ARABIC SIGN SAMVAT;Cf;0;AN;;;;;N;;;;; +0606;ARABIC-INDIC CUBE ROOT;Sm;0;ON;;;;;N;;;;; +0607;ARABIC-INDIC FOURTH ROOT;Sm;0;ON;;;;;N;;;;; +0608;ARABIC RAY;Sm;0;AL;;;;;N;;;;; +0609;ARABIC-INDIC PER MILLE SIGN;Po;0;ET;;;;;N;;;;; +060A;ARABIC-INDIC PER TEN THOUSAND SIGN;Po;0;ET;;;;;N;;;;; +060B;AFGHANI SIGN;Sc;0;AL;;;;;N;;;;; +060C;ARABIC COMMA;Po;0;CS;;;;;N;;;;; +060D;ARABIC DATE SEPARATOR;Po;0;AL;;;;;N;;;;; +060E;ARABIC POETIC VERSE SIGN;So;0;ON;;;;;N;;;;; +060F;ARABIC SIGN MISRA;So;0;ON;;;;;N;;;;; +0610;ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM;Mn;230;NSM;;;;;N;;;;; +0611;ARABIC SIGN ALAYHE ASSALLAM;Mn;230;NSM;;;;;N;;;;; +0612;ARABIC SIGN RAHMATULLAH ALAYHE;Mn;230;NSM;;;;;N;;;;; +0613;ARABIC SIGN RADI ALLAHOU ANHU;Mn;230;NSM;;;;;N;;;;; +0614;ARABIC SIGN TAKHALLUS;Mn;230;NSM;;;;;N;;;;; +0615;ARABIC SMALL HIGH TAH;Mn;230;NSM;;;;;N;;;;; +0616;ARABIC SMALL HIGH LIGATURE ALEF WITH LAM WITH YEH;Mn;230;NSM;;;;;N;;;;; +0617;ARABIC SMALL HIGH ZAIN;Mn;230;NSM;;;;;N;;;;; +0618;ARABIC SMALL FATHA;Mn;30;NSM;;;;;N;;;;; +0619;ARABIC SMALL DAMMA;Mn;31;NSM;;;;;N;;;;; +061A;ARABIC SMALL KASRA;Mn;32;NSM;;;;;N;;;;; +061B;ARABIC SEMICOLON;Po;0;AL;;;;;N;;;;; +061E;ARABIC TRIPLE DOT PUNCTUATION MARK;Po;0;AL;;;;;N;;;;; +061F;ARABIC QUESTION MARK;Po;0;AL;;;;;N;;;;; +0620;ARABIC LETTER KASHMIRI YEH;Lo;0;AL;;;;;N;;;;; +0621;ARABIC LETTER HAMZA;Lo;0;AL;;;;;N;ARABIC LETTER HAMZAH;;;; +0622;ARABIC LETTER ALEF WITH MADDA ABOVE;Lo;0;AL;0627 0653;;;;N;ARABIC LETTER MADDAH ON ALEF;;;; +0623;ARABIC LETTER ALEF WITH HAMZA ABOVE;Lo;0;AL;0627 0654;;;;N;ARABIC LETTER HAMZAH ON ALEF;;;; +0624;ARABIC LETTER WAW WITH HAMZA ABOVE;Lo;0;AL;0648 0654;;;;N;ARABIC LETTER HAMZAH ON WAW;;;; +0625;ARABIC LETTER ALEF WITH HAMZA BELOW;Lo;0;AL;0627 0655;;;;N;ARABIC LETTER HAMZAH UNDER ALEF;;;; +0626;ARABIC LETTER YEH WITH HAMZA ABOVE;Lo;0;AL;064A 0654;;;;N;ARABIC LETTER HAMZAH ON YA;;;; +0627;ARABIC LETTER ALEF;Lo;0;AL;;;;;N;;;;; +0628;ARABIC LETTER BEH;Lo;0;AL;;;;;N;ARABIC LETTER BAA;;;; +0629;ARABIC LETTER TEH MARBUTA;Lo;0;AL;;;;;N;ARABIC LETTER TAA MARBUTAH;;;; +062A;ARABIC LETTER TEH;Lo;0;AL;;;;;N;ARABIC LETTER TAA;;;; +062B;ARABIC LETTER THEH;Lo;0;AL;;;;;N;ARABIC LETTER THAA;;;; +062C;ARABIC LETTER JEEM;Lo;0;AL;;;;;N;;;;; +062D;ARABIC LETTER HAH;Lo;0;AL;;;;;N;ARABIC LETTER HAA;;;; +062E;ARABIC LETTER KHAH;Lo;0;AL;;;;;N;ARABIC LETTER KHAA;;;; +062F;ARABIC LETTER DAL;Lo;0;AL;;;;;N;;;;; +0630;ARABIC LETTER THAL;Lo;0;AL;;;;;N;;;;; +0631;ARABIC LETTER REH;Lo;0;AL;;;;;N;ARABIC LETTER RA;;;; +0632;ARABIC LETTER ZAIN;Lo;0;AL;;;;;N;;;;; +0633;ARABIC LETTER SEEN;Lo;0;AL;;;;;N;;;;; +0634;ARABIC LETTER SHEEN;Lo;0;AL;;;;;N;;;;; +0635;ARABIC LETTER SAD;Lo;0;AL;;;;;N;;;;; +0636;ARABIC LETTER DAD;Lo;0;AL;;;;;N;;;;; +0637;ARABIC LETTER TAH;Lo;0;AL;;;;;N;;;;; +0638;ARABIC LETTER ZAH;Lo;0;AL;;;;;N;ARABIC LETTER DHAH;;;; +0639;ARABIC LETTER AIN;Lo;0;AL;;;;;N;;;;; +063A;ARABIC LETTER GHAIN;Lo;0;AL;;;;;N;;;;; +063B;ARABIC LETTER KEHEH WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +063C;ARABIC LETTER KEHEH WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;;;;; +063D;ARABIC LETTER FARSI YEH WITH INVERTED V;Lo;0;AL;;;;;N;;;;; +063E;ARABIC LETTER FARSI YEH WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +063F;ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0640;ARABIC TATWEEL;Lm;0;AL;;;;;N;;;;; +0641;ARABIC LETTER FEH;Lo;0;AL;;;;;N;ARABIC LETTER FA;;;; +0642;ARABIC LETTER QAF;Lo;0;AL;;;;;N;;;;; +0643;ARABIC LETTER KAF;Lo;0;AL;;;;;N;ARABIC LETTER CAF;;;; +0644;ARABIC LETTER LAM;Lo;0;AL;;;;;N;;;;; +0645;ARABIC LETTER MEEM;Lo;0;AL;;;;;N;;;;; +0646;ARABIC LETTER NOON;Lo;0;AL;;;;;N;;;;; +0647;ARABIC LETTER HEH;Lo;0;AL;;;;;N;ARABIC LETTER HA;;;; +0648;ARABIC LETTER WAW;Lo;0;AL;;;;;N;;;;; +0649;ARABIC LETTER ALEF MAKSURA;Lo;0;AL;;;;;N;ARABIC LETTER ALEF MAQSURAH;;;; +064A;ARABIC LETTER YEH;Lo;0;AL;;;;;N;ARABIC LETTER YA;;;; +064B;ARABIC FATHATAN;Mn;27;NSM;;;;;N;;;;; +064C;ARABIC DAMMATAN;Mn;28;NSM;;;;;N;;;;; +064D;ARABIC KASRATAN;Mn;29;NSM;;;;;N;;;;; +064E;ARABIC FATHA;Mn;30;NSM;;;;;N;ARABIC FATHAH;;;; +064F;ARABIC DAMMA;Mn;31;NSM;;;;;N;ARABIC DAMMAH;;;; +0650;ARABIC KASRA;Mn;32;NSM;;;;;N;ARABIC KASRAH;;;; +0651;ARABIC SHADDA;Mn;33;NSM;;;;;N;ARABIC SHADDAH;;;; +0652;ARABIC SUKUN;Mn;34;NSM;;;;;N;;;;; +0653;ARABIC MADDAH ABOVE;Mn;230;NSM;;;;;N;;;;; +0654;ARABIC HAMZA ABOVE;Mn;230;NSM;;;;;N;;;;; +0655;ARABIC HAMZA BELOW;Mn;220;NSM;;;;;N;;;;; +0656;ARABIC SUBSCRIPT ALEF;Mn;220;NSM;;;;;N;;;;; +0657;ARABIC INVERTED DAMMA;Mn;230;NSM;;;;;N;;;;; +0658;ARABIC MARK NOON GHUNNA;Mn;230;NSM;;;;;N;;;;; +0659;ARABIC ZWARAKAY;Mn;230;NSM;;;;;N;;;;; +065A;ARABIC VOWEL SIGN SMALL V ABOVE;Mn;230;NSM;;;;;N;;;;; +065B;ARABIC VOWEL SIGN INVERTED SMALL V ABOVE;Mn;230;NSM;;;;;N;;;;; +065C;ARABIC VOWEL SIGN DOT BELOW;Mn;220;NSM;;;;;N;;;;; +065D;ARABIC REVERSED DAMMA;Mn;230;NSM;;;;;N;;;;; +065E;ARABIC FATHA WITH TWO DOTS;Mn;230;NSM;;;;;N;;;;; +065F;ARABIC WAVY HAMZA BELOW;Mn;220;NSM;;;;;N;;;;; +0660;ARABIC-INDIC DIGIT ZERO;Nd;0;AN;;0;0;0;N;;;;; +0661;ARABIC-INDIC DIGIT ONE;Nd;0;AN;;1;1;1;N;;;;; +0662;ARABIC-INDIC DIGIT TWO;Nd;0;AN;;2;2;2;N;;;;; +0663;ARABIC-INDIC DIGIT THREE;Nd;0;AN;;3;3;3;N;;;;; +0664;ARABIC-INDIC DIGIT FOUR;Nd;0;AN;;4;4;4;N;;;;; +0665;ARABIC-INDIC DIGIT FIVE;Nd;0;AN;;5;5;5;N;;;;; +0666;ARABIC-INDIC DIGIT SIX;Nd;0;AN;;6;6;6;N;;;;; +0667;ARABIC-INDIC DIGIT SEVEN;Nd;0;AN;;7;7;7;N;;;;; +0668;ARABIC-INDIC DIGIT EIGHT;Nd;0;AN;;8;8;8;N;;;;; +0669;ARABIC-INDIC DIGIT NINE;Nd;0;AN;;9;9;9;N;;;;; +066A;ARABIC PERCENT SIGN;Po;0;ET;;;;;N;;;;; +066B;ARABIC DECIMAL SEPARATOR;Po;0;AN;;;;;N;;;;; +066C;ARABIC THOUSANDS SEPARATOR;Po;0;AN;;;;;N;;;;; +066D;ARABIC FIVE POINTED STAR;Po;0;AL;;;;;N;;;;; +066E;ARABIC LETTER DOTLESS BEH;Lo;0;AL;;;;;N;;;;; +066F;ARABIC LETTER DOTLESS QAF;Lo;0;AL;;;;;N;;;;; +0670;ARABIC LETTER SUPERSCRIPT ALEF;Mn;35;NSM;;;;;N;ARABIC ALEF ABOVE;;;; +0671;ARABIC LETTER ALEF WASLA;Lo;0;AL;;;;;N;ARABIC LETTER HAMZAT WASL ON ALEF;;;; +0672;ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER WAVY HAMZAH ON ALEF;;;; +0673;ARABIC LETTER ALEF WITH WAVY HAMZA BELOW;Lo;0;AL;;;;;N;ARABIC LETTER WAVY HAMZAH UNDER ALEF;;;; +0674;ARABIC LETTER HIGH HAMZA;Lo;0;AL;;;;;N;ARABIC LETTER HIGH HAMZAH;;;; +0675;ARABIC LETTER HIGH HAMZA ALEF;Lo;0;AL; 0627 0674;;;;N;ARABIC LETTER HIGH HAMZAH ALEF;;;; +0676;ARABIC LETTER HIGH HAMZA WAW;Lo;0;AL; 0648 0674;;;;N;ARABIC LETTER HIGH HAMZAH WAW;;;; +0677;ARABIC LETTER U WITH HAMZA ABOVE;Lo;0;AL; 06C7 0674;;;;N;ARABIC LETTER HIGH HAMZAH WAW WITH DAMMAH;;;; +0678;ARABIC LETTER HIGH HAMZA YEH;Lo;0;AL; 064A 0674;;;;N;ARABIC LETTER HIGH HAMZAH YA;;;; +0679;ARABIC LETTER TTEH;Lo;0;AL;;;;;N;ARABIC LETTER TAA WITH SMALL TAH;;;; +067A;ARABIC LETTER TTEHEH;Lo;0;AL;;;;;N;ARABIC LETTER TAA WITH TWO DOTS VERTICAL ABOVE;;;; +067B;ARABIC LETTER BEEH;Lo;0;AL;;;;;N;ARABIC LETTER BAA WITH TWO DOTS VERTICAL BELOW;;;; +067C;ARABIC LETTER TEH WITH RING;Lo;0;AL;;;;;N;ARABIC LETTER TAA WITH RING;;;; +067D;ARABIC LETTER TEH WITH THREE DOTS ABOVE DOWNWARDS;Lo;0;AL;;;;;N;ARABIC LETTER TAA WITH THREE DOTS ABOVE DOWNWARD;;;; +067E;ARABIC LETTER PEH;Lo;0;AL;;;;;N;ARABIC LETTER TAA WITH THREE DOTS BELOW;;;; +067F;ARABIC LETTER TEHEH;Lo;0;AL;;;;;N;ARABIC LETTER TAA WITH FOUR DOTS ABOVE;;;; +0680;ARABIC LETTER BEHEH;Lo;0;AL;;;;;N;ARABIC LETTER BAA WITH FOUR DOTS BELOW;;;; +0681;ARABIC LETTER HAH WITH HAMZA ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER HAMZAH ON HAA;;;; +0682;ARABIC LETTER HAH WITH TWO DOTS VERTICAL ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER HAA WITH TWO DOTS VERTICAL ABOVE;;;; +0683;ARABIC LETTER NYEH;Lo;0;AL;;;;;N;ARABIC LETTER HAA WITH MIDDLE TWO DOTS;;;; +0684;ARABIC LETTER DYEH;Lo;0;AL;;;;;N;ARABIC LETTER HAA WITH MIDDLE TWO DOTS VERTICAL;;;; +0685;ARABIC LETTER HAH WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER HAA WITH THREE DOTS ABOVE;;;; +0686;ARABIC LETTER TCHEH;Lo;0;AL;;;;;N;ARABIC LETTER HAA WITH MIDDLE THREE DOTS DOWNWARD;;;; +0687;ARABIC LETTER TCHEHEH;Lo;0;AL;;;;;N;ARABIC LETTER HAA WITH MIDDLE FOUR DOTS;;;; +0688;ARABIC LETTER DDAL;Lo;0;AL;;;;;N;ARABIC LETTER DAL WITH SMALL TAH;;;; +0689;ARABIC LETTER DAL WITH RING;Lo;0;AL;;;;;N;;;;; +068A;ARABIC LETTER DAL WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +068B;ARABIC LETTER DAL WITH DOT BELOW AND SMALL TAH;Lo;0;AL;;;;;N;;;;; +068C;ARABIC LETTER DAHAL;Lo;0;AL;;;;;N;ARABIC LETTER DAL WITH TWO DOTS ABOVE;;;; +068D;ARABIC LETTER DDAHAL;Lo;0;AL;;;;;N;ARABIC LETTER DAL WITH TWO DOTS BELOW;;;; +068E;ARABIC LETTER DUL;Lo;0;AL;;;;;N;ARABIC LETTER DAL WITH THREE DOTS ABOVE;;;; +068F;ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS;Lo;0;AL;;;;;N;ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARD;;;; +0690;ARABIC LETTER DAL WITH FOUR DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0691;ARABIC LETTER RREH;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH SMALL TAH;;;; +0692;ARABIC LETTER REH WITH SMALL V;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH SMALL V;;;; +0693;ARABIC LETTER REH WITH RING;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH RING;;;; +0694;ARABIC LETTER REH WITH DOT BELOW;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH DOT BELOW;;;; +0695;ARABIC LETTER REH WITH SMALL V BELOW;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH SMALL V BELOW;;;; +0696;ARABIC LETTER REH WITH DOT BELOW AND DOT ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH DOT BELOW AND DOT ABOVE;;;; +0697;ARABIC LETTER REH WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH TWO DOTS ABOVE;;;; +0698;ARABIC LETTER JEH;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH THREE DOTS ABOVE;;;; +0699;ARABIC LETTER REH WITH FOUR DOTS ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER RA WITH FOUR DOTS ABOVE;;;; +069A;ARABIC LETTER SEEN WITH DOT BELOW AND DOT ABOVE;Lo;0;AL;;;;;N;;;;; +069B;ARABIC LETTER SEEN WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;;;;; +069C;ARABIC LETTER SEEN WITH THREE DOTS BELOW AND THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +069D;ARABIC LETTER SAD WITH TWO DOTS BELOW;Lo;0;AL;;;;;N;;;;; +069E;ARABIC LETTER SAD WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +069F;ARABIC LETTER TAH WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06A0;ARABIC LETTER AIN WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06A1;ARABIC LETTER DOTLESS FEH;Lo;0;AL;;;;;N;ARABIC LETTER DOTLESS FA;;;; +06A2;ARABIC LETTER FEH WITH DOT MOVED BELOW;Lo;0;AL;;;;;N;ARABIC LETTER FA WITH DOT MOVED BELOW;;;; +06A3;ARABIC LETTER FEH WITH DOT BELOW;Lo;0;AL;;;;;N;ARABIC LETTER FA WITH DOT BELOW;;;; +06A4;ARABIC LETTER VEH;Lo;0;AL;;;;;N;ARABIC LETTER FA WITH THREE DOTS ABOVE;;;; +06A5;ARABIC LETTER FEH WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;ARABIC LETTER FA WITH THREE DOTS BELOW;;;; +06A6;ARABIC LETTER PEHEH;Lo;0;AL;;;;;N;ARABIC LETTER FA WITH FOUR DOTS ABOVE;;;; +06A7;ARABIC LETTER QAF WITH DOT ABOVE;Lo;0;AL;;;;;N;;;;; +06A8;ARABIC LETTER QAF WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06A9;ARABIC LETTER KEHEH;Lo;0;AL;;;;;N;ARABIC LETTER OPEN CAF;;;; +06AA;ARABIC LETTER SWASH KAF;Lo;0;AL;;;;;N;ARABIC LETTER SWASH CAF;;;; +06AB;ARABIC LETTER KAF WITH RING;Lo;0;AL;;;;;N;ARABIC LETTER CAF WITH RING;;;; +06AC;ARABIC LETTER KAF WITH DOT ABOVE;Lo;0;AL;;;;;N;ARABIC LETTER CAF WITH DOT ABOVE;;;; +06AD;ARABIC LETTER NG;Lo;0;AL;;;;;N;ARABIC LETTER CAF WITH THREE DOTS ABOVE;;;; +06AE;ARABIC LETTER KAF WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;ARABIC LETTER CAF WITH THREE DOTS BELOW;;;; +06AF;ARABIC LETTER GAF;Lo;0;AL;;;;;N;;;;; +06B0;ARABIC LETTER GAF WITH RING;Lo;0;AL;;;;;N;;;;; +06B1;ARABIC LETTER NGOEH;Lo;0;AL;;;;;N;ARABIC LETTER GAF WITH TWO DOTS ABOVE;;;; +06B2;ARABIC LETTER GAF WITH TWO DOTS BELOW;Lo;0;AL;;;;;N;;;;; +06B3;ARABIC LETTER GUEH;Lo;0;AL;;;;;N;ARABIC LETTER GAF WITH TWO DOTS VERTICAL BELOW;;;; +06B4;ARABIC LETTER GAF WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06B5;ARABIC LETTER LAM WITH SMALL V;Lo;0;AL;;;;;N;;;;; +06B6;ARABIC LETTER LAM WITH DOT ABOVE;Lo;0;AL;;;;;N;;;;; +06B7;ARABIC LETTER LAM WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06B8;ARABIC LETTER LAM WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;;;;; +06B9;ARABIC LETTER NOON WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +06BA;ARABIC LETTER NOON GHUNNA;Lo;0;AL;;;;;N;ARABIC LETTER DOTLESS NOON;;;; +06BB;ARABIC LETTER RNOON;Lo;0;AL;;;;;N;ARABIC LETTER DOTLESS NOON WITH SMALL TAH;;;; +06BC;ARABIC LETTER NOON WITH RING;Lo;0;AL;;;;;N;;;;; +06BD;ARABIC LETTER NOON WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06BE;ARABIC LETTER HEH DOACHASHMEE;Lo;0;AL;;;;;N;ARABIC LETTER KNOTTED HA;;;; +06BF;ARABIC LETTER TCHEH WITH DOT ABOVE;Lo;0;AL;;;;;N;;;;; +06C0;ARABIC LETTER HEH WITH YEH ABOVE;Lo;0;AL;06D5 0654;;;;N;ARABIC LETTER HAMZAH ON HA;;;; +06C1;ARABIC LETTER HEH GOAL;Lo;0;AL;;;;;N;ARABIC LETTER HA GOAL;;;; +06C2;ARABIC LETTER HEH GOAL WITH HAMZA ABOVE;Lo;0;AL;06C1 0654;;;;N;ARABIC LETTER HAMZAH ON HA GOAL;;;; +06C3;ARABIC LETTER TEH MARBUTA GOAL;Lo;0;AL;;;;;N;ARABIC LETTER TAA MARBUTAH GOAL;;;; +06C4;ARABIC LETTER WAW WITH RING;Lo;0;AL;;;;;N;;;;; +06C5;ARABIC LETTER KIRGHIZ OE;Lo;0;AL;;;;;N;ARABIC LETTER WAW WITH BAR;;;; +06C6;ARABIC LETTER OE;Lo;0;AL;;;;;N;ARABIC LETTER WAW WITH SMALL V;;;; +06C7;ARABIC LETTER U;Lo;0;AL;;;;;N;ARABIC LETTER WAW WITH DAMMAH;;;; +06C8;ARABIC LETTER YU;Lo;0;AL;;;;;N;ARABIC LETTER WAW WITH ALEF ABOVE;;;; +06C9;ARABIC LETTER KIRGHIZ YU;Lo;0;AL;;;;;N;ARABIC LETTER WAW WITH INVERTED SMALL V;;;; +06CA;ARABIC LETTER WAW WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +06CB;ARABIC LETTER VE;Lo;0;AL;;;;;N;ARABIC LETTER WAW WITH THREE DOTS ABOVE;;;; +06CC;ARABIC LETTER FARSI YEH;Lo;0;AL;;;;;N;ARABIC LETTER DOTLESS YA;;;; +06CD;ARABIC LETTER YEH WITH TAIL;Lo;0;AL;;;;;N;ARABIC LETTER YA WITH TAIL;;;; +06CE;ARABIC LETTER YEH WITH SMALL V;Lo;0;AL;;;;;N;ARABIC LETTER YA WITH SMALL V;;;; +06CF;ARABIC LETTER WAW WITH DOT ABOVE;Lo;0;AL;;;;;N;;;;; +06D0;ARABIC LETTER E;Lo;0;AL;;;;;N;ARABIC LETTER YA WITH TWO DOTS VERTICAL BELOW;;;; +06D1;ARABIC LETTER YEH WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;ARABIC LETTER YA WITH THREE DOTS BELOW;;;; +06D2;ARABIC LETTER YEH BARREE;Lo;0;AL;;;;;N;ARABIC LETTER YA BARREE;;;; +06D3;ARABIC LETTER YEH BARREE WITH HAMZA ABOVE;Lo;0;AL;06D2 0654;;;;N;ARABIC LETTER HAMZAH ON YA BARREE;;;; +06D4;ARABIC FULL STOP;Po;0;AL;;;;;N;ARABIC PERIOD;;;; +06D5;ARABIC LETTER AE;Lo;0;AL;;;;;N;;;;; +06D6;ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA;Mn;230;NSM;;;;;N;;;;; +06D7;ARABIC SMALL HIGH LIGATURE QAF WITH LAM WITH ALEF MAKSURA;Mn;230;NSM;;;;;N;;;;; +06D8;ARABIC SMALL HIGH MEEM INITIAL FORM;Mn;230;NSM;;;;;N;;;;; +06D9;ARABIC SMALL HIGH LAM ALEF;Mn;230;NSM;;;;;N;;;;; +06DA;ARABIC SMALL HIGH JEEM;Mn;230;NSM;;;;;N;;;;; +06DB;ARABIC SMALL HIGH THREE DOTS;Mn;230;NSM;;;;;N;;;;; +06DC;ARABIC SMALL HIGH SEEN;Mn;230;NSM;;;;;N;;;;; +06DD;ARABIC END OF AYAH;Cf;0;AN;;;;;N;;;;; +06DE;ARABIC START OF RUB EL HIZB;So;0;ON;;;;;N;;;;; +06DF;ARABIC SMALL HIGH ROUNDED ZERO;Mn;230;NSM;;;;;N;;;;; +06E0;ARABIC SMALL HIGH UPRIGHT RECTANGULAR ZERO;Mn;230;NSM;;;;;N;;;;; +06E1;ARABIC SMALL HIGH DOTLESS HEAD OF KHAH;Mn;230;NSM;;;;;N;;;;; +06E2;ARABIC SMALL HIGH MEEM ISOLATED FORM;Mn;230;NSM;;;;;N;;;;; +06E3;ARABIC SMALL LOW SEEN;Mn;220;NSM;;;;;N;;;;; +06E4;ARABIC SMALL HIGH MADDA;Mn;230;NSM;;;;;N;;;;; +06E5;ARABIC SMALL WAW;Lm;0;AL;;;;;N;;;;; +06E6;ARABIC SMALL YEH;Lm;0;AL;;;;;N;;;;; +06E7;ARABIC SMALL HIGH YEH;Mn;230;NSM;;;;;N;;;;; +06E8;ARABIC SMALL HIGH NOON;Mn;230;NSM;;;;;N;;;;; +06E9;ARABIC PLACE OF SAJDAH;So;0;ON;;;;;N;;;;; +06EA;ARABIC EMPTY CENTRE LOW STOP;Mn;220;NSM;;;;;N;;;;; +06EB;ARABIC EMPTY CENTRE HIGH STOP;Mn;230;NSM;;;;;N;;;;; +06EC;ARABIC ROUNDED HIGH STOP WITH FILLED CENTRE;Mn;230;NSM;;;;;N;;;;; +06ED;ARABIC SMALL LOW MEEM;Mn;220;NSM;;;;;N;;;;; +06EE;ARABIC LETTER DAL WITH INVERTED V;Lo;0;AL;;;;;N;;;;; +06EF;ARABIC LETTER REH WITH INVERTED V;Lo;0;AL;;;;;N;;;;; +06F0;EXTENDED ARABIC-INDIC DIGIT ZERO;Nd;0;EN;;0;0;0;N;EASTERN ARABIC-INDIC DIGIT ZERO;;;; +06F1;EXTENDED ARABIC-INDIC DIGIT ONE;Nd;0;EN;;1;1;1;N;EASTERN ARABIC-INDIC DIGIT ONE;;;; +06F2;EXTENDED ARABIC-INDIC DIGIT TWO;Nd;0;EN;;2;2;2;N;EASTERN ARABIC-INDIC DIGIT TWO;;;; +06F3;EXTENDED ARABIC-INDIC DIGIT THREE;Nd;0;EN;;3;3;3;N;EASTERN ARABIC-INDIC DIGIT THREE;;;; +06F4;EXTENDED ARABIC-INDIC DIGIT FOUR;Nd;0;EN;;4;4;4;N;EASTERN ARABIC-INDIC DIGIT FOUR;;;; +06F5;EXTENDED ARABIC-INDIC DIGIT FIVE;Nd;0;EN;;5;5;5;N;EASTERN ARABIC-INDIC DIGIT FIVE;;;; +06F6;EXTENDED ARABIC-INDIC DIGIT SIX;Nd;0;EN;;6;6;6;N;EASTERN ARABIC-INDIC DIGIT SIX;;;; +06F7;EXTENDED ARABIC-INDIC DIGIT SEVEN;Nd;0;EN;;7;7;7;N;EASTERN ARABIC-INDIC DIGIT SEVEN;;;; +06F8;EXTENDED ARABIC-INDIC DIGIT EIGHT;Nd;0;EN;;8;8;8;N;EASTERN ARABIC-INDIC DIGIT EIGHT;;;; +06F9;EXTENDED ARABIC-INDIC DIGIT NINE;Nd;0;EN;;9;9;9;N;EASTERN ARABIC-INDIC DIGIT NINE;;;; +06FA;ARABIC LETTER SHEEN WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +06FB;ARABIC LETTER DAD WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +06FC;ARABIC LETTER GHAIN WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +06FD;ARABIC SIGN SINDHI AMPERSAND;So;0;AL;;;;;N;;;;; +06FE;ARABIC SIGN SINDHI POSTPOSITION MEN;So;0;AL;;;;;N;;;;; +06FF;ARABIC LETTER HEH WITH INVERTED V;Lo;0;AL;;;;;N;;;;; +0700;SYRIAC END OF PARAGRAPH;Po;0;AL;;;;;N;;;;; +0701;SYRIAC SUPRALINEAR FULL STOP;Po;0;AL;;;;;N;;;;; +0702;SYRIAC SUBLINEAR FULL STOP;Po;0;AL;;;;;N;;;;; +0703;SYRIAC SUPRALINEAR COLON;Po;0;AL;;;;;N;;;;; +0704;SYRIAC SUBLINEAR COLON;Po;0;AL;;;;;N;;;;; +0705;SYRIAC HORIZONTAL COLON;Po;0;AL;;;;;N;;;;; +0706;SYRIAC COLON SKEWED LEFT;Po;0;AL;;;;;N;;;;; +0707;SYRIAC COLON SKEWED RIGHT;Po;0;AL;;;;;N;;;;; +0708;SYRIAC SUPRALINEAR COLON SKEWED LEFT;Po;0;AL;;;;;N;;;;; +0709;SYRIAC SUBLINEAR COLON SKEWED RIGHT;Po;0;AL;;;;;N;;;;; +070A;SYRIAC CONTRACTION;Po;0;AL;;;;;N;;;;; +070B;SYRIAC HARKLEAN OBELUS;Po;0;AL;;;;;N;;;;; +070C;SYRIAC HARKLEAN METOBELUS;Po;0;AL;;;;;N;;;;; +070D;SYRIAC HARKLEAN ASTERISCUS;Po;0;AL;;;;;N;;;;; +070F;SYRIAC ABBREVIATION MARK;Cf;0;AL;;;;;N;;;;; +0710;SYRIAC LETTER ALAPH;Lo;0;AL;;;;;N;;;;; +0711;SYRIAC LETTER SUPERSCRIPT ALAPH;Mn;36;NSM;;;;;N;;;;; +0712;SYRIAC LETTER BETH;Lo;0;AL;;;;;N;;;;; +0713;SYRIAC LETTER GAMAL;Lo;0;AL;;;;;N;;;;; +0714;SYRIAC LETTER GAMAL GARSHUNI;Lo;0;AL;;;;;N;;;;; +0715;SYRIAC LETTER DALATH;Lo;0;AL;;;;;N;;;;; +0716;SYRIAC LETTER DOTLESS DALATH RISH;Lo;0;AL;;;;;N;;;;; +0717;SYRIAC LETTER HE;Lo;0;AL;;;;;N;;;;; +0718;SYRIAC LETTER WAW;Lo;0;AL;;;;;N;;;;; +0719;SYRIAC LETTER ZAIN;Lo;0;AL;;;;;N;;;;; +071A;SYRIAC LETTER HETH;Lo;0;AL;;;;;N;;;;; +071B;SYRIAC LETTER TETH;Lo;0;AL;;;;;N;;;;; +071C;SYRIAC LETTER TETH GARSHUNI;Lo;0;AL;;;;;N;;;;; +071D;SYRIAC LETTER YUDH;Lo;0;AL;;;;;N;;;;; +071E;SYRIAC LETTER YUDH HE;Lo;0;AL;;;;;N;;;;; +071F;SYRIAC LETTER KAPH;Lo;0;AL;;;;;N;;;;; +0720;SYRIAC LETTER LAMADH;Lo;0;AL;;;;;N;;;;; +0721;SYRIAC LETTER MIM;Lo;0;AL;;;;;N;;;;; +0722;SYRIAC LETTER NUN;Lo;0;AL;;;;;N;;;;; +0723;SYRIAC LETTER SEMKATH;Lo;0;AL;;;;;N;;;;; +0724;SYRIAC LETTER FINAL SEMKATH;Lo;0;AL;;;;;N;;;;; +0725;SYRIAC LETTER E;Lo;0;AL;;;;;N;;;;; +0726;SYRIAC LETTER PE;Lo;0;AL;;;;;N;;;;; +0727;SYRIAC LETTER REVERSED PE;Lo;0;AL;;;;;N;;;;; +0728;SYRIAC LETTER SADHE;Lo;0;AL;;;;;N;;;;; +0729;SYRIAC LETTER QAPH;Lo;0;AL;;;;;N;;;;; +072A;SYRIAC LETTER RISH;Lo;0;AL;;;;;N;;;;; +072B;SYRIAC LETTER SHIN;Lo;0;AL;;;;;N;;;;; +072C;SYRIAC LETTER TAW;Lo;0;AL;;;;;N;;;;; +072D;SYRIAC LETTER PERSIAN BHETH;Lo;0;AL;;;;;N;;;;; +072E;SYRIAC LETTER PERSIAN GHAMAL;Lo;0;AL;;;;;N;;;;; +072F;SYRIAC LETTER PERSIAN DHALATH;Lo;0;AL;;;;;N;;;;; +0730;SYRIAC PTHAHA ABOVE;Mn;230;NSM;;;;;N;;;;; +0731;SYRIAC PTHAHA BELOW;Mn;220;NSM;;;;;N;;;;; +0732;SYRIAC PTHAHA DOTTED;Mn;230;NSM;;;;;N;;;;; +0733;SYRIAC ZQAPHA ABOVE;Mn;230;NSM;;;;;N;;;;; +0734;SYRIAC ZQAPHA BELOW;Mn;220;NSM;;;;;N;;;;; +0735;SYRIAC ZQAPHA DOTTED;Mn;230;NSM;;;;;N;;;;; +0736;SYRIAC RBASA ABOVE;Mn;230;NSM;;;;;N;;;;; +0737;SYRIAC RBASA BELOW;Mn;220;NSM;;;;;N;;;;; +0738;SYRIAC DOTTED ZLAMA HORIZONTAL;Mn;220;NSM;;;;;N;;;;; +0739;SYRIAC DOTTED ZLAMA ANGULAR;Mn;220;NSM;;;;;N;;;;; +073A;SYRIAC HBASA ABOVE;Mn;230;NSM;;;;;N;;;;; +073B;SYRIAC HBASA BELOW;Mn;220;NSM;;;;;N;;;;; +073C;SYRIAC HBASA-ESASA DOTTED;Mn;220;NSM;;;;;N;;;;; +073D;SYRIAC ESASA ABOVE;Mn;230;NSM;;;;;N;;;;; +073E;SYRIAC ESASA BELOW;Mn;220;NSM;;;;;N;;;;; +073F;SYRIAC RWAHA;Mn;230;NSM;;;;;N;;;;; +0740;SYRIAC FEMININE DOT;Mn;230;NSM;;;;;N;;;;; +0741;SYRIAC QUSHSHAYA;Mn;230;NSM;;;;;N;;;;; +0742;SYRIAC RUKKAKHA;Mn;220;NSM;;;;;N;;;;; +0743;SYRIAC TWO VERTICAL DOTS ABOVE;Mn;230;NSM;;;;;N;;;;; +0744;SYRIAC TWO VERTICAL DOTS BELOW;Mn;220;NSM;;;;;N;;;;; +0745;SYRIAC THREE DOTS ABOVE;Mn;230;NSM;;;;;N;;;;; +0746;SYRIAC THREE DOTS BELOW;Mn;220;NSM;;;;;N;;;;; +0747;SYRIAC OBLIQUE LINE ABOVE;Mn;230;NSM;;;;;N;;;;; +0748;SYRIAC OBLIQUE LINE BELOW;Mn;220;NSM;;;;;N;;;;; +0749;SYRIAC MUSIC;Mn;230;NSM;;;;;N;;;;; +074A;SYRIAC BARREKH;Mn;230;NSM;;;;;N;;;;; +074D;SYRIAC LETTER SOGDIAN ZHAIN;Lo;0;AL;;;;;N;;;;; +074E;SYRIAC LETTER SOGDIAN KHAPH;Lo;0;AL;;;;;N;;;;; +074F;SYRIAC LETTER SOGDIAN FE;Lo;0;AL;;;;;N;;;;; +0750;ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW;Lo;0;AL;;;;;N;;;;; +0751;ARABIC LETTER BEH WITH DOT BELOW AND THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0752;ARABIC LETTER BEH WITH THREE DOTS POINTING UPWARDS BELOW;Lo;0;AL;;;;;N;;;;; +0753;ARABIC LETTER BEH WITH THREE DOTS POINTING UPWARDS BELOW AND TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0754;ARABIC LETTER BEH WITH TWO DOTS BELOW AND DOT ABOVE;Lo;0;AL;;;;;N;;;;; +0755;ARABIC LETTER BEH WITH INVERTED SMALL V BELOW;Lo;0;AL;;;;;N;;;;; +0756;ARABIC LETTER BEH WITH SMALL V;Lo;0;AL;;;;;N;;;;; +0757;ARABIC LETTER HAH WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0758;ARABIC LETTER HAH WITH THREE DOTS POINTING UPWARDS BELOW;Lo;0;AL;;;;;N;;;;; +0759;ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW AND SMALL TAH;Lo;0;AL;;;;;N;;;;; +075A;ARABIC LETTER DAL WITH INVERTED SMALL V BELOW;Lo;0;AL;;;;;N;;;;; +075B;ARABIC LETTER REH WITH STROKE;Lo;0;AL;;;;;N;;;;; +075C;ARABIC LETTER SEEN WITH FOUR DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +075D;ARABIC LETTER AIN WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +075E;ARABIC LETTER AIN WITH THREE DOTS POINTING DOWNWARDS ABOVE;Lo;0;AL;;;;;N;;;;; +075F;ARABIC LETTER AIN WITH TWO DOTS VERTICALLY ABOVE;Lo;0;AL;;;;;N;;;;; +0760;ARABIC LETTER FEH WITH TWO DOTS BELOW;Lo;0;AL;;;;;N;;;;; +0761;ARABIC LETTER FEH WITH THREE DOTS POINTING UPWARDS BELOW;Lo;0;AL;;;;;N;;;;; +0762;ARABIC LETTER KEHEH WITH DOT ABOVE;Lo;0;AL;;;;;N;;;;; +0763;ARABIC LETTER KEHEH WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0764;ARABIC LETTER KEHEH WITH THREE DOTS POINTING UPWARDS BELOW;Lo;0;AL;;;;;N;;;;; +0765;ARABIC LETTER MEEM WITH DOT ABOVE;Lo;0;AL;;;;;N;;;;; +0766;ARABIC LETTER MEEM WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +0767;ARABIC LETTER NOON WITH TWO DOTS BELOW;Lo;0;AL;;;;;N;;;;; +0768;ARABIC LETTER NOON WITH SMALL TAH;Lo;0;AL;;;;;N;;;;; +0769;ARABIC LETTER NOON WITH SMALL V;Lo;0;AL;;;;;N;;;;; +076A;ARABIC LETTER LAM WITH BAR;Lo;0;AL;;;;;N;;;;; +076B;ARABIC LETTER REH WITH TWO DOTS VERTICALLY ABOVE;Lo;0;AL;;;;;N;;;;; +076C;ARABIC LETTER REH WITH HAMZA ABOVE;Lo;0;AL;;;;;N;;;;; +076D;ARABIC LETTER SEEN WITH TWO DOTS VERTICALLY ABOVE;Lo;0;AL;;;;;N;;;;; +076E;ARABIC LETTER HAH WITH SMALL ARABIC LETTER TAH BELOW;Lo;0;AL;;;;;N;;;;; +076F;ARABIC LETTER HAH WITH SMALL ARABIC LETTER TAH AND TWO DOTS;Lo;0;AL;;;;;N;;;;; +0770;ARABIC LETTER SEEN WITH SMALL ARABIC LETTER TAH AND TWO DOTS;Lo;0;AL;;;;;N;;;;; +0771;ARABIC LETTER REH WITH SMALL ARABIC LETTER TAH AND TWO DOTS;Lo;0;AL;;;;;N;;;;; +0772;ARABIC LETTER HAH WITH SMALL ARABIC LETTER TAH ABOVE;Lo;0;AL;;;;;N;;;;; +0773;ARABIC LETTER ALEF WITH EXTENDED ARABIC-INDIC DIGIT TWO ABOVE;Lo;0;AL;;;;;N;;;;; +0774;ARABIC LETTER ALEF WITH EXTENDED ARABIC-INDIC DIGIT THREE ABOVE;Lo;0;AL;;;;;N;;;;; +0775;ARABIC LETTER FARSI YEH WITH EXTENDED ARABIC-INDIC DIGIT TWO ABOVE;Lo;0;AL;;;;;N;;;;; +0776;ARABIC LETTER FARSI YEH WITH EXTENDED ARABIC-INDIC DIGIT THREE ABOVE;Lo;0;AL;;;;;N;;;;; +0777;ARABIC LETTER FARSI YEH WITH EXTENDED ARABIC-INDIC DIGIT FOUR BELOW;Lo;0;AL;;;;;N;;;;; +0778;ARABIC LETTER WAW WITH EXTENDED ARABIC-INDIC DIGIT TWO ABOVE;Lo;0;AL;;;;;N;;;;; +0779;ARABIC LETTER WAW WITH EXTENDED ARABIC-INDIC DIGIT THREE ABOVE;Lo;0;AL;;;;;N;;;;; +077A;ARABIC LETTER YEH BARREE WITH EXTENDED ARABIC-INDIC DIGIT TWO ABOVE;Lo;0;AL;;;;;N;;;;; +077B;ARABIC LETTER YEH BARREE WITH EXTENDED ARABIC-INDIC DIGIT THREE ABOVE;Lo;0;AL;;;;;N;;;;; +077C;ARABIC LETTER HAH WITH EXTENDED ARABIC-INDIC DIGIT FOUR BELOW;Lo;0;AL;;;;;N;;;;; +077D;ARABIC LETTER SEEN WITH EXTENDED ARABIC-INDIC DIGIT FOUR ABOVE;Lo;0;AL;;;;;N;;;;; +077E;ARABIC LETTER SEEN WITH INVERTED V;Lo;0;AL;;;;;N;;;;; +077F;ARABIC LETTER KAF WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +0780;THAANA LETTER HAA;Lo;0;AL;;;;;N;;;;; +0781;THAANA LETTER SHAVIYANI;Lo;0;AL;;;;;N;;;;; +0782;THAANA LETTER NOONU;Lo;0;AL;;;;;N;;;;; +0783;THAANA LETTER RAA;Lo;0;AL;;;;;N;;;;; +0784;THAANA LETTER BAA;Lo;0;AL;;;;;N;;;;; +0785;THAANA LETTER LHAVIYANI;Lo;0;AL;;;;;N;;;;; +0786;THAANA LETTER KAAFU;Lo;0;AL;;;;;N;;;;; +0787;THAANA LETTER ALIFU;Lo;0;AL;;;;;N;;;;; +0788;THAANA LETTER VAAVU;Lo;0;AL;;;;;N;;;;; +0789;THAANA LETTER MEEMU;Lo;0;AL;;;;;N;;;;; +078A;THAANA LETTER FAAFU;Lo;0;AL;;;;;N;;;;; +078B;THAANA LETTER DHAALU;Lo;0;AL;;;;;N;;;;; +078C;THAANA LETTER THAA;Lo;0;AL;;;;;N;;;;; +078D;THAANA LETTER LAAMU;Lo;0;AL;;;;;N;;;;; +078E;THAANA LETTER GAAFU;Lo;0;AL;;;;;N;;;;; +078F;THAANA LETTER GNAVIYANI;Lo;0;AL;;;;;N;;;;; +0790;THAANA LETTER SEENU;Lo;0;AL;;;;;N;;;;; +0791;THAANA LETTER DAVIYANI;Lo;0;AL;;;;;N;;;;; +0792;THAANA LETTER ZAVIYANI;Lo;0;AL;;;;;N;;;;; +0793;THAANA LETTER TAVIYANI;Lo;0;AL;;;;;N;;;;; +0794;THAANA LETTER YAA;Lo;0;AL;;;;;N;;;;; +0795;THAANA LETTER PAVIYANI;Lo;0;AL;;;;;N;;;;; +0796;THAANA LETTER JAVIYANI;Lo;0;AL;;;;;N;;;;; +0797;THAANA LETTER CHAVIYANI;Lo;0;AL;;;;;N;;;;; +0798;THAANA LETTER TTAA;Lo;0;AL;;;;;N;;;;; +0799;THAANA LETTER HHAA;Lo;0;AL;;;;;N;;;;; +079A;THAANA LETTER KHAA;Lo;0;AL;;;;;N;;;;; +079B;THAANA LETTER THAALU;Lo;0;AL;;;;;N;;;;; +079C;THAANA LETTER ZAA;Lo;0;AL;;;;;N;;;;; +079D;THAANA LETTER SHEENU;Lo;0;AL;;;;;N;;;;; +079E;THAANA LETTER SAADHU;Lo;0;AL;;;;;N;;;;; +079F;THAANA LETTER DAADHU;Lo;0;AL;;;;;N;;;;; +07A0;THAANA LETTER TO;Lo;0;AL;;;;;N;;;;; +07A1;THAANA LETTER ZO;Lo;0;AL;;;;;N;;;;; +07A2;THAANA LETTER AINU;Lo;0;AL;;;;;N;;;;; +07A3;THAANA LETTER GHAINU;Lo;0;AL;;;;;N;;;;; +07A4;THAANA LETTER QAAFU;Lo;0;AL;;;;;N;;;;; +07A5;THAANA LETTER WAAVU;Lo;0;AL;;;;;N;;;;; +07A6;THAANA ABAFILI;Mn;0;NSM;;;;;N;;;;; +07A7;THAANA AABAAFILI;Mn;0;NSM;;;;;N;;;;; +07A8;THAANA IBIFILI;Mn;0;NSM;;;;;N;;;;; +07A9;THAANA EEBEEFILI;Mn;0;NSM;;;;;N;;;;; +07AA;THAANA UBUFILI;Mn;0;NSM;;;;;N;;;;; +07AB;THAANA OOBOOFILI;Mn;0;NSM;;;;;N;;;;; +07AC;THAANA EBEFILI;Mn;0;NSM;;;;;N;;;;; +07AD;THAANA EYBEYFILI;Mn;0;NSM;;;;;N;;;;; +07AE;THAANA OBOFILI;Mn;0;NSM;;;;;N;;;;; +07AF;THAANA OABOAFILI;Mn;0;NSM;;;;;N;;;;; +07B0;THAANA SUKUN;Mn;0;NSM;;;;;N;;;;; +07B1;THAANA LETTER NAA;Lo;0;AL;;;;;N;;;;; +07C0;NKO DIGIT ZERO;Nd;0;R;;0;0;0;N;;;;; +07C1;NKO DIGIT ONE;Nd;0;R;;1;1;1;N;;;;; +07C2;NKO DIGIT TWO;Nd;0;R;;2;2;2;N;;;;; +07C3;NKO DIGIT THREE;Nd;0;R;;3;3;3;N;;;;; +07C4;NKO DIGIT FOUR;Nd;0;R;;4;4;4;N;;;;; +07C5;NKO DIGIT FIVE;Nd;0;R;;5;5;5;N;;;;; +07C6;NKO DIGIT SIX;Nd;0;R;;6;6;6;N;;;;; +07C7;NKO DIGIT SEVEN;Nd;0;R;;7;7;7;N;;;;; +07C8;NKO DIGIT EIGHT;Nd;0;R;;8;8;8;N;;;;; +07C9;NKO DIGIT NINE;Nd;0;R;;9;9;9;N;;;;; +07CA;NKO LETTER A;Lo;0;R;;;;;N;;;;; +07CB;NKO LETTER EE;Lo;0;R;;;;;N;;;;; +07CC;NKO LETTER I;Lo;0;R;;;;;N;;;;; +07CD;NKO LETTER E;Lo;0;R;;;;;N;;;;; +07CE;NKO LETTER U;Lo;0;R;;;;;N;;;;; +07CF;NKO LETTER OO;Lo;0;R;;;;;N;;;;; +07D0;NKO LETTER O;Lo;0;R;;;;;N;;;;; +07D1;NKO LETTER DAGBASINNA;Lo;0;R;;;;;N;;;;; +07D2;NKO LETTER N;Lo;0;R;;;;;N;;;;; +07D3;NKO LETTER BA;Lo;0;R;;;;;N;;;;; +07D4;NKO LETTER PA;Lo;0;R;;;;;N;;;;; +07D5;NKO LETTER TA;Lo;0;R;;;;;N;;;;; +07D6;NKO LETTER JA;Lo;0;R;;;;;N;;;;; +07D7;NKO LETTER CHA;Lo;0;R;;;;;N;;;;; +07D8;NKO LETTER DA;Lo;0;R;;;;;N;;;;; +07D9;NKO LETTER RA;Lo;0;R;;;;;N;;;;; +07DA;NKO LETTER RRA;Lo;0;R;;;;;N;;;;; +07DB;NKO LETTER SA;Lo;0;R;;;;;N;;;;; +07DC;NKO LETTER GBA;Lo;0;R;;;;;N;;;;; +07DD;NKO LETTER FA;Lo;0;R;;;;;N;;;;; +07DE;NKO LETTER KA;Lo;0;R;;;;;N;;;;; +07DF;NKO LETTER LA;Lo;0;R;;;;;N;;;;; +07E0;NKO LETTER NA WOLOSO;Lo;0;R;;;;;N;;;;; +07E1;NKO LETTER MA;Lo;0;R;;;;;N;;;;; +07E2;NKO LETTER NYA;Lo;0;R;;;;;N;;;;; +07E3;NKO LETTER NA;Lo;0;R;;;;;N;;;;; +07E4;NKO LETTER HA;Lo;0;R;;;;;N;;;;; +07E5;NKO LETTER WA;Lo;0;R;;;;;N;;;;; +07E6;NKO LETTER YA;Lo;0;R;;;;;N;;;;; +07E7;NKO LETTER NYA WOLOSO;Lo;0;R;;;;;N;;;;; +07E8;NKO LETTER JONA JA;Lo;0;R;;;;;N;;;;; +07E9;NKO LETTER JONA CHA;Lo;0;R;;;;;N;;;;; +07EA;NKO LETTER JONA RA;Lo;0;R;;;;;N;;;;; +07EB;NKO COMBINING SHORT HIGH TONE;Mn;230;NSM;;;;;N;;;;; +07EC;NKO COMBINING SHORT LOW TONE;Mn;230;NSM;;;;;N;;;;; +07ED;NKO COMBINING SHORT RISING TONE;Mn;230;NSM;;;;;N;;;;; +07EE;NKO COMBINING LONG DESCENDING TONE;Mn;230;NSM;;;;;N;;;;; +07EF;NKO COMBINING LONG HIGH TONE;Mn;230;NSM;;;;;N;;;;; +07F0;NKO COMBINING LONG LOW TONE;Mn;230;NSM;;;;;N;;;;; +07F1;NKO COMBINING LONG RISING TONE;Mn;230;NSM;;;;;N;;;;; +07F2;NKO COMBINING NASALIZATION MARK;Mn;220;NSM;;;;;N;;;;; +07F3;NKO COMBINING DOUBLE DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +07F4;NKO HIGH TONE APOSTROPHE;Lm;0;R;;;;;N;;;;; +07F5;NKO LOW TONE APOSTROPHE;Lm;0;R;;;;;N;;;;; +07F6;NKO SYMBOL OO DENNEN;So;0;ON;;;;;N;;;;; +07F7;NKO SYMBOL GBAKURUNEN;Po;0;ON;;;;;N;;;;; +07F8;NKO COMMA;Po;0;ON;;;;;N;;;;; +07F9;NKO EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; +07FA;NKO LAJANYALAN;Lm;0;R;;;;;N;;;;; +0800;SAMARITAN LETTER ALAF;Lo;0;R;;;;;N;;;;; +0801;SAMARITAN LETTER BIT;Lo;0;R;;;;;N;;;;; +0802;SAMARITAN LETTER GAMAN;Lo;0;R;;;;;N;;;;; +0803;SAMARITAN LETTER DALAT;Lo;0;R;;;;;N;;;;; +0804;SAMARITAN LETTER IY;Lo;0;R;;;;;N;;;;; +0805;SAMARITAN LETTER BAA;Lo;0;R;;;;;N;;;;; +0806;SAMARITAN LETTER ZEN;Lo;0;R;;;;;N;;;;; +0807;SAMARITAN LETTER IT;Lo;0;R;;;;;N;;;;; +0808;SAMARITAN LETTER TIT;Lo;0;R;;;;;N;;;;; +0809;SAMARITAN LETTER YUT;Lo;0;R;;;;;N;;;;; +080A;SAMARITAN LETTER KAAF;Lo;0;R;;;;;N;;;;; +080B;SAMARITAN LETTER LABAT;Lo;0;R;;;;;N;;;;; +080C;SAMARITAN LETTER MIM;Lo;0;R;;;;;N;;;;; +080D;SAMARITAN LETTER NUN;Lo;0;R;;;;;N;;;;; +080E;SAMARITAN LETTER SINGAAT;Lo;0;R;;;;;N;;;;; +080F;SAMARITAN LETTER IN;Lo;0;R;;;;;N;;;;; +0810;SAMARITAN LETTER FI;Lo;0;R;;;;;N;;;;; +0811;SAMARITAN LETTER TSAADIY;Lo;0;R;;;;;N;;;;; +0812;SAMARITAN LETTER QUF;Lo;0;R;;;;;N;;;;; +0813;SAMARITAN LETTER RISH;Lo;0;R;;;;;N;;;;; +0814;SAMARITAN LETTER SHAN;Lo;0;R;;;;;N;;;;; +0815;SAMARITAN LETTER TAAF;Lo;0;R;;;;;N;;;;; +0816;SAMARITAN MARK IN;Mn;230;NSM;;;;;N;;;;; +0817;SAMARITAN MARK IN-ALAF;Mn;230;NSM;;;;;N;;;;; +0818;SAMARITAN MARK OCCLUSION;Mn;230;NSM;;;;;N;;;;; +0819;SAMARITAN MARK DAGESH;Mn;230;NSM;;;;;N;;;;; +081A;SAMARITAN MODIFIER LETTER EPENTHETIC YUT;Lm;0;R;;;;;N;;;;; +081B;SAMARITAN MARK EPENTHETIC YUT;Mn;230;NSM;;;;;N;;;;; +081C;SAMARITAN VOWEL SIGN LONG E;Mn;230;NSM;;;;;N;;;;; +081D;SAMARITAN VOWEL SIGN E;Mn;230;NSM;;;;;N;;;;; +081E;SAMARITAN VOWEL SIGN OVERLONG AA;Mn;230;NSM;;;;;N;;;;; +081F;SAMARITAN VOWEL SIGN LONG AA;Mn;230;NSM;;;;;N;;;;; +0820;SAMARITAN VOWEL SIGN AA;Mn;230;NSM;;;;;N;;;;; +0821;SAMARITAN VOWEL SIGN OVERLONG A;Mn;230;NSM;;;;;N;;;;; +0822;SAMARITAN VOWEL SIGN LONG A;Mn;230;NSM;;;;;N;;;;; +0823;SAMARITAN VOWEL SIGN A;Mn;230;NSM;;;;;N;;;;; +0824;SAMARITAN MODIFIER LETTER SHORT A;Lm;0;R;;;;;N;;;;; +0825;SAMARITAN VOWEL SIGN SHORT A;Mn;230;NSM;;;;;N;;;;; +0826;SAMARITAN VOWEL SIGN LONG U;Mn;230;NSM;;;;;N;;;;; +0827;SAMARITAN VOWEL SIGN U;Mn;230;NSM;;;;;N;;;;; +0828;SAMARITAN MODIFIER LETTER I;Lm;0;R;;;;;N;;;;; +0829;SAMARITAN VOWEL SIGN LONG I;Mn;230;NSM;;;;;N;;;;; +082A;SAMARITAN VOWEL SIGN I;Mn;230;NSM;;;;;N;;;;; +082B;SAMARITAN VOWEL SIGN O;Mn;230;NSM;;;;;N;;;;; +082C;SAMARITAN VOWEL SIGN SUKUN;Mn;230;NSM;;;;;N;;;;; +082D;SAMARITAN MARK NEQUDAA;Mn;230;NSM;;;;;N;;;;; +0830;SAMARITAN PUNCTUATION NEQUDAA;Po;0;R;;;;;N;;;;; +0831;SAMARITAN PUNCTUATION AFSAAQ;Po;0;R;;;;;N;;;;; +0832;SAMARITAN PUNCTUATION ANGED;Po;0;R;;;;;N;;;;; +0833;SAMARITAN PUNCTUATION BAU;Po;0;R;;;;;N;;;;; +0834;SAMARITAN PUNCTUATION ATMAAU;Po;0;R;;;;;N;;;;; +0835;SAMARITAN PUNCTUATION SHIYYAALAA;Po;0;R;;;;;N;;;;; +0836;SAMARITAN ABBREVIATION MARK;Po;0;R;;;;;N;;;;; +0837;SAMARITAN PUNCTUATION MELODIC QITSA;Po;0;R;;;;;N;;;;; +0838;SAMARITAN PUNCTUATION ZIQAA;Po;0;R;;;;;N;;;;; +0839;SAMARITAN PUNCTUATION QITSA;Po;0;R;;;;;N;;;;; +083A;SAMARITAN PUNCTUATION ZAEF;Po;0;R;;;;;N;;;;; +083B;SAMARITAN PUNCTUATION TURU;Po;0;R;;;;;N;;;;; +083C;SAMARITAN PUNCTUATION ARKAANU;Po;0;R;;;;;N;;;;; +083D;SAMARITAN PUNCTUATION SOF MASHFAAT;Po;0;R;;;;;N;;;;; +083E;SAMARITAN PUNCTUATION ANNAAU;Po;0;R;;;;;N;;;;; +0840;MANDAIC LETTER HALQA;Lo;0;R;;;;;N;;;;; +0841;MANDAIC LETTER AB;Lo;0;R;;;;;N;;;;; +0842;MANDAIC LETTER AG;Lo;0;R;;;;;N;;;;; +0843;MANDAIC LETTER AD;Lo;0;R;;;;;N;;;;; +0844;MANDAIC LETTER AH;Lo;0;R;;;;;N;;;;; +0845;MANDAIC LETTER USHENNA;Lo;0;R;;;;;N;;;;; +0846;MANDAIC LETTER AZ;Lo;0;R;;;;;N;;;;; +0847;MANDAIC LETTER IT;Lo;0;R;;;;;N;;;;; +0848;MANDAIC LETTER ATT;Lo;0;R;;;;;N;;;;; +0849;MANDAIC LETTER AKSA;Lo;0;R;;;;;N;;;;; +084A;MANDAIC LETTER AK;Lo;0;R;;;;;N;;;;; +084B;MANDAIC LETTER AL;Lo;0;R;;;;;N;;;;; +084C;MANDAIC LETTER AM;Lo;0;R;;;;;N;;;;; +084D;MANDAIC LETTER AN;Lo;0;R;;;;;N;;;;; +084E;MANDAIC LETTER AS;Lo;0;R;;;;;N;;;;; +084F;MANDAIC LETTER IN;Lo;0;R;;;;;N;;;;; +0850;MANDAIC LETTER AP;Lo;0;R;;;;;N;;;;; +0851;MANDAIC LETTER ASZ;Lo;0;R;;;;;N;;;;; +0852;MANDAIC LETTER AQ;Lo;0;R;;;;;N;;;;; +0853;MANDAIC LETTER AR;Lo;0;R;;;;;N;;;;; +0854;MANDAIC LETTER ASH;Lo;0;R;;;;;N;;;;; +0855;MANDAIC LETTER AT;Lo;0;R;;;;;N;;;;; +0856;MANDAIC LETTER DUSHENNA;Lo;0;R;;;;;N;;;;; +0857;MANDAIC LETTER KAD;Lo;0;R;;;;;N;;;;; +0858;MANDAIC LETTER AIN;Lo;0;R;;;;;N;;;;; +0859;MANDAIC AFFRICATION MARK;Mn;220;NSM;;;;;N;;;;; +085A;MANDAIC VOCALIZATION MARK;Mn;220;NSM;;;;;N;;;;; +085B;MANDAIC GEMINATION MARK;Mn;220;NSM;;;;;N;;;;; +085E;MANDAIC PUNCTUATION;Po;0;R;;;;;N;;;;; +08A0;ARABIC LETTER BEH WITH SMALL V BELOW;Lo;0;AL;;;;;N;;;;; +08A2;ARABIC LETTER JEEM WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +08A3;ARABIC LETTER TAH WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +08A4;ARABIC LETTER FEH WITH DOT BELOW AND THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +08A5;ARABIC LETTER QAF WITH DOT BELOW;Lo;0;AL;;;;;N;;;;; +08A6;ARABIC LETTER LAM WITH DOUBLE BAR;Lo;0;AL;;;;;N;;;;; +08A7;ARABIC LETTER MEEM WITH THREE DOTS ABOVE;Lo;0;AL;;;;;N;;;;; +08A8;ARABIC LETTER YEH WITH TWO DOTS BELOW AND HAMZA ABOVE;Lo;0;AL;;;;;N;;;;; +08A9;ARABIC LETTER YEH WITH TWO DOTS BELOW AND DOT ABOVE;Lo;0;AL;;;;;N;;;;; +08AA;ARABIC LETTER REH WITH LOOP;Lo;0;AL;;;;;N;;;;; +08AB;ARABIC LETTER WAW WITH DOT WITHIN;Lo;0;AL;;;;;N;;;;; +08AC;ARABIC LETTER ROHINGYA YEH;Lo;0;AL;;;;;N;;;;; +08E4;ARABIC CURLY FATHA;Mn;230;NSM;;;;;N;;;;; +08E5;ARABIC CURLY DAMMA;Mn;230;NSM;;;;;N;;;;; +08E6;ARABIC CURLY KASRA;Mn;220;NSM;;;;;N;;;;; +08E7;ARABIC CURLY FATHATAN;Mn;230;NSM;;;;;N;;;;; +08E8;ARABIC CURLY DAMMATAN;Mn;230;NSM;;;;;N;;;;; +08E9;ARABIC CURLY KASRATAN;Mn;220;NSM;;;;;N;;;;; +08EA;ARABIC TONE ONE DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +08EB;ARABIC TONE TWO DOTS ABOVE;Mn;230;NSM;;;;;N;;;;; +08EC;ARABIC TONE LOOP ABOVE;Mn;230;NSM;;;;;N;;;;; +08ED;ARABIC TONE ONE DOT BELOW;Mn;220;NSM;;;;;N;;;;; +08EE;ARABIC TONE TWO DOTS BELOW;Mn;220;NSM;;;;;N;;;;; +08EF;ARABIC TONE LOOP BELOW;Mn;220;NSM;;;;;N;;;;; +08F0;ARABIC OPEN FATHATAN;Mn;27;NSM;;;;;N;;;;; +08F1;ARABIC OPEN DAMMATAN;Mn;28;NSM;;;;;N;;;;; +08F2;ARABIC OPEN KASRATAN;Mn;29;NSM;;;;;N;;;;; +08F3;ARABIC SMALL HIGH WAW;Mn;230;NSM;;;;;N;;;;; +08F4;ARABIC FATHA WITH RING;Mn;230;NSM;;;;;N;;;;; +08F5;ARABIC FATHA WITH DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +08F6;ARABIC KASRA WITH DOT BELOW;Mn;220;NSM;;;;;N;;;;; +08F7;ARABIC LEFT ARROWHEAD ABOVE;Mn;230;NSM;;;;;N;;;;; +08F8;ARABIC RIGHT ARROWHEAD ABOVE;Mn;230;NSM;;;;;N;;;;; +08F9;ARABIC LEFT ARROWHEAD BELOW;Mn;220;NSM;;;;;N;;;;; +08FA;ARABIC RIGHT ARROWHEAD BELOW;Mn;220;NSM;;;;;N;;;;; +08FB;ARABIC DOUBLE RIGHT ARROWHEAD ABOVE;Mn;230;NSM;;;;;N;;;;; +08FC;ARABIC DOUBLE RIGHT ARROWHEAD ABOVE WITH DOT;Mn;230;NSM;;;;;N;;;;; +08FD;ARABIC RIGHT ARROWHEAD ABOVE WITH DOT;Mn;230;NSM;;;;;N;;;;; +08FE;ARABIC DAMMA WITH DOT;Mn;230;NSM;;;;;N;;;;; +0900;DEVANAGARI SIGN INVERTED CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +0901;DEVANAGARI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +0902;DEVANAGARI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +0903;DEVANAGARI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0904;DEVANAGARI LETTER SHORT A;Lo;0;L;;;;;N;;;;; +0905;DEVANAGARI LETTER A;Lo;0;L;;;;;N;;;;; +0906;DEVANAGARI LETTER AA;Lo;0;L;;;;;N;;;;; +0907;DEVANAGARI LETTER I;Lo;0;L;;;;;N;;;;; +0908;DEVANAGARI LETTER II;Lo;0;L;;;;;N;;;;; +0909;DEVANAGARI LETTER U;Lo;0;L;;;;;N;;;;; +090A;DEVANAGARI LETTER UU;Lo;0;L;;;;;N;;;;; +090B;DEVANAGARI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +090C;DEVANAGARI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +090D;DEVANAGARI LETTER CANDRA E;Lo;0;L;;;;;N;;;;; +090E;DEVANAGARI LETTER SHORT E;Lo;0;L;;;;;N;;;;; +090F;DEVANAGARI LETTER E;Lo;0;L;;;;;N;;;;; +0910;DEVANAGARI LETTER AI;Lo;0;L;;;;;N;;;;; +0911;DEVANAGARI LETTER CANDRA O;Lo;0;L;;;;;N;;;;; +0912;DEVANAGARI LETTER SHORT O;Lo;0;L;;;;;N;;;;; +0913;DEVANAGARI LETTER O;Lo;0;L;;;;;N;;;;; +0914;DEVANAGARI LETTER AU;Lo;0;L;;;;;N;;;;; +0915;DEVANAGARI LETTER KA;Lo;0;L;;;;;N;;;;; +0916;DEVANAGARI LETTER KHA;Lo;0;L;;;;;N;;;;; +0917;DEVANAGARI LETTER GA;Lo;0;L;;;;;N;;;;; +0918;DEVANAGARI LETTER GHA;Lo;0;L;;;;;N;;;;; +0919;DEVANAGARI LETTER NGA;Lo;0;L;;;;;N;;;;; +091A;DEVANAGARI LETTER CA;Lo;0;L;;;;;N;;;;; +091B;DEVANAGARI LETTER CHA;Lo;0;L;;;;;N;;;;; +091C;DEVANAGARI LETTER JA;Lo;0;L;;;;;N;;;;; +091D;DEVANAGARI LETTER JHA;Lo;0;L;;;;;N;;;;; +091E;DEVANAGARI LETTER NYA;Lo;0;L;;;;;N;;;;; +091F;DEVANAGARI LETTER TTA;Lo;0;L;;;;;N;;;;; +0920;DEVANAGARI LETTER TTHA;Lo;0;L;;;;;N;;;;; +0921;DEVANAGARI LETTER DDA;Lo;0;L;;;;;N;;;;; +0922;DEVANAGARI LETTER DDHA;Lo;0;L;;;;;N;;;;; +0923;DEVANAGARI LETTER NNA;Lo;0;L;;;;;N;;;;; +0924;DEVANAGARI LETTER TA;Lo;0;L;;;;;N;;;;; +0925;DEVANAGARI LETTER THA;Lo;0;L;;;;;N;;;;; +0926;DEVANAGARI LETTER DA;Lo;0;L;;;;;N;;;;; +0927;DEVANAGARI LETTER DHA;Lo;0;L;;;;;N;;;;; +0928;DEVANAGARI LETTER NA;Lo;0;L;;;;;N;;;;; +0929;DEVANAGARI LETTER NNNA;Lo;0;L;0928 093C;;;;N;;;;; +092A;DEVANAGARI LETTER PA;Lo;0;L;;;;;N;;;;; +092B;DEVANAGARI LETTER PHA;Lo;0;L;;;;;N;;;;; +092C;DEVANAGARI LETTER BA;Lo;0;L;;;;;N;;;;; +092D;DEVANAGARI LETTER BHA;Lo;0;L;;;;;N;;;;; +092E;DEVANAGARI LETTER MA;Lo;0;L;;;;;N;;;;; +092F;DEVANAGARI LETTER YA;Lo;0;L;;;;;N;;;;; +0930;DEVANAGARI LETTER RA;Lo;0;L;;;;;N;;;;; +0931;DEVANAGARI LETTER RRA;Lo;0;L;0930 093C;;;;N;;;;; +0932;DEVANAGARI LETTER LA;Lo;0;L;;;;;N;;;;; +0933;DEVANAGARI LETTER LLA;Lo;0;L;;;;;N;;;;; +0934;DEVANAGARI LETTER LLLA;Lo;0;L;0933 093C;;;;N;;;;; +0935;DEVANAGARI LETTER VA;Lo;0;L;;;;;N;;;;; +0936;DEVANAGARI LETTER SHA;Lo;0;L;;;;;N;;;;; +0937;DEVANAGARI LETTER SSA;Lo;0;L;;;;;N;;;;; +0938;DEVANAGARI LETTER SA;Lo;0;L;;;;;N;;;;; +0939;DEVANAGARI LETTER HA;Lo;0;L;;;;;N;;;;; +093A;DEVANAGARI VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; +093B;DEVANAGARI VOWEL SIGN OOE;Mc;0;L;;;;;N;;;;; +093C;DEVANAGARI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +093D;DEVANAGARI SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +093E;DEVANAGARI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +093F;DEVANAGARI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +0940;DEVANAGARI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +0941;DEVANAGARI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +0942;DEVANAGARI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +0943;DEVANAGARI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +0944;DEVANAGARI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +0945;DEVANAGARI VOWEL SIGN CANDRA E;Mn;0;NSM;;;;;N;;;;; +0946;DEVANAGARI VOWEL SIGN SHORT E;Mn;0;NSM;;;;;N;;;;; +0947;DEVANAGARI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +0948;DEVANAGARI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +0949;DEVANAGARI VOWEL SIGN CANDRA O;Mc;0;L;;;;;N;;;;; +094A;DEVANAGARI VOWEL SIGN SHORT O;Mc;0;L;;;;;N;;;;; +094B;DEVANAGARI VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +094C;DEVANAGARI VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +094D;DEVANAGARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +094E;DEVANAGARI VOWEL SIGN PRISHTHAMATRA E;Mc;0;L;;;;;N;;;;; +094F;DEVANAGARI VOWEL SIGN AW;Mc;0;L;;;;;N;;;;; +0950;DEVANAGARI OM;Lo;0;L;;;;;N;;;;; +0951;DEVANAGARI STRESS SIGN UDATTA;Mn;230;NSM;;;;;N;;;;; +0952;DEVANAGARI STRESS SIGN ANUDATTA;Mn;220;NSM;;;;;N;;;;; +0953;DEVANAGARI GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;; +0954;DEVANAGARI ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;; +0955;DEVANAGARI VOWEL SIGN CANDRA LONG E;Mn;0;NSM;;;;;N;;;;; +0956;DEVANAGARI VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +0957;DEVANAGARI VOWEL SIGN UUE;Mn;0;NSM;;;;;N;;;;; +0958;DEVANAGARI LETTER QA;Lo;0;L;0915 093C;;;;N;;;;; +0959;DEVANAGARI LETTER KHHA;Lo;0;L;0916 093C;;;;N;;;;; +095A;DEVANAGARI LETTER GHHA;Lo;0;L;0917 093C;;;;N;;;;; +095B;DEVANAGARI LETTER ZA;Lo;0;L;091C 093C;;;;N;;;;; +095C;DEVANAGARI LETTER DDDHA;Lo;0;L;0921 093C;;;;N;;;;; +095D;DEVANAGARI LETTER RHA;Lo;0;L;0922 093C;;;;N;;;;; +095E;DEVANAGARI LETTER FA;Lo;0;L;092B 093C;;;;N;;;;; +095F;DEVANAGARI LETTER YYA;Lo;0;L;092F 093C;;;;N;;;;; +0960;DEVANAGARI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +0961;DEVANAGARI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +0962;DEVANAGARI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +0963;DEVANAGARI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +0964;DEVANAGARI DANDA;Po;0;L;;;;;N;;;;; +0965;DEVANAGARI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +0966;DEVANAGARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0967;DEVANAGARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0968;DEVANAGARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0969;DEVANAGARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +096A;DEVANAGARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +096B;DEVANAGARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +096C;DEVANAGARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +096D;DEVANAGARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +096E;DEVANAGARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +096F;DEVANAGARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0970;DEVANAGARI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +0971;DEVANAGARI SIGN HIGH SPACING DOT;Lm;0;L;;;;;N;;;;; +0972;DEVANAGARI LETTER CANDRA A;Lo;0;L;;;;;N;;;;; +0973;DEVANAGARI LETTER OE;Lo;0;L;;;;;N;;;;; +0974;DEVANAGARI LETTER OOE;Lo;0;L;;;;;N;;;;; +0975;DEVANAGARI LETTER AW;Lo;0;L;;;;;N;;;;; +0976;DEVANAGARI LETTER UE;Lo;0;L;;;;;N;;;;; +0977;DEVANAGARI LETTER UUE;Lo;0;L;;;;;N;;;;; +0979;DEVANAGARI LETTER ZHA;Lo;0;L;;;;;N;;;;; +097A;DEVANAGARI LETTER HEAVY YA;Lo;0;L;;;;;N;;;;; +097B;DEVANAGARI LETTER GGA;Lo;0;L;;;;;N;;;;; +097C;DEVANAGARI LETTER JJA;Lo;0;L;;;;;N;;;;; +097D;DEVANAGARI LETTER GLOTTAL STOP;Lo;0;L;;;;;N;;;;; +097E;DEVANAGARI LETTER DDDA;Lo;0;L;;;;;N;;;;; +097F;DEVANAGARI LETTER BBA;Lo;0;L;;;;;N;;;;; +0981;BENGALI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +0982;BENGALI SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +0983;BENGALI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0985;BENGALI LETTER A;Lo;0;L;;;;;N;;;;; +0986;BENGALI LETTER AA;Lo;0;L;;;;;N;;;;; +0987;BENGALI LETTER I;Lo;0;L;;;;;N;;;;; +0988;BENGALI LETTER II;Lo;0;L;;;;;N;;;;; +0989;BENGALI LETTER U;Lo;0;L;;;;;N;;;;; +098A;BENGALI LETTER UU;Lo;0;L;;;;;N;;;;; +098B;BENGALI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +098C;BENGALI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +098F;BENGALI LETTER E;Lo;0;L;;;;;N;;;;; +0990;BENGALI LETTER AI;Lo;0;L;;;;;N;;;;; +0993;BENGALI LETTER O;Lo;0;L;;;;;N;;;;; +0994;BENGALI LETTER AU;Lo;0;L;;;;;N;;;;; +0995;BENGALI LETTER KA;Lo;0;L;;;;;N;;;;; +0996;BENGALI LETTER KHA;Lo;0;L;;;;;N;;;;; +0997;BENGALI LETTER GA;Lo;0;L;;;;;N;;;;; +0998;BENGALI LETTER GHA;Lo;0;L;;;;;N;;;;; +0999;BENGALI LETTER NGA;Lo;0;L;;;;;N;;;;; +099A;BENGALI LETTER CA;Lo;0;L;;;;;N;;;;; +099B;BENGALI LETTER CHA;Lo;0;L;;;;;N;;;;; +099C;BENGALI LETTER JA;Lo;0;L;;;;;N;;;;; +099D;BENGALI LETTER JHA;Lo;0;L;;;;;N;;;;; +099E;BENGALI LETTER NYA;Lo;0;L;;;;;N;;;;; +099F;BENGALI LETTER TTA;Lo;0;L;;;;;N;;;;; +09A0;BENGALI LETTER TTHA;Lo;0;L;;;;;N;;;;; +09A1;BENGALI LETTER DDA;Lo;0;L;;;;;N;;;;; +09A2;BENGALI LETTER DDHA;Lo;0;L;;;;;N;;;;; +09A3;BENGALI LETTER NNA;Lo;0;L;;;;;N;;;;; +09A4;BENGALI LETTER TA;Lo;0;L;;;;;N;;;;; +09A5;BENGALI LETTER THA;Lo;0;L;;;;;N;;;;; +09A6;BENGALI LETTER DA;Lo;0;L;;;;;N;;;;; +09A7;BENGALI LETTER DHA;Lo;0;L;;;;;N;;;;; +09A8;BENGALI LETTER NA;Lo;0;L;;;;;N;;;;; +09AA;BENGALI LETTER PA;Lo;0;L;;;;;N;;;;; +09AB;BENGALI LETTER PHA;Lo;0;L;;;;;N;;;;; +09AC;BENGALI LETTER BA;Lo;0;L;;;;;N;;;;; +09AD;BENGALI LETTER BHA;Lo;0;L;;;;;N;;;;; +09AE;BENGALI LETTER MA;Lo;0;L;;;;;N;;;;; +09AF;BENGALI LETTER YA;Lo;0;L;;;;;N;;;;; +09B0;BENGALI LETTER RA;Lo;0;L;;;;;N;;;;; +09B2;BENGALI LETTER LA;Lo;0;L;;;;;N;;;;; +09B6;BENGALI LETTER SHA;Lo;0;L;;;;;N;;;;; +09B7;BENGALI LETTER SSA;Lo;0;L;;;;;N;;;;; +09B8;BENGALI LETTER SA;Lo;0;L;;;;;N;;;;; +09B9;BENGALI LETTER HA;Lo;0;L;;;;;N;;;;; +09BC;BENGALI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +09BD;BENGALI SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +09BE;BENGALI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +09BF;BENGALI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +09C0;BENGALI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +09C1;BENGALI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +09C2;BENGALI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +09C3;BENGALI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +09C4;BENGALI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +09C7;BENGALI VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +09C8;BENGALI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +09CB;BENGALI VOWEL SIGN O;Mc;0;L;09C7 09BE;;;;N;;;;; +09CC;BENGALI VOWEL SIGN AU;Mc;0;L;09C7 09D7;;;;N;;;;; +09CD;BENGALI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +09CE;BENGALI LETTER KHANDA TA;Lo;0;L;;;;;N;;;;; +09D7;BENGALI AU LENGTH MARK;Mc;0;L;;;;;N;;;;; +09DC;BENGALI LETTER RRA;Lo;0;L;09A1 09BC;;;;N;;;;; +09DD;BENGALI LETTER RHA;Lo;0;L;09A2 09BC;;;;N;;;;; +09DF;BENGALI LETTER YYA;Lo;0;L;09AF 09BC;;;;N;;;;; +09E0;BENGALI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +09E1;BENGALI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +09E2;BENGALI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +09E3;BENGALI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +09E6;BENGALI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +09E7;BENGALI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +09E8;BENGALI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +09E9;BENGALI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +09EA;BENGALI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +09EB;BENGALI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +09EC;BENGALI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +09ED;BENGALI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +09EE;BENGALI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +09EF;BENGALI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +09F0;BENGALI LETTER RA WITH MIDDLE DIAGONAL;Lo;0;L;;;;;N;;;;; +09F1;BENGALI LETTER RA WITH LOWER DIAGONAL;Lo;0;L;;;;;N;BENGALI LETTER VA WITH LOWER DIAGONAL;;;; +09F2;BENGALI RUPEE MARK;Sc;0;ET;;;;;N;;;;; +09F3;BENGALI RUPEE SIGN;Sc;0;ET;;;;;N;;;;; +09F4;BENGALI CURRENCY NUMERATOR ONE;No;0;L;;;;1/16;N;;;;; +09F5;BENGALI CURRENCY NUMERATOR TWO;No;0;L;;;;1/8;N;;;;; +09F6;BENGALI CURRENCY NUMERATOR THREE;No;0;L;;;;3/16;N;;;;; +09F7;BENGALI CURRENCY NUMERATOR FOUR;No;0;L;;;;1/4;N;;;;; +09F8;BENGALI CURRENCY NUMERATOR ONE LESS THAN THE DENOMINATOR;No;0;L;;;;3/4;N;;;;; +09F9;BENGALI CURRENCY DENOMINATOR SIXTEEN;No;0;L;;;;16;N;;;;; +09FA;BENGALI ISSHAR;So;0;L;;;;;N;;;;; +09FB;BENGALI GANDA MARK;Sc;0;ET;;;;;N;;;;; +0A01;GURMUKHI SIGN ADAK BINDI;Mn;0;NSM;;;;;N;;;;; +0A02;GURMUKHI SIGN BINDI;Mn;0;NSM;;;;;N;;;;; +0A03;GURMUKHI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0A05;GURMUKHI LETTER A;Lo;0;L;;;;;N;;;;; +0A06;GURMUKHI LETTER AA;Lo;0;L;;;;;N;;;;; +0A07;GURMUKHI LETTER I;Lo;0;L;;;;;N;;;;; +0A08;GURMUKHI LETTER II;Lo;0;L;;;;;N;;;;; +0A09;GURMUKHI LETTER U;Lo;0;L;;;;;N;;;;; +0A0A;GURMUKHI LETTER UU;Lo;0;L;;;;;N;;;;; +0A0F;GURMUKHI LETTER EE;Lo;0;L;;;;;N;;;;; +0A10;GURMUKHI LETTER AI;Lo;0;L;;;;;N;;;;; +0A13;GURMUKHI LETTER OO;Lo;0;L;;;;;N;;;;; +0A14;GURMUKHI LETTER AU;Lo;0;L;;;;;N;;;;; +0A15;GURMUKHI LETTER KA;Lo;0;L;;;;;N;;;;; +0A16;GURMUKHI LETTER KHA;Lo;0;L;;;;;N;;;;; +0A17;GURMUKHI LETTER GA;Lo;0;L;;;;;N;;;;; +0A18;GURMUKHI LETTER GHA;Lo;0;L;;;;;N;;;;; +0A19;GURMUKHI LETTER NGA;Lo;0;L;;;;;N;;;;; +0A1A;GURMUKHI LETTER CA;Lo;0;L;;;;;N;;;;; +0A1B;GURMUKHI LETTER CHA;Lo;0;L;;;;;N;;;;; +0A1C;GURMUKHI LETTER JA;Lo;0;L;;;;;N;;;;; +0A1D;GURMUKHI LETTER JHA;Lo;0;L;;;;;N;;;;; +0A1E;GURMUKHI LETTER NYA;Lo;0;L;;;;;N;;;;; +0A1F;GURMUKHI LETTER TTA;Lo;0;L;;;;;N;;;;; +0A20;GURMUKHI LETTER TTHA;Lo;0;L;;;;;N;;;;; +0A21;GURMUKHI LETTER DDA;Lo;0;L;;;;;N;;;;; +0A22;GURMUKHI LETTER DDHA;Lo;0;L;;;;;N;;;;; +0A23;GURMUKHI LETTER NNA;Lo;0;L;;;;;N;;;;; +0A24;GURMUKHI LETTER TA;Lo;0;L;;;;;N;;;;; +0A25;GURMUKHI LETTER THA;Lo;0;L;;;;;N;;;;; +0A26;GURMUKHI LETTER DA;Lo;0;L;;;;;N;;;;; +0A27;GURMUKHI LETTER DHA;Lo;0;L;;;;;N;;;;; +0A28;GURMUKHI LETTER NA;Lo;0;L;;;;;N;;;;; +0A2A;GURMUKHI LETTER PA;Lo;0;L;;;;;N;;;;; +0A2B;GURMUKHI LETTER PHA;Lo;0;L;;;;;N;;;;; +0A2C;GURMUKHI LETTER BA;Lo;0;L;;;;;N;;;;; +0A2D;GURMUKHI LETTER BHA;Lo;0;L;;;;;N;;;;; +0A2E;GURMUKHI LETTER MA;Lo;0;L;;;;;N;;;;; +0A2F;GURMUKHI LETTER YA;Lo;0;L;;;;;N;;;;; +0A30;GURMUKHI LETTER RA;Lo;0;L;;;;;N;;;;; +0A32;GURMUKHI LETTER LA;Lo;0;L;;;;;N;;;;; +0A33;GURMUKHI LETTER LLA;Lo;0;L;0A32 0A3C;;;;N;;;;; +0A35;GURMUKHI LETTER VA;Lo;0;L;;;;;N;;;;; +0A36;GURMUKHI LETTER SHA;Lo;0;L;0A38 0A3C;;;;N;;;;; +0A38;GURMUKHI LETTER SA;Lo;0;L;;;;;N;;;;; +0A39;GURMUKHI LETTER HA;Lo;0;L;;;;;N;;;;; +0A3C;GURMUKHI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +0A3E;GURMUKHI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +0A3F;GURMUKHI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +0A40;GURMUKHI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +0A41;GURMUKHI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +0A42;GURMUKHI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +0A47;GURMUKHI VOWEL SIGN EE;Mn;0;NSM;;;;;N;;;;; +0A48;GURMUKHI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +0A4B;GURMUKHI VOWEL SIGN OO;Mn;0;NSM;;;;;N;;;;; +0A4C;GURMUKHI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +0A4D;GURMUKHI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0A51;GURMUKHI SIGN UDAAT;Mn;0;NSM;;;;;N;;;;; +0A59;GURMUKHI LETTER KHHA;Lo;0;L;0A16 0A3C;;;;N;;;;; +0A5A;GURMUKHI LETTER GHHA;Lo;0;L;0A17 0A3C;;;;N;;;;; +0A5B;GURMUKHI LETTER ZA;Lo;0;L;0A1C 0A3C;;;;N;;;;; +0A5C;GURMUKHI LETTER RRA;Lo;0;L;;;;;N;;;;; +0A5E;GURMUKHI LETTER FA;Lo;0;L;0A2B 0A3C;;;;N;;;;; +0A66;GURMUKHI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0A67;GURMUKHI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0A68;GURMUKHI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0A69;GURMUKHI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0A6A;GURMUKHI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0A6B;GURMUKHI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0A6C;GURMUKHI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0A6D;GURMUKHI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0A6E;GURMUKHI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0A6F;GURMUKHI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0A70;GURMUKHI TIPPI;Mn;0;NSM;;;;;N;;;;; +0A71;GURMUKHI ADDAK;Mn;0;NSM;;;;;N;;;;; +0A72;GURMUKHI IRI;Lo;0;L;;;;;N;;;;; +0A73;GURMUKHI URA;Lo;0;L;;;;;N;;;;; +0A74;GURMUKHI EK ONKAR;Lo;0;L;;;;;N;;;;; +0A75;GURMUKHI SIGN YAKASH;Mn;0;NSM;;;;;N;;;;; +0A81;GUJARATI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +0A82;GUJARATI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +0A83;GUJARATI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0A85;GUJARATI LETTER A;Lo;0;L;;;;;N;;;;; +0A86;GUJARATI LETTER AA;Lo;0;L;;;;;N;;;;; +0A87;GUJARATI LETTER I;Lo;0;L;;;;;N;;;;; +0A88;GUJARATI LETTER II;Lo;0;L;;;;;N;;;;; +0A89;GUJARATI LETTER U;Lo;0;L;;;;;N;;;;; +0A8A;GUJARATI LETTER UU;Lo;0;L;;;;;N;;;;; +0A8B;GUJARATI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +0A8C;GUJARATI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +0A8D;GUJARATI VOWEL CANDRA E;Lo;0;L;;;;;N;;;;; +0A8F;GUJARATI LETTER E;Lo;0;L;;;;;N;;;;; +0A90;GUJARATI LETTER AI;Lo;0;L;;;;;N;;;;; +0A91;GUJARATI VOWEL CANDRA O;Lo;0;L;;;;;N;;;;; +0A93;GUJARATI LETTER O;Lo;0;L;;;;;N;;;;; +0A94;GUJARATI LETTER AU;Lo;0;L;;;;;N;;;;; +0A95;GUJARATI LETTER KA;Lo;0;L;;;;;N;;;;; +0A96;GUJARATI LETTER KHA;Lo;0;L;;;;;N;;;;; +0A97;GUJARATI LETTER GA;Lo;0;L;;;;;N;;;;; +0A98;GUJARATI LETTER GHA;Lo;0;L;;;;;N;;;;; +0A99;GUJARATI LETTER NGA;Lo;0;L;;;;;N;;;;; +0A9A;GUJARATI LETTER CA;Lo;0;L;;;;;N;;;;; +0A9B;GUJARATI LETTER CHA;Lo;0;L;;;;;N;;;;; +0A9C;GUJARATI LETTER JA;Lo;0;L;;;;;N;;;;; +0A9D;GUJARATI LETTER JHA;Lo;0;L;;;;;N;;;;; +0A9E;GUJARATI LETTER NYA;Lo;0;L;;;;;N;;;;; +0A9F;GUJARATI LETTER TTA;Lo;0;L;;;;;N;;;;; +0AA0;GUJARATI LETTER TTHA;Lo;0;L;;;;;N;;;;; +0AA1;GUJARATI LETTER DDA;Lo;0;L;;;;;N;;;;; +0AA2;GUJARATI LETTER DDHA;Lo;0;L;;;;;N;;;;; +0AA3;GUJARATI LETTER NNA;Lo;0;L;;;;;N;;;;; +0AA4;GUJARATI LETTER TA;Lo;0;L;;;;;N;;;;; +0AA5;GUJARATI LETTER THA;Lo;0;L;;;;;N;;;;; +0AA6;GUJARATI LETTER DA;Lo;0;L;;;;;N;;;;; +0AA7;GUJARATI LETTER DHA;Lo;0;L;;;;;N;;;;; +0AA8;GUJARATI LETTER NA;Lo;0;L;;;;;N;;;;; +0AAA;GUJARATI LETTER PA;Lo;0;L;;;;;N;;;;; +0AAB;GUJARATI LETTER PHA;Lo;0;L;;;;;N;;;;; +0AAC;GUJARATI LETTER BA;Lo;0;L;;;;;N;;;;; +0AAD;GUJARATI LETTER BHA;Lo;0;L;;;;;N;;;;; +0AAE;GUJARATI LETTER MA;Lo;0;L;;;;;N;;;;; +0AAF;GUJARATI LETTER YA;Lo;0;L;;;;;N;;;;; +0AB0;GUJARATI LETTER RA;Lo;0;L;;;;;N;;;;; +0AB2;GUJARATI LETTER LA;Lo;0;L;;;;;N;;;;; +0AB3;GUJARATI LETTER LLA;Lo;0;L;;;;;N;;;;; +0AB5;GUJARATI LETTER VA;Lo;0;L;;;;;N;;;;; +0AB6;GUJARATI LETTER SHA;Lo;0;L;;;;;N;;;;; +0AB7;GUJARATI LETTER SSA;Lo;0;L;;;;;N;;;;; +0AB8;GUJARATI LETTER SA;Lo;0;L;;;;;N;;;;; +0AB9;GUJARATI LETTER HA;Lo;0;L;;;;;N;;;;; +0ABC;GUJARATI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +0ABD;GUJARATI SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +0ABE;GUJARATI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +0ABF;GUJARATI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +0AC0;GUJARATI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +0AC1;GUJARATI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +0AC2;GUJARATI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +0AC3;GUJARATI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +0AC4;GUJARATI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +0AC5;GUJARATI VOWEL SIGN CANDRA E;Mn;0;NSM;;;;;N;;;;; +0AC7;GUJARATI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +0AC8;GUJARATI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +0AC9;GUJARATI VOWEL SIGN CANDRA O;Mc;0;L;;;;;N;;;;; +0ACB;GUJARATI VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +0ACC;GUJARATI VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +0ACD;GUJARATI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0AD0;GUJARATI OM;Lo;0;L;;;;;N;;;;; +0AE0;GUJARATI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +0AE1;GUJARATI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +0AE2;GUJARATI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +0AE3;GUJARATI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +0AE6;GUJARATI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0AE7;GUJARATI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0AE8;GUJARATI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0AE9;GUJARATI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0AEA;GUJARATI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0AEB;GUJARATI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0AEC;GUJARATI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0AED;GUJARATI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0AEE;GUJARATI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0AEF;GUJARATI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0AF0;GUJARATI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +0AF1;GUJARATI RUPEE SIGN;Sc;0;ET;;;;;N;;;;; +0B01;ORIYA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +0B02;ORIYA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +0B03;ORIYA SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0B05;ORIYA LETTER A;Lo;0;L;;;;;N;;;;; +0B06;ORIYA LETTER AA;Lo;0;L;;;;;N;;;;; +0B07;ORIYA LETTER I;Lo;0;L;;;;;N;;;;; +0B08;ORIYA LETTER II;Lo;0;L;;;;;N;;;;; +0B09;ORIYA LETTER U;Lo;0;L;;;;;N;;;;; +0B0A;ORIYA LETTER UU;Lo;0;L;;;;;N;;;;; +0B0B;ORIYA LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +0B0C;ORIYA LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +0B0F;ORIYA LETTER E;Lo;0;L;;;;;N;;;;; +0B10;ORIYA LETTER AI;Lo;0;L;;;;;N;;;;; +0B13;ORIYA LETTER O;Lo;0;L;;;;;N;;;;; +0B14;ORIYA LETTER AU;Lo;0;L;;;;;N;;;;; +0B15;ORIYA LETTER KA;Lo;0;L;;;;;N;;;;; +0B16;ORIYA LETTER KHA;Lo;0;L;;;;;N;;;;; +0B17;ORIYA LETTER GA;Lo;0;L;;;;;N;;;;; +0B18;ORIYA LETTER GHA;Lo;0;L;;;;;N;;;;; +0B19;ORIYA LETTER NGA;Lo;0;L;;;;;N;;;;; +0B1A;ORIYA LETTER CA;Lo;0;L;;;;;N;;;;; +0B1B;ORIYA LETTER CHA;Lo;0;L;;;;;N;;;;; +0B1C;ORIYA LETTER JA;Lo;0;L;;;;;N;;;;; +0B1D;ORIYA LETTER JHA;Lo;0;L;;;;;N;;;;; +0B1E;ORIYA LETTER NYA;Lo;0;L;;;;;N;;;;; +0B1F;ORIYA LETTER TTA;Lo;0;L;;;;;N;;;;; +0B20;ORIYA LETTER TTHA;Lo;0;L;;;;;N;;;;; +0B21;ORIYA LETTER DDA;Lo;0;L;;;;;N;;;;; +0B22;ORIYA LETTER DDHA;Lo;0;L;;;;;N;;;;; +0B23;ORIYA LETTER NNA;Lo;0;L;;;;;N;;;;; +0B24;ORIYA LETTER TA;Lo;0;L;;;;;N;;;;; +0B25;ORIYA LETTER THA;Lo;0;L;;;;;N;;;;; +0B26;ORIYA LETTER DA;Lo;0;L;;;;;N;;;;; +0B27;ORIYA LETTER DHA;Lo;0;L;;;;;N;;;;; +0B28;ORIYA LETTER NA;Lo;0;L;;;;;N;;;;; +0B2A;ORIYA LETTER PA;Lo;0;L;;;;;N;;;;; +0B2B;ORIYA LETTER PHA;Lo;0;L;;;;;N;;;;; +0B2C;ORIYA LETTER BA;Lo;0;L;;;;;N;;;;; +0B2D;ORIYA LETTER BHA;Lo;0;L;;;;;N;;;;; +0B2E;ORIYA LETTER MA;Lo;0;L;;;;;N;;;;; +0B2F;ORIYA LETTER YA;Lo;0;L;;;;;N;;;;; +0B30;ORIYA LETTER RA;Lo;0;L;;;;;N;;;;; +0B32;ORIYA LETTER LA;Lo;0;L;;;;;N;;;;; +0B33;ORIYA LETTER LLA;Lo;0;L;;;;;N;;;;; +0B35;ORIYA LETTER VA;Lo;0;L;;;;;N;;;;; +0B36;ORIYA LETTER SHA;Lo;0;L;;;;;N;;;;; +0B37;ORIYA LETTER SSA;Lo;0;L;;;;;N;;;;; +0B38;ORIYA LETTER SA;Lo;0;L;;;;;N;;;;; +0B39;ORIYA LETTER HA;Lo;0;L;;;;;N;;;;; +0B3C;ORIYA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +0B3D;ORIYA SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +0B3E;ORIYA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +0B3F;ORIYA VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +0B40;ORIYA VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +0B41;ORIYA VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +0B42;ORIYA VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +0B43;ORIYA VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +0B44;ORIYA VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +0B47;ORIYA VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +0B48;ORIYA VOWEL SIGN AI;Mc;0;L;0B47 0B56;;;;N;;;;; +0B4B;ORIYA VOWEL SIGN O;Mc;0;L;0B47 0B3E;;;;N;;;;; +0B4C;ORIYA VOWEL SIGN AU;Mc;0;L;0B47 0B57;;;;N;;;;; +0B4D;ORIYA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0B56;ORIYA AI LENGTH MARK;Mn;0;NSM;;;;;N;;;;; +0B57;ORIYA AU LENGTH MARK;Mc;0;L;;;;;N;;;;; +0B5C;ORIYA LETTER RRA;Lo;0;L;0B21 0B3C;;;;N;;;;; +0B5D;ORIYA LETTER RHA;Lo;0;L;0B22 0B3C;;;;N;;;;; +0B5F;ORIYA LETTER YYA;Lo;0;L;;;;;N;;;;; +0B60;ORIYA LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +0B61;ORIYA LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +0B62;ORIYA VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +0B63;ORIYA VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +0B66;ORIYA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0B67;ORIYA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0B68;ORIYA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0B69;ORIYA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0B6A;ORIYA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0B6B;ORIYA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0B6C;ORIYA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0B6D;ORIYA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0B6E;ORIYA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0B6F;ORIYA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0B70;ORIYA ISSHAR;So;0;L;;;;;N;;;;; +0B71;ORIYA LETTER WA;Lo;0;L;;;;;N;;;;; +0B72;ORIYA FRACTION ONE QUARTER;No;0;L;;;;1/4;N;;;;; +0B73;ORIYA FRACTION ONE HALF;No;0;L;;;;1/2;N;;;;; +0B74;ORIYA FRACTION THREE QUARTERS;No;0;L;;;;3/4;N;;;;; +0B75;ORIYA FRACTION ONE SIXTEENTH;No;0;L;;;;1/16;N;;;;; +0B76;ORIYA FRACTION ONE EIGHTH;No;0;L;;;;1/8;N;;;;; +0B77;ORIYA FRACTION THREE SIXTEENTHS;No;0;L;;;;3/16;N;;;;; +0B82;TAMIL SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +0B83;TAMIL SIGN VISARGA;Lo;0;L;;;;;N;;;;; +0B85;TAMIL LETTER A;Lo;0;L;;;;;N;;;;; +0B86;TAMIL LETTER AA;Lo;0;L;;;;;N;;;;; +0B87;TAMIL LETTER I;Lo;0;L;;;;;N;;;;; +0B88;TAMIL LETTER II;Lo;0;L;;;;;N;;;;; +0B89;TAMIL LETTER U;Lo;0;L;;;;;N;;;;; +0B8A;TAMIL LETTER UU;Lo;0;L;;;;;N;;;;; +0B8E;TAMIL LETTER E;Lo;0;L;;;;;N;;;;; +0B8F;TAMIL LETTER EE;Lo;0;L;;;;;N;;;;; +0B90;TAMIL LETTER AI;Lo;0;L;;;;;N;;;;; +0B92;TAMIL LETTER O;Lo;0;L;;;;;N;;;;; +0B93;TAMIL LETTER OO;Lo;0;L;;;;;N;;;;; +0B94;TAMIL LETTER AU;Lo;0;L;0B92 0BD7;;;;N;;;;; +0B95;TAMIL LETTER KA;Lo;0;L;;;;;N;;;;; +0B99;TAMIL LETTER NGA;Lo;0;L;;;;;N;;;;; +0B9A;TAMIL LETTER CA;Lo;0;L;;;;;N;;;;; +0B9C;TAMIL LETTER JA;Lo;0;L;;;;;N;;;;; +0B9E;TAMIL LETTER NYA;Lo;0;L;;;;;N;;;;; +0B9F;TAMIL LETTER TTA;Lo;0;L;;;;;N;;;;; +0BA3;TAMIL LETTER NNA;Lo;0;L;;;;;N;;;;; +0BA4;TAMIL LETTER TA;Lo;0;L;;;;;N;;;;; +0BA8;TAMIL LETTER NA;Lo;0;L;;;;;N;;;;; +0BA9;TAMIL LETTER NNNA;Lo;0;L;;;;;N;;;;; +0BAA;TAMIL LETTER PA;Lo;0;L;;;;;N;;;;; +0BAE;TAMIL LETTER MA;Lo;0;L;;;;;N;;;;; +0BAF;TAMIL LETTER YA;Lo;0;L;;;;;N;;;;; +0BB0;TAMIL LETTER RA;Lo;0;L;;;;;N;;;;; +0BB1;TAMIL LETTER RRA;Lo;0;L;;;;;N;;;;; +0BB2;TAMIL LETTER LA;Lo;0;L;;;;;N;;;;; +0BB3;TAMIL LETTER LLA;Lo;0;L;;;;;N;;;;; +0BB4;TAMIL LETTER LLLA;Lo;0;L;;;;;N;;;;; +0BB5;TAMIL LETTER VA;Lo;0;L;;;;;N;;;;; +0BB6;TAMIL LETTER SHA;Lo;0;L;;;;;N;;;;; +0BB7;TAMIL LETTER SSA;Lo;0;L;;;;;N;;;;; +0BB8;TAMIL LETTER SA;Lo;0;L;;;;;N;;;;; +0BB9;TAMIL LETTER HA;Lo;0;L;;;;;N;;;;; +0BBE;TAMIL VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +0BBF;TAMIL VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +0BC0;TAMIL VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +0BC1;TAMIL VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +0BC2;TAMIL VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +0BC6;TAMIL VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +0BC7;TAMIL VOWEL SIGN EE;Mc;0;L;;;;;N;;;;; +0BC8;TAMIL VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +0BCA;TAMIL VOWEL SIGN O;Mc;0;L;0BC6 0BBE;;;;N;;;;; +0BCB;TAMIL VOWEL SIGN OO;Mc;0;L;0BC7 0BBE;;;;N;;;;; +0BCC;TAMIL VOWEL SIGN AU;Mc;0;L;0BC6 0BD7;;;;N;;;;; +0BCD;TAMIL SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0BD0;TAMIL OM;Lo;0;L;;;;;N;;;;; +0BD7;TAMIL AU LENGTH MARK;Mc;0;L;;;;;N;;;;; +0BE6;TAMIL DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0BE7;TAMIL DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0BE8;TAMIL DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0BE9;TAMIL DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0BEA;TAMIL DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0BEB;TAMIL DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0BEC;TAMIL DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0BED;TAMIL DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0BEE;TAMIL DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0BEF;TAMIL DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0BF0;TAMIL NUMBER TEN;No;0;L;;;;10;N;;;;; +0BF1;TAMIL NUMBER ONE HUNDRED;No;0;L;;;;100;N;;;;; +0BF2;TAMIL NUMBER ONE THOUSAND;No;0;L;;;;1000;N;;;;; +0BF3;TAMIL DAY SIGN;So;0;ON;;;;;N;;;;; +0BF4;TAMIL MONTH SIGN;So;0;ON;;;;;N;;;;; +0BF5;TAMIL YEAR SIGN;So;0;ON;;;;;N;;;;; +0BF6;TAMIL DEBIT SIGN;So;0;ON;;;;;N;;;;; +0BF7;TAMIL CREDIT SIGN;So;0;ON;;;;;N;;;;; +0BF8;TAMIL AS ABOVE SIGN;So;0;ON;;;;;N;;;;; +0BF9;TAMIL RUPEE SIGN;Sc;0;ET;;;;;N;;;;; +0BFA;TAMIL NUMBER SIGN;So;0;ON;;;;;N;;;;; +0C01;TELUGU SIGN CANDRABINDU;Mc;0;L;;;;;N;;;;; +0C02;TELUGU SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +0C03;TELUGU SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0C05;TELUGU LETTER A;Lo;0;L;;;;;N;;;;; +0C06;TELUGU LETTER AA;Lo;0;L;;;;;N;;;;; +0C07;TELUGU LETTER I;Lo;0;L;;;;;N;;;;; +0C08;TELUGU LETTER II;Lo;0;L;;;;;N;;;;; +0C09;TELUGU LETTER U;Lo;0;L;;;;;N;;;;; +0C0A;TELUGU LETTER UU;Lo;0;L;;;;;N;;;;; +0C0B;TELUGU LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +0C0C;TELUGU LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +0C0E;TELUGU LETTER E;Lo;0;L;;;;;N;;;;; +0C0F;TELUGU LETTER EE;Lo;0;L;;;;;N;;;;; +0C10;TELUGU LETTER AI;Lo;0;L;;;;;N;;;;; +0C12;TELUGU LETTER O;Lo;0;L;;;;;N;;;;; +0C13;TELUGU LETTER OO;Lo;0;L;;;;;N;;;;; +0C14;TELUGU LETTER AU;Lo;0;L;;;;;N;;;;; +0C15;TELUGU LETTER KA;Lo;0;L;;;;;N;;;;; +0C16;TELUGU LETTER KHA;Lo;0;L;;;;;N;;;;; +0C17;TELUGU LETTER GA;Lo;0;L;;;;;N;;;;; +0C18;TELUGU LETTER GHA;Lo;0;L;;;;;N;;;;; +0C19;TELUGU LETTER NGA;Lo;0;L;;;;;N;;;;; +0C1A;TELUGU LETTER CA;Lo;0;L;;;;;N;;;;; +0C1B;TELUGU LETTER CHA;Lo;0;L;;;;;N;;;;; +0C1C;TELUGU LETTER JA;Lo;0;L;;;;;N;;;;; +0C1D;TELUGU LETTER JHA;Lo;0;L;;;;;N;;;;; +0C1E;TELUGU LETTER NYA;Lo;0;L;;;;;N;;;;; +0C1F;TELUGU LETTER TTA;Lo;0;L;;;;;N;;;;; +0C20;TELUGU LETTER TTHA;Lo;0;L;;;;;N;;;;; +0C21;TELUGU LETTER DDA;Lo;0;L;;;;;N;;;;; +0C22;TELUGU LETTER DDHA;Lo;0;L;;;;;N;;;;; +0C23;TELUGU LETTER NNA;Lo;0;L;;;;;N;;;;; +0C24;TELUGU LETTER TA;Lo;0;L;;;;;N;;;;; +0C25;TELUGU LETTER THA;Lo;0;L;;;;;N;;;;; +0C26;TELUGU LETTER DA;Lo;0;L;;;;;N;;;;; +0C27;TELUGU LETTER DHA;Lo;0;L;;;;;N;;;;; +0C28;TELUGU LETTER NA;Lo;0;L;;;;;N;;;;; +0C2A;TELUGU LETTER PA;Lo;0;L;;;;;N;;;;; +0C2B;TELUGU LETTER PHA;Lo;0;L;;;;;N;;;;; +0C2C;TELUGU LETTER BA;Lo;0;L;;;;;N;;;;; +0C2D;TELUGU LETTER BHA;Lo;0;L;;;;;N;;;;; +0C2E;TELUGU LETTER MA;Lo;0;L;;;;;N;;;;; +0C2F;TELUGU LETTER YA;Lo;0;L;;;;;N;;;;; +0C30;TELUGU LETTER RA;Lo;0;L;;;;;N;;;;; +0C31;TELUGU LETTER RRA;Lo;0;L;;;;;N;;;;; +0C32;TELUGU LETTER LA;Lo;0;L;;;;;N;;;;; +0C33;TELUGU LETTER LLA;Lo;0;L;;;;;N;;;;; +0C35;TELUGU LETTER VA;Lo;0;L;;;;;N;;;;; +0C36;TELUGU LETTER SHA;Lo;0;L;;;;;N;;;;; +0C37;TELUGU LETTER SSA;Lo;0;L;;;;;N;;;;; +0C38;TELUGU LETTER SA;Lo;0;L;;;;;N;;;;; +0C39;TELUGU LETTER HA;Lo;0;L;;;;;N;;;;; +0C3D;TELUGU SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +0C3E;TELUGU VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +0C3F;TELUGU VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +0C40;TELUGU VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +0C41;TELUGU VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +0C42;TELUGU VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +0C43;TELUGU VOWEL SIGN VOCALIC R;Mc;0;L;;;;;N;;;;; +0C44;TELUGU VOWEL SIGN VOCALIC RR;Mc;0;L;;;;;N;;;;; +0C46;TELUGU VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +0C47;TELUGU VOWEL SIGN EE;Mn;0;NSM;;;;;N;;;;; +0C48;TELUGU VOWEL SIGN AI;Mn;0;NSM;0C46 0C56;;;;N;;;;; +0C4A;TELUGU VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +0C4B;TELUGU VOWEL SIGN OO;Mn;0;NSM;;;;;N;;;;; +0C4C;TELUGU VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +0C4D;TELUGU SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0C55;TELUGU LENGTH MARK;Mn;84;NSM;;;;;N;;;;; +0C56;TELUGU AI LENGTH MARK;Mn;91;NSM;;;;;N;;;;; +0C58;TELUGU LETTER TSA;Lo;0;L;;;;;N;;;;; +0C59;TELUGU LETTER DZA;Lo;0;L;;;;;N;;;;; +0C60;TELUGU LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +0C61;TELUGU LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +0C62;TELUGU VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +0C63;TELUGU VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +0C66;TELUGU DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0C67;TELUGU DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0C68;TELUGU DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0C69;TELUGU DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0C6A;TELUGU DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0C6B;TELUGU DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0C6C;TELUGU DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0C6D;TELUGU DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0C6E;TELUGU DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0C6F;TELUGU DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0C78;TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR;No;0;ON;;;;0;N;;;;; +0C79;TELUGU FRACTION DIGIT ONE FOR ODD POWERS OF FOUR;No;0;ON;;;;1;N;;;;; +0C7A;TELUGU FRACTION DIGIT TWO FOR ODD POWERS OF FOUR;No;0;ON;;;;2;N;;;;; +0C7B;TELUGU FRACTION DIGIT THREE FOR ODD POWERS OF FOUR;No;0;ON;;;;3;N;;;;; +0C7C;TELUGU FRACTION DIGIT ONE FOR EVEN POWERS OF FOUR;No;0;ON;;;;1;N;;;;; +0C7D;TELUGU FRACTION DIGIT TWO FOR EVEN POWERS OF FOUR;No;0;ON;;;;2;N;;;;; +0C7E;TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR;No;0;ON;;;;3;N;;;;; +0C7F;TELUGU SIGN TUUMU;So;0;L;;;;;N;;;;; +0C82;KANNADA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +0C83;KANNADA SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0C85;KANNADA LETTER A;Lo;0;L;;;;;N;;;;; +0C86;KANNADA LETTER AA;Lo;0;L;;;;;N;;;;; +0C87;KANNADA LETTER I;Lo;0;L;;;;;N;;;;; +0C88;KANNADA LETTER II;Lo;0;L;;;;;N;;;;; +0C89;KANNADA LETTER U;Lo;0;L;;;;;N;;;;; +0C8A;KANNADA LETTER UU;Lo;0;L;;;;;N;;;;; +0C8B;KANNADA LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +0C8C;KANNADA LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +0C8E;KANNADA LETTER E;Lo;0;L;;;;;N;;;;; +0C8F;KANNADA LETTER EE;Lo;0;L;;;;;N;;;;; +0C90;KANNADA LETTER AI;Lo;0;L;;;;;N;;;;; +0C92;KANNADA LETTER O;Lo;0;L;;;;;N;;;;; +0C93;KANNADA LETTER OO;Lo;0;L;;;;;N;;;;; +0C94;KANNADA LETTER AU;Lo;0;L;;;;;N;;;;; +0C95;KANNADA LETTER KA;Lo;0;L;;;;;N;;;;; +0C96;KANNADA LETTER KHA;Lo;0;L;;;;;N;;;;; +0C97;KANNADA LETTER GA;Lo;0;L;;;;;N;;;;; +0C98;KANNADA LETTER GHA;Lo;0;L;;;;;N;;;;; +0C99;KANNADA LETTER NGA;Lo;0;L;;;;;N;;;;; +0C9A;KANNADA LETTER CA;Lo;0;L;;;;;N;;;;; +0C9B;KANNADA LETTER CHA;Lo;0;L;;;;;N;;;;; +0C9C;KANNADA LETTER JA;Lo;0;L;;;;;N;;;;; +0C9D;KANNADA LETTER JHA;Lo;0;L;;;;;N;;;;; +0C9E;KANNADA LETTER NYA;Lo;0;L;;;;;N;;;;; +0C9F;KANNADA LETTER TTA;Lo;0;L;;;;;N;;;;; +0CA0;KANNADA LETTER TTHA;Lo;0;L;;;;;N;;;;; +0CA1;KANNADA LETTER DDA;Lo;0;L;;;;;N;;;;; +0CA2;KANNADA LETTER DDHA;Lo;0;L;;;;;N;;;;; +0CA3;KANNADA LETTER NNA;Lo;0;L;;;;;N;;;;; +0CA4;KANNADA LETTER TA;Lo;0;L;;;;;N;;;;; +0CA5;KANNADA LETTER THA;Lo;0;L;;;;;N;;;;; +0CA6;KANNADA LETTER DA;Lo;0;L;;;;;N;;;;; +0CA7;KANNADA LETTER DHA;Lo;0;L;;;;;N;;;;; +0CA8;KANNADA LETTER NA;Lo;0;L;;;;;N;;;;; +0CAA;KANNADA LETTER PA;Lo;0;L;;;;;N;;;;; +0CAB;KANNADA LETTER PHA;Lo;0;L;;;;;N;;;;; +0CAC;KANNADA LETTER BA;Lo;0;L;;;;;N;;;;; +0CAD;KANNADA LETTER BHA;Lo;0;L;;;;;N;;;;; +0CAE;KANNADA LETTER MA;Lo;0;L;;;;;N;;;;; +0CAF;KANNADA LETTER YA;Lo;0;L;;;;;N;;;;; +0CB0;KANNADA LETTER RA;Lo;0;L;;;;;N;;;;; +0CB1;KANNADA LETTER RRA;Lo;0;L;;;;;N;;;;; +0CB2;KANNADA LETTER LA;Lo;0;L;;;;;N;;;;; +0CB3;KANNADA LETTER LLA;Lo;0;L;;;;;N;;;;; +0CB5;KANNADA LETTER VA;Lo;0;L;;;;;N;;;;; +0CB6;KANNADA LETTER SHA;Lo;0;L;;;;;N;;;;; +0CB7;KANNADA LETTER SSA;Lo;0;L;;;;;N;;;;; +0CB8;KANNADA LETTER SA;Lo;0;L;;;;;N;;;;; +0CB9;KANNADA LETTER HA;Lo;0;L;;;;;N;;;;; +0CBC;KANNADA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +0CBD;KANNADA SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +0CBE;KANNADA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +0CBF;KANNADA VOWEL SIGN I;Mn;0;L;;;;;N;;;;; +0CC0;KANNADA VOWEL SIGN II;Mc;0;L;0CBF 0CD5;;;;N;;;;; +0CC1;KANNADA VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +0CC2;KANNADA VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +0CC3;KANNADA VOWEL SIGN VOCALIC R;Mc;0;L;;;;;N;;;;; +0CC4;KANNADA VOWEL SIGN VOCALIC RR;Mc;0;L;;;;;N;;;;; +0CC6;KANNADA VOWEL SIGN E;Mn;0;L;;;;;N;;;;; +0CC7;KANNADA VOWEL SIGN EE;Mc;0;L;0CC6 0CD5;;;;N;;;;; +0CC8;KANNADA VOWEL SIGN AI;Mc;0;L;0CC6 0CD6;;;;N;;;;; +0CCA;KANNADA VOWEL SIGN O;Mc;0;L;0CC6 0CC2;;;;N;;;;; +0CCB;KANNADA VOWEL SIGN OO;Mc;0;L;0CCA 0CD5;;;;N;;;;; +0CCC;KANNADA VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +0CCD;KANNADA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0CD5;KANNADA LENGTH MARK;Mc;0;L;;;;;N;;;;; +0CD6;KANNADA AI LENGTH MARK;Mc;0;L;;;;;N;;;;; +0CDE;KANNADA LETTER FA;Lo;0;L;;;;;N;;;;; +0CE0;KANNADA LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +0CE1;KANNADA LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +0CE2;KANNADA VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +0CE3;KANNADA VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +0CE6;KANNADA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0CE7;KANNADA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0CE8;KANNADA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0CE9;KANNADA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0CEA;KANNADA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0CEB;KANNADA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0CEC;KANNADA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0CED;KANNADA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0CEE;KANNADA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; +0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +0D03;MALAYALAM SIGN VISARGA;Mc;0;L;;;;;N;;;;; +0D05;MALAYALAM LETTER A;Lo;0;L;;;;;N;;;;; +0D06;MALAYALAM LETTER AA;Lo;0;L;;;;;N;;;;; +0D07;MALAYALAM LETTER I;Lo;0;L;;;;;N;;;;; +0D08;MALAYALAM LETTER II;Lo;0;L;;;;;N;;;;; +0D09;MALAYALAM LETTER U;Lo;0;L;;;;;N;;;;; +0D0A;MALAYALAM LETTER UU;Lo;0;L;;;;;N;;;;; +0D0B;MALAYALAM LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +0D0C;MALAYALAM LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +0D0E;MALAYALAM LETTER E;Lo;0;L;;;;;N;;;;; +0D0F;MALAYALAM LETTER EE;Lo;0;L;;;;;N;;;;; +0D10;MALAYALAM LETTER AI;Lo;0;L;;;;;N;;;;; +0D12;MALAYALAM LETTER O;Lo;0;L;;;;;N;;;;; +0D13;MALAYALAM LETTER OO;Lo;0;L;;;;;N;;;;; +0D14;MALAYALAM LETTER AU;Lo;0;L;;;;;N;;;;; +0D15;MALAYALAM LETTER KA;Lo;0;L;;;;;N;;;;; +0D16;MALAYALAM LETTER KHA;Lo;0;L;;;;;N;;;;; +0D17;MALAYALAM LETTER GA;Lo;0;L;;;;;N;;;;; +0D18;MALAYALAM LETTER GHA;Lo;0;L;;;;;N;;;;; +0D19;MALAYALAM LETTER NGA;Lo;0;L;;;;;N;;;;; +0D1A;MALAYALAM LETTER CA;Lo;0;L;;;;;N;;;;; +0D1B;MALAYALAM LETTER CHA;Lo;0;L;;;;;N;;;;; +0D1C;MALAYALAM LETTER JA;Lo;0;L;;;;;N;;;;; +0D1D;MALAYALAM LETTER JHA;Lo;0;L;;;;;N;;;;; +0D1E;MALAYALAM LETTER NYA;Lo;0;L;;;;;N;;;;; +0D1F;MALAYALAM LETTER TTA;Lo;0;L;;;;;N;;;;; +0D20;MALAYALAM LETTER TTHA;Lo;0;L;;;;;N;;;;; +0D21;MALAYALAM LETTER DDA;Lo;0;L;;;;;N;;;;; +0D22;MALAYALAM LETTER DDHA;Lo;0;L;;;;;N;;;;; +0D23;MALAYALAM LETTER NNA;Lo;0;L;;;;;N;;;;; +0D24;MALAYALAM LETTER TA;Lo;0;L;;;;;N;;;;; +0D25;MALAYALAM LETTER THA;Lo;0;L;;;;;N;;;;; +0D26;MALAYALAM LETTER DA;Lo;0;L;;;;;N;;;;; +0D27;MALAYALAM LETTER DHA;Lo;0;L;;;;;N;;;;; +0D28;MALAYALAM LETTER NA;Lo;0;L;;;;;N;;;;; +0D29;MALAYALAM LETTER NNNA;Lo;0;L;;;;;N;;;;; +0D2A;MALAYALAM LETTER PA;Lo;0;L;;;;;N;;;;; +0D2B;MALAYALAM LETTER PHA;Lo;0;L;;;;;N;;;;; +0D2C;MALAYALAM LETTER BA;Lo;0;L;;;;;N;;;;; +0D2D;MALAYALAM LETTER BHA;Lo;0;L;;;;;N;;;;; +0D2E;MALAYALAM LETTER MA;Lo;0;L;;;;;N;;;;; +0D2F;MALAYALAM LETTER YA;Lo;0;L;;;;;N;;;;; +0D30;MALAYALAM LETTER RA;Lo;0;L;;;;;N;;;;; +0D31;MALAYALAM LETTER RRA;Lo;0;L;;;;;N;;;;; +0D32;MALAYALAM LETTER LA;Lo;0;L;;;;;N;;;;; +0D33;MALAYALAM LETTER LLA;Lo;0;L;;;;;N;;;;; +0D34;MALAYALAM LETTER LLLA;Lo;0;L;;;;;N;;;;; +0D35;MALAYALAM LETTER VA;Lo;0;L;;;;;N;;;;; +0D36;MALAYALAM LETTER SHA;Lo;0;L;;;;;N;;;;; +0D37;MALAYALAM LETTER SSA;Lo;0;L;;;;;N;;;;; +0D38;MALAYALAM LETTER SA;Lo;0;L;;;;;N;;;;; +0D39;MALAYALAM LETTER HA;Lo;0;L;;;;;N;;;;; +0D3A;MALAYALAM LETTER TTTA;Lo;0;L;;;;;N;;;;; +0D3D;MALAYALAM SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +0D3E;MALAYALAM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +0D3F;MALAYALAM VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +0D40;MALAYALAM VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +0D41;MALAYALAM VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +0D42;MALAYALAM VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +0D43;MALAYALAM VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +0D44;MALAYALAM VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +0D46;MALAYALAM VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +0D47;MALAYALAM VOWEL SIGN EE;Mc;0;L;;;;;N;;;;; +0D48;MALAYALAM VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +0D4A;MALAYALAM VOWEL SIGN O;Mc;0;L;0D46 0D3E;;;;N;;;;; +0D4B;MALAYALAM VOWEL SIGN OO;Mc;0;L;0D47 0D3E;;;;N;;;;; +0D4C;MALAYALAM VOWEL SIGN AU;Mc;0;L;0D46 0D57;;;;N;;;;; +0D4D;MALAYALAM SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +0D4E;MALAYALAM LETTER DOT REPH;Lo;0;L;;;;;N;;;;; +0D57;MALAYALAM AU LENGTH MARK;Mc;0;L;;;;;N;;;;; +0D60;MALAYALAM LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +0D61;MALAYALAM LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +0D62;MALAYALAM VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +0D63;MALAYALAM VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +0D66;MALAYALAM DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0D67;MALAYALAM DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0D68;MALAYALAM DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0D69;MALAYALAM DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0D6A;MALAYALAM DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0D6B;MALAYALAM DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0D6C;MALAYALAM DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0D6D;MALAYALAM DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0D6E;MALAYALAM DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0D6F;MALAYALAM DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0D70;MALAYALAM NUMBER TEN;No;0;L;;;;10;N;;;;; +0D71;MALAYALAM NUMBER ONE HUNDRED;No;0;L;;;;100;N;;;;; +0D72;MALAYALAM NUMBER ONE THOUSAND;No;0;L;;;;1000;N;;;;; +0D73;MALAYALAM FRACTION ONE QUARTER;No;0;L;;;;1/4;N;;;;; +0D74;MALAYALAM FRACTION ONE HALF;No;0;L;;;;1/2;N;;;;; +0D75;MALAYALAM FRACTION THREE QUARTERS;No;0;L;;;;3/4;N;;;;; +0D79;MALAYALAM DATE MARK;So;0;L;;;;;N;;;;; +0D7A;MALAYALAM LETTER CHILLU NN;Lo;0;L;;;;;N;;;;; +0D7B;MALAYALAM LETTER CHILLU N;Lo;0;L;;;;;N;;;;; +0D7C;MALAYALAM LETTER CHILLU RR;Lo;0;L;;;;;N;;;;; +0D7D;MALAYALAM LETTER CHILLU L;Lo;0;L;;;;;N;;;;; +0D7E;MALAYALAM LETTER CHILLU LL;Lo;0;L;;;;;N;;;;; +0D7F;MALAYALAM LETTER CHILLU K;Lo;0;L;;;;;N;;;;; +0D82;SINHALA SIGN ANUSVARAYA;Mc;0;L;;;;;N;;;;; +0D83;SINHALA SIGN VISARGAYA;Mc;0;L;;;;;N;;;;; +0D85;SINHALA LETTER AYANNA;Lo;0;L;;;;;N;;;;; +0D86;SINHALA LETTER AAYANNA;Lo;0;L;;;;;N;;;;; +0D87;SINHALA LETTER AEYANNA;Lo;0;L;;;;;N;;;;; +0D88;SINHALA LETTER AEEYANNA;Lo;0;L;;;;;N;;;;; +0D89;SINHALA LETTER IYANNA;Lo;0;L;;;;;N;;;;; +0D8A;SINHALA LETTER IIYANNA;Lo;0;L;;;;;N;;;;; +0D8B;SINHALA LETTER UYANNA;Lo;0;L;;;;;N;;;;; +0D8C;SINHALA LETTER UUYANNA;Lo;0;L;;;;;N;;;;; +0D8D;SINHALA LETTER IRUYANNA;Lo;0;L;;;;;N;;;;; +0D8E;SINHALA LETTER IRUUYANNA;Lo;0;L;;;;;N;;;;; +0D8F;SINHALA LETTER ILUYANNA;Lo;0;L;;;;;N;;;;; +0D90;SINHALA LETTER ILUUYANNA;Lo;0;L;;;;;N;;;;; +0D91;SINHALA LETTER EYANNA;Lo;0;L;;;;;N;;;;; +0D92;SINHALA LETTER EEYANNA;Lo;0;L;;;;;N;;;;; +0D93;SINHALA LETTER AIYANNA;Lo;0;L;;;;;N;;;;; +0D94;SINHALA LETTER OYANNA;Lo;0;L;;;;;N;;;;; +0D95;SINHALA LETTER OOYANNA;Lo;0;L;;;;;N;;;;; +0D96;SINHALA LETTER AUYANNA;Lo;0;L;;;;;N;;;;; +0D9A;SINHALA LETTER ALPAPRAANA KAYANNA;Lo;0;L;;;;;N;;;;; +0D9B;SINHALA LETTER MAHAAPRAANA KAYANNA;Lo;0;L;;;;;N;;;;; +0D9C;SINHALA LETTER ALPAPRAANA GAYANNA;Lo;0;L;;;;;N;;;;; +0D9D;SINHALA LETTER MAHAAPRAANA GAYANNA;Lo;0;L;;;;;N;;;;; +0D9E;SINHALA LETTER KANTAJA NAASIKYAYA;Lo;0;L;;;;;N;;;;; +0D9F;SINHALA LETTER SANYAKA GAYANNA;Lo;0;L;;;;;N;;;;; +0DA0;SINHALA LETTER ALPAPRAANA CAYANNA;Lo;0;L;;;;;N;;;;; +0DA1;SINHALA LETTER MAHAAPRAANA CAYANNA;Lo;0;L;;;;;N;;;;; +0DA2;SINHALA LETTER ALPAPRAANA JAYANNA;Lo;0;L;;;;;N;;;;; +0DA3;SINHALA LETTER MAHAAPRAANA JAYANNA;Lo;0;L;;;;;N;;;;; +0DA4;SINHALA LETTER TAALUJA NAASIKYAYA;Lo;0;L;;;;;N;;;;; +0DA5;SINHALA LETTER TAALUJA SANYOOGA NAAKSIKYAYA;Lo;0;L;;;;;N;;;;; +0DA6;SINHALA LETTER SANYAKA JAYANNA;Lo;0;L;;;;;N;;;;; +0DA7;SINHALA LETTER ALPAPRAANA TTAYANNA;Lo;0;L;;;;;N;;;;; +0DA8;SINHALA LETTER MAHAAPRAANA TTAYANNA;Lo;0;L;;;;;N;;;;; +0DA9;SINHALA LETTER ALPAPRAANA DDAYANNA;Lo;0;L;;;;;N;;;;; +0DAA;SINHALA LETTER MAHAAPRAANA DDAYANNA;Lo;0;L;;;;;N;;;;; +0DAB;SINHALA LETTER MUURDHAJA NAYANNA;Lo;0;L;;;;;N;;;;; +0DAC;SINHALA LETTER SANYAKA DDAYANNA;Lo;0;L;;;;;N;;;;; +0DAD;SINHALA LETTER ALPAPRAANA TAYANNA;Lo;0;L;;;;;N;;;;; +0DAE;SINHALA LETTER MAHAAPRAANA TAYANNA;Lo;0;L;;;;;N;;;;; +0DAF;SINHALA LETTER ALPAPRAANA DAYANNA;Lo;0;L;;;;;N;;;;; +0DB0;SINHALA LETTER MAHAAPRAANA DAYANNA;Lo;0;L;;;;;N;;;;; +0DB1;SINHALA LETTER DANTAJA NAYANNA;Lo;0;L;;;;;N;;;;; +0DB3;SINHALA LETTER SANYAKA DAYANNA;Lo;0;L;;;;;N;;;;; +0DB4;SINHALA LETTER ALPAPRAANA PAYANNA;Lo;0;L;;;;;N;;;;; +0DB5;SINHALA LETTER MAHAAPRAANA PAYANNA;Lo;0;L;;;;;N;;;;; +0DB6;SINHALA LETTER ALPAPRAANA BAYANNA;Lo;0;L;;;;;N;;;;; +0DB7;SINHALA LETTER MAHAAPRAANA BAYANNA;Lo;0;L;;;;;N;;;;; +0DB8;SINHALA LETTER MAYANNA;Lo;0;L;;;;;N;;;;; +0DB9;SINHALA LETTER AMBA BAYANNA;Lo;0;L;;;;;N;;;;; +0DBA;SINHALA LETTER YAYANNA;Lo;0;L;;;;;N;;;;; +0DBB;SINHALA LETTER RAYANNA;Lo;0;L;;;;;N;;;;; +0DBD;SINHALA LETTER DANTAJA LAYANNA;Lo;0;L;;;;;N;;;;; +0DC0;SINHALA LETTER VAYANNA;Lo;0;L;;;;;N;;;;; +0DC1;SINHALA LETTER TAALUJA SAYANNA;Lo;0;L;;;;;N;;;;; +0DC2;SINHALA LETTER MUURDHAJA SAYANNA;Lo;0;L;;;;;N;;;;; +0DC3;SINHALA LETTER DANTAJA SAYANNA;Lo;0;L;;;;;N;;;;; +0DC4;SINHALA LETTER HAYANNA;Lo;0;L;;;;;N;;;;; +0DC5;SINHALA LETTER MUURDHAJA LAYANNA;Lo;0;L;;;;;N;;;;; +0DC6;SINHALA LETTER FAYANNA;Lo;0;L;;;;;N;;;;; +0DCA;SINHALA SIGN AL-LAKUNA;Mn;9;NSM;;;;;N;;;;; +0DCF;SINHALA VOWEL SIGN AELA-PILLA;Mc;0;L;;;;;N;;;;; +0DD0;SINHALA VOWEL SIGN KETTI AEDA-PILLA;Mc;0;L;;;;;N;;;;; +0DD1;SINHALA VOWEL SIGN DIGA AEDA-PILLA;Mc;0;L;;;;;N;;;;; +0DD2;SINHALA VOWEL SIGN KETTI IS-PILLA;Mn;0;NSM;;;;;N;;;;; +0DD3;SINHALA VOWEL SIGN DIGA IS-PILLA;Mn;0;NSM;;;;;N;;;;; +0DD4;SINHALA VOWEL SIGN KETTI PAA-PILLA;Mn;0;NSM;;;;;N;;;;; +0DD6;SINHALA VOWEL SIGN DIGA PAA-PILLA;Mn;0;NSM;;;;;N;;;;; +0DD8;SINHALA VOWEL SIGN GAETTA-PILLA;Mc;0;L;;;;;N;;;;; +0DD9;SINHALA VOWEL SIGN KOMBUVA;Mc;0;L;;;;;N;;;;; +0DDA;SINHALA VOWEL SIGN DIGA KOMBUVA;Mc;0;L;0DD9 0DCA;;;;N;;;;; +0DDB;SINHALA VOWEL SIGN KOMBU DEKA;Mc;0;L;;;;;N;;;;; +0DDC;SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA;Mc;0;L;0DD9 0DCF;;;;N;;;;; +0DDD;SINHALA VOWEL SIGN KOMBUVA HAA DIGA AELA-PILLA;Mc;0;L;0DDC 0DCA;;;;N;;;;; +0DDE;SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA;Mc;0;L;0DD9 0DDF;;;;N;;;;; +0DDF;SINHALA VOWEL SIGN GAYANUKITTA;Mc;0;L;;;;;N;;;;; +0DF2;SINHALA VOWEL SIGN DIGA GAETTA-PILLA;Mc;0;L;;;;;N;;;;; +0DF3;SINHALA VOWEL SIGN DIGA GAYANUKITTA;Mc;0;L;;;;;N;;;;; +0DF4;SINHALA PUNCTUATION KUNDDALIYA;Po;0;L;;;;;N;;;;; +0E01;THAI CHARACTER KO KAI;Lo;0;L;;;;;N;THAI LETTER KO KAI;;;; +0E02;THAI CHARACTER KHO KHAI;Lo;0;L;;;;;N;THAI LETTER KHO KHAI;;;; +0E03;THAI CHARACTER KHO KHUAT;Lo;0;L;;;;;N;THAI LETTER KHO KHUAT;;;; +0E04;THAI CHARACTER KHO KHWAI;Lo;0;L;;;;;N;THAI LETTER KHO KHWAI;;;; +0E05;THAI CHARACTER KHO KHON;Lo;0;L;;;;;N;THAI LETTER KHO KHON;;;; +0E06;THAI CHARACTER KHO RAKHANG;Lo;0;L;;;;;N;THAI LETTER KHO RAKHANG;;;; +0E07;THAI CHARACTER NGO NGU;Lo;0;L;;;;;N;THAI LETTER NGO NGU;;;; +0E08;THAI CHARACTER CHO CHAN;Lo;0;L;;;;;N;THAI LETTER CHO CHAN;;;; +0E09;THAI CHARACTER CHO CHING;Lo;0;L;;;;;N;THAI LETTER CHO CHING;;;; +0E0A;THAI CHARACTER CHO CHANG;Lo;0;L;;;;;N;THAI LETTER CHO CHANG;;;; +0E0B;THAI CHARACTER SO SO;Lo;0;L;;;;;N;THAI LETTER SO SO;;;; +0E0C;THAI CHARACTER CHO CHOE;Lo;0;L;;;;;N;THAI LETTER CHO CHOE;;;; +0E0D;THAI CHARACTER YO YING;Lo;0;L;;;;;N;THAI LETTER YO YING;;;; +0E0E;THAI CHARACTER DO CHADA;Lo;0;L;;;;;N;THAI LETTER DO CHADA;;;; +0E0F;THAI CHARACTER TO PATAK;Lo;0;L;;;;;N;THAI LETTER TO PATAK;;;; +0E10;THAI CHARACTER THO THAN;Lo;0;L;;;;;N;THAI LETTER THO THAN;;;; +0E11;THAI CHARACTER THO NANGMONTHO;Lo;0;L;;;;;N;THAI LETTER THO NANGMONTHO;;;; +0E12;THAI CHARACTER THO PHUTHAO;Lo;0;L;;;;;N;THAI LETTER THO PHUTHAO;;;; +0E13;THAI CHARACTER NO NEN;Lo;0;L;;;;;N;THAI LETTER NO NEN;;;; +0E14;THAI CHARACTER DO DEK;Lo;0;L;;;;;N;THAI LETTER DO DEK;;;; +0E15;THAI CHARACTER TO TAO;Lo;0;L;;;;;N;THAI LETTER TO TAO;;;; +0E16;THAI CHARACTER THO THUNG;Lo;0;L;;;;;N;THAI LETTER THO THUNG;;;; +0E17;THAI CHARACTER THO THAHAN;Lo;0;L;;;;;N;THAI LETTER THO THAHAN;;;; +0E18;THAI CHARACTER THO THONG;Lo;0;L;;;;;N;THAI LETTER THO THONG;;;; +0E19;THAI CHARACTER NO NU;Lo;0;L;;;;;N;THAI LETTER NO NU;;;; +0E1A;THAI CHARACTER BO BAIMAI;Lo;0;L;;;;;N;THAI LETTER BO BAIMAI;;;; +0E1B;THAI CHARACTER PO PLA;Lo;0;L;;;;;N;THAI LETTER PO PLA;;;; +0E1C;THAI CHARACTER PHO PHUNG;Lo;0;L;;;;;N;THAI LETTER PHO PHUNG;;;; +0E1D;THAI CHARACTER FO FA;Lo;0;L;;;;;N;THAI LETTER FO FA;;;; +0E1E;THAI CHARACTER PHO PHAN;Lo;0;L;;;;;N;THAI LETTER PHO PHAN;;;; +0E1F;THAI CHARACTER FO FAN;Lo;0;L;;;;;N;THAI LETTER FO FAN;;;; +0E20;THAI CHARACTER PHO SAMPHAO;Lo;0;L;;;;;N;THAI LETTER PHO SAMPHAO;;;; +0E21;THAI CHARACTER MO MA;Lo;0;L;;;;;N;THAI LETTER MO MA;;;; +0E22;THAI CHARACTER YO YAK;Lo;0;L;;;;;N;THAI LETTER YO YAK;;;; +0E23;THAI CHARACTER RO RUA;Lo;0;L;;;;;N;THAI LETTER RO RUA;;;; +0E24;THAI CHARACTER RU;Lo;0;L;;;;;N;THAI LETTER RU;;;; +0E25;THAI CHARACTER LO LING;Lo;0;L;;;;;N;THAI LETTER LO LING;;;; +0E26;THAI CHARACTER LU;Lo;0;L;;;;;N;THAI LETTER LU;;;; +0E27;THAI CHARACTER WO WAEN;Lo;0;L;;;;;N;THAI LETTER WO WAEN;;;; +0E28;THAI CHARACTER SO SALA;Lo;0;L;;;;;N;THAI LETTER SO SALA;;;; +0E29;THAI CHARACTER SO RUSI;Lo;0;L;;;;;N;THAI LETTER SO RUSI;;;; +0E2A;THAI CHARACTER SO SUA;Lo;0;L;;;;;N;THAI LETTER SO SUA;;;; +0E2B;THAI CHARACTER HO HIP;Lo;0;L;;;;;N;THAI LETTER HO HIP;;;; +0E2C;THAI CHARACTER LO CHULA;Lo;0;L;;;;;N;THAI LETTER LO CHULA;;;; +0E2D;THAI CHARACTER O ANG;Lo;0;L;;;;;N;THAI LETTER O ANG;;;; +0E2E;THAI CHARACTER HO NOKHUK;Lo;0;L;;;;;N;THAI LETTER HO NOK HUK;;;; +0E2F;THAI CHARACTER PAIYANNOI;Lo;0;L;;;;;N;THAI PAI YAN NOI;;;; +0E30;THAI CHARACTER SARA A;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA A;;;; +0E31;THAI CHARACTER MAI HAN-AKAT;Mn;0;NSM;;;;;N;THAI VOWEL SIGN MAI HAN-AKAT;;;; +0E32;THAI CHARACTER SARA AA;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA AA;;;; +0E33;THAI CHARACTER SARA AM;Lo;0;L; 0E4D 0E32;;;;N;THAI VOWEL SIGN SARA AM;;;; +0E34;THAI CHARACTER SARA I;Mn;0;NSM;;;;;N;THAI VOWEL SIGN SARA I;;;; +0E35;THAI CHARACTER SARA II;Mn;0;NSM;;;;;N;THAI VOWEL SIGN SARA II;;;; +0E36;THAI CHARACTER SARA UE;Mn;0;NSM;;;;;N;THAI VOWEL SIGN SARA UE;;;; +0E37;THAI CHARACTER SARA UEE;Mn;0;NSM;;;;;N;THAI VOWEL SIGN SARA UEE;;;; +0E38;THAI CHARACTER SARA U;Mn;103;NSM;;;;;N;THAI VOWEL SIGN SARA U;;;; +0E39;THAI CHARACTER SARA UU;Mn;103;NSM;;;;;N;THAI VOWEL SIGN SARA UU;;;; +0E3A;THAI CHARACTER PHINTHU;Mn;9;NSM;;;;;N;THAI VOWEL SIGN PHINTHU;;;; +0E3F;THAI CURRENCY SYMBOL BAHT;Sc;0;ET;;;;;N;THAI BAHT SIGN;;;; +0E40;THAI CHARACTER SARA E;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA E;;;; +0E41;THAI CHARACTER SARA AE;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA AE;;;; +0E42;THAI CHARACTER SARA O;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA O;;;; +0E43;THAI CHARACTER SARA AI MAIMUAN;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA MAI MUAN;;;; +0E44;THAI CHARACTER SARA AI MAIMALAI;Lo;0;L;;;;;N;THAI VOWEL SIGN SARA MAI MALAI;;;; +0E45;THAI CHARACTER LAKKHANGYAO;Lo;0;L;;;;;N;THAI LAK KHANG YAO;;;; +0E46;THAI CHARACTER MAIYAMOK;Lm;0;L;;;;;N;THAI MAI YAMOK;;;; +0E47;THAI CHARACTER MAITAIKHU;Mn;0;NSM;;;;;N;THAI VOWEL SIGN MAI TAI KHU;;;; +0E48;THAI CHARACTER MAI EK;Mn;107;NSM;;;;;N;THAI TONE MAI EK;;;; +0E49;THAI CHARACTER MAI THO;Mn;107;NSM;;;;;N;THAI TONE MAI THO;;;; +0E4A;THAI CHARACTER MAI TRI;Mn;107;NSM;;;;;N;THAI TONE MAI TRI;;;; +0E4B;THAI CHARACTER MAI CHATTAWA;Mn;107;NSM;;;;;N;THAI TONE MAI CHATTAWA;;;; +0E4C;THAI CHARACTER THANTHAKHAT;Mn;0;NSM;;;;;N;THAI THANTHAKHAT;;;; +0E4D;THAI CHARACTER NIKHAHIT;Mn;0;NSM;;;;;N;THAI NIKKHAHIT;;;; +0E4E;THAI CHARACTER YAMAKKAN;Mn;0;NSM;;;;;N;THAI YAMAKKAN;;;; +0E4F;THAI CHARACTER FONGMAN;Po;0;L;;;;;N;THAI FONGMAN;;;; +0E50;THAI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0E51;THAI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0E52;THAI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0E53;THAI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0E54;THAI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0E55;THAI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0E56;THAI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0E57;THAI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0E58;THAI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0E59;THAI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0E5A;THAI CHARACTER ANGKHANKHU;Po;0;L;;;;;N;THAI ANGKHANKHU;;;; +0E5B;THAI CHARACTER KHOMUT;Po;0;L;;;;;N;THAI KHOMUT;;;; +0E81;LAO LETTER KO;Lo;0;L;;;;;N;;;;; +0E82;LAO LETTER KHO SUNG;Lo;0;L;;;;;N;;;;; +0E84;LAO LETTER KHO TAM;Lo;0;L;;;;;N;;;;; +0E87;LAO LETTER NGO;Lo;0;L;;;;;N;;;;; +0E88;LAO LETTER CO;Lo;0;L;;;;;N;;;;; +0E8A;LAO LETTER SO TAM;Lo;0;L;;;;;N;;;;; +0E8D;LAO LETTER NYO;Lo;0;L;;;;;N;;;;; +0E94;LAO LETTER DO;Lo;0;L;;;;;N;;;;; +0E95;LAO LETTER TO;Lo;0;L;;;;;N;;;;; +0E96;LAO LETTER THO SUNG;Lo;0;L;;;;;N;;;;; +0E97;LAO LETTER THO TAM;Lo;0;L;;;;;N;;;;; +0E99;LAO LETTER NO;Lo;0;L;;;;;N;;;;; +0E9A;LAO LETTER BO;Lo;0;L;;;;;N;;;;; +0E9B;LAO LETTER PO;Lo;0;L;;;;;N;;;;; +0E9C;LAO LETTER PHO SUNG;Lo;0;L;;;;;N;;;;; +0E9D;LAO LETTER FO TAM;Lo;0;L;;;;;N;;;;; +0E9E;LAO LETTER PHO TAM;Lo;0;L;;;;;N;;;;; +0E9F;LAO LETTER FO SUNG;Lo;0;L;;;;;N;;;;; +0EA1;LAO LETTER MO;Lo;0;L;;;;;N;;;;; +0EA2;LAO LETTER YO;Lo;0;L;;;;;N;;;;; +0EA3;LAO LETTER LO LING;Lo;0;L;;;;;N;;;;; +0EA5;LAO LETTER LO LOOT;Lo;0;L;;;;;N;;;;; +0EA7;LAO LETTER WO;Lo;0;L;;;;;N;;;;; +0EAA;LAO LETTER SO SUNG;Lo;0;L;;;;;N;;;;; +0EAB;LAO LETTER HO SUNG;Lo;0;L;;;;;N;;;;; +0EAD;LAO LETTER O;Lo;0;L;;;;;N;;;;; +0EAE;LAO LETTER HO TAM;Lo;0;L;;;;;N;;;;; +0EAF;LAO ELLIPSIS;Lo;0;L;;;;;N;;;;; +0EB0;LAO VOWEL SIGN A;Lo;0;L;;;;;N;;;;; +0EB1;LAO VOWEL SIGN MAI KAN;Mn;0;NSM;;;;;N;;;;; +0EB2;LAO VOWEL SIGN AA;Lo;0;L;;;;;N;;;;; +0EB3;LAO VOWEL SIGN AM;Lo;0;L; 0ECD 0EB2;;;;N;;;;; +0EB4;LAO VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +0EB5;LAO VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +0EB6;LAO VOWEL SIGN Y;Mn;0;NSM;;;;;N;;;;; +0EB7;LAO VOWEL SIGN YY;Mn;0;NSM;;;;;N;;;;; +0EB8;LAO VOWEL SIGN U;Mn;118;NSM;;;;;N;;;;; +0EB9;LAO VOWEL SIGN UU;Mn;118;NSM;;;;;N;;;;; +0EBB;LAO VOWEL SIGN MAI KON;Mn;0;NSM;;;;;N;;;;; +0EBC;LAO SEMIVOWEL SIGN LO;Mn;0;NSM;;;;;N;;;;; +0EBD;LAO SEMIVOWEL SIGN NYO;Lo;0;L;;;;;N;;;;; +0EC0;LAO VOWEL SIGN E;Lo;0;L;;;;;N;;;;; +0EC1;LAO VOWEL SIGN EI;Lo;0;L;;;;;N;;;;; +0EC2;LAO VOWEL SIGN O;Lo;0;L;;;;;N;;;;; +0EC3;LAO VOWEL SIGN AY;Lo;0;L;;;;;N;;;;; +0EC4;LAO VOWEL SIGN AI;Lo;0;L;;;;;N;;;;; +0EC6;LAO KO LA;Lm;0;L;;;;;N;;;;; +0EC8;LAO TONE MAI EK;Mn;122;NSM;;;;;N;;;;; +0EC9;LAO TONE MAI THO;Mn;122;NSM;;;;;N;;;;; +0ECA;LAO TONE MAI TI;Mn;122;NSM;;;;;N;;;;; +0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;; +0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;; +0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;; +0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0ED3;LAO DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0ED4;LAO DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0ED5;LAO DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0ED6;LAO DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0ED7;LAO DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0ED8;LAO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0ED9;LAO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0EDC;LAO HO NO;Lo;0;L; 0EAB 0E99;;;;N;;;;; +0EDD;LAO HO MO;Lo;0;L; 0EAB 0EA1;;;;N;;;;; +0EDE;LAO LETTER KHMU GO;Lo;0;L;;;;;N;;;;; +0EDF;LAO LETTER KHMU NYO;Lo;0;L;;;;;N;;;;; +0F00;TIBETAN SYLLABLE OM;Lo;0;L;;;;;N;;;;; +0F01;TIBETAN MARK GTER YIG MGO TRUNCATED A;So;0;L;;;;;N;;;;; +0F02;TIBETAN MARK GTER YIG MGO -UM RNAM BCAD MA;So;0;L;;;;;N;;;;; +0F03;TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA;So;0;L;;;;;N;;;;; +0F04;TIBETAN MARK INITIAL YIG MGO MDUN MA;Po;0;L;;;;;N;TIBETAN SINGLE ORNAMENT;;;; +0F05;TIBETAN MARK CLOSING YIG MGO SGAB MA;Po;0;L;;;;;N;;;;; +0F06;TIBETAN MARK CARET YIG MGO PHUR SHAD MA;Po;0;L;;;;;N;;;;; +0F07;TIBETAN MARK YIG MGO TSHEG SHAD MA;Po;0;L;;;;;N;;;;; +0F08;TIBETAN MARK SBRUL SHAD;Po;0;L;;;;;N;TIBETAN RGYANSHAD;;;; +0F09;TIBETAN MARK BSKUR YIG MGO;Po;0;L;;;;;N;;;;; +0F0A;TIBETAN MARK BKA- SHOG YIG MGO;Po;0;L;;;;;N;;;;; +0F0B;TIBETAN MARK INTERSYLLABIC TSHEG;Po;0;L;;;;;N;TIBETAN TSEG;;;; +0F0C;TIBETAN MARK DELIMITER TSHEG BSTAR;Po;0;L; 0F0B;;;;N;;;;; +0F0D;TIBETAN MARK SHAD;Po;0;L;;;;;N;TIBETAN SHAD;;;; +0F0E;TIBETAN MARK NYIS SHAD;Po;0;L;;;;;N;TIBETAN DOUBLE SHAD;;;; +0F0F;TIBETAN MARK TSHEG SHAD;Po;0;L;;;;;N;;;;; +0F10;TIBETAN MARK NYIS TSHEG SHAD;Po;0;L;;;;;N;;;;; +0F11;TIBETAN MARK RIN CHEN SPUNGS SHAD;Po;0;L;;;;;N;TIBETAN RINCHANPHUNGSHAD;;;; +0F12;TIBETAN MARK RGYA GRAM SHAD;Po;0;L;;;;;N;;;;; +0F13;TIBETAN MARK CARET -DZUD RTAGS ME LONG CAN;So;0;L;;;;;N;;;;; +0F14;TIBETAN MARK GTER TSHEG;Po;0;L;;;;;N;TIBETAN COMMA;;;; +0F15;TIBETAN LOGOTYPE SIGN CHAD RTAGS;So;0;L;;;;;N;;;;; +0F16;TIBETAN LOGOTYPE SIGN LHAG RTAGS;So;0;L;;;;;N;;;;; +0F17;TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS;So;0;L;;;;;N;;;;; +0F18;TIBETAN ASTROLOGICAL SIGN -KHYUD PA;Mn;220;NSM;;;;;N;;;;; +0F19;TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS;Mn;220;NSM;;;;;N;;;;; +0F1A;TIBETAN SIGN RDEL DKAR GCIG;So;0;L;;;;;N;;;;; +0F1B;TIBETAN SIGN RDEL DKAR GNYIS;So;0;L;;;;;N;;;;; +0F1C;TIBETAN SIGN RDEL DKAR GSUM;So;0;L;;;;;N;;;;; +0F1D;TIBETAN SIGN RDEL NAG GCIG;So;0;L;;;;;N;;;;; +0F1E;TIBETAN SIGN RDEL NAG GNYIS;So;0;L;;;;;N;;;;; +0F1F;TIBETAN SIGN RDEL DKAR RDEL NAG;So;0;L;;;;;N;;;;; +0F20;TIBETAN DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +0F21;TIBETAN DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +0F22;TIBETAN DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +0F23;TIBETAN DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +0F24;TIBETAN DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +0F25;TIBETAN DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +0F26;TIBETAN DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +0F27;TIBETAN DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +0F28;TIBETAN DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +0F29;TIBETAN DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0F2A;TIBETAN DIGIT HALF ONE;No;0;L;;;;1/2;N;;;;; +0F2B;TIBETAN DIGIT HALF TWO;No;0;L;;;;3/2;N;;;;; +0F2C;TIBETAN DIGIT HALF THREE;No;0;L;;;;5/2;N;;;;; +0F2D;TIBETAN DIGIT HALF FOUR;No;0;L;;;;7/2;N;;;;; +0F2E;TIBETAN DIGIT HALF FIVE;No;0;L;;;;9/2;N;;;;; +0F2F;TIBETAN DIGIT HALF SIX;No;0;L;;;;11/2;N;;;;; +0F30;TIBETAN DIGIT HALF SEVEN;No;0;L;;;;13/2;N;;;;; +0F31;TIBETAN DIGIT HALF EIGHT;No;0;L;;;;15/2;N;;;;; +0F32;TIBETAN DIGIT HALF NINE;No;0;L;;;;17/2;N;;;;; +0F33;TIBETAN DIGIT HALF ZERO;No;0;L;;;;-1/2;N;;;;; +0F34;TIBETAN MARK BSDUS RTAGS;So;0;L;;;;;N;;;;; +0F35;TIBETAN MARK NGAS BZUNG NYI ZLA;Mn;220;NSM;;;;;N;TIBETAN HONORIFIC UNDER RING;;;; +0F36;TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN;So;0;L;;;;;N;;;;; +0F37;TIBETAN MARK NGAS BZUNG SGOR RTAGS;Mn;220;NSM;;;;;N;TIBETAN UNDER RING;;;; +0F38;TIBETAN MARK CHE MGO;So;0;L;;;;;N;;;;; +0F39;TIBETAN MARK TSA -PHRU;Mn;216;NSM;;;;;N;TIBETAN LENITION MARK;;;; +0F3A;TIBETAN MARK GUG RTAGS GYON;Ps;0;ON;;;;;Y;;;;; +0F3B;TIBETAN MARK GUG RTAGS GYAS;Pe;0;ON;;;;;Y;;;;; +0F3C;TIBETAN MARK ANG KHANG GYON;Ps;0;ON;;;;;Y;TIBETAN LEFT BRACE;;;; +0F3D;TIBETAN MARK ANG KHANG GYAS;Pe;0;ON;;;;;Y;TIBETAN RIGHT BRACE;;;; +0F3E;TIBETAN SIGN YAR TSHES;Mc;0;L;;;;;N;;;;; +0F3F;TIBETAN SIGN MAR TSHES;Mc;0;L;;;;;N;;;;; +0F40;TIBETAN LETTER KA;Lo;0;L;;;;;N;;;;; +0F41;TIBETAN LETTER KHA;Lo;0;L;;;;;N;;;;; +0F42;TIBETAN LETTER GA;Lo;0;L;;;;;N;;;;; +0F43;TIBETAN LETTER GHA;Lo;0;L;0F42 0FB7;;;;N;;;;; +0F44;TIBETAN LETTER NGA;Lo;0;L;;;;;N;;;;; +0F45;TIBETAN LETTER CA;Lo;0;L;;;;;N;;;;; +0F46;TIBETAN LETTER CHA;Lo;0;L;;;;;N;;;;; +0F47;TIBETAN LETTER JA;Lo;0;L;;;;;N;;;;; +0F49;TIBETAN LETTER NYA;Lo;0;L;;;;;N;;;;; +0F4A;TIBETAN LETTER TTA;Lo;0;L;;;;;N;TIBETAN LETTER REVERSED TA;;;; +0F4B;TIBETAN LETTER TTHA;Lo;0;L;;;;;N;TIBETAN LETTER REVERSED THA;;;; +0F4C;TIBETAN LETTER DDA;Lo;0;L;;;;;N;TIBETAN LETTER REVERSED DA;;;; +0F4D;TIBETAN LETTER DDHA;Lo;0;L;0F4C 0FB7;;;;N;;;;; +0F4E;TIBETAN LETTER NNA;Lo;0;L;;;;;N;TIBETAN LETTER REVERSED NA;;;; +0F4F;TIBETAN LETTER TA;Lo;0;L;;;;;N;;;;; +0F50;TIBETAN LETTER THA;Lo;0;L;;;;;N;;;;; +0F51;TIBETAN LETTER DA;Lo;0;L;;;;;N;;;;; +0F52;TIBETAN LETTER DHA;Lo;0;L;0F51 0FB7;;;;N;;;;; +0F53;TIBETAN LETTER NA;Lo;0;L;;;;;N;;;;; +0F54;TIBETAN LETTER PA;Lo;0;L;;;;;N;;;;; +0F55;TIBETAN LETTER PHA;Lo;0;L;;;;;N;;;;; +0F56;TIBETAN LETTER BA;Lo;0;L;;;;;N;;;;; +0F57;TIBETAN LETTER BHA;Lo;0;L;0F56 0FB7;;;;N;;;;; +0F58;TIBETAN LETTER MA;Lo;0;L;;;;;N;;;;; +0F59;TIBETAN LETTER TSA;Lo;0;L;;;;;N;;;;; +0F5A;TIBETAN LETTER TSHA;Lo;0;L;;;;;N;;;;; +0F5B;TIBETAN LETTER DZA;Lo;0;L;;;;;N;;;;; +0F5C;TIBETAN LETTER DZHA;Lo;0;L;0F5B 0FB7;;;;N;;;;; +0F5D;TIBETAN LETTER WA;Lo;0;L;;;;;N;;;;; +0F5E;TIBETAN LETTER ZHA;Lo;0;L;;;;;N;;;;; +0F5F;TIBETAN LETTER ZA;Lo;0;L;;;;;N;;;;; +0F60;TIBETAN LETTER -A;Lo;0;L;;;;;N;TIBETAN LETTER AA;;;; +0F61;TIBETAN LETTER YA;Lo;0;L;;;;;N;;;;; +0F62;TIBETAN LETTER RA;Lo;0;L;;;;;N;;;;; +0F63;TIBETAN LETTER LA;Lo;0;L;;;;;N;;;;; +0F64;TIBETAN LETTER SHA;Lo;0;L;;;;;N;;;;; +0F65;TIBETAN LETTER SSA;Lo;0;L;;;;;N;TIBETAN LETTER REVERSED SHA;;;; +0F66;TIBETAN LETTER SA;Lo;0;L;;;;;N;;;;; +0F67;TIBETAN LETTER HA;Lo;0;L;;;;;N;;;;; +0F68;TIBETAN LETTER A;Lo;0;L;;;;;N;;;;; +0F69;TIBETAN LETTER KSSA;Lo;0;L;0F40 0FB5;;;;N;;;;; +0F6A;TIBETAN LETTER FIXED-FORM RA;Lo;0;L;;;;;N;;;;; +0F6B;TIBETAN LETTER KKA;Lo;0;L;;;;;N;;;;; +0F6C;TIBETAN LETTER RRA;Lo;0;L;;;;;N;;;;; +0F71;TIBETAN VOWEL SIGN AA;Mn;129;NSM;;;;;N;;;;; +0F72;TIBETAN VOWEL SIGN I;Mn;130;NSM;;;;;N;;;;; +0F73;TIBETAN VOWEL SIGN II;Mn;0;NSM;0F71 0F72;;;;N;;;;; +0F74;TIBETAN VOWEL SIGN U;Mn;132;NSM;;;;;N;;;;; +0F75;TIBETAN VOWEL SIGN UU;Mn;0;NSM;0F71 0F74;;;;N;;;;; +0F76;TIBETAN VOWEL SIGN VOCALIC R;Mn;0;NSM;0FB2 0F80;;;;N;;;;; +0F77;TIBETAN VOWEL SIGN VOCALIC RR;Mn;0;NSM; 0FB2 0F81;;;;N;;;;; +0F78;TIBETAN VOWEL SIGN VOCALIC L;Mn;0;NSM;0FB3 0F80;;;;N;;;;; +0F79;TIBETAN VOWEL SIGN VOCALIC LL;Mn;0;NSM; 0FB3 0F81;;;;N;;;;; +0F7A;TIBETAN VOWEL SIGN E;Mn;130;NSM;;;;;N;;;;; +0F7B;TIBETAN VOWEL SIGN EE;Mn;130;NSM;;;;;N;TIBETAN VOWEL SIGN AI;;;; +0F7C;TIBETAN VOWEL SIGN O;Mn;130;NSM;;;;;N;;;;; +0F7D;TIBETAN VOWEL SIGN OO;Mn;130;NSM;;;;;N;TIBETAN VOWEL SIGN AU;;;; +0F7E;TIBETAN SIGN RJES SU NGA RO;Mn;0;NSM;;;;;N;TIBETAN ANUSVARA;;;; +0F7F;TIBETAN SIGN RNAM BCAD;Mc;0;L;;;;;N;TIBETAN VISARGA;;;; +0F80;TIBETAN VOWEL SIGN REVERSED I;Mn;130;NSM;;;;;N;TIBETAN VOWEL SIGN SHORT I;;;; +0F81;TIBETAN VOWEL SIGN REVERSED II;Mn;0;NSM;0F71 0F80;;;;N;;;;; +0F82;TIBETAN SIGN NYI ZLA NAA DA;Mn;230;NSM;;;;;N;TIBETAN CANDRABINDU WITH ORNAMENT;;;; +0F83;TIBETAN SIGN SNA LDAN;Mn;230;NSM;;;;;N;TIBETAN CANDRABINDU;;;; +0F84;TIBETAN MARK HALANTA;Mn;9;NSM;;;;;N;TIBETAN VIRAMA;;;; +0F85;TIBETAN MARK PALUTA;Po;0;L;;;;;N;TIBETAN CHUCHENYIGE;;;; +0F86;TIBETAN SIGN LCI RTAGS;Mn;230;NSM;;;;;N;;;;; +0F87;TIBETAN SIGN YANG RTAGS;Mn;230;NSM;;;;;N;;;;; +0F88;TIBETAN SIGN LCE TSA CAN;Lo;0;L;;;;;N;;;;; +0F89;TIBETAN SIGN MCHU CAN;Lo;0;L;;;;;N;;;;; +0F8A;TIBETAN SIGN GRU CAN RGYINGS;Lo;0;L;;;;;N;;;;; +0F8B;TIBETAN SIGN GRU MED RGYINGS;Lo;0;L;;;;;N;;;;; +0F8C;TIBETAN SIGN INVERTED MCHU CAN;Lo;0;L;;;;;N;;;;; +0F8D;TIBETAN SUBJOINED SIGN LCE TSA CAN;Mn;0;NSM;;;;;N;;;;; +0F8E;TIBETAN SUBJOINED SIGN MCHU CAN;Mn;0;NSM;;;;;N;;;;; +0F8F;TIBETAN SUBJOINED SIGN INVERTED MCHU CAN;Mn;0;NSM;;;;;N;;;;; +0F90;TIBETAN SUBJOINED LETTER KA;Mn;0;NSM;;;;;N;;;;; +0F91;TIBETAN SUBJOINED LETTER KHA;Mn;0;NSM;;;;;N;;;;; +0F92;TIBETAN SUBJOINED LETTER GA;Mn;0;NSM;;;;;N;;;;; +0F93;TIBETAN SUBJOINED LETTER GHA;Mn;0;NSM;0F92 0FB7;;;;N;;;;; +0F94;TIBETAN SUBJOINED LETTER NGA;Mn;0;NSM;;;;;N;;;;; +0F95;TIBETAN SUBJOINED LETTER CA;Mn;0;NSM;;;;;N;;;;; +0F96;TIBETAN SUBJOINED LETTER CHA;Mn;0;NSM;;;;;N;;;;; +0F97;TIBETAN SUBJOINED LETTER JA;Mn;0;NSM;;;;;N;;;;; +0F99;TIBETAN SUBJOINED LETTER NYA;Mn;0;NSM;;;;;N;;;;; +0F9A;TIBETAN SUBJOINED LETTER TTA;Mn;0;NSM;;;;;N;;;;; +0F9B;TIBETAN SUBJOINED LETTER TTHA;Mn;0;NSM;;;;;N;;;;; +0F9C;TIBETAN SUBJOINED LETTER DDA;Mn;0;NSM;;;;;N;;;;; +0F9D;TIBETAN SUBJOINED LETTER DDHA;Mn;0;NSM;0F9C 0FB7;;;;N;;;;; +0F9E;TIBETAN SUBJOINED LETTER NNA;Mn;0;NSM;;;;;N;;;;; +0F9F;TIBETAN SUBJOINED LETTER TA;Mn;0;NSM;;;;;N;;;;; +0FA0;TIBETAN SUBJOINED LETTER THA;Mn;0;NSM;;;;;N;;;;; +0FA1;TIBETAN SUBJOINED LETTER DA;Mn;0;NSM;;;;;N;;;;; +0FA2;TIBETAN SUBJOINED LETTER DHA;Mn;0;NSM;0FA1 0FB7;;;;N;;;;; +0FA3;TIBETAN SUBJOINED LETTER NA;Mn;0;NSM;;;;;N;;;;; +0FA4;TIBETAN SUBJOINED LETTER PA;Mn;0;NSM;;;;;N;;;;; +0FA5;TIBETAN SUBJOINED LETTER PHA;Mn;0;NSM;;;;;N;;;;; +0FA6;TIBETAN SUBJOINED LETTER BA;Mn;0;NSM;;;;;N;;;;; +0FA7;TIBETAN SUBJOINED LETTER BHA;Mn;0;NSM;0FA6 0FB7;;;;N;;;;; +0FA8;TIBETAN SUBJOINED LETTER MA;Mn;0;NSM;;;;;N;;;;; +0FA9;TIBETAN SUBJOINED LETTER TSA;Mn;0;NSM;;;;;N;;;;; +0FAA;TIBETAN SUBJOINED LETTER TSHA;Mn;0;NSM;;;;;N;;;;; +0FAB;TIBETAN SUBJOINED LETTER DZA;Mn;0;NSM;;;;;N;;;;; +0FAC;TIBETAN SUBJOINED LETTER DZHA;Mn;0;NSM;0FAB 0FB7;;;;N;;;;; +0FAD;TIBETAN SUBJOINED LETTER WA;Mn;0;NSM;;;;;N;;;;; +0FAE;TIBETAN SUBJOINED LETTER ZHA;Mn;0;NSM;;;;;N;;;;; +0FAF;TIBETAN SUBJOINED LETTER ZA;Mn;0;NSM;;;;;N;;;;; +0FB0;TIBETAN SUBJOINED LETTER -A;Mn;0;NSM;;;;;N;;;;; +0FB1;TIBETAN SUBJOINED LETTER YA;Mn;0;NSM;;;;;N;;;;; +0FB2;TIBETAN SUBJOINED LETTER RA;Mn;0;NSM;;;;;N;;;;; +0FB3;TIBETAN SUBJOINED LETTER LA;Mn;0;NSM;;;;;N;;;;; +0FB4;TIBETAN SUBJOINED LETTER SHA;Mn;0;NSM;;;;;N;;;;; +0FB5;TIBETAN SUBJOINED LETTER SSA;Mn;0;NSM;;;;;N;;;;; +0FB6;TIBETAN SUBJOINED LETTER SA;Mn;0;NSM;;;;;N;;;;; +0FB7;TIBETAN SUBJOINED LETTER HA;Mn;0;NSM;;;;;N;;;;; +0FB8;TIBETAN SUBJOINED LETTER A;Mn;0;NSM;;;;;N;;;;; +0FB9;TIBETAN SUBJOINED LETTER KSSA;Mn;0;NSM;0F90 0FB5;;;;N;;;;; +0FBA;TIBETAN SUBJOINED LETTER FIXED-FORM WA;Mn;0;NSM;;;;;N;;;;; +0FBB;TIBETAN SUBJOINED LETTER FIXED-FORM YA;Mn;0;NSM;;;;;N;;;;; +0FBC;TIBETAN SUBJOINED LETTER FIXED-FORM RA;Mn;0;NSM;;;;;N;;;;; +0FBE;TIBETAN KU RU KHA;So;0;L;;;;;N;;;;; +0FBF;TIBETAN KU RU KHA BZHI MIG CAN;So;0;L;;;;;N;;;;; +0FC0;TIBETAN CANTILLATION SIGN HEAVY BEAT;So;0;L;;;;;N;;;;; +0FC1;TIBETAN CANTILLATION SIGN LIGHT BEAT;So;0;L;;;;;N;;;;; +0FC2;TIBETAN CANTILLATION SIGN CANG TE-U;So;0;L;;;;;N;;;;; +0FC3;TIBETAN CANTILLATION SIGN SBUB -CHAL;So;0;L;;;;;N;;;;; +0FC4;TIBETAN SYMBOL DRIL BU;So;0;L;;;;;N;;;;; +0FC5;TIBETAN SYMBOL RDO RJE;So;0;L;;;;;N;;;;; +0FC6;TIBETAN SYMBOL PADMA GDAN;Mn;220;NSM;;;;;N;;;;; +0FC7;TIBETAN SYMBOL RDO RJE RGYA GRAM;So;0;L;;;;;N;;;;; +0FC8;TIBETAN SYMBOL PHUR PA;So;0;L;;;;;N;;;;; +0FC9;TIBETAN SYMBOL NOR BU;So;0;L;;;;;N;;;;; +0FCA;TIBETAN SYMBOL NOR BU NYIS -KHYIL;So;0;L;;;;;N;;;;; +0FCB;TIBETAN SYMBOL NOR BU GSUM -KHYIL;So;0;L;;;;;N;;;;; +0FCC;TIBETAN SYMBOL NOR BU BZHI -KHYIL;So;0;L;;;;;N;;;;; +0FCE;TIBETAN SIGN RDEL NAG RDEL DKAR;So;0;L;;;;;N;;;;; +0FCF;TIBETAN SIGN RDEL NAG GSUM;So;0;L;;;;;N;;;;; +0FD0;TIBETAN MARK BSKA- SHOG GI MGO RGYAN;Po;0;L;;;;;N;;;;; +0FD1;TIBETAN MARK MNYAM YIG GI MGO RGYAN;Po;0;L;;;;;N;;;;; +0FD2;TIBETAN MARK NYIS TSHEG;Po;0;L;;;;;N;;;;; +0FD3;TIBETAN MARK INITIAL BRDA RNYING YIG MGO MDUN MA;Po;0;L;;;;;N;;;;; +0FD4;TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA;Po;0;L;;;;;N;;;;; +0FD5;RIGHT-FACING SVASTI SIGN;So;0;L;;;;;N;;;;; +0FD6;LEFT-FACING SVASTI SIGN;So;0;L;;;;;N;;;;; +0FD7;RIGHT-FACING SVASTI SIGN WITH DOTS;So;0;L;;;;;N;;;;; +0FD8;LEFT-FACING SVASTI SIGN WITH DOTS;So;0;L;;;;;N;;;;; +0FD9;TIBETAN MARK LEADING MCHAN RTAGS;Po;0;L;;;;;N;;;;; +0FDA;TIBETAN MARK TRAILING MCHAN RTAGS;Po;0;L;;;;;N;;;;; +1000;MYANMAR LETTER KA;Lo;0;L;;;;;N;;;;; +1001;MYANMAR LETTER KHA;Lo;0;L;;;;;N;;;;; +1002;MYANMAR LETTER GA;Lo;0;L;;;;;N;;;;; +1003;MYANMAR LETTER GHA;Lo;0;L;;;;;N;;;;; +1004;MYANMAR LETTER NGA;Lo;0;L;;;;;N;;;;; +1005;MYANMAR LETTER CA;Lo;0;L;;;;;N;;;;; +1006;MYANMAR LETTER CHA;Lo;0;L;;;;;N;;;;; +1007;MYANMAR LETTER JA;Lo;0;L;;;;;N;;;;; +1008;MYANMAR LETTER JHA;Lo;0;L;;;;;N;;;;; +1009;MYANMAR LETTER NYA;Lo;0;L;;;;;N;;;;; +100A;MYANMAR LETTER NNYA;Lo;0;L;;;;;N;;;;; +100B;MYANMAR LETTER TTA;Lo;0;L;;;;;N;;;;; +100C;MYANMAR LETTER TTHA;Lo;0;L;;;;;N;;;;; +100D;MYANMAR LETTER DDA;Lo;0;L;;;;;N;;;;; +100E;MYANMAR LETTER DDHA;Lo;0;L;;;;;N;;;;; +100F;MYANMAR LETTER NNA;Lo;0;L;;;;;N;;;;; +1010;MYANMAR LETTER TA;Lo;0;L;;;;;N;;;;; +1011;MYANMAR LETTER THA;Lo;0;L;;;;;N;;;;; +1012;MYANMAR LETTER DA;Lo;0;L;;;;;N;;;;; +1013;MYANMAR LETTER DHA;Lo;0;L;;;;;N;;;;; +1014;MYANMAR LETTER NA;Lo;0;L;;;;;N;;;;; +1015;MYANMAR LETTER PA;Lo;0;L;;;;;N;;;;; +1016;MYANMAR LETTER PHA;Lo;0;L;;;;;N;;;;; +1017;MYANMAR LETTER BA;Lo;0;L;;;;;N;;;;; +1018;MYANMAR LETTER BHA;Lo;0;L;;;;;N;;;;; +1019;MYANMAR LETTER MA;Lo;0;L;;;;;N;;;;; +101A;MYANMAR LETTER YA;Lo;0;L;;;;;N;;;;; +101B;MYANMAR LETTER RA;Lo;0;L;;;;;N;;;;; +101C;MYANMAR LETTER LA;Lo;0;L;;;;;N;;;;; +101D;MYANMAR LETTER WA;Lo;0;L;;;;;N;;;;; +101E;MYANMAR LETTER SA;Lo;0;L;;;;;N;;;;; +101F;MYANMAR LETTER HA;Lo;0;L;;;;;N;;;;; +1020;MYANMAR LETTER LLA;Lo;0;L;;;;;N;;;;; +1021;MYANMAR LETTER A;Lo;0;L;;;;;N;;;;; +1022;MYANMAR LETTER SHAN A;Lo;0;L;;;;;N;;;;; +1023;MYANMAR LETTER I;Lo;0;L;;;;;N;;;;; +1024;MYANMAR LETTER II;Lo;0;L;;;;;N;;;;; +1025;MYANMAR LETTER U;Lo;0;L;;;;;N;;;;; +1026;MYANMAR LETTER UU;Lo;0;L;1025 102E;;;;N;;;;; +1027;MYANMAR LETTER E;Lo;0;L;;;;;N;;;;; +1028;MYANMAR LETTER MON E;Lo;0;L;;;;;N;;;;; +1029;MYANMAR LETTER O;Lo;0;L;;;;;N;;;;; +102A;MYANMAR LETTER AU;Lo;0;L;;;;;N;;;;; +102B;MYANMAR VOWEL SIGN TALL AA;Mc;0;L;;;;;N;;;;; +102C;MYANMAR VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +102D;MYANMAR VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +102E;MYANMAR VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +102F;MYANMAR VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1030;MYANMAR VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +1031;MYANMAR VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +1032;MYANMAR VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +1033;MYANMAR VOWEL SIGN MON II;Mn;0;NSM;;;;;N;;;;; +1034;MYANMAR VOWEL SIGN MON O;Mn;0;NSM;;;;;N;;;;; +1035;MYANMAR VOWEL SIGN E ABOVE;Mn;0;NSM;;;;;N;;;;; +1036;MYANMAR SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +1037;MYANMAR SIGN DOT BELOW;Mn;7;NSM;;;;;N;;;;; +1038;MYANMAR SIGN VISARGA;Mc;0;L;;;;;N;;;;; +1039;MYANMAR SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +103A;MYANMAR SIGN ASAT;Mn;9;NSM;;;;;N;;;;; +103B;MYANMAR CONSONANT SIGN MEDIAL YA;Mc;0;L;;;;;N;;;;; +103C;MYANMAR CONSONANT SIGN MEDIAL RA;Mc;0;L;;;;;N;;;;; +103D;MYANMAR CONSONANT SIGN MEDIAL WA;Mn;0;NSM;;;;;N;;;;; +103E;MYANMAR CONSONANT SIGN MEDIAL HA;Mn;0;NSM;;;;;N;;;;; +103F;MYANMAR LETTER GREAT SA;Lo;0;L;;;;;N;;;;; +1040;MYANMAR DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1041;MYANMAR DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1042;MYANMAR DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1043;MYANMAR DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1044;MYANMAR DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1045;MYANMAR DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1046;MYANMAR DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1047;MYANMAR DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1048;MYANMAR DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1049;MYANMAR DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +104A;MYANMAR SIGN LITTLE SECTION;Po;0;L;;;;;N;;;;; +104B;MYANMAR SIGN SECTION;Po;0;L;;;;;N;;;;; +104C;MYANMAR SYMBOL LOCATIVE;Po;0;L;;;;;N;;;;; +104D;MYANMAR SYMBOL COMPLETED;Po;0;L;;;;;N;;;;; +104E;MYANMAR SYMBOL AFOREMENTIONED;Po;0;L;;;;;N;;;;; +104F;MYANMAR SYMBOL GENITIVE;Po;0;L;;;;;N;;;;; +1050;MYANMAR LETTER SHA;Lo;0;L;;;;;N;;;;; +1051;MYANMAR LETTER SSA;Lo;0;L;;;;;N;;;;; +1052;MYANMAR LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +1053;MYANMAR LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +1054;MYANMAR LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +1055;MYANMAR LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +1056;MYANMAR VOWEL SIGN VOCALIC R;Mc;0;L;;;;;N;;;;; +1057;MYANMAR VOWEL SIGN VOCALIC RR;Mc;0;L;;;;;N;;;;; +1058;MYANMAR VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +1059;MYANMAR VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +105A;MYANMAR LETTER MON NGA;Lo;0;L;;;;;N;;;;; +105B;MYANMAR LETTER MON JHA;Lo;0;L;;;;;N;;;;; +105C;MYANMAR LETTER MON BBA;Lo;0;L;;;;;N;;;;; +105D;MYANMAR LETTER MON BBE;Lo;0;L;;;;;N;;;;; +105E;MYANMAR CONSONANT SIGN MON MEDIAL NA;Mn;0;NSM;;;;;N;;;;; +105F;MYANMAR CONSONANT SIGN MON MEDIAL MA;Mn;0;NSM;;;;;N;;;;; +1060;MYANMAR CONSONANT SIGN MON MEDIAL LA;Mn;0;NSM;;;;;N;;;;; +1061;MYANMAR LETTER SGAW KAREN SHA;Lo;0;L;;;;;N;;;;; +1062;MYANMAR VOWEL SIGN SGAW KAREN EU;Mc;0;L;;;;;N;;;;; +1063;MYANMAR TONE MARK SGAW KAREN HATHI;Mc;0;L;;;;;N;;;;; +1064;MYANMAR TONE MARK SGAW KAREN KE PHO;Mc;0;L;;;;;N;;;;; +1065;MYANMAR LETTER WESTERN PWO KAREN THA;Lo;0;L;;;;;N;;;;; +1066;MYANMAR LETTER WESTERN PWO KAREN PWA;Lo;0;L;;;;;N;;;;; +1067;MYANMAR VOWEL SIGN WESTERN PWO KAREN EU;Mc;0;L;;;;;N;;;;; +1068;MYANMAR VOWEL SIGN WESTERN PWO KAREN UE;Mc;0;L;;;;;N;;;;; +1069;MYANMAR SIGN WESTERN PWO KAREN TONE-1;Mc;0;L;;;;;N;;;;; +106A;MYANMAR SIGN WESTERN PWO KAREN TONE-2;Mc;0;L;;;;;N;;;;; +106B;MYANMAR SIGN WESTERN PWO KAREN TONE-3;Mc;0;L;;;;;N;;;;; +106C;MYANMAR SIGN WESTERN PWO KAREN TONE-4;Mc;0;L;;;;;N;;;;; +106D;MYANMAR SIGN WESTERN PWO KAREN TONE-5;Mc;0;L;;;;;N;;;;; +106E;MYANMAR LETTER EASTERN PWO KAREN NNA;Lo;0;L;;;;;N;;;;; +106F;MYANMAR LETTER EASTERN PWO KAREN YWA;Lo;0;L;;;;;N;;;;; +1070;MYANMAR LETTER EASTERN PWO KAREN GHWA;Lo;0;L;;;;;N;;;;; +1071;MYANMAR VOWEL SIGN GEBA KAREN I;Mn;0;NSM;;;;;N;;;;; +1072;MYANMAR VOWEL SIGN KAYAH OE;Mn;0;NSM;;;;;N;;;;; +1073;MYANMAR VOWEL SIGN KAYAH U;Mn;0;NSM;;;;;N;;;;; +1074;MYANMAR VOWEL SIGN KAYAH EE;Mn;0;NSM;;;;;N;;;;; +1075;MYANMAR LETTER SHAN KA;Lo;0;L;;;;;N;;;;; +1076;MYANMAR LETTER SHAN KHA;Lo;0;L;;;;;N;;;;; +1077;MYANMAR LETTER SHAN GA;Lo;0;L;;;;;N;;;;; +1078;MYANMAR LETTER SHAN CA;Lo;0;L;;;;;N;;;;; +1079;MYANMAR LETTER SHAN ZA;Lo;0;L;;;;;N;;;;; +107A;MYANMAR LETTER SHAN NYA;Lo;0;L;;;;;N;;;;; +107B;MYANMAR LETTER SHAN DA;Lo;0;L;;;;;N;;;;; +107C;MYANMAR LETTER SHAN NA;Lo;0;L;;;;;N;;;;; +107D;MYANMAR LETTER SHAN PHA;Lo;0;L;;;;;N;;;;; +107E;MYANMAR LETTER SHAN FA;Lo;0;L;;;;;N;;;;; +107F;MYANMAR LETTER SHAN BA;Lo;0;L;;;;;N;;;;; +1080;MYANMAR LETTER SHAN THA;Lo;0;L;;;;;N;;;;; +1081;MYANMAR LETTER SHAN HA;Lo;0;L;;;;;N;;;;; +1082;MYANMAR CONSONANT SIGN SHAN MEDIAL WA;Mn;0;NSM;;;;;N;;;;; +1083;MYANMAR VOWEL SIGN SHAN AA;Mc;0;L;;;;;N;;;;; +1084;MYANMAR VOWEL SIGN SHAN E;Mc;0;L;;;;;N;;;;; +1085;MYANMAR VOWEL SIGN SHAN E ABOVE;Mn;0;NSM;;;;;N;;;;; +1086;MYANMAR VOWEL SIGN SHAN FINAL Y;Mn;0;NSM;;;;;N;;;;; +1087;MYANMAR SIGN SHAN TONE-2;Mc;0;L;;;;;N;;;;; +1088;MYANMAR SIGN SHAN TONE-3;Mc;0;L;;;;;N;;;;; +1089;MYANMAR SIGN SHAN TONE-5;Mc;0;L;;;;;N;;;;; +108A;MYANMAR SIGN SHAN TONE-6;Mc;0;L;;;;;N;;;;; +108B;MYANMAR SIGN SHAN COUNCIL TONE-2;Mc;0;L;;;;;N;;;;; +108C;MYANMAR SIGN SHAN COUNCIL TONE-3;Mc;0;L;;;;;N;;;;; +108D;MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE;Mn;220;NSM;;;;;N;;;;; +108E;MYANMAR LETTER RUMAI PALAUNG FA;Lo;0;L;;;;;N;;;;; +108F;MYANMAR SIGN RUMAI PALAUNG TONE-5;Mc;0;L;;;;;N;;;;; +1090;MYANMAR SHAN DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1091;MYANMAR SHAN DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1092;MYANMAR SHAN DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1093;MYANMAR SHAN DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1094;MYANMAR SHAN DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1095;MYANMAR SHAN DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1096;MYANMAR SHAN DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1097;MYANMAR SHAN DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1098;MYANMAR SHAN DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1099;MYANMAR SHAN DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +109A;MYANMAR SIGN KHAMTI TONE-1;Mc;0;L;;;;;N;;;;; +109B;MYANMAR SIGN KHAMTI TONE-3;Mc;0;L;;;;;N;;;;; +109C;MYANMAR VOWEL SIGN AITON A;Mc;0;L;;;;;N;;;;; +109D;MYANMAR VOWEL SIGN AITON AI;Mn;0;NSM;;;;;N;;;;; +109E;MYANMAR SYMBOL SHAN ONE;So;0;L;;;;;N;;;;; +109F;MYANMAR SYMBOL SHAN EXCLAMATION;So;0;L;;;;;N;;;;; +10A0;GEORGIAN CAPITAL LETTER AN;Lu;0;L;;;;;N;;;;2D00; +10A1;GEORGIAN CAPITAL LETTER BAN;Lu;0;L;;;;;N;;;;2D01; +10A2;GEORGIAN CAPITAL LETTER GAN;Lu;0;L;;;;;N;;;;2D02; +10A3;GEORGIAN CAPITAL LETTER DON;Lu;0;L;;;;;N;;;;2D03; +10A4;GEORGIAN CAPITAL LETTER EN;Lu;0;L;;;;;N;;;;2D04; +10A5;GEORGIAN CAPITAL LETTER VIN;Lu;0;L;;;;;N;;;;2D05; +10A6;GEORGIAN CAPITAL LETTER ZEN;Lu;0;L;;;;;N;;;;2D06; +10A7;GEORGIAN CAPITAL LETTER TAN;Lu;0;L;;;;;N;;;;2D07; +10A8;GEORGIAN CAPITAL LETTER IN;Lu;0;L;;;;;N;;;;2D08; +10A9;GEORGIAN CAPITAL LETTER KAN;Lu;0;L;;;;;N;;;;2D09; +10AA;GEORGIAN CAPITAL LETTER LAS;Lu;0;L;;;;;N;;;;2D0A; +10AB;GEORGIAN CAPITAL LETTER MAN;Lu;0;L;;;;;N;;;;2D0B; +10AC;GEORGIAN CAPITAL LETTER NAR;Lu;0;L;;;;;N;;;;2D0C; +10AD;GEORGIAN CAPITAL LETTER ON;Lu;0;L;;;;;N;;;;2D0D; +10AE;GEORGIAN CAPITAL LETTER PAR;Lu;0;L;;;;;N;;;;2D0E; +10AF;GEORGIAN CAPITAL LETTER ZHAR;Lu;0;L;;;;;N;;;;2D0F; +10B0;GEORGIAN CAPITAL LETTER RAE;Lu;0;L;;;;;N;;;;2D10; +10B1;GEORGIAN CAPITAL LETTER SAN;Lu;0;L;;;;;N;;;;2D11; +10B2;GEORGIAN CAPITAL LETTER TAR;Lu;0;L;;;;;N;;;;2D12; +10B3;GEORGIAN CAPITAL LETTER UN;Lu;0;L;;;;;N;;;;2D13; +10B4;GEORGIAN CAPITAL LETTER PHAR;Lu;0;L;;;;;N;;;;2D14; +10B5;GEORGIAN CAPITAL LETTER KHAR;Lu;0;L;;;;;N;;;;2D15; +10B6;GEORGIAN CAPITAL LETTER GHAN;Lu;0;L;;;;;N;;;;2D16; +10B7;GEORGIAN CAPITAL LETTER QAR;Lu;0;L;;;;;N;;;;2D17; +10B8;GEORGIAN CAPITAL LETTER SHIN;Lu;0;L;;;;;N;;;;2D18; +10B9;GEORGIAN CAPITAL LETTER CHIN;Lu;0;L;;;;;N;;;;2D19; +10BA;GEORGIAN CAPITAL LETTER CAN;Lu;0;L;;;;;N;;;;2D1A; +10BB;GEORGIAN CAPITAL LETTER JIL;Lu;0;L;;;;;N;;;;2D1B; +10BC;GEORGIAN CAPITAL LETTER CIL;Lu;0;L;;;;;N;;;;2D1C; +10BD;GEORGIAN CAPITAL LETTER CHAR;Lu;0;L;;;;;N;;;;2D1D; +10BE;GEORGIAN CAPITAL LETTER XAN;Lu;0;L;;;;;N;;;;2D1E; +10BF;GEORGIAN CAPITAL LETTER JHAN;Lu;0;L;;;;;N;;;;2D1F; +10C0;GEORGIAN CAPITAL LETTER HAE;Lu;0;L;;;;;N;;;;2D20; +10C1;GEORGIAN CAPITAL LETTER HE;Lu;0;L;;;;;N;;;;2D21; +10C2;GEORGIAN CAPITAL LETTER HIE;Lu;0;L;;;;;N;;;;2D22; +10C3;GEORGIAN CAPITAL LETTER WE;Lu;0;L;;;;;N;;;;2D23; +10C4;GEORGIAN CAPITAL LETTER HAR;Lu;0;L;;;;;N;;;;2D24; +10C5;GEORGIAN CAPITAL LETTER HOE;Lu;0;L;;;;;N;;;;2D25; +10C7;GEORGIAN CAPITAL LETTER YN;Lu;0;L;;;;;N;;;;2D27; +10CD;GEORGIAN CAPITAL LETTER AEN;Lu;0;L;;;;;N;;;;2D2D; +10D0;GEORGIAN LETTER AN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER AN;;;; +10D1;GEORGIAN LETTER BAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER BAN;;;; +10D2;GEORGIAN LETTER GAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER GAN;;;; +10D3;GEORGIAN LETTER DON;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER DON;;;; +10D4;GEORGIAN LETTER EN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER EN;;;; +10D5;GEORGIAN LETTER VIN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER VIN;;;; +10D6;GEORGIAN LETTER ZEN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER ZEN;;;; +10D7;GEORGIAN LETTER TAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER TAN;;;; +10D8;GEORGIAN LETTER IN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER IN;;;; +10D9;GEORGIAN LETTER KAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER KAN;;;; +10DA;GEORGIAN LETTER LAS;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER LAS;;;; +10DB;GEORGIAN LETTER MAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER MAN;;;; +10DC;GEORGIAN LETTER NAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER NAR;;;; +10DD;GEORGIAN LETTER ON;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER ON;;;; +10DE;GEORGIAN LETTER PAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER PAR;;;; +10DF;GEORGIAN LETTER ZHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER ZHAR;;;; +10E0;GEORGIAN LETTER RAE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER RAE;;;; +10E1;GEORGIAN LETTER SAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER SAN;;;; +10E2;GEORGIAN LETTER TAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER TAR;;;; +10E3;GEORGIAN LETTER UN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER UN;;;; +10E4;GEORGIAN LETTER PHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER PHAR;;;; +10E5;GEORGIAN LETTER KHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER KHAR;;;; +10E6;GEORGIAN LETTER GHAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER GHAN;;;; +10E7;GEORGIAN LETTER QAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER QAR;;;; +10E8;GEORGIAN LETTER SHIN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER SHIN;;;; +10E9;GEORGIAN LETTER CHIN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CHIN;;;; +10EA;GEORGIAN LETTER CAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CAN;;;; +10EB;GEORGIAN LETTER JIL;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER JIL;;;; +10EC;GEORGIAN LETTER CIL;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CIL;;;; +10ED;GEORGIAN LETTER CHAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER CHAR;;;; +10EE;GEORGIAN LETTER XAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER XAN;;;; +10EF;GEORGIAN LETTER JHAN;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER JHAN;;;; +10F0;GEORGIAN LETTER HAE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HAE;;;; +10F1;GEORGIAN LETTER HE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HE;;;; +10F2;GEORGIAN LETTER HIE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HIE;;;; +10F3;GEORGIAN LETTER WE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER WE;;;; +10F4;GEORGIAN LETTER HAR;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HAR;;;; +10F5;GEORGIAN LETTER HOE;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER HOE;;;; +10F6;GEORGIAN LETTER FI;Lo;0;L;;;;;N;GEORGIAN SMALL LETTER FI;;;; +10F7;GEORGIAN LETTER YN;Lo;0;L;;;;;N;;;;; +10F8;GEORGIAN LETTER ELIFI;Lo;0;L;;;;;N;;;;; +10F9;GEORGIAN LETTER TURNED GAN;Lo;0;L;;;;;N;;;;; +10FA;GEORGIAN LETTER AIN;Lo;0;L;;;;;N;;;;; +10FB;GEORGIAN PARAGRAPH SEPARATOR;Po;0;L;;;;;N;;;;; +10FC;MODIFIER LETTER GEORGIAN NAR;Lm;0;L; 10DC;;;;N;;;;; +10FD;GEORGIAN LETTER AEN;Lo;0;L;;;;;N;;;;; +10FE;GEORGIAN LETTER HARD SIGN;Lo;0;L;;;;;N;;;;; +10FF;GEORGIAN LETTER LABIAL SIGN;Lo;0;L;;;;;N;;;;; +1100;HANGUL CHOSEONG KIYEOK;Lo;0;L;;;;;N;;;;; +1101;HANGUL CHOSEONG SSANGKIYEOK;Lo;0;L;;;;;N;;;;; +1102;HANGUL CHOSEONG NIEUN;Lo;0;L;;;;;N;;;;; +1103;HANGUL CHOSEONG TIKEUT;Lo;0;L;;;;;N;;;;; +1104;HANGUL CHOSEONG SSANGTIKEUT;Lo;0;L;;;;;N;;;;; +1105;HANGUL CHOSEONG RIEUL;Lo;0;L;;;;;N;;;;; +1106;HANGUL CHOSEONG MIEUM;Lo;0;L;;;;;N;;;;; +1107;HANGUL CHOSEONG PIEUP;Lo;0;L;;;;;N;;;;; +1108;HANGUL CHOSEONG SSANGPIEUP;Lo;0;L;;;;;N;;;;; +1109;HANGUL CHOSEONG SIOS;Lo;0;L;;;;;N;;;;; +110A;HANGUL CHOSEONG SSANGSIOS;Lo;0;L;;;;;N;;;;; +110B;HANGUL CHOSEONG IEUNG;Lo;0;L;;;;;N;;;;; +110C;HANGUL CHOSEONG CIEUC;Lo;0;L;;;;;N;;;;; +110D;HANGUL CHOSEONG SSANGCIEUC;Lo;0;L;;;;;N;;;;; +110E;HANGUL CHOSEONG CHIEUCH;Lo;0;L;;;;;N;;;;; +110F;HANGUL CHOSEONG KHIEUKH;Lo;0;L;;;;;N;;;;; +1110;HANGUL CHOSEONG THIEUTH;Lo;0;L;;;;;N;;;;; +1111;HANGUL CHOSEONG PHIEUPH;Lo;0;L;;;;;N;;;;; +1112;HANGUL CHOSEONG HIEUH;Lo;0;L;;;;;N;;;;; +1113;HANGUL CHOSEONG NIEUN-KIYEOK;Lo;0;L;;;;;N;;;;; +1114;HANGUL CHOSEONG SSANGNIEUN;Lo;0;L;;;;;N;;;;; +1115;HANGUL CHOSEONG NIEUN-TIKEUT;Lo;0;L;;;;;N;;;;; +1116;HANGUL CHOSEONG NIEUN-PIEUP;Lo;0;L;;;;;N;;;;; +1117;HANGUL CHOSEONG TIKEUT-KIYEOK;Lo;0;L;;;;;N;;;;; +1118;HANGUL CHOSEONG RIEUL-NIEUN;Lo;0;L;;;;;N;;;;; +1119;HANGUL CHOSEONG SSANGRIEUL;Lo;0;L;;;;;N;;;;; +111A;HANGUL CHOSEONG RIEUL-HIEUH;Lo;0;L;;;;;N;;;;; +111B;HANGUL CHOSEONG KAPYEOUNRIEUL;Lo;0;L;;;;;N;;;;; +111C;HANGUL CHOSEONG MIEUM-PIEUP;Lo;0;L;;;;;N;;;;; +111D;HANGUL CHOSEONG KAPYEOUNMIEUM;Lo;0;L;;;;;N;;;;; +111E;HANGUL CHOSEONG PIEUP-KIYEOK;Lo;0;L;;;;;N;;;;; +111F;HANGUL CHOSEONG PIEUP-NIEUN;Lo;0;L;;;;;N;;;;; +1120;HANGUL CHOSEONG PIEUP-TIKEUT;Lo;0;L;;;;;N;;;;; +1121;HANGUL CHOSEONG PIEUP-SIOS;Lo;0;L;;;;;N;;;;; +1122;HANGUL CHOSEONG PIEUP-SIOS-KIYEOK;Lo;0;L;;;;;N;;;;; +1123;HANGUL CHOSEONG PIEUP-SIOS-TIKEUT;Lo;0;L;;;;;N;;;;; +1124;HANGUL CHOSEONG PIEUP-SIOS-PIEUP;Lo;0;L;;;;;N;;;;; +1125;HANGUL CHOSEONG PIEUP-SSANGSIOS;Lo;0;L;;;;;N;;;;; +1126;HANGUL CHOSEONG PIEUP-SIOS-CIEUC;Lo;0;L;;;;;N;;;;; +1127;HANGUL CHOSEONG PIEUP-CIEUC;Lo;0;L;;;;;N;;;;; +1128;HANGUL CHOSEONG PIEUP-CHIEUCH;Lo;0;L;;;;;N;;;;; +1129;HANGUL CHOSEONG PIEUP-THIEUTH;Lo;0;L;;;;;N;;;;; +112A;HANGUL CHOSEONG PIEUP-PHIEUPH;Lo;0;L;;;;;N;;;;; +112B;HANGUL CHOSEONG KAPYEOUNPIEUP;Lo;0;L;;;;;N;;;;; +112C;HANGUL CHOSEONG KAPYEOUNSSANGPIEUP;Lo;0;L;;;;;N;;;;; +112D;HANGUL CHOSEONG SIOS-KIYEOK;Lo;0;L;;;;;N;;;;; +112E;HANGUL CHOSEONG SIOS-NIEUN;Lo;0;L;;;;;N;;;;; +112F;HANGUL CHOSEONG SIOS-TIKEUT;Lo;0;L;;;;;N;;;;; +1130;HANGUL CHOSEONG SIOS-RIEUL;Lo;0;L;;;;;N;;;;; +1131;HANGUL CHOSEONG SIOS-MIEUM;Lo;0;L;;;;;N;;;;; +1132;HANGUL CHOSEONG SIOS-PIEUP;Lo;0;L;;;;;N;;;;; +1133;HANGUL CHOSEONG SIOS-PIEUP-KIYEOK;Lo;0;L;;;;;N;;;;; +1134;HANGUL CHOSEONG SIOS-SSANGSIOS;Lo;0;L;;;;;N;;;;; +1135;HANGUL CHOSEONG SIOS-IEUNG;Lo;0;L;;;;;N;;;;; +1136;HANGUL CHOSEONG SIOS-CIEUC;Lo;0;L;;;;;N;;;;; +1137;HANGUL CHOSEONG SIOS-CHIEUCH;Lo;0;L;;;;;N;;;;; +1138;HANGUL CHOSEONG SIOS-KHIEUKH;Lo;0;L;;;;;N;;;;; +1139;HANGUL CHOSEONG SIOS-THIEUTH;Lo;0;L;;;;;N;;;;; +113A;HANGUL CHOSEONG SIOS-PHIEUPH;Lo;0;L;;;;;N;;;;; +113B;HANGUL CHOSEONG SIOS-HIEUH;Lo;0;L;;;;;N;;;;; +113C;HANGUL CHOSEONG CHITUEUMSIOS;Lo;0;L;;;;;N;;;;; +113D;HANGUL CHOSEONG CHITUEUMSSANGSIOS;Lo;0;L;;;;;N;;;;; +113E;HANGUL CHOSEONG CEONGCHIEUMSIOS;Lo;0;L;;;;;N;;;;; +113F;HANGUL CHOSEONG CEONGCHIEUMSSANGSIOS;Lo;0;L;;;;;N;;;;; +1140;HANGUL CHOSEONG PANSIOS;Lo;0;L;;;;;N;;;;; +1141;HANGUL CHOSEONG IEUNG-KIYEOK;Lo;0;L;;;;;N;;;;; +1142;HANGUL CHOSEONG IEUNG-TIKEUT;Lo;0;L;;;;;N;;;;; +1143;HANGUL CHOSEONG IEUNG-MIEUM;Lo;0;L;;;;;N;;;;; +1144;HANGUL CHOSEONG IEUNG-PIEUP;Lo;0;L;;;;;N;;;;; +1145;HANGUL CHOSEONG IEUNG-SIOS;Lo;0;L;;;;;N;;;;; +1146;HANGUL CHOSEONG IEUNG-PANSIOS;Lo;0;L;;;;;N;;;;; +1147;HANGUL CHOSEONG SSANGIEUNG;Lo;0;L;;;;;N;;;;; +1148;HANGUL CHOSEONG IEUNG-CIEUC;Lo;0;L;;;;;N;;;;; +1149;HANGUL CHOSEONG IEUNG-CHIEUCH;Lo;0;L;;;;;N;;;;; +114A;HANGUL CHOSEONG IEUNG-THIEUTH;Lo;0;L;;;;;N;;;;; +114B;HANGUL CHOSEONG IEUNG-PHIEUPH;Lo;0;L;;;;;N;;;;; +114C;HANGUL CHOSEONG YESIEUNG;Lo;0;L;;;;;N;;;;; +114D;HANGUL CHOSEONG CIEUC-IEUNG;Lo;0;L;;;;;N;;;;; +114E;HANGUL CHOSEONG CHITUEUMCIEUC;Lo;0;L;;;;;N;;;;; +114F;HANGUL CHOSEONG CHITUEUMSSANGCIEUC;Lo;0;L;;;;;N;;;;; +1150;HANGUL CHOSEONG CEONGCHIEUMCIEUC;Lo;0;L;;;;;N;;;;; +1151;HANGUL CHOSEONG CEONGCHIEUMSSANGCIEUC;Lo;0;L;;;;;N;;;;; +1152;HANGUL CHOSEONG CHIEUCH-KHIEUKH;Lo;0;L;;;;;N;;;;; +1153;HANGUL CHOSEONG CHIEUCH-HIEUH;Lo;0;L;;;;;N;;;;; +1154;HANGUL CHOSEONG CHITUEUMCHIEUCH;Lo;0;L;;;;;N;;;;; +1155;HANGUL CHOSEONG CEONGCHIEUMCHIEUCH;Lo;0;L;;;;;N;;;;; +1156;HANGUL CHOSEONG PHIEUPH-PIEUP;Lo;0;L;;;;;N;;;;; +1157;HANGUL CHOSEONG KAPYEOUNPHIEUPH;Lo;0;L;;;;;N;;;;; +1158;HANGUL CHOSEONG SSANGHIEUH;Lo;0;L;;;;;N;;;;; +1159;HANGUL CHOSEONG YEORINHIEUH;Lo;0;L;;;;;N;;;;; +115A;HANGUL CHOSEONG KIYEOK-TIKEUT;Lo;0;L;;;;;N;;;;; +115B;HANGUL CHOSEONG NIEUN-SIOS;Lo;0;L;;;;;N;;;;; +115C;HANGUL CHOSEONG NIEUN-CIEUC;Lo;0;L;;;;;N;;;;; +115D;HANGUL CHOSEONG NIEUN-HIEUH;Lo;0;L;;;;;N;;;;; +115E;HANGUL CHOSEONG TIKEUT-RIEUL;Lo;0;L;;;;;N;;;;; +115F;HANGUL CHOSEONG FILLER;Lo;0;L;;;;;N;;;;; +1160;HANGUL JUNGSEONG FILLER;Lo;0;L;;;;;N;;;;; +1161;HANGUL JUNGSEONG A;Lo;0;L;;;;;N;;;;; +1162;HANGUL JUNGSEONG AE;Lo;0;L;;;;;N;;;;; +1163;HANGUL JUNGSEONG YA;Lo;0;L;;;;;N;;;;; +1164;HANGUL JUNGSEONG YAE;Lo;0;L;;;;;N;;;;; +1165;HANGUL JUNGSEONG EO;Lo;0;L;;;;;N;;;;; +1166;HANGUL JUNGSEONG E;Lo;0;L;;;;;N;;;;; +1167;HANGUL JUNGSEONG YEO;Lo;0;L;;;;;N;;;;; +1168;HANGUL JUNGSEONG YE;Lo;0;L;;;;;N;;;;; +1169;HANGUL JUNGSEONG O;Lo;0;L;;;;;N;;;;; +116A;HANGUL JUNGSEONG WA;Lo;0;L;;;;;N;;;;; +116B;HANGUL JUNGSEONG WAE;Lo;0;L;;;;;N;;;;; +116C;HANGUL JUNGSEONG OE;Lo;0;L;;;;;N;;;;; +116D;HANGUL JUNGSEONG YO;Lo;0;L;;;;;N;;;;; +116E;HANGUL JUNGSEONG U;Lo;0;L;;;;;N;;;;; +116F;HANGUL JUNGSEONG WEO;Lo;0;L;;;;;N;;;;; +1170;HANGUL JUNGSEONG WE;Lo;0;L;;;;;N;;;;; +1171;HANGUL JUNGSEONG WI;Lo;0;L;;;;;N;;;;; +1172;HANGUL JUNGSEONG YU;Lo;0;L;;;;;N;;;;; +1173;HANGUL JUNGSEONG EU;Lo;0;L;;;;;N;;;;; +1174;HANGUL JUNGSEONG YI;Lo;0;L;;;;;N;;;;; +1175;HANGUL JUNGSEONG I;Lo;0;L;;;;;N;;;;; +1176;HANGUL JUNGSEONG A-O;Lo;0;L;;;;;N;;;;; +1177;HANGUL JUNGSEONG A-U;Lo;0;L;;;;;N;;;;; +1178;HANGUL JUNGSEONG YA-O;Lo;0;L;;;;;N;;;;; +1179;HANGUL JUNGSEONG YA-YO;Lo;0;L;;;;;N;;;;; +117A;HANGUL JUNGSEONG EO-O;Lo;0;L;;;;;N;;;;; +117B;HANGUL JUNGSEONG EO-U;Lo;0;L;;;;;N;;;;; +117C;HANGUL JUNGSEONG EO-EU;Lo;0;L;;;;;N;;;;; +117D;HANGUL JUNGSEONG YEO-O;Lo;0;L;;;;;N;;;;; +117E;HANGUL JUNGSEONG YEO-U;Lo;0;L;;;;;N;;;;; +117F;HANGUL JUNGSEONG O-EO;Lo;0;L;;;;;N;;;;; +1180;HANGUL JUNGSEONG O-E;Lo;0;L;;;;;N;;;;; +1181;HANGUL JUNGSEONG O-YE;Lo;0;L;;;;;N;;;;; +1182;HANGUL JUNGSEONG O-O;Lo;0;L;;;;;N;;;;; +1183;HANGUL JUNGSEONG O-U;Lo;0;L;;;;;N;;;;; +1184;HANGUL JUNGSEONG YO-YA;Lo;0;L;;;;;N;;;;; +1185;HANGUL JUNGSEONG YO-YAE;Lo;0;L;;;;;N;;;;; +1186;HANGUL JUNGSEONG YO-YEO;Lo;0;L;;;;;N;;;;; +1187;HANGUL JUNGSEONG YO-O;Lo;0;L;;;;;N;;;;; +1188;HANGUL JUNGSEONG YO-I;Lo;0;L;;;;;N;;;;; +1189;HANGUL JUNGSEONG U-A;Lo;0;L;;;;;N;;;;; +118A;HANGUL JUNGSEONG U-AE;Lo;0;L;;;;;N;;;;; +118B;HANGUL JUNGSEONG U-EO-EU;Lo;0;L;;;;;N;;;;; +118C;HANGUL JUNGSEONG U-YE;Lo;0;L;;;;;N;;;;; +118D;HANGUL JUNGSEONG U-U;Lo;0;L;;;;;N;;;;; +118E;HANGUL JUNGSEONG YU-A;Lo;0;L;;;;;N;;;;; +118F;HANGUL JUNGSEONG YU-EO;Lo;0;L;;;;;N;;;;; +1190;HANGUL JUNGSEONG YU-E;Lo;0;L;;;;;N;;;;; +1191;HANGUL JUNGSEONG YU-YEO;Lo;0;L;;;;;N;;;;; +1192;HANGUL JUNGSEONG YU-YE;Lo;0;L;;;;;N;;;;; +1193;HANGUL JUNGSEONG YU-U;Lo;0;L;;;;;N;;;;; +1194;HANGUL JUNGSEONG YU-I;Lo;0;L;;;;;N;;;;; +1195;HANGUL JUNGSEONG EU-U;Lo;0;L;;;;;N;;;;; +1196;HANGUL JUNGSEONG EU-EU;Lo;0;L;;;;;N;;;;; +1197;HANGUL JUNGSEONG YI-U;Lo;0;L;;;;;N;;;;; +1198;HANGUL JUNGSEONG I-A;Lo;0;L;;;;;N;;;;; +1199;HANGUL JUNGSEONG I-YA;Lo;0;L;;;;;N;;;;; +119A;HANGUL JUNGSEONG I-O;Lo;0;L;;;;;N;;;;; +119B;HANGUL JUNGSEONG I-U;Lo;0;L;;;;;N;;;;; +119C;HANGUL JUNGSEONG I-EU;Lo;0;L;;;;;N;;;;; +119D;HANGUL JUNGSEONG I-ARAEA;Lo;0;L;;;;;N;;;;; +119E;HANGUL JUNGSEONG ARAEA;Lo;0;L;;;;;N;;;;; +119F;HANGUL JUNGSEONG ARAEA-EO;Lo;0;L;;;;;N;;;;; +11A0;HANGUL JUNGSEONG ARAEA-U;Lo;0;L;;;;;N;;;;; +11A1;HANGUL JUNGSEONG ARAEA-I;Lo;0;L;;;;;N;;;;; +11A2;HANGUL JUNGSEONG SSANGARAEA;Lo;0;L;;;;;N;;;;; +11A3;HANGUL JUNGSEONG A-EU;Lo;0;L;;;;;N;;;;; +11A4;HANGUL JUNGSEONG YA-U;Lo;0;L;;;;;N;;;;; +11A5;HANGUL JUNGSEONG YEO-YA;Lo;0;L;;;;;N;;;;; +11A6;HANGUL JUNGSEONG O-YA;Lo;0;L;;;;;N;;;;; +11A7;HANGUL JUNGSEONG O-YAE;Lo;0;L;;;;;N;;;;; +11A8;HANGUL JONGSEONG KIYEOK;Lo;0;L;;;;;N;;;;; +11A9;HANGUL JONGSEONG SSANGKIYEOK;Lo;0;L;;;;;N;;;;; +11AA;HANGUL JONGSEONG KIYEOK-SIOS;Lo;0;L;;;;;N;;;;; +11AB;HANGUL JONGSEONG NIEUN;Lo;0;L;;;;;N;;;;; +11AC;HANGUL JONGSEONG NIEUN-CIEUC;Lo;0;L;;;;;N;;;;; +11AD;HANGUL JONGSEONG NIEUN-HIEUH;Lo;0;L;;;;;N;;;;; +11AE;HANGUL JONGSEONG TIKEUT;Lo;0;L;;;;;N;;;;; +11AF;HANGUL JONGSEONG RIEUL;Lo;0;L;;;;;N;;;;; +11B0;HANGUL JONGSEONG RIEUL-KIYEOK;Lo;0;L;;;;;N;;;;; +11B1;HANGUL JONGSEONG RIEUL-MIEUM;Lo;0;L;;;;;N;;;;; +11B2;HANGUL JONGSEONG RIEUL-PIEUP;Lo;0;L;;;;;N;;;;; +11B3;HANGUL JONGSEONG RIEUL-SIOS;Lo;0;L;;;;;N;;;;; +11B4;HANGUL JONGSEONG RIEUL-THIEUTH;Lo;0;L;;;;;N;;;;; +11B5;HANGUL JONGSEONG RIEUL-PHIEUPH;Lo;0;L;;;;;N;;;;; +11B6;HANGUL JONGSEONG RIEUL-HIEUH;Lo;0;L;;;;;N;;;;; +11B7;HANGUL JONGSEONG MIEUM;Lo;0;L;;;;;N;;;;; +11B8;HANGUL JONGSEONG PIEUP;Lo;0;L;;;;;N;;;;; +11B9;HANGUL JONGSEONG PIEUP-SIOS;Lo;0;L;;;;;N;;;;; +11BA;HANGUL JONGSEONG SIOS;Lo;0;L;;;;;N;;;;; +11BB;HANGUL JONGSEONG SSANGSIOS;Lo;0;L;;;;;N;;;;; +11BC;HANGUL JONGSEONG IEUNG;Lo;0;L;;;;;N;;;;; +11BD;HANGUL JONGSEONG CIEUC;Lo;0;L;;;;;N;;;;; +11BE;HANGUL JONGSEONG CHIEUCH;Lo;0;L;;;;;N;;;;; +11BF;HANGUL JONGSEONG KHIEUKH;Lo;0;L;;;;;N;;;;; +11C0;HANGUL JONGSEONG THIEUTH;Lo;0;L;;;;;N;;;;; +11C1;HANGUL JONGSEONG PHIEUPH;Lo;0;L;;;;;N;;;;; +11C2;HANGUL JONGSEONG HIEUH;Lo;0;L;;;;;N;;;;; +11C3;HANGUL JONGSEONG KIYEOK-RIEUL;Lo;0;L;;;;;N;;;;; +11C4;HANGUL JONGSEONG KIYEOK-SIOS-KIYEOK;Lo;0;L;;;;;N;;;;; +11C5;HANGUL JONGSEONG NIEUN-KIYEOK;Lo;0;L;;;;;N;;;;; +11C6;HANGUL JONGSEONG NIEUN-TIKEUT;Lo;0;L;;;;;N;;;;; +11C7;HANGUL JONGSEONG NIEUN-SIOS;Lo;0;L;;;;;N;;;;; +11C8;HANGUL JONGSEONG NIEUN-PANSIOS;Lo;0;L;;;;;N;;;;; +11C9;HANGUL JONGSEONG NIEUN-THIEUTH;Lo;0;L;;;;;N;;;;; +11CA;HANGUL JONGSEONG TIKEUT-KIYEOK;Lo;0;L;;;;;N;;;;; +11CB;HANGUL JONGSEONG TIKEUT-RIEUL;Lo;0;L;;;;;N;;;;; +11CC;HANGUL JONGSEONG RIEUL-KIYEOK-SIOS;Lo;0;L;;;;;N;;;;; +11CD;HANGUL JONGSEONG RIEUL-NIEUN;Lo;0;L;;;;;N;;;;; +11CE;HANGUL JONGSEONG RIEUL-TIKEUT;Lo;0;L;;;;;N;;;;; +11CF;HANGUL JONGSEONG RIEUL-TIKEUT-HIEUH;Lo;0;L;;;;;N;;;;; +11D0;HANGUL JONGSEONG SSANGRIEUL;Lo;0;L;;;;;N;;;;; +11D1;HANGUL JONGSEONG RIEUL-MIEUM-KIYEOK;Lo;0;L;;;;;N;;;;; +11D2;HANGUL JONGSEONG RIEUL-MIEUM-SIOS;Lo;0;L;;;;;N;;;;; +11D3;HANGUL JONGSEONG RIEUL-PIEUP-SIOS;Lo;0;L;;;;;N;;;;; +11D4;HANGUL JONGSEONG RIEUL-PIEUP-HIEUH;Lo;0;L;;;;;N;;;;; +11D5;HANGUL JONGSEONG RIEUL-KAPYEOUNPIEUP;Lo;0;L;;;;;N;;;;; +11D6;HANGUL JONGSEONG RIEUL-SSANGSIOS;Lo;0;L;;;;;N;;;;; +11D7;HANGUL JONGSEONG RIEUL-PANSIOS;Lo;0;L;;;;;N;;;;; +11D8;HANGUL JONGSEONG RIEUL-KHIEUKH;Lo;0;L;;;;;N;;;;; +11D9;HANGUL JONGSEONG RIEUL-YEORINHIEUH;Lo;0;L;;;;;N;;;;; +11DA;HANGUL JONGSEONG MIEUM-KIYEOK;Lo;0;L;;;;;N;;;;; +11DB;HANGUL JONGSEONG MIEUM-RIEUL;Lo;0;L;;;;;N;;;;; +11DC;HANGUL JONGSEONG MIEUM-PIEUP;Lo;0;L;;;;;N;;;;; +11DD;HANGUL JONGSEONG MIEUM-SIOS;Lo;0;L;;;;;N;;;;; +11DE;HANGUL JONGSEONG MIEUM-SSANGSIOS;Lo;0;L;;;;;N;;;;; +11DF;HANGUL JONGSEONG MIEUM-PANSIOS;Lo;0;L;;;;;N;;;;; +11E0;HANGUL JONGSEONG MIEUM-CHIEUCH;Lo;0;L;;;;;N;;;;; +11E1;HANGUL JONGSEONG MIEUM-HIEUH;Lo;0;L;;;;;N;;;;; +11E2;HANGUL JONGSEONG KAPYEOUNMIEUM;Lo;0;L;;;;;N;;;;; +11E3;HANGUL JONGSEONG PIEUP-RIEUL;Lo;0;L;;;;;N;;;;; +11E4;HANGUL JONGSEONG PIEUP-PHIEUPH;Lo;0;L;;;;;N;;;;; +11E5;HANGUL JONGSEONG PIEUP-HIEUH;Lo;0;L;;;;;N;;;;; +11E6;HANGUL JONGSEONG KAPYEOUNPIEUP;Lo;0;L;;;;;N;;;;; +11E7;HANGUL JONGSEONG SIOS-KIYEOK;Lo;0;L;;;;;N;;;;; +11E8;HANGUL JONGSEONG SIOS-TIKEUT;Lo;0;L;;;;;N;;;;; +11E9;HANGUL JONGSEONG SIOS-RIEUL;Lo;0;L;;;;;N;;;;; +11EA;HANGUL JONGSEONG SIOS-PIEUP;Lo;0;L;;;;;N;;;;; +11EB;HANGUL JONGSEONG PANSIOS;Lo;0;L;;;;;N;;;;; +11EC;HANGUL JONGSEONG IEUNG-KIYEOK;Lo;0;L;;;;;N;;;;; +11ED;HANGUL JONGSEONG IEUNG-SSANGKIYEOK;Lo;0;L;;;;;N;;;;; +11EE;HANGUL JONGSEONG SSANGIEUNG;Lo;0;L;;;;;N;;;;; +11EF;HANGUL JONGSEONG IEUNG-KHIEUKH;Lo;0;L;;;;;N;;;;; +11F0;HANGUL JONGSEONG YESIEUNG;Lo;0;L;;;;;N;;;;; +11F1;HANGUL JONGSEONG YESIEUNG-SIOS;Lo;0;L;;;;;N;;;;; +11F2;HANGUL JONGSEONG YESIEUNG-PANSIOS;Lo;0;L;;;;;N;;;;; +11F3;HANGUL JONGSEONG PHIEUPH-PIEUP;Lo;0;L;;;;;N;;;;; +11F4;HANGUL JONGSEONG KAPYEOUNPHIEUPH;Lo;0;L;;;;;N;;;;; +11F5;HANGUL JONGSEONG HIEUH-NIEUN;Lo;0;L;;;;;N;;;;; +11F6;HANGUL JONGSEONG HIEUH-RIEUL;Lo;0;L;;;;;N;;;;; +11F7;HANGUL JONGSEONG HIEUH-MIEUM;Lo;0;L;;;;;N;;;;; +11F8;HANGUL JONGSEONG HIEUH-PIEUP;Lo;0;L;;;;;N;;;;; +11F9;HANGUL JONGSEONG YEORINHIEUH;Lo;0;L;;;;;N;;;;; +11FA;HANGUL JONGSEONG KIYEOK-NIEUN;Lo;0;L;;;;;N;;;;; +11FB;HANGUL JONGSEONG KIYEOK-PIEUP;Lo;0;L;;;;;N;;;;; +11FC;HANGUL JONGSEONG KIYEOK-CHIEUCH;Lo;0;L;;;;;N;;;;; +11FD;HANGUL JONGSEONG KIYEOK-KHIEUKH;Lo;0;L;;;;;N;;;;; +11FE;HANGUL JONGSEONG KIYEOK-HIEUH;Lo;0;L;;;;;N;;;;; +11FF;HANGUL JONGSEONG SSANGNIEUN;Lo;0;L;;;;;N;;;;; +1200;ETHIOPIC SYLLABLE HA;Lo;0;L;;;;;N;;;;; +1201;ETHIOPIC SYLLABLE HU;Lo;0;L;;;;;N;;;;; +1202;ETHIOPIC SYLLABLE HI;Lo;0;L;;;;;N;;;;; +1203;ETHIOPIC SYLLABLE HAA;Lo;0;L;;;;;N;;;;; +1204;ETHIOPIC SYLLABLE HEE;Lo;0;L;;;;;N;;;;; +1205;ETHIOPIC SYLLABLE HE;Lo;0;L;;;;;N;;;;; +1206;ETHIOPIC SYLLABLE HO;Lo;0;L;;;;;N;;;;; +1207;ETHIOPIC SYLLABLE HOA;Lo;0;L;;;;;N;;;;; +1208;ETHIOPIC SYLLABLE LA;Lo;0;L;;;;;N;;;;; +1209;ETHIOPIC SYLLABLE LU;Lo;0;L;;;;;N;;;;; +120A;ETHIOPIC SYLLABLE LI;Lo;0;L;;;;;N;;;;; +120B;ETHIOPIC SYLLABLE LAA;Lo;0;L;;;;;N;;;;; +120C;ETHIOPIC SYLLABLE LEE;Lo;0;L;;;;;N;;;;; +120D;ETHIOPIC SYLLABLE LE;Lo;0;L;;;;;N;;;;; +120E;ETHIOPIC SYLLABLE LO;Lo;0;L;;;;;N;;;;; +120F;ETHIOPIC SYLLABLE LWA;Lo;0;L;;;;;N;;;;; +1210;ETHIOPIC SYLLABLE HHA;Lo;0;L;;;;;N;;;;; +1211;ETHIOPIC SYLLABLE HHU;Lo;0;L;;;;;N;;;;; +1212;ETHIOPIC SYLLABLE HHI;Lo;0;L;;;;;N;;;;; +1213;ETHIOPIC SYLLABLE HHAA;Lo;0;L;;;;;N;;;;; +1214;ETHIOPIC SYLLABLE HHEE;Lo;0;L;;;;;N;;;;; +1215;ETHIOPIC SYLLABLE HHE;Lo;0;L;;;;;N;;;;; +1216;ETHIOPIC SYLLABLE HHO;Lo;0;L;;;;;N;;;;; +1217;ETHIOPIC SYLLABLE HHWA;Lo;0;L;;;;;N;;;;; +1218;ETHIOPIC SYLLABLE MA;Lo;0;L;;;;;N;;;;; +1219;ETHIOPIC SYLLABLE MU;Lo;0;L;;;;;N;;;;; +121A;ETHIOPIC SYLLABLE MI;Lo;0;L;;;;;N;;;;; +121B;ETHIOPIC SYLLABLE MAA;Lo;0;L;;;;;N;;;;; +121C;ETHIOPIC SYLLABLE MEE;Lo;0;L;;;;;N;;;;; +121D;ETHIOPIC SYLLABLE ME;Lo;0;L;;;;;N;;;;; +121E;ETHIOPIC SYLLABLE MO;Lo;0;L;;;;;N;;;;; +121F;ETHIOPIC SYLLABLE MWA;Lo;0;L;;;;;N;;;;; +1220;ETHIOPIC SYLLABLE SZA;Lo;0;L;;;;;N;;;;; +1221;ETHIOPIC SYLLABLE SZU;Lo;0;L;;;;;N;;;;; +1222;ETHIOPIC SYLLABLE SZI;Lo;0;L;;;;;N;;;;; +1223;ETHIOPIC SYLLABLE SZAA;Lo;0;L;;;;;N;;;;; +1224;ETHIOPIC SYLLABLE SZEE;Lo;0;L;;;;;N;;;;; +1225;ETHIOPIC SYLLABLE SZE;Lo;0;L;;;;;N;;;;; +1226;ETHIOPIC SYLLABLE SZO;Lo;0;L;;;;;N;;;;; +1227;ETHIOPIC SYLLABLE SZWA;Lo;0;L;;;;;N;;;;; +1228;ETHIOPIC SYLLABLE RA;Lo;0;L;;;;;N;;;;; +1229;ETHIOPIC SYLLABLE RU;Lo;0;L;;;;;N;;;;; +122A;ETHIOPIC SYLLABLE RI;Lo;0;L;;;;;N;;;;; +122B;ETHIOPIC SYLLABLE RAA;Lo;0;L;;;;;N;;;;; +122C;ETHIOPIC SYLLABLE REE;Lo;0;L;;;;;N;;;;; +122D;ETHIOPIC SYLLABLE RE;Lo;0;L;;;;;N;;;;; +122E;ETHIOPIC SYLLABLE RO;Lo;0;L;;;;;N;;;;; +122F;ETHIOPIC SYLLABLE RWA;Lo;0;L;;;;;N;;;;; +1230;ETHIOPIC SYLLABLE SA;Lo;0;L;;;;;N;;;;; +1231;ETHIOPIC SYLLABLE SU;Lo;0;L;;;;;N;;;;; +1232;ETHIOPIC SYLLABLE SI;Lo;0;L;;;;;N;;;;; +1233;ETHIOPIC SYLLABLE SAA;Lo;0;L;;;;;N;;;;; +1234;ETHIOPIC SYLLABLE SEE;Lo;0;L;;;;;N;;;;; +1235;ETHIOPIC SYLLABLE SE;Lo;0;L;;;;;N;;;;; +1236;ETHIOPIC SYLLABLE SO;Lo;0;L;;;;;N;;;;; +1237;ETHIOPIC SYLLABLE SWA;Lo;0;L;;;;;N;;;;; +1238;ETHIOPIC SYLLABLE SHA;Lo;0;L;;;;;N;;;;; +1239;ETHIOPIC SYLLABLE SHU;Lo;0;L;;;;;N;;;;; +123A;ETHIOPIC SYLLABLE SHI;Lo;0;L;;;;;N;;;;; +123B;ETHIOPIC SYLLABLE SHAA;Lo;0;L;;;;;N;;;;; +123C;ETHIOPIC SYLLABLE SHEE;Lo;0;L;;;;;N;;;;; +123D;ETHIOPIC SYLLABLE SHE;Lo;0;L;;;;;N;;;;; +123E;ETHIOPIC SYLLABLE SHO;Lo;0;L;;;;;N;;;;; +123F;ETHIOPIC SYLLABLE SHWA;Lo;0;L;;;;;N;;;;; +1240;ETHIOPIC SYLLABLE QA;Lo;0;L;;;;;N;;;;; +1241;ETHIOPIC SYLLABLE QU;Lo;0;L;;;;;N;;;;; +1242;ETHIOPIC SYLLABLE QI;Lo;0;L;;;;;N;;;;; +1243;ETHIOPIC SYLLABLE QAA;Lo;0;L;;;;;N;;;;; +1244;ETHIOPIC SYLLABLE QEE;Lo;0;L;;;;;N;;;;; +1245;ETHIOPIC SYLLABLE QE;Lo;0;L;;;;;N;;;;; +1246;ETHIOPIC SYLLABLE QO;Lo;0;L;;;;;N;;;;; +1247;ETHIOPIC SYLLABLE QOA;Lo;0;L;;;;;N;;;;; +1248;ETHIOPIC SYLLABLE QWA;Lo;0;L;;;;;N;;;;; +124A;ETHIOPIC SYLLABLE QWI;Lo;0;L;;;;;N;;;;; +124B;ETHIOPIC SYLLABLE QWAA;Lo;0;L;;;;;N;;;;; +124C;ETHIOPIC SYLLABLE QWEE;Lo;0;L;;;;;N;;;;; +124D;ETHIOPIC SYLLABLE QWE;Lo;0;L;;;;;N;;;;; +1250;ETHIOPIC SYLLABLE QHA;Lo;0;L;;;;;N;;;;; +1251;ETHIOPIC SYLLABLE QHU;Lo;0;L;;;;;N;;;;; +1252;ETHIOPIC SYLLABLE QHI;Lo;0;L;;;;;N;;;;; +1253;ETHIOPIC SYLLABLE QHAA;Lo;0;L;;;;;N;;;;; +1254;ETHIOPIC SYLLABLE QHEE;Lo;0;L;;;;;N;;;;; +1255;ETHIOPIC SYLLABLE QHE;Lo;0;L;;;;;N;;;;; +1256;ETHIOPIC SYLLABLE QHO;Lo;0;L;;;;;N;;;;; +1258;ETHIOPIC SYLLABLE QHWA;Lo;0;L;;;;;N;;;;; +125A;ETHIOPIC SYLLABLE QHWI;Lo;0;L;;;;;N;;;;; +125B;ETHIOPIC SYLLABLE QHWAA;Lo;0;L;;;;;N;;;;; +125C;ETHIOPIC SYLLABLE QHWEE;Lo;0;L;;;;;N;;;;; +125D;ETHIOPIC SYLLABLE QHWE;Lo;0;L;;;;;N;;;;; +1260;ETHIOPIC SYLLABLE BA;Lo;0;L;;;;;N;;;;; +1261;ETHIOPIC SYLLABLE BU;Lo;0;L;;;;;N;;;;; +1262;ETHIOPIC SYLLABLE BI;Lo;0;L;;;;;N;;;;; +1263;ETHIOPIC SYLLABLE BAA;Lo;0;L;;;;;N;;;;; +1264;ETHIOPIC SYLLABLE BEE;Lo;0;L;;;;;N;;;;; +1265;ETHIOPIC SYLLABLE BE;Lo;0;L;;;;;N;;;;; +1266;ETHIOPIC SYLLABLE BO;Lo;0;L;;;;;N;;;;; +1267;ETHIOPIC SYLLABLE BWA;Lo;0;L;;;;;N;;;;; +1268;ETHIOPIC SYLLABLE VA;Lo;0;L;;;;;N;;;;; +1269;ETHIOPIC SYLLABLE VU;Lo;0;L;;;;;N;;;;; +126A;ETHIOPIC SYLLABLE VI;Lo;0;L;;;;;N;;;;; +126B;ETHIOPIC SYLLABLE VAA;Lo;0;L;;;;;N;;;;; +126C;ETHIOPIC SYLLABLE VEE;Lo;0;L;;;;;N;;;;; +126D;ETHIOPIC SYLLABLE VE;Lo;0;L;;;;;N;;;;; +126E;ETHIOPIC SYLLABLE VO;Lo;0;L;;;;;N;;;;; +126F;ETHIOPIC SYLLABLE VWA;Lo;0;L;;;;;N;;;;; +1270;ETHIOPIC SYLLABLE TA;Lo;0;L;;;;;N;;;;; +1271;ETHIOPIC SYLLABLE TU;Lo;0;L;;;;;N;;;;; +1272;ETHIOPIC SYLLABLE TI;Lo;0;L;;;;;N;;;;; +1273;ETHIOPIC SYLLABLE TAA;Lo;0;L;;;;;N;;;;; +1274;ETHIOPIC SYLLABLE TEE;Lo;0;L;;;;;N;;;;; +1275;ETHIOPIC SYLLABLE TE;Lo;0;L;;;;;N;;;;; +1276;ETHIOPIC SYLLABLE TO;Lo;0;L;;;;;N;;;;; +1277;ETHIOPIC SYLLABLE TWA;Lo;0;L;;;;;N;;;;; +1278;ETHIOPIC SYLLABLE CA;Lo;0;L;;;;;N;;;;; +1279;ETHIOPIC SYLLABLE CU;Lo;0;L;;;;;N;;;;; +127A;ETHIOPIC SYLLABLE CI;Lo;0;L;;;;;N;;;;; +127B;ETHIOPIC SYLLABLE CAA;Lo;0;L;;;;;N;;;;; +127C;ETHIOPIC SYLLABLE CEE;Lo;0;L;;;;;N;;;;; +127D;ETHIOPIC SYLLABLE CE;Lo;0;L;;;;;N;;;;; +127E;ETHIOPIC SYLLABLE CO;Lo;0;L;;;;;N;;;;; +127F;ETHIOPIC SYLLABLE CWA;Lo;0;L;;;;;N;;;;; +1280;ETHIOPIC SYLLABLE XA;Lo;0;L;;;;;N;;;;; +1281;ETHIOPIC SYLLABLE XU;Lo;0;L;;;;;N;;;;; +1282;ETHIOPIC SYLLABLE XI;Lo;0;L;;;;;N;;;;; +1283;ETHIOPIC SYLLABLE XAA;Lo;0;L;;;;;N;;;;; +1284;ETHIOPIC SYLLABLE XEE;Lo;0;L;;;;;N;;;;; +1285;ETHIOPIC SYLLABLE XE;Lo;0;L;;;;;N;;;;; +1286;ETHIOPIC SYLLABLE XO;Lo;0;L;;;;;N;;;;; +1287;ETHIOPIC SYLLABLE XOA;Lo;0;L;;;;;N;;;;; +1288;ETHIOPIC SYLLABLE XWA;Lo;0;L;;;;;N;;;;; +128A;ETHIOPIC SYLLABLE XWI;Lo;0;L;;;;;N;;;;; +128B;ETHIOPIC SYLLABLE XWAA;Lo;0;L;;;;;N;;;;; +128C;ETHIOPIC SYLLABLE XWEE;Lo;0;L;;;;;N;;;;; +128D;ETHIOPIC SYLLABLE XWE;Lo;0;L;;;;;N;;;;; +1290;ETHIOPIC SYLLABLE NA;Lo;0;L;;;;;N;;;;; +1291;ETHIOPIC SYLLABLE NU;Lo;0;L;;;;;N;;;;; +1292;ETHIOPIC SYLLABLE NI;Lo;0;L;;;;;N;;;;; +1293;ETHIOPIC SYLLABLE NAA;Lo;0;L;;;;;N;;;;; +1294;ETHIOPIC SYLLABLE NEE;Lo;0;L;;;;;N;;;;; +1295;ETHIOPIC SYLLABLE NE;Lo;0;L;;;;;N;;;;; +1296;ETHIOPIC SYLLABLE NO;Lo;0;L;;;;;N;;;;; +1297;ETHIOPIC SYLLABLE NWA;Lo;0;L;;;;;N;;;;; +1298;ETHIOPIC SYLLABLE NYA;Lo;0;L;;;;;N;;;;; +1299;ETHIOPIC SYLLABLE NYU;Lo;0;L;;;;;N;;;;; +129A;ETHIOPIC SYLLABLE NYI;Lo;0;L;;;;;N;;;;; +129B;ETHIOPIC SYLLABLE NYAA;Lo;0;L;;;;;N;;;;; +129C;ETHIOPIC SYLLABLE NYEE;Lo;0;L;;;;;N;;;;; +129D;ETHIOPIC SYLLABLE NYE;Lo;0;L;;;;;N;;;;; +129E;ETHIOPIC SYLLABLE NYO;Lo;0;L;;;;;N;;;;; +129F;ETHIOPIC SYLLABLE NYWA;Lo;0;L;;;;;N;;;;; +12A0;ETHIOPIC SYLLABLE GLOTTAL A;Lo;0;L;;;;;N;;;;; +12A1;ETHIOPIC SYLLABLE GLOTTAL U;Lo;0;L;;;;;N;;;;; +12A2;ETHIOPIC SYLLABLE GLOTTAL I;Lo;0;L;;;;;N;;;;; +12A3;ETHIOPIC SYLLABLE GLOTTAL AA;Lo;0;L;;;;;N;;;;; +12A4;ETHIOPIC SYLLABLE GLOTTAL EE;Lo;0;L;;;;;N;;;;; +12A5;ETHIOPIC SYLLABLE GLOTTAL E;Lo;0;L;;;;;N;;;;; +12A6;ETHIOPIC SYLLABLE GLOTTAL O;Lo;0;L;;;;;N;;;;; +12A7;ETHIOPIC SYLLABLE GLOTTAL WA;Lo;0;L;;;;;N;;;;; +12A8;ETHIOPIC SYLLABLE KA;Lo;0;L;;;;;N;;;;; +12A9;ETHIOPIC SYLLABLE KU;Lo;0;L;;;;;N;;;;; +12AA;ETHIOPIC SYLLABLE KI;Lo;0;L;;;;;N;;;;; +12AB;ETHIOPIC SYLLABLE KAA;Lo;0;L;;;;;N;;;;; +12AC;ETHIOPIC SYLLABLE KEE;Lo;0;L;;;;;N;;;;; +12AD;ETHIOPIC SYLLABLE KE;Lo;0;L;;;;;N;;;;; +12AE;ETHIOPIC SYLLABLE KO;Lo;0;L;;;;;N;;;;; +12AF;ETHIOPIC SYLLABLE KOA;Lo;0;L;;;;;N;;;;; +12B0;ETHIOPIC SYLLABLE KWA;Lo;0;L;;;;;N;;;;; +12B2;ETHIOPIC SYLLABLE KWI;Lo;0;L;;;;;N;;;;; +12B3;ETHIOPIC SYLLABLE KWAA;Lo;0;L;;;;;N;;;;; +12B4;ETHIOPIC SYLLABLE KWEE;Lo;0;L;;;;;N;;;;; +12B5;ETHIOPIC SYLLABLE KWE;Lo;0;L;;;;;N;;;;; +12B8;ETHIOPIC SYLLABLE KXA;Lo;0;L;;;;;N;;;;; +12B9;ETHIOPIC SYLLABLE KXU;Lo;0;L;;;;;N;;;;; +12BA;ETHIOPIC SYLLABLE KXI;Lo;0;L;;;;;N;;;;; +12BB;ETHIOPIC SYLLABLE KXAA;Lo;0;L;;;;;N;;;;; +12BC;ETHIOPIC SYLLABLE KXEE;Lo;0;L;;;;;N;;;;; +12BD;ETHIOPIC SYLLABLE KXE;Lo;0;L;;;;;N;;;;; +12BE;ETHIOPIC SYLLABLE KXO;Lo;0;L;;;;;N;;;;; +12C0;ETHIOPIC SYLLABLE KXWA;Lo;0;L;;;;;N;;;;; +12C2;ETHIOPIC SYLLABLE KXWI;Lo;0;L;;;;;N;;;;; +12C3;ETHIOPIC SYLLABLE KXWAA;Lo;0;L;;;;;N;;;;; +12C4;ETHIOPIC SYLLABLE KXWEE;Lo;0;L;;;;;N;;;;; +12C5;ETHIOPIC SYLLABLE KXWE;Lo;0;L;;;;;N;;;;; +12C8;ETHIOPIC SYLLABLE WA;Lo;0;L;;;;;N;;;;; +12C9;ETHIOPIC SYLLABLE WU;Lo;0;L;;;;;N;;;;; +12CA;ETHIOPIC SYLLABLE WI;Lo;0;L;;;;;N;;;;; +12CB;ETHIOPIC SYLLABLE WAA;Lo;0;L;;;;;N;;;;; +12CC;ETHIOPIC SYLLABLE WEE;Lo;0;L;;;;;N;;;;; +12CD;ETHIOPIC SYLLABLE WE;Lo;0;L;;;;;N;;;;; +12CE;ETHIOPIC SYLLABLE WO;Lo;0;L;;;;;N;;;;; +12CF;ETHIOPIC SYLLABLE WOA;Lo;0;L;;;;;N;;;;; +12D0;ETHIOPIC SYLLABLE PHARYNGEAL A;Lo;0;L;;;;;N;;;;; +12D1;ETHIOPIC SYLLABLE PHARYNGEAL U;Lo;0;L;;;;;N;;;;; +12D2;ETHIOPIC SYLLABLE PHARYNGEAL I;Lo;0;L;;;;;N;;;;; +12D3;ETHIOPIC SYLLABLE PHARYNGEAL AA;Lo;0;L;;;;;N;;;;; +12D4;ETHIOPIC SYLLABLE PHARYNGEAL EE;Lo;0;L;;;;;N;;;;; +12D5;ETHIOPIC SYLLABLE PHARYNGEAL E;Lo;0;L;;;;;N;;;;; +12D6;ETHIOPIC SYLLABLE PHARYNGEAL O;Lo;0;L;;;;;N;;;;; +12D8;ETHIOPIC SYLLABLE ZA;Lo;0;L;;;;;N;;;;; +12D9;ETHIOPIC SYLLABLE ZU;Lo;0;L;;;;;N;;;;; +12DA;ETHIOPIC SYLLABLE ZI;Lo;0;L;;;;;N;;;;; +12DB;ETHIOPIC SYLLABLE ZAA;Lo;0;L;;;;;N;;;;; +12DC;ETHIOPIC SYLLABLE ZEE;Lo;0;L;;;;;N;;;;; +12DD;ETHIOPIC SYLLABLE ZE;Lo;0;L;;;;;N;;;;; +12DE;ETHIOPIC SYLLABLE ZO;Lo;0;L;;;;;N;;;;; +12DF;ETHIOPIC SYLLABLE ZWA;Lo;0;L;;;;;N;;;;; +12E0;ETHIOPIC SYLLABLE ZHA;Lo;0;L;;;;;N;;;;; +12E1;ETHIOPIC SYLLABLE ZHU;Lo;0;L;;;;;N;;;;; +12E2;ETHIOPIC SYLLABLE ZHI;Lo;0;L;;;;;N;;;;; +12E3;ETHIOPIC SYLLABLE ZHAA;Lo;0;L;;;;;N;;;;; +12E4;ETHIOPIC SYLLABLE ZHEE;Lo;0;L;;;;;N;;;;; +12E5;ETHIOPIC SYLLABLE ZHE;Lo;0;L;;;;;N;;;;; +12E6;ETHIOPIC SYLLABLE ZHO;Lo;0;L;;;;;N;;;;; +12E7;ETHIOPIC SYLLABLE ZHWA;Lo;0;L;;;;;N;;;;; +12E8;ETHIOPIC SYLLABLE YA;Lo;0;L;;;;;N;;;;; +12E9;ETHIOPIC SYLLABLE YU;Lo;0;L;;;;;N;;;;; +12EA;ETHIOPIC SYLLABLE YI;Lo;0;L;;;;;N;;;;; +12EB;ETHIOPIC SYLLABLE YAA;Lo;0;L;;;;;N;;;;; +12EC;ETHIOPIC SYLLABLE YEE;Lo;0;L;;;;;N;;;;; +12ED;ETHIOPIC SYLLABLE YE;Lo;0;L;;;;;N;;;;; +12EE;ETHIOPIC SYLLABLE YO;Lo;0;L;;;;;N;;;;; +12EF;ETHIOPIC SYLLABLE YOA;Lo;0;L;;;;;N;;;;; +12F0;ETHIOPIC SYLLABLE DA;Lo;0;L;;;;;N;;;;; +12F1;ETHIOPIC SYLLABLE DU;Lo;0;L;;;;;N;;;;; +12F2;ETHIOPIC SYLLABLE DI;Lo;0;L;;;;;N;;;;; +12F3;ETHIOPIC SYLLABLE DAA;Lo;0;L;;;;;N;;;;; +12F4;ETHIOPIC SYLLABLE DEE;Lo;0;L;;;;;N;;;;; +12F5;ETHIOPIC SYLLABLE DE;Lo;0;L;;;;;N;;;;; +12F6;ETHIOPIC SYLLABLE DO;Lo;0;L;;;;;N;;;;; +12F7;ETHIOPIC SYLLABLE DWA;Lo;0;L;;;;;N;;;;; +12F8;ETHIOPIC SYLLABLE DDA;Lo;0;L;;;;;N;;;;; +12F9;ETHIOPIC SYLLABLE DDU;Lo;0;L;;;;;N;;;;; +12FA;ETHIOPIC SYLLABLE DDI;Lo;0;L;;;;;N;;;;; +12FB;ETHIOPIC SYLLABLE DDAA;Lo;0;L;;;;;N;;;;; +12FC;ETHIOPIC SYLLABLE DDEE;Lo;0;L;;;;;N;;;;; +12FD;ETHIOPIC SYLLABLE DDE;Lo;0;L;;;;;N;;;;; +12FE;ETHIOPIC SYLLABLE DDO;Lo;0;L;;;;;N;;;;; +12FF;ETHIOPIC SYLLABLE DDWA;Lo;0;L;;;;;N;;;;; +1300;ETHIOPIC SYLLABLE JA;Lo;0;L;;;;;N;;;;; +1301;ETHIOPIC SYLLABLE JU;Lo;0;L;;;;;N;;;;; +1302;ETHIOPIC SYLLABLE JI;Lo;0;L;;;;;N;;;;; +1303;ETHIOPIC SYLLABLE JAA;Lo;0;L;;;;;N;;;;; +1304;ETHIOPIC SYLLABLE JEE;Lo;0;L;;;;;N;;;;; +1305;ETHIOPIC SYLLABLE JE;Lo;0;L;;;;;N;;;;; +1306;ETHIOPIC SYLLABLE JO;Lo;0;L;;;;;N;;;;; +1307;ETHIOPIC SYLLABLE JWA;Lo;0;L;;;;;N;;;;; +1308;ETHIOPIC SYLLABLE GA;Lo;0;L;;;;;N;;;;; +1309;ETHIOPIC SYLLABLE GU;Lo;0;L;;;;;N;;;;; +130A;ETHIOPIC SYLLABLE GI;Lo;0;L;;;;;N;;;;; +130B;ETHIOPIC SYLLABLE GAA;Lo;0;L;;;;;N;;;;; +130C;ETHIOPIC SYLLABLE GEE;Lo;0;L;;;;;N;;;;; +130D;ETHIOPIC SYLLABLE GE;Lo;0;L;;;;;N;;;;; +130E;ETHIOPIC SYLLABLE GO;Lo;0;L;;;;;N;;;;; +130F;ETHIOPIC SYLLABLE GOA;Lo;0;L;;;;;N;;;;; +1310;ETHIOPIC SYLLABLE GWA;Lo;0;L;;;;;N;;;;; +1312;ETHIOPIC SYLLABLE GWI;Lo;0;L;;;;;N;;;;; +1313;ETHIOPIC SYLLABLE GWAA;Lo;0;L;;;;;N;;;;; +1314;ETHIOPIC SYLLABLE GWEE;Lo;0;L;;;;;N;;;;; +1315;ETHIOPIC SYLLABLE GWE;Lo;0;L;;;;;N;;;;; +1318;ETHIOPIC SYLLABLE GGA;Lo;0;L;;;;;N;;;;; +1319;ETHIOPIC SYLLABLE GGU;Lo;0;L;;;;;N;;;;; +131A;ETHIOPIC SYLLABLE GGI;Lo;0;L;;;;;N;;;;; +131B;ETHIOPIC SYLLABLE GGAA;Lo;0;L;;;;;N;;;;; +131C;ETHIOPIC SYLLABLE GGEE;Lo;0;L;;;;;N;;;;; +131D;ETHIOPIC SYLLABLE GGE;Lo;0;L;;;;;N;;;;; +131E;ETHIOPIC SYLLABLE GGO;Lo;0;L;;;;;N;;;;; +131F;ETHIOPIC SYLLABLE GGWAA;Lo;0;L;;;;;N;;;;; +1320;ETHIOPIC SYLLABLE THA;Lo;0;L;;;;;N;;;;; +1321;ETHIOPIC SYLLABLE THU;Lo;0;L;;;;;N;;;;; +1322;ETHIOPIC SYLLABLE THI;Lo;0;L;;;;;N;;;;; +1323;ETHIOPIC SYLLABLE THAA;Lo;0;L;;;;;N;;;;; +1324;ETHIOPIC SYLLABLE THEE;Lo;0;L;;;;;N;;;;; +1325;ETHIOPIC SYLLABLE THE;Lo;0;L;;;;;N;;;;; +1326;ETHIOPIC SYLLABLE THO;Lo;0;L;;;;;N;;;;; +1327;ETHIOPIC SYLLABLE THWA;Lo;0;L;;;;;N;;;;; +1328;ETHIOPIC SYLLABLE CHA;Lo;0;L;;;;;N;;;;; +1329;ETHIOPIC SYLLABLE CHU;Lo;0;L;;;;;N;;;;; +132A;ETHIOPIC SYLLABLE CHI;Lo;0;L;;;;;N;;;;; +132B;ETHIOPIC SYLLABLE CHAA;Lo;0;L;;;;;N;;;;; +132C;ETHIOPIC SYLLABLE CHEE;Lo;0;L;;;;;N;;;;; +132D;ETHIOPIC SYLLABLE CHE;Lo;0;L;;;;;N;;;;; +132E;ETHIOPIC SYLLABLE CHO;Lo;0;L;;;;;N;;;;; +132F;ETHIOPIC SYLLABLE CHWA;Lo;0;L;;;;;N;;;;; +1330;ETHIOPIC SYLLABLE PHA;Lo;0;L;;;;;N;;;;; +1331;ETHIOPIC SYLLABLE PHU;Lo;0;L;;;;;N;;;;; +1332;ETHIOPIC SYLLABLE PHI;Lo;0;L;;;;;N;;;;; +1333;ETHIOPIC SYLLABLE PHAA;Lo;0;L;;;;;N;;;;; +1334;ETHIOPIC SYLLABLE PHEE;Lo;0;L;;;;;N;;;;; +1335;ETHIOPIC SYLLABLE PHE;Lo;0;L;;;;;N;;;;; +1336;ETHIOPIC SYLLABLE PHO;Lo;0;L;;;;;N;;;;; +1337;ETHIOPIC SYLLABLE PHWA;Lo;0;L;;;;;N;;;;; +1338;ETHIOPIC SYLLABLE TSA;Lo;0;L;;;;;N;;;;; +1339;ETHIOPIC SYLLABLE TSU;Lo;0;L;;;;;N;;;;; +133A;ETHIOPIC SYLLABLE TSI;Lo;0;L;;;;;N;;;;; +133B;ETHIOPIC SYLLABLE TSAA;Lo;0;L;;;;;N;;;;; +133C;ETHIOPIC SYLLABLE TSEE;Lo;0;L;;;;;N;;;;; +133D;ETHIOPIC SYLLABLE TSE;Lo;0;L;;;;;N;;;;; +133E;ETHIOPIC SYLLABLE TSO;Lo;0;L;;;;;N;;;;; +133F;ETHIOPIC SYLLABLE TSWA;Lo;0;L;;;;;N;;;;; +1340;ETHIOPIC SYLLABLE TZA;Lo;0;L;;;;;N;;;;; +1341;ETHIOPIC SYLLABLE TZU;Lo;0;L;;;;;N;;;;; +1342;ETHIOPIC SYLLABLE TZI;Lo;0;L;;;;;N;;;;; +1343;ETHIOPIC SYLLABLE TZAA;Lo;0;L;;;;;N;;;;; +1344;ETHIOPIC SYLLABLE TZEE;Lo;0;L;;;;;N;;;;; +1345;ETHIOPIC SYLLABLE TZE;Lo;0;L;;;;;N;;;;; +1346;ETHIOPIC SYLLABLE TZO;Lo;0;L;;;;;N;;;;; +1347;ETHIOPIC SYLLABLE TZOA;Lo;0;L;;;;;N;;;;; +1348;ETHIOPIC SYLLABLE FA;Lo;0;L;;;;;N;;;;; +1349;ETHIOPIC SYLLABLE FU;Lo;0;L;;;;;N;;;;; +134A;ETHIOPIC SYLLABLE FI;Lo;0;L;;;;;N;;;;; +134B;ETHIOPIC SYLLABLE FAA;Lo;0;L;;;;;N;;;;; +134C;ETHIOPIC SYLLABLE FEE;Lo;0;L;;;;;N;;;;; +134D;ETHIOPIC SYLLABLE FE;Lo;0;L;;;;;N;;;;; +134E;ETHIOPIC SYLLABLE FO;Lo;0;L;;;;;N;;;;; +134F;ETHIOPIC SYLLABLE FWA;Lo;0;L;;;;;N;;;;; +1350;ETHIOPIC SYLLABLE PA;Lo;0;L;;;;;N;;;;; +1351;ETHIOPIC SYLLABLE PU;Lo;0;L;;;;;N;;;;; +1352;ETHIOPIC SYLLABLE PI;Lo;0;L;;;;;N;;;;; +1353;ETHIOPIC SYLLABLE PAA;Lo;0;L;;;;;N;;;;; +1354;ETHIOPIC SYLLABLE PEE;Lo;0;L;;;;;N;;;;; +1355;ETHIOPIC SYLLABLE PE;Lo;0;L;;;;;N;;;;; +1356;ETHIOPIC SYLLABLE PO;Lo;0;L;;;;;N;;;;; +1357;ETHIOPIC SYLLABLE PWA;Lo;0;L;;;;;N;;;;; +1358;ETHIOPIC SYLLABLE RYA;Lo;0;L;;;;;N;;;;; +1359;ETHIOPIC SYLLABLE MYA;Lo;0;L;;;;;N;;;;; +135A;ETHIOPIC SYLLABLE FYA;Lo;0;L;;;;;N;;;;; +135D;ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK;Mn;230;NSM;;;;;N;;;;; +135E;ETHIOPIC COMBINING VOWEL LENGTH MARK;Mn;230;NSM;;;;;N;;;;; +135F;ETHIOPIC COMBINING GEMINATION MARK;Mn;230;NSM;;;;;N;;;;; +1360;ETHIOPIC SECTION MARK;Po;0;L;;;;;N;;;;; +1361;ETHIOPIC WORDSPACE;Po;0;L;;;;;N;;;;; +1362;ETHIOPIC FULL STOP;Po;0;L;;;;;N;;;;; +1363;ETHIOPIC COMMA;Po;0;L;;;;;N;;;;; +1364;ETHIOPIC SEMICOLON;Po;0;L;;;;;N;;;;; +1365;ETHIOPIC COLON;Po;0;L;;;;;N;;;;; +1366;ETHIOPIC PREFACE COLON;Po;0;L;;;;;N;;;;; +1367;ETHIOPIC QUESTION MARK;Po;0;L;;;;;N;;;;; +1368;ETHIOPIC PARAGRAPH SEPARATOR;Po;0;L;;;;;N;;;;; +1369;ETHIOPIC DIGIT ONE;No;0;L;;;1;1;N;;;;; +136A;ETHIOPIC DIGIT TWO;No;0;L;;;2;2;N;;;;; +136B;ETHIOPIC DIGIT THREE;No;0;L;;;3;3;N;;;;; +136C;ETHIOPIC DIGIT FOUR;No;0;L;;;4;4;N;;;;; +136D;ETHIOPIC DIGIT FIVE;No;0;L;;;5;5;N;;;;; +136E;ETHIOPIC DIGIT SIX;No;0;L;;;6;6;N;;;;; +136F;ETHIOPIC DIGIT SEVEN;No;0;L;;;7;7;N;;;;; +1370;ETHIOPIC DIGIT EIGHT;No;0;L;;;8;8;N;;;;; +1371;ETHIOPIC DIGIT NINE;No;0;L;;;9;9;N;;;;; +1372;ETHIOPIC NUMBER TEN;No;0;L;;;;10;N;;;;; +1373;ETHIOPIC NUMBER TWENTY;No;0;L;;;;20;N;;;;; +1374;ETHIOPIC NUMBER THIRTY;No;0;L;;;;30;N;;;;; +1375;ETHIOPIC NUMBER FORTY;No;0;L;;;;40;N;;;;; +1376;ETHIOPIC NUMBER FIFTY;No;0;L;;;;50;N;;;;; +1377;ETHIOPIC NUMBER SIXTY;No;0;L;;;;60;N;;;;; +1378;ETHIOPIC NUMBER SEVENTY;No;0;L;;;;70;N;;;;; +1379;ETHIOPIC NUMBER EIGHTY;No;0;L;;;;80;N;;;;; +137A;ETHIOPIC NUMBER NINETY;No;0;L;;;;90;N;;;;; +137B;ETHIOPIC NUMBER HUNDRED;No;0;L;;;;100;N;;;;; +137C;ETHIOPIC NUMBER TEN THOUSAND;No;0;L;;;;10000;N;;;;; +1380;ETHIOPIC SYLLABLE SEBATBEIT MWA;Lo;0;L;;;;;N;;;;; +1381;ETHIOPIC SYLLABLE MWI;Lo;0;L;;;;;N;;;;; +1382;ETHIOPIC SYLLABLE MWEE;Lo;0;L;;;;;N;;;;; +1383;ETHIOPIC SYLLABLE MWE;Lo;0;L;;;;;N;;;;; +1384;ETHIOPIC SYLLABLE SEBATBEIT BWA;Lo;0;L;;;;;N;;;;; +1385;ETHIOPIC SYLLABLE BWI;Lo;0;L;;;;;N;;;;; +1386;ETHIOPIC SYLLABLE BWEE;Lo;0;L;;;;;N;;;;; +1387;ETHIOPIC SYLLABLE BWE;Lo;0;L;;;;;N;;;;; +1388;ETHIOPIC SYLLABLE SEBATBEIT FWA;Lo;0;L;;;;;N;;;;; +1389;ETHIOPIC SYLLABLE FWI;Lo;0;L;;;;;N;;;;; +138A;ETHIOPIC SYLLABLE FWEE;Lo;0;L;;;;;N;;;;; +138B;ETHIOPIC SYLLABLE FWE;Lo;0;L;;;;;N;;;;; +138C;ETHIOPIC SYLLABLE SEBATBEIT PWA;Lo;0;L;;;;;N;;;;; +138D;ETHIOPIC SYLLABLE PWI;Lo;0;L;;;;;N;;;;; +138E;ETHIOPIC SYLLABLE PWEE;Lo;0;L;;;;;N;;;;; +138F;ETHIOPIC SYLLABLE PWE;Lo;0;L;;;;;N;;;;; +1390;ETHIOPIC TONAL MARK YIZET;So;0;ON;;;;;N;;;;; +1391;ETHIOPIC TONAL MARK DERET;So;0;ON;;;;;N;;;;; +1392;ETHIOPIC TONAL MARK RIKRIK;So;0;ON;;;;;N;;;;; +1393;ETHIOPIC TONAL MARK SHORT RIKRIK;So;0;ON;;;;;N;;;;; +1394;ETHIOPIC TONAL MARK DIFAT;So;0;ON;;;;;N;;;;; +1395;ETHIOPIC TONAL MARK KENAT;So;0;ON;;;;;N;;;;; +1396;ETHIOPIC TONAL MARK CHIRET;So;0;ON;;;;;N;;;;; +1397;ETHIOPIC TONAL MARK HIDET;So;0;ON;;;;;N;;;;; +1398;ETHIOPIC TONAL MARK DERET-HIDET;So;0;ON;;;;;N;;;;; +1399;ETHIOPIC TONAL MARK KURT;So;0;ON;;;;;N;;;;; +13A0;CHEROKEE LETTER A;Lo;0;L;;;;;N;;;;; +13A1;CHEROKEE LETTER E;Lo;0;L;;;;;N;;;;; +13A2;CHEROKEE LETTER I;Lo;0;L;;;;;N;;;;; +13A3;CHEROKEE LETTER O;Lo;0;L;;;;;N;;;;; +13A4;CHEROKEE LETTER U;Lo;0;L;;;;;N;;;;; +13A5;CHEROKEE LETTER V;Lo;0;L;;;;;N;;;;; +13A6;CHEROKEE LETTER GA;Lo;0;L;;;;;N;;;;; +13A7;CHEROKEE LETTER KA;Lo;0;L;;;;;N;;;;; +13A8;CHEROKEE LETTER GE;Lo;0;L;;;;;N;;;;; +13A9;CHEROKEE LETTER GI;Lo;0;L;;;;;N;;;;; +13AA;CHEROKEE LETTER GO;Lo;0;L;;;;;N;;;;; +13AB;CHEROKEE LETTER GU;Lo;0;L;;;;;N;;;;; +13AC;CHEROKEE LETTER GV;Lo;0;L;;;;;N;;;;; +13AD;CHEROKEE LETTER HA;Lo;0;L;;;;;N;;;;; +13AE;CHEROKEE LETTER HE;Lo;0;L;;;;;N;;;;; +13AF;CHEROKEE LETTER HI;Lo;0;L;;;;;N;;;;; +13B0;CHEROKEE LETTER HO;Lo;0;L;;;;;N;;;;; +13B1;CHEROKEE LETTER HU;Lo;0;L;;;;;N;;;;; +13B2;CHEROKEE LETTER HV;Lo;0;L;;;;;N;;;;; +13B3;CHEROKEE LETTER LA;Lo;0;L;;;;;N;;;;; +13B4;CHEROKEE LETTER LE;Lo;0;L;;;;;N;;;;; +13B5;CHEROKEE LETTER LI;Lo;0;L;;;;;N;;;;; +13B6;CHEROKEE LETTER LO;Lo;0;L;;;;;N;;;;; +13B7;CHEROKEE LETTER LU;Lo;0;L;;;;;N;;;;; +13B8;CHEROKEE LETTER LV;Lo;0;L;;;;;N;;;;; +13B9;CHEROKEE LETTER MA;Lo;0;L;;;;;N;;;;; +13BA;CHEROKEE LETTER ME;Lo;0;L;;;;;N;;;;; +13BB;CHEROKEE LETTER MI;Lo;0;L;;;;;N;;;;; +13BC;CHEROKEE LETTER MO;Lo;0;L;;;;;N;;;;; +13BD;CHEROKEE LETTER MU;Lo;0;L;;;;;N;;;;; +13BE;CHEROKEE LETTER NA;Lo;0;L;;;;;N;;;;; +13BF;CHEROKEE LETTER HNA;Lo;0;L;;;;;N;;;;; +13C0;CHEROKEE LETTER NAH;Lo;0;L;;;;;N;;;;; +13C1;CHEROKEE LETTER NE;Lo;0;L;;;;;N;;;;; +13C2;CHEROKEE LETTER NI;Lo;0;L;;;;;N;;;;; +13C3;CHEROKEE LETTER NO;Lo;0;L;;;;;N;;;;; +13C4;CHEROKEE LETTER NU;Lo;0;L;;;;;N;;;;; +13C5;CHEROKEE LETTER NV;Lo;0;L;;;;;N;;;;; +13C6;CHEROKEE LETTER QUA;Lo;0;L;;;;;N;;;;; +13C7;CHEROKEE LETTER QUE;Lo;0;L;;;;;N;;;;; +13C8;CHEROKEE LETTER QUI;Lo;0;L;;;;;N;;;;; +13C9;CHEROKEE LETTER QUO;Lo;0;L;;;;;N;;;;; +13CA;CHEROKEE LETTER QUU;Lo;0;L;;;;;N;;;;; +13CB;CHEROKEE LETTER QUV;Lo;0;L;;;;;N;;;;; +13CC;CHEROKEE LETTER SA;Lo;0;L;;;;;N;;;;; +13CD;CHEROKEE LETTER S;Lo;0;L;;;;;N;;;;; +13CE;CHEROKEE LETTER SE;Lo;0;L;;;;;N;;;;; +13CF;CHEROKEE LETTER SI;Lo;0;L;;;;;N;;;;; +13D0;CHEROKEE LETTER SO;Lo;0;L;;;;;N;;;;; +13D1;CHEROKEE LETTER SU;Lo;0;L;;;;;N;;;;; +13D2;CHEROKEE LETTER SV;Lo;0;L;;;;;N;;;;; +13D3;CHEROKEE LETTER DA;Lo;0;L;;;;;N;;;;; +13D4;CHEROKEE LETTER TA;Lo;0;L;;;;;N;;;;; +13D5;CHEROKEE LETTER DE;Lo;0;L;;;;;N;;;;; +13D6;CHEROKEE LETTER TE;Lo;0;L;;;;;N;;;;; +13D7;CHEROKEE LETTER DI;Lo;0;L;;;;;N;;;;; +13D8;CHEROKEE LETTER TI;Lo;0;L;;;;;N;;;;; +13D9;CHEROKEE LETTER DO;Lo;0;L;;;;;N;;;;; +13DA;CHEROKEE LETTER DU;Lo;0;L;;;;;N;;;;; +13DB;CHEROKEE LETTER DV;Lo;0;L;;;;;N;;;;; +13DC;CHEROKEE LETTER DLA;Lo;0;L;;;;;N;;;;; +13DD;CHEROKEE LETTER TLA;Lo;0;L;;;;;N;;;;; +13DE;CHEROKEE LETTER TLE;Lo;0;L;;;;;N;;;;; +13DF;CHEROKEE LETTER TLI;Lo;0;L;;;;;N;;;;; +13E0;CHEROKEE LETTER TLO;Lo;0;L;;;;;N;;;;; +13E1;CHEROKEE LETTER TLU;Lo;0;L;;;;;N;;;;; +13E2;CHEROKEE LETTER TLV;Lo;0;L;;;;;N;;;;; +13E3;CHEROKEE LETTER TSA;Lo;0;L;;;;;N;;;;; +13E4;CHEROKEE LETTER TSE;Lo;0;L;;;;;N;;;;; +13E5;CHEROKEE LETTER TSI;Lo;0;L;;;;;N;;;;; +13E6;CHEROKEE LETTER TSO;Lo;0;L;;;;;N;;;;; +13E7;CHEROKEE LETTER TSU;Lo;0;L;;;;;N;;;;; +13E8;CHEROKEE LETTER TSV;Lo;0;L;;;;;N;;;;; +13E9;CHEROKEE LETTER WA;Lo;0;L;;;;;N;;;;; +13EA;CHEROKEE LETTER WE;Lo;0;L;;;;;N;;;;; +13EB;CHEROKEE LETTER WI;Lo;0;L;;;;;N;;;;; +13EC;CHEROKEE LETTER WO;Lo;0;L;;;;;N;;;;; +13ED;CHEROKEE LETTER WU;Lo;0;L;;;;;N;;;;; +13EE;CHEROKEE LETTER WV;Lo;0;L;;;;;N;;;;; +13EF;CHEROKEE LETTER YA;Lo;0;L;;;;;N;;;;; +13F0;CHEROKEE LETTER YE;Lo;0;L;;;;;N;;;;; +13F1;CHEROKEE LETTER YI;Lo;0;L;;;;;N;;;;; +13F2;CHEROKEE LETTER YO;Lo;0;L;;;;;N;;;;; +13F3;CHEROKEE LETTER YU;Lo;0;L;;;;;N;;;;; +13F4;CHEROKEE LETTER YV;Lo;0;L;;;;;N;;;;; +1400;CANADIAN SYLLABICS HYPHEN;Pd;0;ON;;;;;N;;;;; +1401;CANADIAN SYLLABICS E;Lo;0;L;;;;;N;;;;; +1402;CANADIAN SYLLABICS AAI;Lo;0;L;;;;;N;;;;; +1403;CANADIAN SYLLABICS I;Lo;0;L;;;;;N;;;;; +1404;CANADIAN SYLLABICS II;Lo;0;L;;;;;N;;;;; +1405;CANADIAN SYLLABICS O;Lo;0;L;;;;;N;;;;; +1406;CANADIAN SYLLABICS OO;Lo;0;L;;;;;N;;;;; +1407;CANADIAN SYLLABICS Y-CREE OO;Lo;0;L;;;;;N;;;;; +1408;CANADIAN SYLLABICS CARRIER EE;Lo;0;L;;;;;N;;;;; +1409;CANADIAN SYLLABICS CARRIER I;Lo;0;L;;;;;N;;;;; +140A;CANADIAN SYLLABICS A;Lo;0;L;;;;;N;;;;; +140B;CANADIAN SYLLABICS AA;Lo;0;L;;;;;N;;;;; +140C;CANADIAN SYLLABICS WE;Lo;0;L;;;;;N;;;;; +140D;CANADIAN SYLLABICS WEST-CREE WE;Lo;0;L;;;;;N;;;;; +140E;CANADIAN SYLLABICS WI;Lo;0;L;;;;;N;;;;; +140F;CANADIAN SYLLABICS WEST-CREE WI;Lo;0;L;;;;;N;;;;; +1410;CANADIAN SYLLABICS WII;Lo;0;L;;;;;N;;;;; +1411;CANADIAN SYLLABICS WEST-CREE WII;Lo;0;L;;;;;N;;;;; +1412;CANADIAN SYLLABICS WO;Lo;0;L;;;;;N;;;;; +1413;CANADIAN SYLLABICS WEST-CREE WO;Lo;0;L;;;;;N;;;;; +1414;CANADIAN SYLLABICS WOO;Lo;0;L;;;;;N;;;;; +1415;CANADIAN SYLLABICS WEST-CREE WOO;Lo;0;L;;;;;N;;;;; +1416;CANADIAN SYLLABICS NASKAPI WOO;Lo;0;L;;;;;N;;;;; +1417;CANADIAN SYLLABICS WA;Lo;0;L;;;;;N;;;;; +1418;CANADIAN SYLLABICS WEST-CREE WA;Lo;0;L;;;;;N;;;;; +1419;CANADIAN SYLLABICS WAA;Lo;0;L;;;;;N;;;;; +141A;CANADIAN SYLLABICS WEST-CREE WAA;Lo;0;L;;;;;N;;;;; +141B;CANADIAN SYLLABICS NASKAPI WAA;Lo;0;L;;;;;N;;;;; +141C;CANADIAN SYLLABICS AI;Lo;0;L;;;;;N;;;;; +141D;CANADIAN SYLLABICS Y-CREE W;Lo;0;L;;;;;N;;;;; +141E;CANADIAN SYLLABICS GLOTTAL STOP;Lo;0;L;;;;;N;;;;; +141F;CANADIAN SYLLABICS FINAL ACUTE;Lo;0;L;;;;;N;;;;; +1420;CANADIAN SYLLABICS FINAL GRAVE;Lo;0;L;;;;;N;;;;; +1421;CANADIAN SYLLABICS FINAL BOTTOM HALF RING;Lo;0;L;;;;;N;;;;; +1422;CANADIAN SYLLABICS FINAL TOP HALF RING;Lo;0;L;;;;;N;;;;; +1423;CANADIAN SYLLABICS FINAL RIGHT HALF RING;Lo;0;L;;;;;N;;;;; +1424;CANADIAN SYLLABICS FINAL RING;Lo;0;L;;;;;N;;;;; +1425;CANADIAN SYLLABICS FINAL DOUBLE ACUTE;Lo;0;L;;;;;N;;;;; +1426;CANADIAN SYLLABICS FINAL DOUBLE SHORT VERTICAL STROKES;Lo;0;L;;;;;N;;;;; +1427;CANADIAN SYLLABICS FINAL MIDDLE DOT;Lo;0;L;;;;;N;;;;; +1428;CANADIAN SYLLABICS FINAL SHORT HORIZONTAL STROKE;Lo;0;L;;;;;N;;;;; +1429;CANADIAN SYLLABICS FINAL PLUS;Lo;0;L;;;;;N;;;;; +142A;CANADIAN SYLLABICS FINAL DOWN TACK;Lo;0;L;;;;;N;;;;; +142B;CANADIAN SYLLABICS EN;Lo;0;L;;;;;N;;;;; +142C;CANADIAN SYLLABICS IN;Lo;0;L;;;;;N;;;;; +142D;CANADIAN SYLLABICS ON;Lo;0;L;;;;;N;;;;; +142E;CANADIAN SYLLABICS AN;Lo;0;L;;;;;N;;;;; +142F;CANADIAN SYLLABICS PE;Lo;0;L;;;;;N;;;;; +1430;CANADIAN SYLLABICS PAAI;Lo;0;L;;;;;N;;;;; +1431;CANADIAN SYLLABICS PI;Lo;0;L;;;;;N;;;;; +1432;CANADIAN SYLLABICS PII;Lo;0;L;;;;;N;;;;; +1433;CANADIAN SYLLABICS PO;Lo;0;L;;;;;N;;;;; +1434;CANADIAN SYLLABICS POO;Lo;0;L;;;;;N;;;;; +1435;CANADIAN SYLLABICS Y-CREE POO;Lo;0;L;;;;;N;;;;; +1436;CANADIAN SYLLABICS CARRIER HEE;Lo;0;L;;;;;N;;;;; +1437;CANADIAN SYLLABICS CARRIER HI;Lo;0;L;;;;;N;;;;; +1438;CANADIAN SYLLABICS PA;Lo;0;L;;;;;N;;;;; +1439;CANADIAN SYLLABICS PAA;Lo;0;L;;;;;N;;;;; +143A;CANADIAN SYLLABICS PWE;Lo;0;L;;;;;N;;;;; +143B;CANADIAN SYLLABICS WEST-CREE PWE;Lo;0;L;;;;;N;;;;; +143C;CANADIAN SYLLABICS PWI;Lo;0;L;;;;;N;;;;; +143D;CANADIAN SYLLABICS WEST-CREE PWI;Lo;0;L;;;;;N;;;;; +143E;CANADIAN SYLLABICS PWII;Lo;0;L;;;;;N;;;;; +143F;CANADIAN SYLLABICS WEST-CREE PWII;Lo;0;L;;;;;N;;;;; +1440;CANADIAN SYLLABICS PWO;Lo;0;L;;;;;N;;;;; +1441;CANADIAN SYLLABICS WEST-CREE PWO;Lo;0;L;;;;;N;;;;; +1442;CANADIAN SYLLABICS PWOO;Lo;0;L;;;;;N;;;;; +1443;CANADIAN SYLLABICS WEST-CREE PWOO;Lo;0;L;;;;;N;;;;; +1444;CANADIAN SYLLABICS PWA;Lo;0;L;;;;;N;;;;; +1445;CANADIAN SYLLABICS WEST-CREE PWA;Lo;0;L;;;;;N;;;;; +1446;CANADIAN SYLLABICS PWAA;Lo;0;L;;;;;N;;;;; +1447;CANADIAN SYLLABICS WEST-CREE PWAA;Lo;0;L;;;;;N;;;;; +1448;CANADIAN SYLLABICS Y-CREE PWAA;Lo;0;L;;;;;N;;;;; +1449;CANADIAN SYLLABICS P;Lo;0;L;;;;;N;;;;; +144A;CANADIAN SYLLABICS WEST-CREE P;Lo;0;L;;;;;N;;;;; +144B;CANADIAN SYLLABICS CARRIER H;Lo;0;L;;;;;N;;;;; +144C;CANADIAN SYLLABICS TE;Lo;0;L;;;;;N;;;;; +144D;CANADIAN SYLLABICS TAAI;Lo;0;L;;;;;N;;;;; +144E;CANADIAN SYLLABICS TI;Lo;0;L;;;;;N;;;;; +144F;CANADIAN SYLLABICS TII;Lo;0;L;;;;;N;;;;; +1450;CANADIAN SYLLABICS TO;Lo;0;L;;;;;N;;;;; +1451;CANADIAN SYLLABICS TOO;Lo;0;L;;;;;N;;;;; +1452;CANADIAN SYLLABICS Y-CREE TOO;Lo;0;L;;;;;N;;;;; +1453;CANADIAN SYLLABICS CARRIER DEE;Lo;0;L;;;;;N;;;;; +1454;CANADIAN SYLLABICS CARRIER DI;Lo;0;L;;;;;N;;;;; +1455;CANADIAN SYLLABICS TA;Lo;0;L;;;;;N;;;;; +1456;CANADIAN SYLLABICS TAA;Lo;0;L;;;;;N;;;;; +1457;CANADIAN SYLLABICS TWE;Lo;0;L;;;;;N;;;;; +1458;CANADIAN SYLLABICS WEST-CREE TWE;Lo;0;L;;;;;N;;;;; +1459;CANADIAN SYLLABICS TWI;Lo;0;L;;;;;N;;;;; +145A;CANADIAN SYLLABICS WEST-CREE TWI;Lo;0;L;;;;;N;;;;; +145B;CANADIAN SYLLABICS TWII;Lo;0;L;;;;;N;;;;; +145C;CANADIAN SYLLABICS WEST-CREE TWII;Lo;0;L;;;;;N;;;;; +145D;CANADIAN SYLLABICS TWO;Lo;0;L;;;;;N;;;;; +145E;CANADIAN SYLLABICS WEST-CREE TWO;Lo;0;L;;;;;N;;;;; +145F;CANADIAN SYLLABICS TWOO;Lo;0;L;;;;;N;;;;; +1460;CANADIAN SYLLABICS WEST-CREE TWOO;Lo;0;L;;;;;N;;;;; +1461;CANADIAN SYLLABICS TWA;Lo;0;L;;;;;N;;;;; +1462;CANADIAN SYLLABICS WEST-CREE TWA;Lo;0;L;;;;;N;;;;; +1463;CANADIAN SYLLABICS TWAA;Lo;0;L;;;;;N;;;;; +1464;CANADIAN SYLLABICS WEST-CREE TWAA;Lo;0;L;;;;;N;;;;; +1465;CANADIAN SYLLABICS NASKAPI TWAA;Lo;0;L;;;;;N;;;;; +1466;CANADIAN SYLLABICS T;Lo;0;L;;;;;N;;;;; +1467;CANADIAN SYLLABICS TTE;Lo;0;L;;;;;N;;;;; +1468;CANADIAN SYLLABICS TTI;Lo;0;L;;;;;N;;;;; +1469;CANADIAN SYLLABICS TTO;Lo;0;L;;;;;N;;;;; +146A;CANADIAN SYLLABICS TTA;Lo;0;L;;;;;N;;;;; +146B;CANADIAN SYLLABICS KE;Lo;0;L;;;;;N;;;;; +146C;CANADIAN SYLLABICS KAAI;Lo;0;L;;;;;N;;;;; +146D;CANADIAN SYLLABICS KI;Lo;0;L;;;;;N;;;;; +146E;CANADIAN SYLLABICS KII;Lo;0;L;;;;;N;;;;; +146F;CANADIAN SYLLABICS KO;Lo;0;L;;;;;N;;;;; +1470;CANADIAN SYLLABICS KOO;Lo;0;L;;;;;N;;;;; +1471;CANADIAN SYLLABICS Y-CREE KOO;Lo;0;L;;;;;N;;;;; +1472;CANADIAN SYLLABICS KA;Lo;0;L;;;;;N;;;;; +1473;CANADIAN SYLLABICS KAA;Lo;0;L;;;;;N;;;;; +1474;CANADIAN SYLLABICS KWE;Lo;0;L;;;;;N;;;;; +1475;CANADIAN SYLLABICS WEST-CREE KWE;Lo;0;L;;;;;N;;;;; +1476;CANADIAN SYLLABICS KWI;Lo;0;L;;;;;N;;;;; +1477;CANADIAN SYLLABICS WEST-CREE KWI;Lo;0;L;;;;;N;;;;; +1478;CANADIAN SYLLABICS KWII;Lo;0;L;;;;;N;;;;; +1479;CANADIAN SYLLABICS WEST-CREE KWII;Lo;0;L;;;;;N;;;;; +147A;CANADIAN SYLLABICS KWO;Lo;0;L;;;;;N;;;;; +147B;CANADIAN SYLLABICS WEST-CREE KWO;Lo;0;L;;;;;N;;;;; +147C;CANADIAN SYLLABICS KWOO;Lo;0;L;;;;;N;;;;; +147D;CANADIAN SYLLABICS WEST-CREE KWOO;Lo;0;L;;;;;N;;;;; +147E;CANADIAN SYLLABICS KWA;Lo;0;L;;;;;N;;;;; +147F;CANADIAN SYLLABICS WEST-CREE KWA;Lo;0;L;;;;;N;;;;; +1480;CANADIAN SYLLABICS KWAA;Lo;0;L;;;;;N;;;;; +1481;CANADIAN SYLLABICS WEST-CREE KWAA;Lo;0;L;;;;;N;;;;; +1482;CANADIAN SYLLABICS NASKAPI KWAA;Lo;0;L;;;;;N;;;;; +1483;CANADIAN SYLLABICS K;Lo;0;L;;;;;N;;;;; +1484;CANADIAN SYLLABICS KW;Lo;0;L;;;;;N;;;;; +1485;CANADIAN SYLLABICS SOUTH-SLAVEY KEH;Lo;0;L;;;;;N;;;;; +1486;CANADIAN SYLLABICS SOUTH-SLAVEY KIH;Lo;0;L;;;;;N;;;;; +1487;CANADIAN SYLLABICS SOUTH-SLAVEY KOH;Lo;0;L;;;;;N;;;;; +1488;CANADIAN SYLLABICS SOUTH-SLAVEY KAH;Lo;0;L;;;;;N;;;;; +1489;CANADIAN SYLLABICS CE;Lo;0;L;;;;;N;;;;; +148A;CANADIAN SYLLABICS CAAI;Lo;0;L;;;;;N;;;;; +148B;CANADIAN SYLLABICS CI;Lo;0;L;;;;;N;;;;; +148C;CANADIAN SYLLABICS CII;Lo;0;L;;;;;N;;;;; +148D;CANADIAN SYLLABICS CO;Lo;0;L;;;;;N;;;;; +148E;CANADIAN SYLLABICS COO;Lo;0;L;;;;;N;;;;; +148F;CANADIAN SYLLABICS Y-CREE COO;Lo;0;L;;;;;N;;;;; +1490;CANADIAN SYLLABICS CA;Lo;0;L;;;;;N;;;;; +1491;CANADIAN SYLLABICS CAA;Lo;0;L;;;;;N;;;;; +1492;CANADIAN SYLLABICS CWE;Lo;0;L;;;;;N;;;;; +1493;CANADIAN SYLLABICS WEST-CREE CWE;Lo;0;L;;;;;N;;;;; +1494;CANADIAN SYLLABICS CWI;Lo;0;L;;;;;N;;;;; +1495;CANADIAN SYLLABICS WEST-CREE CWI;Lo;0;L;;;;;N;;;;; +1496;CANADIAN SYLLABICS CWII;Lo;0;L;;;;;N;;;;; +1497;CANADIAN SYLLABICS WEST-CREE CWII;Lo;0;L;;;;;N;;;;; +1498;CANADIAN SYLLABICS CWO;Lo;0;L;;;;;N;;;;; +1499;CANADIAN SYLLABICS WEST-CREE CWO;Lo;0;L;;;;;N;;;;; +149A;CANADIAN SYLLABICS CWOO;Lo;0;L;;;;;N;;;;; +149B;CANADIAN SYLLABICS WEST-CREE CWOO;Lo;0;L;;;;;N;;;;; +149C;CANADIAN SYLLABICS CWA;Lo;0;L;;;;;N;;;;; +149D;CANADIAN SYLLABICS WEST-CREE CWA;Lo;0;L;;;;;N;;;;; +149E;CANADIAN SYLLABICS CWAA;Lo;0;L;;;;;N;;;;; +149F;CANADIAN SYLLABICS WEST-CREE CWAA;Lo;0;L;;;;;N;;;;; +14A0;CANADIAN SYLLABICS NASKAPI CWAA;Lo;0;L;;;;;N;;;;; +14A1;CANADIAN SYLLABICS C;Lo;0;L;;;;;N;;;;; +14A2;CANADIAN SYLLABICS SAYISI TH;Lo;0;L;;;;;N;;;;; +14A3;CANADIAN SYLLABICS ME;Lo;0;L;;;;;N;;;;; +14A4;CANADIAN SYLLABICS MAAI;Lo;0;L;;;;;N;;;;; +14A5;CANADIAN SYLLABICS MI;Lo;0;L;;;;;N;;;;; +14A6;CANADIAN SYLLABICS MII;Lo;0;L;;;;;N;;;;; +14A7;CANADIAN SYLLABICS MO;Lo;0;L;;;;;N;;;;; +14A8;CANADIAN SYLLABICS MOO;Lo;0;L;;;;;N;;;;; +14A9;CANADIAN SYLLABICS Y-CREE MOO;Lo;0;L;;;;;N;;;;; +14AA;CANADIAN SYLLABICS MA;Lo;0;L;;;;;N;;;;; +14AB;CANADIAN SYLLABICS MAA;Lo;0;L;;;;;N;;;;; +14AC;CANADIAN SYLLABICS MWE;Lo;0;L;;;;;N;;;;; +14AD;CANADIAN SYLLABICS WEST-CREE MWE;Lo;0;L;;;;;N;;;;; +14AE;CANADIAN SYLLABICS MWI;Lo;0;L;;;;;N;;;;; +14AF;CANADIAN SYLLABICS WEST-CREE MWI;Lo;0;L;;;;;N;;;;; +14B0;CANADIAN SYLLABICS MWII;Lo;0;L;;;;;N;;;;; +14B1;CANADIAN SYLLABICS WEST-CREE MWII;Lo;0;L;;;;;N;;;;; +14B2;CANADIAN SYLLABICS MWO;Lo;0;L;;;;;N;;;;; +14B3;CANADIAN SYLLABICS WEST-CREE MWO;Lo;0;L;;;;;N;;;;; +14B4;CANADIAN SYLLABICS MWOO;Lo;0;L;;;;;N;;;;; +14B5;CANADIAN SYLLABICS WEST-CREE MWOO;Lo;0;L;;;;;N;;;;; +14B6;CANADIAN SYLLABICS MWA;Lo;0;L;;;;;N;;;;; +14B7;CANADIAN SYLLABICS WEST-CREE MWA;Lo;0;L;;;;;N;;;;; +14B8;CANADIAN SYLLABICS MWAA;Lo;0;L;;;;;N;;;;; +14B9;CANADIAN SYLLABICS WEST-CREE MWAA;Lo;0;L;;;;;N;;;;; +14BA;CANADIAN SYLLABICS NASKAPI MWAA;Lo;0;L;;;;;N;;;;; +14BB;CANADIAN SYLLABICS M;Lo;0;L;;;;;N;;;;; +14BC;CANADIAN SYLLABICS WEST-CREE M;Lo;0;L;;;;;N;;;;; +14BD;CANADIAN SYLLABICS MH;Lo;0;L;;;;;N;;;;; +14BE;CANADIAN SYLLABICS ATHAPASCAN M;Lo;0;L;;;;;N;;;;; +14BF;CANADIAN SYLLABICS SAYISI M;Lo;0;L;;;;;N;;;;; +14C0;CANADIAN SYLLABICS NE;Lo;0;L;;;;;N;;;;; +14C1;CANADIAN SYLLABICS NAAI;Lo;0;L;;;;;N;;;;; +14C2;CANADIAN SYLLABICS NI;Lo;0;L;;;;;N;;;;; +14C3;CANADIAN SYLLABICS NII;Lo;0;L;;;;;N;;;;; +14C4;CANADIAN SYLLABICS NO;Lo;0;L;;;;;N;;;;; +14C5;CANADIAN SYLLABICS NOO;Lo;0;L;;;;;N;;;;; +14C6;CANADIAN SYLLABICS Y-CREE NOO;Lo;0;L;;;;;N;;;;; +14C7;CANADIAN SYLLABICS NA;Lo;0;L;;;;;N;;;;; +14C8;CANADIAN SYLLABICS NAA;Lo;0;L;;;;;N;;;;; +14C9;CANADIAN SYLLABICS NWE;Lo;0;L;;;;;N;;;;; +14CA;CANADIAN SYLLABICS WEST-CREE NWE;Lo;0;L;;;;;N;;;;; +14CB;CANADIAN SYLLABICS NWA;Lo;0;L;;;;;N;;;;; +14CC;CANADIAN SYLLABICS WEST-CREE NWA;Lo;0;L;;;;;N;;;;; +14CD;CANADIAN SYLLABICS NWAA;Lo;0;L;;;;;N;;;;; +14CE;CANADIAN SYLLABICS WEST-CREE NWAA;Lo;0;L;;;;;N;;;;; +14CF;CANADIAN SYLLABICS NASKAPI NWAA;Lo;0;L;;;;;N;;;;; +14D0;CANADIAN SYLLABICS N;Lo;0;L;;;;;N;;;;; +14D1;CANADIAN SYLLABICS CARRIER NG;Lo;0;L;;;;;N;;;;; +14D2;CANADIAN SYLLABICS NH;Lo;0;L;;;;;N;;;;; +14D3;CANADIAN SYLLABICS LE;Lo;0;L;;;;;N;;;;; +14D4;CANADIAN SYLLABICS LAAI;Lo;0;L;;;;;N;;;;; +14D5;CANADIAN SYLLABICS LI;Lo;0;L;;;;;N;;;;; +14D6;CANADIAN SYLLABICS LII;Lo;0;L;;;;;N;;;;; +14D7;CANADIAN SYLLABICS LO;Lo;0;L;;;;;N;;;;; +14D8;CANADIAN SYLLABICS LOO;Lo;0;L;;;;;N;;;;; +14D9;CANADIAN SYLLABICS Y-CREE LOO;Lo;0;L;;;;;N;;;;; +14DA;CANADIAN SYLLABICS LA;Lo;0;L;;;;;N;;;;; +14DB;CANADIAN SYLLABICS LAA;Lo;0;L;;;;;N;;;;; +14DC;CANADIAN SYLLABICS LWE;Lo;0;L;;;;;N;;;;; +14DD;CANADIAN SYLLABICS WEST-CREE LWE;Lo;0;L;;;;;N;;;;; +14DE;CANADIAN SYLLABICS LWI;Lo;0;L;;;;;N;;;;; +14DF;CANADIAN SYLLABICS WEST-CREE LWI;Lo;0;L;;;;;N;;;;; +14E0;CANADIAN SYLLABICS LWII;Lo;0;L;;;;;N;;;;; +14E1;CANADIAN SYLLABICS WEST-CREE LWII;Lo;0;L;;;;;N;;;;; +14E2;CANADIAN SYLLABICS LWO;Lo;0;L;;;;;N;;;;; +14E3;CANADIAN SYLLABICS WEST-CREE LWO;Lo;0;L;;;;;N;;;;; +14E4;CANADIAN SYLLABICS LWOO;Lo;0;L;;;;;N;;;;; +14E5;CANADIAN SYLLABICS WEST-CREE LWOO;Lo;0;L;;;;;N;;;;; +14E6;CANADIAN SYLLABICS LWA;Lo;0;L;;;;;N;;;;; +14E7;CANADIAN SYLLABICS WEST-CREE LWA;Lo;0;L;;;;;N;;;;; +14E8;CANADIAN SYLLABICS LWAA;Lo;0;L;;;;;N;;;;; +14E9;CANADIAN SYLLABICS WEST-CREE LWAA;Lo;0;L;;;;;N;;;;; +14EA;CANADIAN SYLLABICS L;Lo;0;L;;;;;N;;;;; +14EB;CANADIAN SYLLABICS WEST-CREE L;Lo;0;L;;;;;N;;;;; +14EC;CANADIAN SYLLABICS MEDIAL L;Lo;0;L;;;;;N;;;;; +14ED;CANADIAN SYLLABICS SE;Lo;0;L;;;;;N;;;;; +14EE;CANADIAN SYLLABICS SAAI;Lo;0;L;;;;;N;;;;; +14EF;CANADIAN SYLLABICS SI;Lo;0;L;;;;;N;;;;; +14F0;CANADIAN SYLLABICS SII;Lo;0;L;;;;;N;;;;; +14F1;CANADIAN SYLLABICS SO;Lo;0;L;;;;;N;;;;; +14F2;CANADIAN SYLLABICS SOO;Lo;0;L;;;;;N;;;;; +14F3;CANADIAN SYLLABICS Y-CREE SOO;Lo;0;L;;;;;N;;;;; +14F4;CANADIAN SYLLABICS SA;Lo;0;L;;;;;N;;;;; +14F5;CANADIAN SYLLABICS SAA;Lo;0;L;;;;;N;;;;; +14F6;CANADIAN SYLLABICS SWE;Lo;0;L;;;;;N;;;;; +14F7;CANADIAN SYLLABICS WEST-CREE SWE;Lo;0;L;;;;;N;;;;; +14F8;CANADIAN SYLLABICS SWI;Lo;0;L;;;;;N;;;;; +14F9;CANADIAN SYLLABICS WEST-CREE SWI;Lo;0;L;;;;;N;;;;; +14FA;CANADIAN SYLLABICS SWII;Lo;0;L;;;;;N;;;;; +14FB;CANADIAN SYLLABICS WEST-CREE SWII;Lo;0;L;;;;;N;;;;; +14FC;CANADIAN SYLLABICS SWO;Lo;0;L;;;;;N;;;;; +14FD;CANADIAN SYLLABICS WEST-CREE SWO;Lo;0;L;;;;;N;;;;; +14FE;CANADIAN SYLLABICS SWOO;Lo;0;L;;;;;N;;;;; +14FF;CANADIAN SYLLABICS WEST-CREE SWOO;Lo;0;L;;;;;N;;;;; +1500;CANADIAN SYLLABICS SWA;Lo;0;L;;;;;N;;;;; +1501;CANADIAN SYLLABICS WEST-CREE SWA;Lo;0;L;;;;;N;;;;; +1502;CANADIAN SYLLABICS SWAA;Lo;0;L;;;;;N;;;;; +1503;CANADIAN SYLLABICS WEST-CREE SWAA;Lo;0;L;;;;;N;;;;; +1504;CANADIAN SYLLABICS NASKAPI SWAA;Lo;0;L;;;;;N;;;;; +1505;CANADIAN SYLLABICS S;Lo;0;L;;;;;N;;;;; +1506;CANADIAN SYLLABICS ATHAPASCAN S;Lo;0;L;;;;;N;;;;; +1507;CANADIAN SYLLABICS SW;Lo;0;L;;;;;N;;;;; +1508;CANADIAN SYLLABICS BLACKFOOT S;Lo;0;L;;;;;N;;;;; +1509;CANADIAN SYLLABICS MOOSE-CREE SK;Lo;0;L;;;;;N;;;;; +150A;CANADIAN SYLLABICS NASKAPI SKW;Lo;0;L;;;;;N;;;;; +150B;CANADIAN SYLLABICS NASKAPI S-W;Lo;0;L;;;;;N;;;;; +150C;CANADIAN SYLLABICS NASKAPI SPWA;Lo;0;L;;;;;N;;;;; +150D;CANADIAN SYLLABICS NASKAPI STWA;Lo;0;L;;;;;N;;;;; +150E;CANADIAN SYLLABICS NASKAPI SKWA;Lo;0;L;;;;;N;;;;; +150F;CANADIAN SYLLABICS NASKAPI SCWA;Lo;0;L;;;;;N;;;;; +1510;CANADIAN SYLLABICS SHE;Lo;0;L;;;;;N;;;;; +1511;CANADIAN SYLLABICS SHI;Lo;0;L;;;;;N;;;;; +1512;CANADIAN SYLLABICS SHII;Lo;0;L;;;;;N;;;;; +1513;CANADIAN SYLLABICS SHO;Lo;0;L;;;;;N;;;;; +1514;CANADIAN SYLLABICS SHOO;Lo;0;L;;;;;N;;;;; +1515;CANADIAN SYLLABICS SHA;Lo;0;L;;;;;N;;;;; +1516;CANADIAN SYLLABICS SHAA;Lo;0;L;;;;;N;;;;; +1517;CANADIAN SYLLABICS SHWE;Lo;0;L;;;;;N;;;;; +1518;CANADIAN SYLLABICS WEST-CREE SHWE;Lo;0;L;;;;;N;;;;; +1519;CANADIAN SYLLABICS SHWI;Lo;0;L;;;;;N;;;;; +151A;CANADIAN SYLLABICS WEST-CREE SHWI;Lo;0;L;;;;;N;;;;; +151B;CANADIAN SYLLABICS SHWII;Lo;0;L;;;;;N;;;;; +151C;CANADIAN SYLLABICS WEST-CREE SHWII;Lo;0;L;;;;;N;;;;; +151D;CANADIAN SYLLABICS SHWO;Lo;0;L;;;;;N;;;;; +151E;CANADIAN SYLLABICS WEST-CREE SHWO;Lo;0;L;;;;;N;;;;; +151F;CANADIAN SYLLABICS SHWOO;Lo;0;L;;;;;N;;;;; +1520;CANADIAN SYLLABICS WEST-CREE SHWOO;Lo;0;L;;;;;N;;;;; +1521;CANADIAN SYLLABICS SHWA;Lo;0;L;;;;;N;;;;; +1522;CANADIAN SYLLABICS WEST-CREE SHWA;Lo;0;L;;;;;N;;;;; +1523;CANADIAN SYLLABICS SHWAA;Lo;0;L;;;;;N;;;;; +1524;CANADIAN SYLLABICS WEST-CREE SHWAA;Lo;0;L;;;;;N;;;;; +1525;CANADIAN SYLLABICS SH;Lo;0;L;;;;;N;;;;; +1526;CANADIAN SYLLABICS YE;Lo;0;L;;;;;N;;;;; +1527;CANADIAN SYLLABICS YAAI;Lo;0;L;;;;;N;;;;; +1528;CANADIAN SYLLABICS YI;Lo;0;L;;;;;N;;;;; +1529;CANADIAN SYLLABICS YII;Lo;0;L;;;;;N;;;;; +152A;CANADIAN SYLLABICS YO;Lo;0;L;;;;;N;;;;; +152B;CANADIAN SYLLABICS YOO;Lo;0;L;;;;;N;;;;; +152C;CANADIAN SYLLABICS Y-CREE YOO;Lo;0;L;;;;;N;;;;; +152D;CANADIAN SYLLABICS YA;Lo;0;L;;;;;N;;;;; +152E;CANADIAN SYLLABICS YAA;Lo;0;L;;;;;N;;;;; +152F;CANADIAN SYLLABICS YWE;Lo;0;L;;;;;N;;;;; +1530;CANADIAN SYLLABICS WEST-CREE YWE;Lo;0;L;;;;;N;;;;; +1531;CANADIAN SYLLABICS YWI;Lo;0;L;;;;;N;;;;; +1532;CANADIAN SYLLABICS WEST-CREE YWI;Lo;0;L;;;;;N;;;;; +1533;CANADIAN SYLLABICS YWII;Lo;0;L;;;;;N;;;;; +1534;CANADIAN SYLLABICS WEST-CREE YWII;Lo;0;L;;;;;N;;;;; +1535;CANADIAN SYLLABICS YWO;Lo;0;L;;;;;N;;;;; +1536;CANADIAN SYLLABICS WEST-CREE YWO;Lo;0;L;;;;;N;;;;; +1537;CANADIAN SYLLABICS YWOO;Lo;0;L;;;;;N;;;;; +1538;CANADIAN SYLLABICS WEST-CREE YWOO;Lo;0;L;;;;;N;;;;; +1539;CANADIAN SYLLABICS YWA;Lo;0;L;;;;;N;;;;; +153A;CANADIAN SYLLABICS WEST-CREE YWA;Lo;0;L;;;;;N;;;;; +153B;CANADIAN SYLLABICS YWAA;Lo;0;L;;;;;N;;;;; +153C;CANADIAN SYLLABICS WEST-CREE YWAA;Lo;0;L;;;;;N;;;;; +153D;CANADIAN SYLLABICS NASKAPI YWAA;Lo;0;L;;;;;N;;;;; +153E;CANADIAN SYLLABICS Y;Lo;0;L;;;;;N;;;;; +153F;CANADIAN SYLLABICS BIBLE-CREE Y;Lo;0;L;;;;;N;;;;; +1540;CANADIAN SYLLABICS WEST-CREE Y;Lo;0;L;;;;;N;;;;; +1541;CANADIAN SYLLABICS SAYISI YI;Lo;0;L;;;;;N;;;;; +1542;CANADIAN SYLLABICS RE;Lo;0;L;;;;;N;;;;; +1543;CANADIAN SYLLABICS R-CREE RE;Lo;0;L;;;;;N;;;;; +1544;CANADIAN SYLLABICS WEST-CREE LE;Lo;0;L;;;;;N;;;;; +1545;CANADIAN SYLLABICS RAAI;Lo;0;L;;;;;N;;;;; +1546;CANADIAN SYLLABICS RI;Lo;0;L;;;;;N;;;;; +1547;CANADIAN SYLLABICS RII;Lo;0;L;;;;;N;;;;; +1548;CANADIAN SYLLABICS RO;Lo;0;L;;;;;N;;;;; +1549;CANADIAN SYLLABICS ROO;Lo;0;L;;;;;N;;;;; +154A;CANADIAN SYLLABICS WEST-CREE LO;Lo;0;L;;;;;N;;;;; +154B;CANADIAN SYLLABICS RA;Lo;0;L;;;;;N;;;;; +154C;CANADIAN SYLLABICS RAA;Lo;0;L;;;;;N;;;;; +154D;CANADIAN SYLLABICS WEST-CREE LA;Lo;0;L;;;;;N;;;;; +154E;CANADIAN SYLLABICS RWAA;Lo;0;L;;;;;N;;;;; +154F;CANADIAN SYLLABICS WEST-CREE RWAA;Lo;0;L;;;;;N;;;;; +1550;CANADIAN SYLLABICS R;Lo;0;L;;;;;N;;;;; +1551;CANADIAN SYLLABICS WEST-CREE R;Lo;0;L;;;;;N;;;;; +1552;CANADIAN SYLLABICS MEDIAL R;Lo;0;L;;;;;N;;;;; +1553;CANADIAN SYLLABICS FE;Lo;0;L;;;;;N;;;;; +1554;CANADIAN SYLLABICS FAAI;Lo;0;L;;;;;N;;;;; +1555;CANADIAN SYLLABICS FI;Lo;0;L;;;;;N;;;;; +1556;CANADIAN SYLLABICS FII;Lo;0;L;;;;;N;;;;; +1557;CANADIAN SYLLABICS FO;Lo;0;L;;;;;N;;;;; +1558;CANADIAN SYLLABICS FOO;Lo;0;L;;;;;N;;;;; +1559;CANADIAN SYLLABICS FA;Lo;0;L;;;;;N;;;;; +155A;CANADIAN SYLLABICS FAA;Lo;0;L;;;;;N;;;;; +155B;CANADIAN SYLLABICS FWAA;Lo;0;L;;;;;N;;;;; +155C;CANADIAN SYLLABICS WEST-CREE FWAA;Lo;0;L;;;;;N;;;;; +155D;CANADIAN SYLLABICS F;Lo;0;L;;;;;N;;;;; +155E;CANADIAN SYLLABICS THE;Lo;0;L;;;;;N;;;;; +155F;CANADIAN SYLLABICS N-CREE THE;Lo;0;L;;;;;N;;;;; +1560;CANADIAN SYLLABICS THI;Lo;0;L;;;;;N;;;;; +1561;CANADIAN SYLLABICS N-CREE THI;Lo;0;L;;;;;N;;;;; +1562;CANADIAN SYLLABICS THII;Lo;0;L;;;;;N;;;;; +1563;CANADIAN SYLLABICS N-CREE THII;Lo;0;L;;;;;N;;;;; +1564;CANADIAN SYLLABICS THO;Lo;0;L;;;;;N;;;;; +1565;CANADIAN SYLLABICS THOO;Lo;0;L;;;;;N;;;;; +1566;CANADIAN SYLLABICS THA;Lo;0;L;;;;;N;;;;; +1567;CANADIAN SYLLABICS THAA;Lo;0;L;;;;;N;;;;; +1568;CANADIAN SYLLABICS THWAA;Lo;0;L;;;;;N;;;;; +1569;CANADIAN SYLLABICS WEST-CREE THWAA;Lo;0;L;;;;;N;;;;; +156A;CANADIAN SYLLABICS TH;Lo;0;L;;;;;N;;;;; +156B;CANADIAN SYLLABICS TTHE;Lo;0;L;;;;;N;;;;; +156C;CANADIAN SYLLABICS TTHI;Lo;0;L;;;;;N;;;;; +156D;CANADIAN SYLLABICS TTHO;Lo;0;L;;;;;N;;;;; +156E;CANADIAN SYLLABICS TTHA;Lo;0;L;;;;;N;;;;; +156F;CANADIAN SYLLABICS TTH;Lo;0;L;;;;;N;;;;; +1570;CANADIAN SYLLABICS TYE;Lo;0;L;;;;;N;;;;; +1571;CANADIAN SYLLABICS TYI;Lo;0;L;;;;;N;;;;; +1572;CANADIAN SYLLABICS TYO;Lo;0;L;;;;;N;;;;; +1573;CANADIAN SYLLABICS TYA;Lo;0;L;;;;;N;;;;; +1574;CANADIAN SYLLABICS NUNAVIK HE;Lo;0;L;;;;;N;;;;; +1575;CANADIAN SYLLABICS NUNAVIK HI;Lo;0;L;;;;;N;;;;; +1576;CANADIAN SYLLABICS NUNAVIK HII;Lo;0;L;;;;;N;;;;; +1577;CANADIAN SYLLABICS NUNAVIK HO;Lo;0;L;;;;;N;;;;; +1578;CANADIAN SYLLABICS NUNAVIK HOO;Lo;0;L;;;;;N;;;;; +1579;CANADIAN SYLLABICS NUNAVIK HA;Lo;0;L;;;;;N;;;;; +157A;CANADIAN SYLLABICS NUNAVIK HAA;Lo;0;L;;;;;N;;;;; +157B;CANADIAN SYLLABICS NUNAVIK H;Lo;0;L;;;;;N;;;;; +157C;CANADIAN SYLLABICS NUNAVUT H;Lo;0;L;;;;;N;;;;; +157D;CANADIAN SYLLABICS HK;Lo;0;L;;;;;N;;;;; +157E;CANADIAN SYLLABICS QAAI;Lo;0;L;;;;;N;;;;; +157F;CANADIAN SYLLABICS QI;Lo;0;L;;;;;N;;;;; +1580;CANADIAN SYLLABICS QII;Lo;0;L;;;;;N;;;;; +1581;CANADIAN SYLLABICS QO;Lo;0;L;;;;;N;;;;; +1582;CANADIAN SYLLABICS QOO;Lo;0;L;;;;;N;;;;; +1583;CANADIAN SYLLABICS QA;Lo;0;L;;;;;N;;;;; +1584;CANADIAN SYLLABICS QAA;Lo;0;L;;;;;N;;;;; +1585;CANADIAN SYLLABICS Q;Lo;0;L;;;;;N;;;;; +1586;CANADIAN SYLLABICS TLHE;Lo;0;L;;;;;N;;;;; +1587;CANADIAN SYLLABICS TLHI;Lo;0;L;;;;;N;;;;; +1588;CANADIAN SYLLABICS TLHO;Lo;0;L;;;;;N;;;;; +1589;CANADIAN SYLLABICS TLHA;Lo;0;L;;;;;N;;;;; +158A;CANADIAN SYLLABICS WEST-CREE RE;Lo;0;L;;;;;N;;;;; +158B;CANADIAN SYLLABICS WEST-CREE RI;Lo;0;L;;;;;N;;;;; +158C;CANADIAN SYLLABICS WEST-CREE RO;Lo;0;L;;;;;N;;;;; +158D;CANADIAN SYLLABICS WEST-CREE RA;Lo;0;L;;;;;N;;;;; +158E;CANADIAN SYLLABICS NGAAI;Lo;0;L;;;;;N;;;;; +158F;CANADIAN SYLLABICS NGI;Lo;0;L;;;;;N;;;;; +1590;CANADIAN SYLLABICS NGII;Lo;0;L;;;;;N;;;;; +1591;CANADIAN SYLLABICS NGO;Lo;0;L;;;;;N;;;;; +1592;CANADIAN SYLLABICS NGOO;Lo;0;L;;;;;N;;;;; +1593;CANADIAN SYLLABICS NGA;Lo;0;L;;;;;N;;;;; +1594;CANADIAN SYLLABICS NGAA;Lo;0;L;;;;;N;;;;; +1595;CANADIAN SYLLABICS NG;Lo;0;L;;;;;N;;;;; +1596;CANADIAN SYLLABICS NNG;Lo;0;L;;;;;N;;;;; +1597;CANADIAN SYLLABICS SAYISI SHE;Lo;0;L;;;;;N;;;;; +1598;CANADIAN SYLLABICS SAYISI SHI;Lo;0;L;;;;;N;;;;; +1599;CANADIAN SYLLABICS SAYISI SHO;Lo;0;L;;;;;N;;;;; +159A;CANADIAN SYLLABICS SAYISI SHA;Lo;0;L;;;;;N;;;;; +159B;CANADIAN SYLLABICS WOODS-CREE THE;Lo;0;L;;;;;N;;;;; +159C;CANADIAN SYLLABICS WOODS-CREE THI;Lo;0;L;;;;;N;;;;; +159D;CANADIAN SYLLABICS WOODS-CREE THO;Lo;0;L;;;;;N;;;;; +159E;CANADIAN SYLLABICS WOODS-CREE THA;Lo;0;L;;;;;N;;;;; +159F;CANADIAN SYLLABICS WOODS-CREE TH;Lo;0;L;;;;;N;;;;; +15A0;CANADIAN SYLLABICS LHI;Lo;0;L;;;;;N;;;;; +15A1;CANADIAN SYLLABICS LHII;Lo;0;L;;;;;N;;;;; +15A2;CANADIAN SYLLABICS LHO;Lo;0;L;;;;;N;;;;; +15A3;CANADIAN SYLLABICS LHOO;Lo;0;L;;;;;N;;;;; +15A4;CANADIAN SYLLABICS LHA;Lo;0;L;;;;;N;;;;; +15A5;CANADIAN SYLLABICS LHAA;Lo;0;L;;;;;N;;;;; +15A6;CANADIAN SYLLABICS LH;Lo;0;L;;;;;N;;;;; +15A7;CANADIAN SYLLABICS TH-CREE THE;Lo;0;L;;;;;N;;;;; +15A8;CANADIAN SYLLABICS TH-CREE THI;Lo;0;L;;;;;N;;;;; +15A9;CANADIAN SYLLABICS TH-CREE THII;Lo;0;L;;;;;N;;;;; +15AA;CANADIAN SYLLABICS TH-CREE THO;Lo;0;L;;;;;N;;;;; +15AB;CANADIAN SYLLABICS TH-CREE THOO;Lo;0;L;;;;;N;;;;; +15AC;CANADIAN SYLLABICS TH-CREE THA;Lo;0;L;;;;;N;;;;; +15AD;CANADIAN SYLLABICS TH-CREE THAA;Lo;0;L;;;;;N;;;;; +15AE;CANADIAN SYLLABICS TH-CREE TH;Lo;0;L;;;;;N;;;;; +15AF;CANADIAN SYLLABICS AIVILIK B;Lo;0;L;;;;;N;;;;; +15B0;CANADIAN SYLLABICS BLACKFOOT E;Lo;0;L;;;;;N;;;;; +15B1;CANADIAN SYLLABICS BLACKFOOT I;Lo;0;L;;;;;N;;;;; +15B2;CANADIAN SYLLABICS BLACKFOOT O;Lo;0;L;;;;;N;;;;; +15B3;CANADIAN SYLLABICS BLACKFOOT A;Lo;0;L;;;;;N;;;;; +15B4;CANADIAN SYLLABICS BLACKFOOT WE;Lo;0;L;;;;;N;;;;; +15B5;CANADIAN SYLLABICS BLACKFOOT WI;Lo;0;L;;;;;N;;;;; +15B6;CANADIAN SYLLABICS BLACKFOOT WO;Lo;0;L;;;;;N;;;;; +15B7;CANADIAN SYLLABICS BLACKFOOT WA;Lo;0;L;;;;;N;;;;; +15B8;CANADIAN SYLLABICS BLACKFOOT NE;Lo;0;L;;;;;N;;;;; +15B9;CANADIAN SYLLABICS BLACKFOOT NI;Lo;0;L;;;;;N;;;;; +15BA;CANADIAN SYLLABICS BLACKFOOT NO;Lo;0;L;;;;;N;;;;; +15BB;CANADIAN SYLLABICS BLACKFOOT NA;Lo;0;L;;;;;N;;;;; +15BC;CANADIAN SYLLABICS BLACKFOOT KE;Lo;0;L;;;;;N;;;;; +15BD;CANADIAN SYLLABICS BLACKFOOT KI;Lo;0;L;;;;;N;;;;; +15BE;CANADIAN SYLLABICS BLACKFOOT KO;Lo;0;L;;;;;N;;;;; +15BF;CANADIAN SYLLABICS BLACKFOOT KA;Lo;0;L;;;;;N;;;;; +15C0;CANADIAN SYLLABICS SAYISI HE;Lo;0;L;;;;;N;;;;; +15C1;CANADIAN SYLLABICS SAYISI HI;Lo;0;L;;;;;N;;;;; +15C2;CANADIAN SYLLABICS SAYISI HO;Lo;0;L;;;;;N;;;;; +15C3;CANADIAN SYLLABICS SAYISI HA;Lo;0;L;;;;;N;;;;; +15C4;CANADIAN SYLLABICS CARRIER GHU;Lo;0;L;;;;;N;;;;; +15C5;CANADIAN SYLLABICS CARRIER GHO;Lo;0;L;;;;;N;;;;; +15C6;CANADIAN SYLLABICS CARRIER GHE;Lo;0;L;;;;;N;;;;; +15C7;CANADIAN SYLLABICS CARRIER GHEE;Lo;0;L;;;;;N;;;;; +15C8;CANADIAN SYLLABICS CARRIER GHI;Lo;0;L;;;;;N;;;;; +15C9;CANADIAN SYLLABICS CARRIER GHA;Lo;0;L;;;;;N;;;;; +15CA;CANADIAN SYLLABICS CARRIER RU;Lo;0;L;;;;;N;;;;; +15CB;CANADIAN SYLLABICS CARRIER RO;Lo;0;L;;;;;N;;;;; +15CC;CANADIAN SYLLABICS CARRIER RE;Lo;0;L;;;;;N;;;;; +15CD;CANADIAN SYLLABICS CARRIER REE;Lo;0;L;;;;;N;;;;; +15CE;CANADIAN SYLLABICS CARRIER RI;Lo;0;L;;;;;N;;;;; +15CF;CANADIAN SYLLABICS CARRIER RA;Lo;0;L;;;;;N;;;;; +15D0;CANADIAN SYLLABICS CARRIER WU;Lo;0;L;;;;;N;;;;; +15D1;CANADIAN SYLLABICS CARRIER WO;Lo;0;L;;;;;N;;;;; +15D2;CANADIAN SYLLABICS CARRIER WE;Lo;0;L;;;;;N;;;;; +15D3;CANADIAN SYLLABICS CARRIER WEE;Lo;0;L;;;;;N;;;;; +15D4;CANADIAN SYLLABICS CARRIER WI;Lo;0;L;;;;;N;;;;; +15D5;CANADIAN SYLLABICS CARRIER WA;Lo;0;L;;;;;N;;;;; +15D6;CANADIAN SYLLABICS CARRIER HWU;Lo;0;L;;;;;N;;;;; +15D7;CANADIAN SYLLABICS CARRIER HWO;Lo;0;L;;;;;N;;;;; +15D8;CANADIAN SYLLABICS CARRIER HWE;Lo;0;L;;;;;N;;;;; +15D9;CANADIAN SYLLABICS CARRIER HWEE;Lo;0;L;;;;;N;;;;; +15DA;CANADIAN SYLLABICS CARRIER HWI;Lo;0;L;;;;;N;;;;; +15DB;CANADIAN SYLLABICS CARRIER HWA;Lo;0;L;;;;;N;;;;; +15DC;CANADIAN SYLLABICS CARRIER THU;Lo;0;L;;;;;N;;;;; +15DD;CANADIAN SYLLABICS CARRIER THO;Lo;0;L;;;;;N;;;;; +15DE;CANADIAN SYLLABICS CARRIER THE;Lo;0;L;;;;;N;;;;; +15DF;CANADIAN SYLLABICS CARRIER THEE;Lo;0;L;;;;;N;;;;; +15E0;CANADIAN SYLLABICS CARRIER THI;Lo;0;L;;;;;N;;;;; +15E1;CANADIAN SYLLABICS CARRIER THA;Lo;0;L;;;;;N;;;;; +15E2;CANADIAN SYLLABICS CARRIER TTU;Lo;0;L;;;;;N;;;;; +15E3;CANADIAN SYLLABICS CARRIER TTO;Lo;0;L;;;;;N;;;;; +15E4;CANADIAN SYLLABICS CARRIER TTE;Lo;0;L;;;;;N;;;;; +15E5;CANADIAN SYLLABICS CARRIER TTEE;Lo;0;L;;;;;N;;;;; +15E6;CANADIAN SYLLABICS CARRIER TTI;Lo;0;L;;;;;N;;;;; +15E7;CANADIAN SYLLABICS CARRIER TTA;Lo;0;L;;;;;N;;;;; +15E8;CANADIAN SYLLABICS CARRIER PU;Lo;0;L;;;;;N;;;;; +15E9;CANADIAN SYLLABICS CARRIER PO;Lo;0;L;;;;;N;;;;; +15EA;CANADIAN SYLLABICS CARRIER PE;Lo;0;L;;;;;N;;;;; +15EB;CANADIAN SYLLABICS CARRIER PEE;Lo;0;L;;;;;N;;;;; +15EC;CANADIAN SYLLABICS CARRIER PI;Lo;0;L;;;;;N;;;;; +15ED;CANADIAN SYLLABICS CARRIER PA;Lo;0;L;;;;;N;;;;; +15EE;CANADIAN SYLLABICS CARRIER P;Lo;0;L;;;;;N;;;;; +15EF;CANADIAN SYLLABICS CARRIER GU;Lo;0;L;;;;;N;;;;; +15F0;CANADIAN SYLLABICS CARRIER GO;Lo;0;L;;;;;N;;;;; +15F1;CANADIAN SYLLABICS CARRIER GE;Lo;0;L;;;;;N;;;;; +15F2;CANADIAN SYLLABICS CARRIER GEE;Lo;0;L;;;;;N;;;;; +15F3;CANADIAN SYLLABICS CARRIER GI;Lo;0;L;;;;;N;;;;; +15F4;CANADIAN SYLLABICS CARRIER GA;Lo;0;L;;;;;N;;;;; +15F5;CANADIAN SYLLABICS CARRIER KHU;Lo;0;L;;;;;N;;;;; +15F6;CANADIAN SYLLABICS CARRIER KHO;Lo;0;L;;;;;N;;;;; +15F7;CANADIAN SYLLABICS CARRIER KHE;Lo;0;L;;;;;N;;;;; +15F8;CANADIAN SYLLABICS CARRIER KHEE;Lo;0;L;;;;;N;;;;; +15F9;CANADIAN SYLLABICS CARRIER KHI;Lo;0;L;;;;;N;;;;; +15FA;CANADIAN SYLLABICS CARRIER KHA;Lo;0;L;;;;;N;;;;; +15FB;CANADIAN SYLLABICS CARRIER KKU;Lo;0;L;;;;;N;;;;; +15FC;CANADIAN SYLLABICS CARRIER KKO;Lo;0;L;;;;;N;;;;; +15FD;CANADIAN SYLLABICS CARRIER KKE;Lo;0;L;;;;;N;;;;; +15FE;CANADIAN SYLLABICS CARRIER KKEE;Lo;0;L;;;;;N;;;;; +15FF;CANADIAN SYLLABICS CARRIER KKI;Lo;0;L;;;;;N;;;;; +1600;CANADIAN SYLLABICS CARRIER KKA;Lo;0;L;;;;;N;;;;; +1601;CANADIAN SYLLABICS CARRIER KK;Lo;0;L;;;;;N;;;;; +1602;CANADIAN SYLLABICS CARRIER NU;Lo;0;L;;;;;N;;;;; +1603;CANADIAN SYLLABICS CARRIER NO;Lo;0;L;;;;;N;;;;; +1604;CANADIAN SYLLABICS CARRIER NE;Lo;0;L;;;;;N;;;;; +1605;CANADIAN SYLLABICS CARRIER NEE;Lo;0;L;;;;;N;;;;; +1606;CANADIAN SYLLABICS CARRIER NI;Lo;0;L;;;;;N;;;;; +1607;CANADIAN SYLLABICS CARRIER NA;Lo;0;L;;;;;N;;;;; +1608;CANADIAN SYLLABICS CARRIER MU;Lo;0;L;;;;;N;;;;; +1609;CANADIAN SYLLABICS CARRIER MO;Lo;0;L;;;;;N;;;;; +160A;CANADIAN SYLLABICS CARRIER ME;Lo;0;L;;;;;N;;;;; +160B;CANADIAN SYLLABICS CARRIER MEE;Lo;0;L;;;;;N;;;;; +160C;CANADIAN SYLLABICS CARRIER MI;Lo;0;L;;;;;N;;;;; +160D;CANADIAN SYLLABICS CARRIER MA;Lo;0;L;;;;;N;;;;; +160E;CANADIAN SYLLABICS CARRIER YU;Lo;0;L;;;;;N;;;;; +160F;CANADIAN SYLLABICS CARRIER YO;Lo;0;L;;;;;N;;;;; +1610;CANADIAN SYLLABICS CARRIER YE;Lo;0;L;;;;;N;;;;; +1611;CANADIAN SYLLABICS CARRIER YEE;Lo;0;L;;;;;N;;;;; +1612;CANADIAN SYLLABICS CARRIER YI;Lo;0;L;;;;;N;;;;; +1613;CANADIAN SYLLABICS CARRIER YA;Lo;0;L;;;;;N;;;;; +1614;CANADIAN SYLLABICS CARRIER JU;Lo;0;L;;;;;N;;;;; +1615;CANADIAN SYLLABICS SAYISI JU;Lo;0;L;;;;;N;;;;; +1616;CANADIAN SYLLABICS CARRIER JO;Lo;0;L;;;;;N;;;;; +1617;CANADIAN SYLLABICS CARRIER JE;Lo;0;L;;;;;N;;;;; +1618;CANADIAN SYLLABICS CARRIER JEE;Lo;0;L;;;;;N;;;;; +1619;CANADIAN SYLLABICS CARRIER JI;Lo;0;L;;;;;N;;;;; +161A;CANADIAN SYLLABICS SAYISI JI;Lo;0;L;;;;;N;;;;; +161B;CANADIAN SYLLABICS CARRIER JA;Lo;0;L;;;;;N;;;;; +161C;CANADIAN SYLLABICS CARRIER JJU;Lo;0;L;;;;;N;;;;; +161D;CANADIAN SYLLABICS CARRIER JJO;Lo;0;L;;;;;N;;;;; +161E;CANADIAN SYLLABICS CARRIER JJE;Lo;0;L;;;;;N;;;;; +161F;CANADIAN SYLLABICS CARRIER JJEE;Lo;0;L;;;;;N;;;;; +1620;CANADIAN SYLLABICS CARRIER JJI;Lo;0;L;;;;;N;;;;; +1621;CANADIAN SYLLABICS CARRIER JJA;Lo;0;L;;;;;N;;;;; +1622;CANADIAN SYLLABICS CARRIER LU;Lo;0;L;;;;;N;;;;; +1623;CANADIAN SYLLABICS CARRIER LO;Lo;0;L;;;;;N;;;;; +1624;CANADIAN SYLLABICS CARRIER LE;Lo;0;L;;;;;N;;;;; +1625;CANADIAN SYLLABICS CARRIER LEE;Lo;0;L;;;;;N;;;;; +1626;CANADIAN SYLLABICS CARRIER LI;Lo;0;L;;;;;N;;;;; +1627;CANADIAN SYLLABICS CARRIER LA;Lo;0;L;;;;;N;;;;; +1628;CANADIAN SYLLABICS CARRIER DLU;Lo;0;L;;;;;N;;;;; +1629;CANADIAN SYLLABICS CARRIER DLO;Lo;0;L;;;;;N;;;;; +162A;CANADIAN SYLLABICS CARRIER DLE;Lo;0;L;;;;;N;;;;; +162B;CANADIAN SYLLABICS CARRIER DLEE;Lo;0;L;;;;;N;;;;; +162C;CANADIAN SYLLABICS CARRIER DLI;Lo;0;L;;;;;N;;;;; +162D;CANADIAN SYLLABICS CARRIER DLA;Lo;0;L;;;;;N;;;;; +162E;CANADIAN SYLLABICS CARRIER LHU;Lo;0;L;;;;;N;;;;; +162F;CANADIAN SYLLABICS CARRIER LHO;Lo;0;L;;;;;N;;;;; +1630;CANADIAN SYLLABICS CARRIER LHE;Lo;0;L;;;;;N;;;;; +1631;CANADIAN SYLLABICS CARRIER LHEE;Lo;0;L;;;;;N;;;;; +1632;CANADIAN SYLLABICS CARRIER LHI;Lo;0;L;;;;;N;;;;; +1633;CANADIAN SYLLABICS CARRIER LHA;Lo;0;L;;;;;N;;;;; +1634;CANADIAN SYLLABICS CARRIER TLHU;Lo;0;L;;;;;N;;;;; +1635;CANADIAN SYLLABICS CARRIER TLHO;Lo;0;L;;;;;N;;;;; +1636;CANADIAN SYLLABICS CARRIER TLHE;Lo;0;L;;;;;N;;;;; +1637;CANADIAN SYLLABICS CARRIER TLHEE;Lo;0;L;;;;;N;;;;; +1638;CANADIAN SYLLABICS CARRIER TLHI;Lo;0;L;;;;;N;;;;; +1639;CANADIAN SYLLABICS CARRIER TLHA;Lo;0;L;;;;;N;;;;; +163A;CANADIAN SYLLABICS CARRIER TLU;Lo;0;L;;;;;N;;;;; +163B;CANADIAN SYLLABICS CARRIER TLO;Lo;0;L;;;;;N;;;;; +163C;CANADIAN SYLLABICS CARRIER TLE;Lo;0;L;;;;;N;;;;; +163D;CANADIAN SYLLABICS CARRIER TLEE;Lo;0;L;;;;;N;;;;; +163E;CANADIAN SYLLABICS CARRIER TLI;Lo;0;L;;;;;N;;;;; +163F;CANADIAN SYLLABICS CARRIER TLA;Lo;0;L;;;;;N;;;;; +1640;CANADIAN SYLLABICS CARRIER ZU;Lo;0;L;;;;;N;;;;; +1641;CANADIAN SYLLABICS CARRIER ZO;Lo;0;L;;;;;N;;;;; +1642;CANADIAN SYLLABICS CARRIER ZE;Lo;0;L;;;;;N;;;;; +1643;CANADIAN SYLLABICS CARRIER ZEE;Lo;0;L;;;;;N;;;;; +1644;CANADIAN SYLLABICS CARRIER ZI;Lo;0;L;;;;;N;;;;; +1645;CANADIAN SYLLABICS CARRIER ZA;Lo;0;L;;;;;N;;;;; +1646;CANADIAN SYLLABICS CARRIER Z;Lo;0;L;;;;;N;;;;; +1647;CANADIAN SYLLABICS CARRIER INITIAL Z;Lo;0;L;;;;;N;;;;; +1648;CANADIAN SYLLABICS CARRIER DZU;Lo;0;L;;;;;N;;;;; +1649;CANADIAN SYLLABICS CARRIER DZO;Lo;0;L;;;;;N;;;;; +164A;CANADIAN SYLLABICS CARRIER DZE;Lo;0;L;;;;;N;;;;; +164B;CANADIAN SYLLABICS CARRIER DZEE;Lo;0;L;;;;;N;;;;; +164C;CANADIAN SYLLABICS CARRIER DZI;Lo;0;L;;;;;N;;;;; +164D;CANADIAN SYLLABICS CARRIER DZA;Lo;0;L;;;;;N;;;;; +164E;CANADIAN SYLLABICS CARRIER SU;Lo;0;L;;;;;N;;;;; +164F;CANADIAN SYLLABICS CARRIER SO;Lo;0;L;;;;;N;;;;; +1650;CANADIAN SYLLABICS CARRIER SE;Lo;0;L;;;;;N;;;;; +1651;CANADIAN SYLLABICS CARRIER SEE;Lo;0;L;;;;;N;;;;; +1652;CANADIAN SYLLABICS CARRIER SI;Lo;0;L;;;;;N;;;;; +1653;CANADIAN SYLLABICS CARRIER SA;Lo;0;L;;;;;N;;;;; +1654;CANADIAN SYLLABICS CARRIER SHU;Lo;0;L;;;;;N;;;;; +1655;CANADIAN SYLLABICS CARRIER SHO;Lo;0;L;;;;;N;;;;; +1656;CANADIAN SYLLABICS CARRIER SHE;Lo;0;L;;;;;N;;;;; +1657;CANADIAN SYLLABICS CARRIER SHEE;Lo;0;L;;;;;N;;;;; +1658;CANADIAN SYLLABICS CARRIER SHI;Lo;0;L;;;;;N;;;;; +1659;CANADIAN SYLLABICS CARRIER SHA;Lo;0;L;;;;;N;;;;; +165A;CANADIAN SYLLABICS CARRIER SH;Lo;0;L;;;;;N;;;;; +165B;CANADIAN SYLLABICS CARRIER TSU;Lo;0;L;;;;;N;;;;; +165C;CANADIAN SYLLABICS CARRIER TSO;Lo;0;L;;;;;N;;;;; +165D;CANADIAN SYLLABICS CARRIER TSE;Lo;0;L;;;;;N;;;;; +165E;CANADIAN SYLLABICS CARRIER TSEE;Lo;0;L;;;;;N;;;;; +165F;CANADIAN SYLLABICS CARRIER TSI;Lo;0;L;;;;;N;;;;; +1660;CANADIAN SYLLABICS CARRIER TSA;Lo;0;L;;;;;N;;;;; +1661;CANADIAN SYLLABICS CARRIER CHU;Lo;0;L;;;;;N;;;;; +1662;CANADIAN SYLLABICS CARRIER CHO;Lo;0;L;;;;;N;;;;; +1663;CANADIAN SYLLABICS CARRIER CHE;Lo;0;L;;;;;N;;;;; +1664;CANADIAN SYLLABICS CARRIER CHEE;Lo;0;L;;;;;N;;;;; +1665;CANADIAN SYLLABICS CARRIER CHI;Lo;0;L;;;;;N;;;;; +1666;CANADIAN SYLLABICS CARRIER CHA;Lo;0;L;;;;;N;;;;; +1667;CANADIAN SYLLABICS CARRIER TTSU;Lo;0;L;;;;;N;;;;; +1668;CANADIAN SYLLABICS CARRIER TTSO;Lo;0;L;;;;;N;;;;; +1669;CANADIAN SYLLABICS CARRIER TTSE;Lo;0;L;;;;;N;;;;; +166A;CANADIAN SYLLABICS CARRIER TTSEE;Lo;0;L;;;;;N;;;;; +166B;CANADIAN SYLLABICS CARRIER TTSI;Lo;0;L;;;;;N;;;;; +166C;CANADIAN SYLLABICS CARRIER TTSA;Lo;0;L;;;;;N;;;;; +166D;CANADIAN SYLLABICS CHI SIGN;Po;0;L;;;;;N;;;;; +166E;CANADIAN SYLLABICS FULL STOP;Po;0;L;;;;;N;;;;; +166F;CANADIAN SYLLABICS QAI;Lo;0;L;;;;;N;;;;; +1670;CANADIAN SYLLABICS NGAI;Lo;0;L;;;;;N;;;;; +1671;CANADIAN SYLLABICS NNGI;Lo;0;L;;;;;N;;;;; +1672;CANADIAN SYLLABICS NNGII;Lo;0;L;;;;;N;;;;; +1673;CANADIAN SYLLABICS NNGO;Lo;0;L;;;;;N;;;;; +1674;CANADIAN SYLLABICS NNGOO;Lo;0;L;;;;;N;;;;; +1675;CANADIAN SYLLABICS NNGA;Lo;0;L;;;;;N;;;;; +1676;CANADIAN SYLLABICS NNGAA;Lo;0;L;;;;;N;;;;; +1677;CANADIAN SYLLABICS WOODS-CREE THWEE;Lo;0;L;;;;;N;;;;; +1678;CANADIAN SYLLABICS WOODS-CREE THWI;Lo;0;L;;;;;N;;;;; +1679;CANADIAN SYLLABICS WOODS-CREE THWII;Lo;0;L;;;;;N;;;;; +167A;CANADIAN SYLLABICS WOODS-CREE THWO;Lo;0;L;;;;;N;;;;; +167B;CANADIAN SYLLABICS WOODS-CREE THWOO;Lo;0;L;;;;;N;;;;; +167C;CANADIAN SYLLABICS WOODS-CREE THWA;Lo;0;L;;;;;N;;;;; +167D;CANADIAN SYLLABICS WOODS-CREE THWAA;Lo;0;L;;;;;N;;;;; +167E;CANADIAN SYLLABICS WOODS-CREE FINAL TH;Lo;0;L;;;;;N;;;;; +167F;CANADIAN SYLLABICS BLACKFOOT W;Lo;0;L;;;;;N;;;;; +1680;OGHAM SPACE MARK;Zs;0;WS;;;;;N;;;;; +1681;OGHAM LETTER BEITH;Lo;0;L;;;;;N;;;;; +1682;OGHAM LETTER LUIS;Lo;0;L;;;;;N;;;;; +1683;OGHAM LETTER FEARN;Lo;0;L;;;;;N;;;;; +1684;OGHAM LETTER SAIL;Lo;0;L;;;;;N;;;;; +1685;OGHAM LETTER NION;Lo;0;L;;;;;N;;;;; +1686;OGHAM LETTER UATH;Lo;0;L;;;;;N;;;;; +1687;OGHAM LETTER DAIR;Lo;0;L;;;;;N;;;;; +1688;OGHAM LETTER TINNE;Lo;0;L;;;;;N;;;;; +1689;OGHAM LETTER COLL;Lo;0;L;;;;;N;;;;; +168A;OGHAM LETTER CEIRT;Lo;0;L;;;;;N;;;;; +168B;OGHAM LETTER MUIN;Lo;0;L;;;;;N;;;;; +168C;OGHAM LETTER GORT;Lo;0;L;;;;;N;;;;; +168D;OGHAM LETTER NGEADAL;Lo;0;L;;;;;N;;;;; +168E;OGHAM LETTER STRAIF;Lo;0;L;;;;;N;;;;; +168F;OGHAM LETTER RUIS;Lo;0;L;;;;;N;;;;; +1690;OGHAM LETTER AILM;Lo;0;L;;;;;N;;;;; +1691;OGHAM LETTER ONN;Lo;0;L;;;;;N;;;;; +1692;OGHAM LETTER UR;Lo;0;L;;;;;N;;;;; +1693;OGHAM LETTER EADHADH;Lo;0;L;;;;;N;;;;; +1694;OGHAM LETTER IODHADH;Lo;0;L;;;;;N;;;;; +1695;OGHAM LETTER EABHADH;Lo;0;L;;;;;N;;;;; +1696;OGHAM LETTER OR;Lo;0;L;;;;;N;;;;; +1697;OGHAM LETTER UILLEANN;Lo;0;L;;;;;N;;;;; +1698;OGHAM LETTER IFIN;Lo;0;L;;;;;N;;;;; +1699;OGHAM LETTER EAMHANCHOLL;Lo;0;L;;;;;N;;;;; +169A;OGHAM LETTER PEITH;Lo;0;L;;;;;N;;;;; +169B;OGHAM FEATHER MARK;Ps;0;ON;;;;;Y;;;;; +169C;OGHAM REVERSED FEATHER MARK;Pe;0;ON;;;;;Y;;;;; +16A0;RUNIC LETTER FEHU FEOH FE F;Lo;0;L;;;;;N;;;;; +16A1;RUNIC LETTER V;Lo;0;L;;;;;N;;;;; +16A2;RUNIC LETTER URUZ UR U;Lo;0;L;;;;;N;;;;; +16A3;RUNIC LETTER YR;Lo;0;L;;;;;N;;;;; +16A4;RUNIC LETTER Y;Lo;0;L;;;;;N;;;;; +16A5;RUNIC LETTER W;Lo;0;L;;;;;N;;;;; +16A6;RUNIC LETTER THURISAZ THURS THORN;Lo;0;L;;;;;N;;;;; +16A7;RUNIC LETTER ETH;Lo;0;L;;;;;N;;;;; +16A8;RUNIC LETTER ANSUZ A;Lo;0;L;;;;;N;;;;; +16A9;RUNIC LETTER OS O;Lo;0;L;;;;;N;;;;; +16AA;RUNIC LETTER AC A;Lo;0;L;;;;;N;;;;; +16AB;RUNIC LETTER AESC;Lo;0;L;;;;;N;;;;; +16AC;RUNIC LETTER LONG-BRANCH-OSS O;Lo;0;L;;;;;N;;;;; +16AD;RUNIC LETTER SHORT-TWIG-OSS O;Lo;0;L;;;;;N;;;;; +16AE;RUNIC LETTER O;Lo;0;L;;;;;N;;;;; +16AF;RUNIC LETTER OE;Lo;0;L;;;;;N;;;;; +16B0;RUNIC LETTER ON;Lo;0;L;;;;;N;;;;; +16B1;RUNIC LETTER RAIDO RAD REID R;Lo;0;L;;;;;N;;;;; +16B2;RUNIC LETTER KAUNA;Lo;0;L;;;;;N;;;;; +16B3;RUNIC LETTER CEN;Lo;0;L;;;;;N;;;;; +16B4;RUNIC LETTER KAUN K;Lo;0;L;;;;;N;;;;; +16B5;RUNIC LETTER G;Lo;0;L;;;;;N;;;;; +16B6;RUNIC LETTER ENG;Lo;0;L;;;;;N;;;;; +16B7;RUNIC LETTER GEBO GYFU G;Lo;0;L;;;;;N;;;;; +16B8;RUNIC LETTER GAR;Lo;0;L;;;;;N;;;;; +16B9;RUNIC LETTER WUNJO WYNN W;Lo;0;L;;;;;N;;;;; +16BA;RUNIC LETTER HAGLAZ H;Lo;0;L;;;;;N;;;;; +16BB;RUNIC LETTER HAEGL H;Lo;0;L;;;;;N;;;;; +16BC;RUNIC LETTER LONG-BRANCH-HAGALL H;Lo;0;L;;;;;N;;;;; +16BD;RUNIC LETTER SHORT-TWIG-HAGALL H;Lo;0;L;;;;;N;;;;; +16BE;RUNIC LETTER NAUDIZ NYD NAUD N;Lo;0;L;;;;;N;;;;; +16BF;RUNIC LETTER SHORT-TWIG-NAUD N;Lo;0;L;;;;;N;;;;; +16C0;RUNIC LETTER DOTTED-N;Lo;0;L;;;;;N;;;;; +16C1;RUNIC LETTER ISAZ IS ISS I;Lo;0;L;;;;;N;;;;; +16C2;RUNIC LETTER E;Lo;0;L;;;;;N;;;;; +16C3;RUNIC LETTER JERAN J;Lo;0;L;;;;;N;;;;; +16C4;RUNIC LETTER GER;Lo;0;L;;;;;N;;;;; +16C5;RUNIC LETTER LONG-BRANCH-AR AE;Lo;0;L;;;;;N;;;;; +16C6;RUNIC LETTER SHORT-TWIG-AR A;Lo;0;L;;;;;N;;;;; +16C7;RUNIC LETTER IWAZ EOH;Lo;0;L;;;;;N;;;;; +16C8;RUNIC LETTER PERTHO PEORTH P;Lo;0;L;;;;;N;;;;; +16C9;RUNIC LETTER ALGIZ EOLHX;Lo;0;L;;;;;N;;;;; +16CA;RUNIC LETTER SOWILO S;Lo;0;L;;;;;N;;;;; +16CB;RUNIC LETTER SIGEL LONG-BRANCH-SOL S;Lo;0;L;;;;;N;;;;; +16CC;RUNIC LETTER SHORT-TWIG-SOL S;Lo;0;L;;;;;N;;;;; +16CD;RUNIC LETTER C;Lo;0;L;;;;;N;;;;; +16CE;RUNIC LETTER Z;Lo;0;L;;;;;N;;;;; +16CF;RUNIC LETTER TIWAZ TIR TYR T;Lo;0;L;;;;;N;;;;; +16D0;RUNIC LETTER SHORT-TWIG-TYR T;Lo;0;L;;;;;N;;;;; +16D1;RUNIC LETTER D;Lo;0;L;;;;;N;;;;; +16D2;RUNIC LETTER BERKANAN BEORC BJARKAN B;Lo;0;L;;;;;N;;;;; +16D3;RUNIC LETTER SHORT-TWIG-BJARKAN B;Lo;0;L;;;;;N;;;;; +16D4;RUNIC LETTER DOTTED-P;Lo;0;L;;;;;N;;;;; +16D5;RUNIC LETTER OPEN-P;Lo;0;L;;;;;N;;;;; +16D6;RUNIC LETTER EHWAZ EH E;Lo;0;L;;;;;N;;;;; +16D7;RUNIC LETTER MANNAZ MAN M;Lo;0;L;;;;;N;;;;; +16D8;RUNIC LETTER LONG-BRANCH-MADR M;Lo;0;L;;;;;N;;;;; +16D9;RUNIC LETTER SHORT-TWIG-MADR M;Lo;0;L;;;;;N;;;;; +16DA;RUNIC LETTER LAUKAZ LAGU LOGR L;Lo;0;L;;;;;N;;;;; +16DB;RUNIC LETTER DOTTED-L;Lo;0;L;;;;;N;;;;; +16DC;RUNIC LETTER INGWAZ;Lo;0;L;;;;;N;;;;; +16DD;RUNIC LETTER ING;Lo;0;L;;;;;N;;;;; +16DE;RUNIC LETTER DAGAZ DAEG D;Lo;0;L;;;;;N;;;;; +16DF;RUNIC LETTER OTHALAN ETHEL O;Lo;0;L;;;;;N;;;;; +16E0;RUNIC LETTER EAR;Lo;0;L;;;;;N;;;;; +16E1;RUNIC LETTER IOR;Lo;0;L;;;;;N;;;;; +16E2;RUNIC LETTER CWEORTH;Lo;0;L;;;;;N;;;;; +16E3;RUNIC LETTER CALC;Lo;0;L;;;;;N;;;;; +16E4;RUNIC LETTER CEALC;Lo;0;L;;;;;N;;;;; +16E5;RUNIC LETTER STAN;Lo;0;L;;;;;N;;;;; +16E6;RUNIC LETTER LONG-BRANCH-YR;Lo;0;L;;;;;N;;;;; +16E7;RUNIC LETTER SHORT-TWIG-YR;Lo;0;L;;;;;N;;;;; +16E8;RUNIC LETTER ICELANDIC-YR;Lo;0;L;;;;;N;;;;; +16E9;RUNIC LETTER Q;Lo;0;L;;;;;N;;;;; +16EA;RUNIC LETTER X;Lo;0;L;;;;;N;;;;; +16EB;RUNIC SINGLE PUNCTUATION;Po;0;L;;;;;N;;;;; +16EC;RUNIC MULTIPLE PUNCTUATION;Po;0;L;;;;;N;;;;; +16ED;RUNIC CROSS PUNCTUATION;Po;0;L;;;;;N;;;;; +16EE;RUNIC ARLAUG SYMBOL;Nl;0;L;;;;17;N;;;;; +16EF;RUNIC TVIMADUR SYMBOL;Nl;0;L;;;;18;N;;;;; +16F0;RUNIC BELGTHOR SYMBOL;Nl;0;L;;;;19;N;;;;; +1700;TAGALOG LETTER A;Lo;0;L;;;;;N;;;;; +1701;TAGALOG LETTER I;Lo;0;L;;;;;N;;;;; +1702;TAGALOG LETTER U;Lo;0;L;;;;;N;;;;; +1703;TAGALOG LETTER KA;Lo;0;L;;;;;N;;;;; +1704;TAGALOG LETTER GA;Lo;0;L;;;;;N;;;;; +1705;TAGALOG LETTER NGA;Lo;0;L;;;;;N;;;;; +1706;TAGALOG LETTER TA;Lo;0;L;;;;;N;;;;; +1707;TAGALOG LETTER DA;Lo;0;L;;;;;N;;;;; +1708;TAGALOG LETTER NA;Lo;0;L;;;;;N;;;;; +1709;TAGALOG LETTER PA;Lo;0;L;;;;;N;;;;; +170A;TAGALOG LETTER BA;Lo;0;L;;;;;N;;;;; +170B;TAGALOG LETTER MA;Lo;0;L;;;;;N;;;;; +170C;TAGALOG LETTER YA;Lo;0;L;;;;;N;;;;; +170E;TAGALOG LETTER LA;Lo;0;L;;;;;N;;;;; +170F;TAGALOG LETTER WA;Lo;0;L;;;;;N;;;;; +1710;TAGALOG LETTER SA;Lo;0;L;;;;;N;;;;; +1711;TAGALOG LETTER HA;Lo;0;L;;;;;N;;;;; +1712;TAGALOG VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1713;TAGALOG VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1714;TAGALOG SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +1720;HANUNOO LETTER A;Lo;0;L;;;;;N;;;;; +1721;HANUNOO LETTER I;Lo;0;L;;;;;N;;;;; +1722;HANUNOO LETTER U;Lo;0;L;;;;;N;;;;; +1723;HANUNOO LETTER KA;Lo;0;L;;;;;N;;;;; +1724;HANUNOO LETTER GA;Lo;0;L;;;;;N;;;;; +1725;HANUNOO LETTER NGA;Lo;0;L;;;;;N;;;;; +1726;HANUNOO LETTER TA;Lo;0;L;;;;;N;;;;; +1727;HANUNOO LETTER DA;Lo;0;L;;;;;N;;;;; +1728;HANUNOO LETTER NA;Lo;0;L;;;;;N;;;;; +1729;HANUNOO LETTER PA;Lo;0;L;;;;;N;;;;; +172A;HANUNOO LETTER BA;Lo;0;L;;;;;N;;;;; +172B;HANUNOO LETTER MA;Lo;0;L;;;;;N;;;;; +172C;HANUNOO LETTER YA;Lo;0;L;;;;;N;;;;; +172D;HANUNOO LETTER RA;Lo;0;L;;;;;N;;;;; +172E;HANUNOO LETTER LA;Lo;0;L;;;;;N;;;;; +172F;HANUNOO LETTER WA;Lo;0;L;;;;;N;;;;; +1730;HANUNOO LETTER SA;Lo;0;L;;;;;N;;;;; +1731;HANUNOO LETTER HA;Lo;0;L;;;;;N;;;;; +1732;HANUNOO VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1733;HANUNOO VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1734;HANUNOO SIGN PAMUDPOD;Mn;9;NSM;;;;;N;;;;; +1735;PHILIPPINE SINGLE PUNCTUATION;Po;0;L;;;;;N;;;;; +1736;PHILIPPINE DOUBLE PUNCTUATION;Po;0;L;;;;;N;;;;; +1740;BUHID LETTER A;Lo;0;L;;;;;N;;;;; +1741;BUHID LETTER I;Lo;0;L;;;;;N;;;;; +1742;BUHID LETTER U;Lo;0;L;;;;;N;;;;; +1743;BUHID LETTER KA;Lo;0;L;;;;;N;;;;; +1744;BUHID LETTER GA;Lo;0;L;;;;;N;;;;; +1745;BUHID LETTER NGA;Lo;0;L;;;;;N;;;;; +1746;BUHID LETTER TA;Lo;0;L;;;;;N;;;;; +1747;BUHID LETTER DA;Lo;0;L;;;;;N;;;;; +1748;BUHID LETTER NA;Lo;0;L;;;;;N;;;;; +1749;BUHID LETTER PA;Lo;0;L;;;;;N;;;;; +174A;BUHID LETTER BA;Lo;0;L;;;;;N;;;;; +174B;BUHID LETTER MA;Lo;0;L;;;;;N;;;;; +174C;BUHID LETTER YA;Lo;0;L;;;;;N;;;;; +174D;BUHID LETTER RA;Lo;0;L;;;;;N;;;;; +174E;BUHID LETTER LA;Lo;0;L;;;;;N;;;;; +174F;BUHID LETTER WA;Lo;0;L;;;;;N;;;;; +1750;BUHID LETTER SA;Lo;0;L;;;;;N;;;;; +1751;BUHID LETTER HA;Lo;0;L;;;;;N;;;;; +1752;BUHID VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1753;BUHID VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1760;TAGBANWA LETTER A;Lo;0;L;;;;;N;;;;; +1761;TAGBANWA LETTER I;Lo;0;L;;;;;N;;;;; +1762;TAGBANWA LETTER U;Lo;0;L;;;;;N;;;;; +1763;TAGBANWA LETTER KA;Lo;0;L;;;;;N;;;;; +1764;TAGBANWA LETTER GA;Lo;0;L;;;;;N;;;;; +1765;TAGBANWA LETTER NGA;Lo;0;L;;;;;N;;;;; +1766;TAGBANWA LETTER TA;Lo;0;L;;;;;N;;;;; +1767;TAGBANWA LETTER DA;Lo;0;L;;;;;N;;;;; +1768;TAGBANWA LETTER NA;Lo;0;L;;;;;N;;;;; +1769;TAGBANWA LETTER PA;Lo;0;L;;;;;N;;;;; +176A;TAGBANWA LETTER BA;Lo;0;L;;;;;N;;;;; +176B;TAGBANWA LETTER MA;Lo;0;L;;;;;N;;;;; +176C;TAGBANWA LETTER YA;Lo;0;L;;;;;N;;;;; +176E;TAGBANWA LETTER LA;Lo;0;L;;;;;N;;;;; +176F;TAGBANWA LETTER WA;Lo;0;L;;;;;N;;;;; +1770;TAGBANWA LETTER SA;Lo;0;L;;;;;N;;;;; +1772;TAGBANWA VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1773;TAGBANWA VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1780;KHMER LETTER KA;Lo;0;L;;;;;N;;;;; +1781;KHMER LETTER KHA;Lo;0;L;;;;;N;;;;; +1782;KHMER LETTER KO;Lo;0;L;;;;;N;;;;; +1783;KHMER LETTER KHO;Lo;0;L;;;;;N;;;;; +1784;KHMER LETTER NGO;Lo;0;L;;;;;N;;;;; +1785;KHMER LETTER CA;Lo;0;L;;;;;N;;;;; +1786;KHMER LETTER CHA;Lo;0;L;;;;;N;;;;; +1787;KHMER LETTER CO;Lo;0;L;;;;;N;;;;; +1788;KHMER LETTER CHO;Lo;0;L;;;;;N;;;;; +1789;KHMER LETTER NYO;Lo;0;L;;;;;N;;;;; +178A;KHMER LETTER DA;Lo;0;L;;;;;N;;;;; +178B;KHMER LETTER TTHA;Lo;0;L;;;;;N;;;;; +178C;KHMER LETTER DO;Lo;0;L;;;;;N;;;;; +178D;KHMER LETTER TTHO;Lo;0;L;;;;;N;;;;; +178E;KHMER LETTER NNO;Lo;0;L;;;;;N;;;;; +178F;KHMER LETTER TA;Lo;0;L;;;;;N;;;;; +1790;KHMER LETTER THA;Lo;0;L;;;;;N;;;;; +1791;KHMER LETTER TO;Lo;0;L;;;;;N;;;;; +1792;KHMER LETTER THO;Lo;0;L;;;;;N;;;;; +1793;KHMER LETTER NO;Lo;0;L;;;;;N;;;;; +1794;KHMER LETTER BA;Lo;0;L;;;;;N;;;;; +1795;KHMER LETTER PHA;Lo;0;L;;;;;N;;;;; +1796;KHMER LETTER PO;Lo;0;L;;;;;N;;;;; +1797;KHMER LETTER PHO;Lo;0;L;;;;;N;;;;; +1798;KHMER LETTER MO;Lo;0;L;;;;;N;;;;; +1799;KHMER LETTER YO;Lo;0;L;;;;;N;;;;; +179A;KHMER LETTER RO;Lo;0;L;;;;;N;;;;; +179B;KHMER LETTER LO;Lo;0;L;;;;;N;;;;; +179C;KHMER LETTER VO;Lo;0;L;;;;;N;;;;; +179D;KHMER LETTER SHA;Lo;0;L;;;;;N;;;;; +179E;KHMER LETTER SSO;Lo;0;L;;;;;N;;;;; +179F;KHMER LETTER SA;Lo;0;L;;;;;N;;;;; +17A0;KHMER LETTER HA;Lo;0;L;;;;;N;;;;; +17A1;KHMER LETTER LA;Lo;0;L;;;;;N;;;;; +17A2;KHMER LETTER QA;Lo;0;L;;;;;N;;;;; +17A3;KHMER INDEPENDENT VOWEL QAQ;Lo;0;L;;;;;N;;;;; +17A4;KHMER INDEPENDENT VOWEL QAA;Lo;0;L;;;;;N;;;;; +17A5;KHMER INDEPENDENT VOWEL QI;Lo;0;L;;;;;N;;;;; +17A6;KHMER INDEPENDENT VOWEL QII;Lo;0;L;;;;;N;;;;; +17A7;KHMER INDEPENDENT VOWEL QU;Lo;0;L;;;;;N;;;;; +17A8;KHMER INDEPENDENT VOWEL QUK;Lo;0;L;;;;;N;;;;; +17A9;KHMER INDEPENDENT VOWEL QUU;Lo;0;L;;;;;N;;;;; +17AA;KHMER INDEPENDENT VOWEL QUUV;Lo;0;L;;;;;N;;;;; +17AB;KHMER INDEPENDENT VOWEL RY;Lo;0;L;;;;;N;;;;; +17AC;KHMER INDEPENDENT VOWEL RYY;Lo;0;L;;;;;N;;;;; +17AD;KHMER INDEPENDENT VOWEL LY;Lo;0;L;;;;;N;;;;; +17AE;KHMER INDEPENDENT VOWEL LYY;Lo;0;L;;;;;N;;;;; +17AF;KHMER INDEPENDENT VOWEL QE;Lo;0;L;;;;;N;;;;; +17B0;KHMER INDEPENDENT VOWEL QAI;Lo;0;L;;;;;N;;;;; +17B1;KHMER INDEPENDENT VOWEL QOO TYPE ONE;Lo;0;L;;;;;N;;;;; +17B2;KHMER INDEPENDENT VOWEL QOO TYPE TWO;Lo;0;L;;;;;N;;;;; +17B3;KHMER INDEPENDENT VOWEL QAU;Lo;0;L;;;;;N;;;;; +17B4;KHMER VOWEL INHERENT AQ;Mn;0;NSM;;;;;N;;;;; +17B5;KHMER VOWEL INHERENT AA;Mn;0;NSM;;;;;N;;;;; +17B6;KHMER VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +17B7;KHMER VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +17B8;KHMER VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +17B9;KHMER VOWEL SIGN Y;Mn;0;NSM;;;;;N;;;;; +17BA;KHMER VOWEL SIGN YY;Mn;0;NSM;;;;;N;;;;; +17BB;KHMER VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +17BC;KHMER VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +17BD;KHMER VOWEL SIGN UA;Mn;0;NSM;;;;;N;;;;; +17BE;KHMER VOWEL SIGN OE;Mc;0;L;;;;;N;;;;; +17BF;KHMER VOWEL SIGN YA;Mc;0;L;;;;;N;;;;; +17C0;KHMER VOWEL SIGN IE;Mc;0;L;;;;;N;;;;; +17C1;KHMER VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +17C2;KHMER VOWEL SIGN AE;Mc;0;L;;;;;N;;;;; +17C3;KHMER VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +17C4;KHMER VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +17C5;KHMER VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +17C6;KHMER SIGN NIKAHIT;Mn;0;NSM;;;;;N;;;;; +17C7;KHMER SIGN REAHMUK;Mc;0;L;;;;;N;;;;; +17C8;KHMER SIGN YUUKALEAPINTU;Mc;0;L;;;;;N;;;;; +17C9;KHMER SIGN MUUSIKATOAN;Mn;0;NSM;;;;;N;;;;; +17CA;KHMER SIGN TRIISAP;Mn;0;NSM;;;;;N;;;;; +17CB;KHMER SIGN BANTOC;Mn;0;NSM;;;;;N;;;;; +17CC;KHMER SIGN ROBAT;Mn;0;NSM;;;;;N;;;;; +17CD;KHMER SIGN TOANDAKHIAT;Mn;0;NSM;;;;;N;;;;; +17CE;KHMER SIGN KAKABAT;Mn;0;NSM;;;;;N;;;;; +17CF;KHMER SIGN AHSDA;Mn;0;NSM;;;;;N;;;;; +17D0;KHMER SIGN SAMYOK SANNYA;Mn;0;NSM;;;;;N;;;;; +17D1;KHMER SIGN VIRIAM;Mn;0;NSM;;;;;N;;;;; +17D2;KHMER SIGN COENG;Mn;9;NSM;;;;;N;;;;; +17D3;KHMER SIGN BATHAMASAT;Mn;0;NSM;;;;;N;;;;; +17D4;KHMER SIGN KHAN;Po;0;L;;;;;N;;;;; +17D5;KHMER SIGN BARIYOOSAN;Po;0;L;;;;;N;;;;; +17D6;KHMER SIGN CAMNUC PII KUUH;Po;0;L;;;;;N;;;;; +17D7;KHMER SIGN LEK TOO;Lm;0;L;;;;;N;;;;; +17D8;KHMER SIGN BEYYAL;Po;0;L;;;;;N;;;;; +17D9;KHMER SIGN PHNAEK MUAN;Po;0;L;;;;;N;;;;; +17DA;KHMER SIGN KOOMUUT;Po;0;L;;;;;N;;;;; +17DB;KHMER CURRENCY SYMBOL RIEL;Sc;0;ET;;;;;N;;;;; +17DC;KHMER SIGN AVAKRAHASANYA;Lo;0;L;;;;;N;;;;; +17DD;KHMER SIGN ATTHACAN;Mn;230;NSM;;;;;N;;;;; +17E0;KHMER DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +17E1;KHMER DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +17E2;KHMER DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +17E3;KHMER DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +17E4;KHMER DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +17E5;KHMER DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +17E6;KHMER DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +17E7;KHMER DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +17E8;KHMER DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +17E9;KHMER DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +17F0;KHMER SYMBOL LEK ATTAK SON;No;0;ON;;;;0;N;;;;; +17F1;KHMER SYMBOL LEK ATTAK MUOY;No;0;ON;;;;1;N;;;;; +17F2;KHMER SYMBOL LEK ATTAK PII;No;0;ON;;;;2;N;;;;; +17F3;KHMER SYMBOL LEK ATTAK BEI;No;0;ON;;;;3;N;;;;; +17F4;KHMER SYMBOL LEK ATTAK BUON;No;0;ON;;;;4;N;;;;; +17F5;KHMER SYMBOL LEK ATTAK PRAM;No;0;ON;;;;5;N;;;;; +17F6;KHMER SYMBOL LEK ATTAK PRAM-MUOY;No;0;ON;;;;6;N;;;;; +17F7;KHMER SYMBOL LEK ATTAK PRAM-PII;No;0;ON;;;;7;N;;;;; +17F8;KHMER SYMBOL LEK ATTAK PRAM-BEI;No;0;ON;;;;8;N;;;;; +17F9;KHMER SYMBOL LEK ATTAK PRAM-BUON;No;0;ON;;;;9;N;;;;; +1800;MONGOLIAN BIRGA;Po;0;ON;;;;;N;;;;; +1801;MONGOLIAN ELLIPSIS;Po;0;ON;;;;;N;;;;; +1802;MONGOLIAN COMMA;Po;0;ON;;;;;N;;;;; +1803;MONGOLIAN FULL STOP;Po;0;ON;;;;;N;;;;; +1804;MONGOLIAN COLON;Po;0;ON;;;;;N;;;;; +1805;MONGOLIAN FOUR DOTS;Po;0;ON;;;;;N;;;;; +1806;MONGOLIAN TODO SOFT HYPHEN;Pd;0;ON;;;;;N;;;;; +1807;MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER;Po;0;ON;;;;;N;;;;; +1808;MONGOLIAN MANCHU COMMA;Po;0;ON;;;;;N;;;;; +1809;MONGOLIAN MANCHU FULL STOP;Po;0;ON;;;;;N;;;;; +180A;MONGOLIAN NIRUGU;Po;0;ON;;;;;N;;;;; +180B;MONGOLIAN FREE VARIATION SELECTOR ONE;Mn;0;NSM;;;;;N;;;;; +180C;MONGOLIAN FREE VARIATION SELECTOR TWO;Mn;0;NSM;;;;;N;;;;; +180D;MONGOLIAN FREE VARIATION SELECTOR THREE;Mn;0;NSM;;;;;N;;;;; +180E;MONGOLIAN VOWEL SEPARATOR;Zs;0;WS;;;;;N;;;;; +1810;MONGOLIAN DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1811;MONGOLIAN DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1812;MONGOLIAN DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1813;MONGOLIAN DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1814;MONGOLIAN DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1815;MONGOLIAN DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1816;MONGOLIAN DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1817;MONGOLIAN DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1818;MONGOLIAN DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1819;MONGOLIAN DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1820;MONGOLIAN LETTER A;Lo;0;L;;;;;N;;;;; +1821;MONGOLIAN LETTER E;Lo;0;L;;;;;N;;;;; +1822;MONGOLIAN LETTER I;Lo;0;L;;;;;N;;;;; +1823;MONGOLIAN LETTER O;Lo;0;L;;;;;N;;;;; +1824;MONGOLIAN LETTER U;Lo;0;L;;;;;N;;;;; +1825;MONGOLIAN LETTER OE;Lo;0;L;;;;;N;;;;; +1826;MONGOLIAN LETTER UE;Lo;0;L;;;;;N;;;;; +1827;MONGOLIAN LETTER EE;Lo;0;L;;;;;N;;;;; +1828;MONGOLIAN LETTER NA;Lo;0;L;;;;;N;;;;; +1829;MONGOLIAN LETTER ANG;Lo;0;L;;;;;N;;;;; +182A;MONGOLIAN LETTER BA;Lo;0;L;;;;;N;;;;; +182B;MONGOLIAN LETTER PA;Lo;0;L;;;;;N;;;;; +182C;MONGOLIAN LETTER QA;Lo;0;L;;;;;N;;;;; +182D;MONGOLIAN LETTER GA;Lo;0;L;;;;;N;;;;; +182E;MONGOLIAN LETTER MA;Lo;0;L;;;;;N;;;;; +182F;MONGOLIAN LETTER LA;Lo;0;L;;;;;N;;;;; +1830;MONGOLIAN LETTER SA;Lo;0;L;;;;;N;;;;; +1831;MONGOLIAN LETTER SHA;Lo;0;L;;;;;N;;;;; +1832;MONGOLIAN LETTER TA;Lo;0;L;;;;;N;;;;; +1833;MONGOLIAN LETTER DA;Lo;0;L;;;;;N;;;;; +1834;MONGOLIAN LETTER CHA;Lo;0;L;;;;;N;;;;; +1835;MONGOLIAN LETTER JA;Lo;0;L;;;;;N;;;;; +1836;MONGOLIAN LETTER YA;Lo;0;L;;;;;N;;;;; +1837;MONGOLIAN LETTER RA;Lo;0;L;;;;;N;;;;; +1838;MONGOLIAN LETTER WA;Lo;0;L;;;;;N;;;;; +1839;MONGOLIAN LETTER FA;Lo;0;L;;;;;N;;;;; +183A;MONGOLIAN LETTER KA;Lo;0;L;;;;;N;;;;; +183B;MONGOLIAN LETTER KHA;Lo;0;L;;;;;N;;;;; +183C;MONGOLIAN LETTER TSA;Lo;0;L;;;;;N;;;;; +183D;MONGOLIAN LETTER ZA;Lo;0;L;;;;;N;;;;; +183E;MONGOLIAN LETTER HAA;Lo;0;L;;;;;N;;;;; +183F;MONGOLIAN LETTER ZRA;Lo;0;L;;;;;N;;;;; +1840;MONGOLIAN LETTER LHA;Lo;0;L;;;;;N;;;;; +1841;MONGOLIAN LETTER ZHI;Lo;0;L;;;;;N;;;;; +1842;MONGOLIAN LETTER CHI;Lo;0;L;;;;;N;;;;; +1843;MONGOLIAN LETTER TODO LONG VOWEL SIGN;Lm;0;L;;;;;N;;;;; +1844;MONGOLIAN LETTER TODO E;Lo;0;L;;;;;N;;;;; +1845;MONGOLIAN LETTER TODO I;Lo;0;L;;;;;N;;;;; +1846;MONGOLIAN LETTER TODO O;Lo;0;L;;;;;N;;;;; +1847;MONGOLIAN LETTER TODO U;Lo;0;L;;;;;N;;;;; +1848;MONGOLIAN LETTER TODO OE;Lo;0;L;;;;;N;;;;; +1849;MONGOLIAN LETTER TODO UE;Lo;0;L;;;;;N;;;;; +184A;MONGOLIAN LETTER TODO ANG;Lo;0;L;;;;;N;;;;; +184B;MONGOLIAN LETTER TODO BA;Lo;0;L;;;;;N;;;;; +184C;MONGOLIAN LETTER TODO PA;Lo;0;L;;;;;N;;;;; +184D;MONGOLIAN LETTER TODO QA;Lo;0;L;;;;;N;;;;; +184E;MONGOLIAN LETTER TODO GA;Lo;0;L;;;;;N;;;;; +184F;MONGOLIAN LETTER TODO MA;Lo;0;L;;;;;N;;;;; +1850;MONGOLIAN LETTER TODO TA;Lo;0;L;;;;;N;;;;; +1851;MONGOLIAN LETTER TODO DA;Lo;0;L;;;;;N;;;;; +1852;MONGOLIAN LETTER TODO CHA;Lo;0;L;;;;;N;;;;; +1853;MONGOLIAN LETTER TODO JA;Lo;0;L;;;;;N;;;;; +1854;MONGOLIAN LETTER TODO TSA;Lo;0;L;;;;;N;;;;; +1855;MONGOLIAN LETTER TODO YA;Lo;0;L;;;;;N;;;;; +1856;MONGOLIAN LETTER TODO WA;Lo;0;L;;;;;N;;;;; +1857;MONGOLIAN LETTER TODO KA;Lo;0;L;;;;;N;;;;; +1858;MONGOLIAN LETTER TODO GAA;Lo;0;L;;;;;N;;;;; +1859;MONGOLIAN LETTER TODO HAA;Lo;0;L;;;;;N;;;;; +185A;MONGOLIAN LETTER TODO JIA;Lo;0;L;;;;;N;;;;; +185B;MONGOLIAN LETTER TODO NIA;Lo;0;L;;;;;N;;;;; +185C;MONGOLIAN LETTER TODO DZA;Lo;0;L;;;;;N;;;;; +185D;MONGOLIAN LETTER SIBE E;Lo;0;L;;;;;N;;;;; +185E;MONGOLIAN LETTER SIBE I;Lo;0;L;;;;;N;;;;; +185F;MONGOLIAN LETTER SIBE IY;Lo;0;L;;;;;N;;;;; +1860;MONGOLIAN LETTER SIBE UE;Lo;0;L;;;;;N;;;;; +1861;MONGOLIAN LETTER SIBE U;Lo;0;L;;;;;N;;;;; +1862;MONGOLIAN LETTER SIBE ANG;Lo;0;L;;;;;N;;;;; +1863;MONGOLIAN LETTER SIBE KA;Lo;0;L;;;;;N;;;;; +1864;MONGOLIAN LETTER SIBE GA;Lo;0;L;;;;;N;;;;; +1865;MONGOLIAN LETTER SIBE HA;Lo;0;L;;;;;N;;;;; +1866;MONGOLIAN LETTER SIBE PA;Lo;0;L;;;;;N;;;;; +1867;MONGOLIAN LETTER SIBE SHA;Lo;0;L;;;;;N;;;;; +1868;MONGOLIAN LETTER SIBE TA;Lo;0;L;;;;;N;;;;; +1869;MONGOLIAN LETTER SIBE DA;Lo;0;L;;;;;N;;;;; +186A;MONGOLIAN LETTER SIBE JA;Lo;0;L;;;;;N;;;;; +186B;MONGOLIAN LETTER SIBE FA;Lo;0;L;;;;;N;;;;; +186C;MONGOLIAN LETTER SIBE GAA;Lo;0;L;;;;;N;;;;; +186D;MONGOLIAN LETTER SIBE HAA;Lo;0;L;;;;;N;;;;; +186E;MONGOLIAN LETTER SIBE TSA;Lo;0;L;;;;;N;;;;; +186F;MONGOLIAN LETTER SIBE ZA;Lo;0;L;;;;;N;;;;; +1870;MONGOLIAN LETTER SIBE RAA;Lo;0;L;;;;;N;;;;; +1871;MONGOLIAN LETTER SIBE CHA;Lo;0;L;;;;;N;;;;; +1872;MONGOLIAN LETTER SIBE ZHA;Lo;0;L;;;;;N;;;;; +1873;MONGOLIAN LETTER MANCHU I;Lo;0;L;;;;;N;;;;; +1874;MONGOLIAN LETTER MANCHU KA;Lo;0;L;;;;;N;;;;; +1875;MONGOLIAN LETTER MANCHU RA;Lo;0;L;;;;;N;;;;; +1876;MONGOLIAN LETTER MANCHU FA;Lo;0;L;;;;;N;;;;; +1877;MONGOLIAN LETTER MANCHU ZHA;Lo;0;L;;;;;N;;;;; +1880;MONGOLIAN LETTER ALI GALI ANUSVARA ONE;Lo;0;L;;;;;N;;;;; +1881;MONGOLIAN LETTER ALI GALI VISARGA ONE;Lo;0;L;;;;;N;;;;; +1882;MONGOLIAN LETTER ALI GALI DAMARU;Lo;0;L;;;;;N;;;;; +1883;MONGOLIAN LETTER ALI GALI UBADAMA;Lo;0;L;;;;;N;;;;; +1884;MONGOLIAN LETTER ALI GALI INVERTED UBADAMA;Lo;0;L;;;;;N;;;;; +1885;MONGOLIAN LETTER ALI GALI BALUDA;Lo;0;L;;;;;N;;;;; +1886;MONGOLIAN LETTER ALI GALI THREE BALUDA;Lo;0;L;;;;;N;;;;; +1887;MONGOLIAN LETTER ALI GALI A;Lo;0;L;;;;;N;;;;; +1888;MONGOLIAN LETTER ALI GALI I;Lo;0;L;;;;;N;;;;; +1889;MONGOLIAN LETTER ALI GALI KA;Lo;0;L;;;;;N;;;;; +188A;MONGOLIAN LETTER ALI GALI NGA;Lo;0;L;;;;;N;;;;; +188B;MONGOLIAN LETTER ALI GALI CA;Lo;0;L;;;;;N;;;;; +188C;MONGOLIAN LETTER ALI GALI TTA;Lo;0;L;;;;;N;;;;; +188D;MONGOLIAN LETTER ALI GALI TTHA;Lo;0;L;;;;;N;;;;; +188E;MONGOLIAN LETTER ALI GALI DDA;Lo;0;L;;;;;N;;;;; +188F;MONGOLIAN LETTER ALI GALI NNA;Lo;0;L;;;;;N;;;;; +1890;MONGOLIAN LETTER ALI GALI TA;Lo;0;L;;;;;N;;;;; +1891;MONGOLIAN LETTER ALI GALI DA;Lo;0;L;;;;;N;;;;; +1892;MONGOLIAN LETTER ALI GALI PA;Lo;0;L;;;;;N;;;;; +1893;MONGOLIAN LETTER ALI GALI PHA;Lo;0;L;;;;;N;;;;; +1894;MONGOLIAN LETTER ALI GALI SSA;Lo;0;L;;;;;N;;;;; +1895;MONGOLIAN LETTER ALI GALI ZHA;Lo;0;L;;;;;N;;;;; +1896;MONGOLIAN LETTER ALI GALI ZA;Lo;0;L;;;;;N;;;;; +1897;MONGOLIAN LETTER ALI GALI AH;Lo;0;L;;;;;N;;;;; +1898;MONGOLIAN LETTER TODO ALI GALI TA;Lo;0;L;;;;;N;;;;; +1899;MONGOLIAN LETTER TODO ALI GALI ZHA;Lo;0;L;;;;;N;;;;; +189A;MONGOLIAN LETTER MANCHU ALI GALI GHA;Lo;0;L;;;;;N;;;;; +189B;MONGOLIAN LETTER MANCHU ALI GALI NGA;Lo;0;L;;;;;N;;;;; +189C;MONGOLIAN LETTER MANCHU ALI GALI CA;Lo;0;L;;;;;N;;;;; +189D;MONGOLIAN LETTER MANCHU ALI GALI JHA;Lo;0;L;;;;;N;;;;; +189E;MONGOLIAN LETTER MANCHU ALI GALI TTA;Lo;0;L;;;;;N;;;;; +189F;MONGOLIAN LETTER MANCHU ALI GALI DDHA;Lo;0;L;;;;;N;;;;; +18A0;MONGOLIAN LETTER MANCHU ALI GALI TA;Lo;0;L;;;;;N;;;;; +18A1;MONGOLIAN LETTER MANCHU ALI GALI DHA;Lo;0;L;;;;;N;;;;; +18A2;MONGOLIAN LETTER MANCHU ALI GALI SSA;Lo;0;L;;;;;N;;;;; +18A3;MONGOLIAN LETTER MANCHU ALI GALI CYA;Lo;0;L;;;;;N;;;;; +18A4;MONGOLIAN LETTER MANCHU ALI GALI ZHA;Lo;0;L;;;;;N;;;;; +18A5;MONGOLIAN LETTER MANCHU ALI GALI ZA;Lo;0;L;;;;;N;;;;; +18A6;MONGOLIAN LETTER ALI GALI HALF U;Lo;0;L;;;;;N;;;;; +18A7;MONGOLIAN LETTER ALI GALI HALF YA;Lo;0;L;;;;;N;;;;; +18A8;MONGOLIAN LETTER MANCHU ALI GALI BHA;Lo;0;L;;;;;N;;;;; +18A9;MONGOLIAN LETTER ALI GALI DAGALGA;Mn;228;NSM;;;;;N;;;;; +18AA;MONGOLIAN LETTER MANCHU ALI GALI LHA;Lo;0;L;;;;;N;;;;; +18B0;CANADIAN SYLLABICS OY;Lo;0;L;;;;;N;;;;; +18B1;CANADIAN SYLLABICS AY;Lo;0;L;;;;;N;;;;; +18B2;CANADIAN SYLLABICS AAY;Lo;0;L;;;;;N;;;;; +18B3;CANADIAN SYLLABICS WAY;Lo;0;L;;;;;N;;;;; +18B4;CANADIAN SYLLABICS POY;Lo;0;L;;;;;N;;;;; +18B5;CANADIAN SYLLABICS PAY;Lo;0;L;;;;;N;;;;; +18B6;CANADIAN SYLLABICS PWOY;Lo;0;L;;;;;N;;;;; +18B7;CANADIAN SYLLABICS TAY;Lo;0;L;;;;;N;;;;; +18B8;CANADIAN SYLLABICS KAY;Lo;0;L;;;;;N;;;;; +18B9;CANADIAN SYLLABICS KWAY;Lo;0;L;;;;;N;;;;; +18BA;CANADIAN SYLLABICS MAY;Lo;0;L;;;;;N;;;;; +18BB;CANADIAN SYLLABICS NOY;Lo;0;L;;;;;N;;;;; +18BC;CANADIAN SYLLABICS NAY;Lo;0;L;;;;;N;;;;; +18BD;CANADIAN SYLLABICS LAY;Lo;0;L;;;;;N;;;;; +18BE;CANADIAN SYLLABICS SOY;Lo;0;L;;;;;N;;;;; +18BF;CANADIAN SYLLABICS SAY;Lo;0;L;;;;;N;;;;; +18C0;CANADIAN SYLLABICS SHOY;Lo;0;L;;;;;N;;;;; +18C1;CANADIAN SYLLABICS SHAY;Lo;0;L;;;;;N;;;;; +18C2;CANADIAN SYLLABICS SHWOY;Lo;0;L;;;;;N;;;;; +18C3;CANADIAN SYLLABICS YOY;Lo;0;L;;;;;N;;;;; +18C4;CANADIAN SYLLABICS YAY;Lo;0;L;;;;;N;;;;; +18C5;CANADIAN SYLLABICS RAY;Lo;0;L;;;;;N;;;;; +18C6;CANADIAN SYLLABICS NWI;Lo;0;L;;;;;N;;;;; +18C7;CANADIAN SYLLABICS OJIBWAY NWI;Lo;0;L;;;;;N;;;;; +18C8;CANADIAN SYLLABICS NWII;Lo;0;L;;;;;N;;;;; +18C9;CANADIAN SYLLABICS OJIBWAY NWII;Lo;0;L;;;;;N;;;;; +18CA;CANADIAN SYLLABICS NWO;Lo;0;L;;;;;N;;;;; +18CB;CANADIAN SYLLABICS OJIBWAY NWO;Lo;0;L;;;;;N;;;;; +18CC;CANADIAN SYLLABICS NWOO;Lo;0;L;;;;;N;;;;; +18CD;CANADIAN SYLLABICS OJIBWAY NWOO;Lo;0;L;;;;;N;;;;; +18CE;CANADIAN SYLLABICS RWEE;Lo;0;L;;;;;N;;;;; +18CF;CANADIAN SYLLABICS RWI;Lo;0;L;;;;;N;;;;; +18D0;CANADIAN SYLLABICS RWII;Lo;0;L;;;;;N;;;;; +18D1;CANADIAN SYLLABICS RWO;Lo;0;L;;;;;N;;;;; +18D2;CANADIAN SYLLABICS RWOO;Lo;0;L;;;;;N;;;;; +18D3;CANADIAN SYLLABICS RWA;Lo;0;L;;;;;N;;;;; +18D4;CANADIAN SYLLABICS OJIBWAY P;Lo;0;L;;;;;N;;;;; +18D5;CANADIAN SYLLABICS OJIBWAY T;Lo;0;L;;;;;N;;;;; +18D6;CANADIAN SYLLABICS OJIBWAY K;Lo;0;L;;;;;N;;;;; +18D7;CANADIAN SYLLABICS OJIBWAY C;Lo;0;L;;;;;N;;;;; +18D8;CANADIAN SYLLABICS OJIBWAY M;Lo;0;L;;;;;N;;;;; +18D9;CANADIAN SYLLABICS OJIBWAY N;Lo;0;L;;;;;N;;;;; +18DA;CANADIAN SYLLABICS OJIBWAY S;Lo;0;L;;;;;N;;;;; +18DB;CANADIAN SYLLABICS OJIBWAY SH;Lo;0;L;;;;;N;;;;; +18DC;CANADIAN SYLLABICS EASTERN W;Lo;0;L;;;;;N;;;;; +18DD;CANADIAN SYLLABICS WESTERN W;Lo;0;L;;;;;N;;;;; +18DE;CANADIAN SYLLABICS FINAL SMALL RING;Lo;0;L;;;;;N;;;;; +18DF;CANADIAN SYLLABICS FINAL RAISED DOT;Lo;0;L;;;;;N;;;;; +18E0;CANADIAN SYLLABICS R-CREE RWE;Lo;0;L;;;;;N;;;;; +18E1;CANADIAN SYLLABICS WEST-CREE LOO;Lo;0;L;;;;;N;;;;; +18E2;CANADIAN SYLLABICS WEST-CREE LAA;Lo;0;L;;;;;N;;;;; +18E3;CANADIAN SYLLABICS THWE;Lo;0;L;;;;;N;;;;; +18E4;CANADIAN SYLLABICS THWA;Lo;0;L;;;;;N;;;;; +18E5;CANADIAN SYLLABICS TTHWE;Lo;0;L;;;;;N;;;;; +18E6;CANADIAN SYLLABICS TTHOO;Lo;0;L;;;;;N;;;;; +18E7;CANADIAN SYLLABICS TTHAA;Lo;0;L;;;;;N;;;;; +18E8;CANADIAN SYLLABICS TLHWE;Lo;0;L;;;;;N;;;;; +18E9;CANADIAN SYLLABICS TLHOO;Lo;0;L;;;;;N;;;;; +18EA;CANADIAN SYLLABICS SAYISI SHWE;Lo;0;L;;;;;N;;;;; +18EB;CANADIAN SYLLABICS SAYISI SHOO;Lo;0;L;;;;;N;;;;; +18EC;CANADIAN SYLLABICS SAYISI HOO;Lo;0;L;;;;;N;;;;; +18ED;CANADIAN SYLLABICS CARRIER GWU;Lo;0;L;;;;;N;;;;; +18EE;CANADIAN SYLLABICS CARRIER DENE GEE;Lo;0;L;;;;;N;;;;; +18EF;CANADIAN SYLLABICS CARRIER GAA;Lo;0;L;;;;;N;;;;; +18F0;CANADIAN SYLLABICS CARRIER GWA;Lo;0;L;;;;;N;;;;; +18F1;CANADIAN SYLLABICS SAYISI JUU;Lo;0;L;;;;;N;;;;; +18F2;CANADIAN SYLLABICS CARRIER JWA;Lo;0;L;;;;;N;;;;; +18F3;CANADIAN SYLLABICS BEAVER DENE L;Lo;0;L;;;;;N;;;;; +18F4;CANADIAN SYLLABICS BEAVER DENE R;Lo;0;L;;;;;N;;;;; +18F5;CANADIAN SYLLABICS CARRIER DENTAL S;Lo;0;L;;;;;N;;;;; +1900;LIMBU VOWEL-CARRIER LETTER;Lo;0;L;;;;;N;;;;; +1901;LIMBU LETTER KA;Lo;0;L;;;;;N;;;;; +1902;LIMBU LETTER KHA;Lo;0;L;;;;;N;;;;; +1903;LIMBU LETTER GA;Lo;0;L;;;;;N;;;;; +1904;LIMBU LETTER GHA;Lo;0;L;;;;;N;;;;; +1905;LIMBU LETTER NGA;Lo;0;L;;;;;N;;;;; +1906;LIMBU LETTER CA;Lo;0;L;;;;;N;;;;; +1907;LIMBU LETTER CHA;Lo;0;L;;;;;N;;;;; +1908;LIMBU LETTER JA;Lo;0;L;;;;;N;;;;; +1909;LIMBU LETTER JHA;Lo;0;L;;;;;N;;;;; +190A;LIMBU LETTER YAN;Lo;0;L;;;;;N;;;;; +190B;LIMBU LETTER TA;Lo;0;L;;;;;N;;;;; +190C;LIMBU LETTER THA;Lo;0;L;;;;;N;;;;; +190D;LIMBU LETTER DA;Lo;0;L;;;;;N;;;;; +190E;LIMBU LETTER DHA;Lo;0;L;;;;;N;;;;; +190F;LIMBU LETTER NA;Lo;0;L;;;;;N;;;;; +1910;LIMBU LETTER PA;Lo;0;L;;;;;N;;;;; +1911;LIMBU LETTER PHA;Lo;0;L;;;;;N;;;;; +1912;LIMBU LETTER BA;Lo;0;L;;;;;N;;;;; +1913;LIMBU LETTER BHA;Lo;0;L;;;;;N;;;;; +1914;LIMBU LETTER MA;Lo;0;L;;;;;N;;;;; +1915;LIMBU LETTER YA;Lo;0;L;;;;;N;;;;; +1916;LIMBU LETTER RA;Lo;0;L;;;;;N;;;;; +1917;LIMBU LETTER LA;Lo;0;L;;;;;N;;;;; +1918;LIMBU LETTER WA;Lo;0;L;;;;;N;;;;; +1919;LIMBU LETTER SHA;Lo;0;L;;;;;N;;;;; +191A;LIMBU LETTER SSA;Lo;0;L;;;;;N;;;;; +191B;LIMBU LETTER SA;Lo;0;L;;;;;N;;;;; +191C;LIMBU LETTER HA;Lo;0;L;;;;;N;;;;; +1920;LIMBU VOWEL SIGN A;Mn;0;NSM;;;;;N;;;;; +1921;LIMBU VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1922;LIMBU VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1923;LIMBU VOWEL SIGN EE;Mc;0;L;;;;;N;;;;; +1924;LIMBU VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +1925;LIMBU VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +1926;LIMBU VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +1927;LIMBU VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +1928;LIMBU VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +1929;LIMBU SUBJOINED LETTER YA;Mc;0;L;;;;;N;;;;; +192A;LIMBU SUBJOINED LETTER RA;Mc;0;L;;;;;N;;;;; +192B;LIMBU SUBJOINED LETTER WA;Mc;0;L;;;;;N;;;;; +1930;LIMBU SMALL LETTER KA;Mc;0;L;;;;;N;;;;; +1931;LIMBU SMALL LETTER NGA;Mc;0;L;;;;;N;;;;; +1932;LIMBU SMALL LETTER ANUSVARA;Mn;0;NSM;;;;;N;;;;; +1933;LIMBU SMALL LETTER TA;Mc;0;L;;;;;N;;;;; +1934;LIMBU SMALL LETTER NA;Mc;0;L;;;;;N;;;;; +1935;LIMBU SMALL LETTER PA;Mc;0;L;;;;;N;;;;; +1936;LIMBU SMALL LETTER MA;Mc;0;L;;;;;N;;;;; +1937;LIMBU SMALL LETTER RA;Mc;0;L;;;;;N;;;;; +1938;LIMBU SMALL LETTER LA;Mc;0;L;;;;;N;;;;; +1939;LIMBU SIGN MUKPHRENG;Mn;222;NSM;;;;;N;;;;; +193A;LIMBU SIGN KEMPHRENG;Mn;230;NSM;;;;;N;;;;; +193B;LIMBU SIGN SA-I;Mn;220;NSM;;;;;N;;;;; +1940;LIMBU SIGN LOO;So;0;ON;;;;;N;;;;; +1944;LIMBU EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; +1945;LIMBU QUESTION MARK;Po;0;ON;;;;;N;;;;; +1946;LIMBU DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1947;LIMBU DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1948;LIMBU DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1949;LIMBU DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +194A;LIMBU DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +194B;LIMBU DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +194C;LIMBU DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +194D;LIMBU DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +194E;LIMBU DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +194F;LIMBU DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1950;TAI LE LETTER KA;Lo;0;L;;;;;N;;;;; +1951;TAI LE LETTER XA;Lo;0;L;;;;;N;;;;; +1952;TAI LE LETTER NGA;Lo;0;L;;;;;N;;;;; +1953;TAI LE LETTER TSA;Lo;0;L;;;;;N;;;;; +1954;TAI LE LETTER SA;Lo;0;L;;;;;N;;;;; +1955;TAI LE LETTER YA;Lo;0;L;;;;;N;;;;; +1956;TAI LE LETTER TA;Lo;0;L;;;;;N;;;;; +1957;TAI LE LETTER THA;Lo;0;L;;;;;N;;;;; +1958;TAI LE LETTER LA;Lo;0;L;;;;;N;;;;; +1959;TAI LE LETTER PA;Lo;0;L;;;;;N;;;;; +195A;TAI LE LETTER PHA;Lo;0;L;;;;;N;;;;; +195B;TAI LE LETTER MA;Lo;0;L;;;;;N;;;;; +195C;TAI LE LETTER FA;Lo;0;L;;;;;N;;;;; +195D;TAI LE LETTER VA;Lo;0;L;;;;;N;;;;; +195E;TAI LE LETTER HA;Lo;0;L;;;;;N;;;;; +195F;TAI LE LETTER QA;Lo;0;L;;;;;N;;;;; +1960;TAI LE LETTER KHA;Lo;0;L;;;;;N;;;;; +1961;TAI LE LETTER TSHA;Lo;0;L;;;;;N;;;;; +1962;TAI LE LETTER NA;Lo;0;L;;;;;N;;;;; +1963;TAI LE LETTER A;Lo;0;L;;;;;N;;;;; +1964;TAI LE LETTER I;Lo;0;L;;;;;N;;;;; +1965;TAI LE LETTER EE;Lo;0;L;;;;;N;;;;; +1966;TAI LE LETTER EH;Lo;0;L;;;;;N;;;;; +1967;TAI LE LETTER U;Lo;0;L;;;;;N;;;;; +1968;TAI LE LETTER OO;Lo;0;L;;;;;N;;;;; +1969;TAI LE LETTER O;Lo;0;L;;;;;N;;;;; +196A;TAI LE LETTER UE;Lo;0;L;;;;;N;;;;; +196B;TAI LE LETTER E;Lo;0;L;;;;;N;;;;; +196C;TAI LE LETTER AUE;Lo;0;L;;;;;N;;;;; +196D;TAI LE LETTER AI;Lo;0;L;;;;;N;;;;; +1970;TAI LE LETTER TONE-2;Lo;0;L;;;;;N;;;;; +1971;TAI LE LETTER TONE-3;Lo;0;L;;;;;N;;;;; +1972;TAI LE LETTER TONE-4;Lo;0;L;;;;;N;;;;; +1973;TAI LE LETTER TONE-5;Lo;0;L;;;;;N;;;;; +1974;TAI LE LETTER TONE-6;Lo;0;L;;;;;N;;;;; +1980;NEW TAI LUE LETTER HIGH QA;Lo;0;L;;;;;N;;;;; +1981;NEW TAI LUE LETTER LOW QA;Lo;0;L;;;;;N;;;;; +1982;NEW TAI LUE LETTER HIGH KA;Lo;0;L;;;;;N;;;;; +1983;NEW TAI LUE LETTER HIGH XA;Lo;0;L;;;;;N;;;;; +1984;NEW TAI LUE LETTER HIGH NGA;Lo;0;L;;;;;N;;;;; +1985;NEW TAI LUE LETTER LOW KA;Lo;0;L;;;;;N;;;;; +1986;NEW TAI LUE LETTER LOW XA;Lo;0;L;;;;;N;;;;; +1987;NEW TAI LUE LETTER LOW NGA;Lo;0;L;;;;;N;;;;; +1988;NEW TAI LUE LETTER HIGH TSA;Lo;0;L;;;;;N;;;;; +1989;NEW TAI LUE LETTER HIGH SA;Lo;0;L;;;;;N;;;;; +198A;NEW TAI LUE LETTER HIGH YA;Lo;0;L;;;;;N;;;;; +198B;NEW TAI LUE LETTER LOW TSA;Lo;0;L;;;;;N;;;;; +198C;NEW TAI LUE LETTER LOW SA;Lo;0;L;;;;;N;;;;; +198D;NEW TAI LUE LETTER LOW YA;Lo;0;L;;;;;N;;;;; +198E;NEW TAI LUE LETTER HIGH TA;Lo;0;L;;;;;N;;;;; +198F;NEW TAI LUE LETTER HIGH THA;Lo;0;L;;;;;N;;;;; +1990;NEW TAI LUE LETTER HIGH NA;Lo;0;L;;;;;N;;;;; +1991;NEW TAI LUE LETTER LOW TA;Lo;0;L;;;;;N;;;;; +1992;NEW TAI LUE LETTER LOW THA;Lo;0;L;;;;;N;;;;; +1993;NEW TAI LUE LETTER LOW NA;Lo;0;L;;;;;N;;;;; +1994;NEW TAI LUE LETTER HIGH PA;Lo;0;L;;;;;N;;;;; +1995;NEW TAI LUE LETTER HIGH PHA;Lo;0;L;;;;;N;;;;; +1996;NEW TAI LUE LETTER HIGH MA;Lo;0;L;;;;;N;;;;; +1997;NEW TAI LUE LETTER LOW PA;Lo;0;L;;;;;N;;;;; +1998;NEW TAI LUE LETTER LOW PHA;Lo;0;L;;;;;N;;;;; +1999;NEW TAI LUE LETTER LOW MA;Lo;0;L;;;;;N;;;;; +199A;NEW TAI LUE LETTER HIGH FA;Lo;0;L;;;;;N;;;;; +199B;NEW TAI LUE LETTER HIGH VA;Lo;0;L;;;;;N;;;;; +199C;NEW TAI LUE LETTER HIGH LA;Lo;0;L;;;;;N;;;;; +199D;NEW TAI LUE LETTER LOW FA;Lo;0;L;;;;;N;;;;; +199E;NEW TAI LUE LETTER LOW VA;Lo;0;L;;;;;N;;;;; +199F;NEW TAI LUE LETTER LOW LA;Lo;0;L;;;;;N;;;;; +19A0;NEW TAI LUE LETTER HIGH HA;Lo;0;L;;;;;N;;;;; +19A1;NEW TAI LUE LETTER HIGH DA;Lo;0;L;;;;;N;;;;; +19A2;NEW TAI LUE LETTER HIGH BA;Lo;0;L;;;;;N;;;;; +19A3;NEW TAI LUE LETTER LOW HA;Lo;0;L;;;;;N;;;;; +19A4;NEW TAI LUE LETTER LOW DA;Lo;0;L;;;;;N;;;;; +19A5;NEW TAI LUE LETTER LOW BA;Lo;0;L;;;;;N;;;;; +19A6;NEW TAI LUE LETTER HIGH KVA;Lo;0;L;;;;;N;;;;; +19A7;NEW TAI LUE LETTER HIGH XVA;Lo;0;L;;;;;N;;;;; +19A8;NEW TAI LUE LETTER LOW KVA;Lo;0;L;;;;;N;;;;; +19A9;NEW TAI LUE LETTER LOW XVA;Lo;0;L;;;;;N;;;;; +19AA;NEW TAI LUE LETTER HIGH SUA;Lo;0;L;;;;;N;;;;; +19AB;NEW TAI LUE LETTER LOW SUA;Lo;0;L;;;;;N;;;;; +19B0;NEW TAI LUE VOWEL SIGN VOWEL SHORTENER;Mc;0;L;;;;;N;;;;; +19B1;NEW TAI LUE VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +19B2;NEW TAI LUE VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +19B3;NEW TAI LUE VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +19B4;NEW TAI LUE VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +19B5;NEW TAI LUE VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +19B6;NEW TAI LUE VOWEL SIGN AE;Mc;0;L;;;;;N;;;;; +19B7;NEW TAI LUE VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +19B8;NEW TAI LUE VOWEL SIGN OA;Mc;0;L;;;;;N;;;;; +19B9;NEW TAI LUE VOWEL SIGN UE;Mc;0;L;;;;;N;;;;; +19BA;NEW TAI LUE VOWEL SIGN AY;Mc;0;L;;;;;N;;;;; +19BB;NEW TAI LUE VOWEL SIGN AAY;Mc;0;L;;;;;N;;;;; +19BC;NEW TAI LUE VOWEL SIGN UY;Mc;0;L;;;;;N;;;;; +19BD;NEW TAI LUE VOWEL SIGN OY;Mc;0;L;;;;;N;;;;; +19BE;NEW TAI LUE VOWEL SIGN OAY;Mc;0;L;;;;;N;;;;; +19BF;NEW TAI LUE VOWEL SIGN UEY;Mc;0;L;;;;;N;;;;; +19C0;NEW TAI LUE VOWEL SIGN IY;Mc;0;L;;;;;N;;;;; +19C1;NEW TAI LUE LETTER FINAL V;Lo;0;L;;;;;N;;;;; +19C2;NEW TAI LUE LETTER FINAL NG;Lo;0;L;;;;;N;;;;; +19C3;NEW TAI LUE LETTER FINAL N;Lo;0;L;;;;;N;;;;; +19C4;NEW TAI LUE LETTER FINAL M;Lo;0;L;;;;;N;;;;; +19C5;NEW TAI LUE LETTER FINAL K;Lo;0;L;;;;;N;;;;; +19C6;NEW TAI LUE LETTER FINAL D;Lo;0;L;;;;;N;;;;; +19C7;NEW TAI LUE LETTER FINAL B;Lo;0;L;;;;;N;;;;; +19C8;NEW TAI LUE TONE MARK-1;Mc;0;L;;;;;N;;;;; +19C9;NEW TAI LUE TONE MARK-2;Mc;0;L;;;;;N;;;;; +19D0;NEW TAI LUE DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +19D1;NEW TAI LUE DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +19D2;NEW TAI LUE DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +19D3;NEW TAI LUE DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +19D4;NEW TAI LUE DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +19D5;NEW TAI LUE DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +19D6;NEW TAI LUE DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +19D7;NEW TAI LUE DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +19D8;NEW TAI LUE DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +19D9;NEW TAI LUE DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +19DA;NEW TAI LUE THAM DIGIT ONE;No;0;L;;;1;1;N;;;;; +19DE;NEW TAI LUE SIGN LAE;So;0;ON;;;;;N;;;;; +19DF;NEW TAI LUE SIGN LAEV;So;0;ON;;;;;N;;;;; +19E0;KHMER SYMBOL PATHAMASAT;So;0;ON;;;;;N;;;;; +19E1;KHMER SYMBOL MUOY KOET;So;0;ON;;;;;N;;;;; +19E2;KHMER SYMBOL PII KOET;So;0;ON;;;;;N;;;;; +19E3;KHMER SYMBOL BEI KOET;So;0;ON;;;;;N;;;;; +19E4;KHMER SYMBOL BUON KOET;So;0;ON;;;;;N;;;;; +19E5;KHMER SYMBOL PRAM KOET;So;0;ON;;;;;N;;;;; +19E6;KHMER SYMBOL PRAM-MUOY KOET;So;0;ON;;;;;N;;;;; +19E7;KHMER SYMBOL PRAM-PII KOET;So;0;ON;;;;;N;;;;; +19E8;KHMER SYMBOL PRAM-BEI KOET;So;0;ON;;;;;N;;;;; +19E9;KHMER SYMBOL PRAM-BUON KOET;So;0;ON;;;;;N;;;;; +19EA;KHMER SYMBOL DAP KOET;So;0;ON;;;;;N;;;;; +19EB;KHMER SYMBOL DAP-MUOY KOET;So;0;ON;;;;;N;;;;; +19EC;KHMER SYMBOL DAP-PII KOET;So;0;ON;;;;;N;;;;; +19ED;KHMER SYMBOL DAP-BEI KOET;So;0;ON;;;;;N;;;;; +19EE;KHMER SYMBOL DAP-BUON KOET;So;0;ON;;;;;N;;;;; +19EF;KHMER SYMBOL DAP-PRAM KOET;So;0;ON;;;;;N;;;;; +19F0;KHMER SYMBOL TUTEYASAT;So;0;ON;;;;;N;;;;; +19F1;KHMER SYMBOL MUOY ROC;So;0;ON;;;;;N;;;;; +19F2;KHMER SYMBOL PII ROC;So;0;ON;;;;;N;;;;; +19F3;KHMER SYMBOL BEI ROC;So;0;ON;;;;;N;;;;; +19F4;KHMER SYMBOL BUON ROC;So;0;ON;;;;;N;;;;; +19F5;KHMER SYMBOL PRAM ROC;So;0;ON;;;;;N;;;;; +19F6;KHMER SYMBOL PRAM-MUOY ROC;So;0;ON;;;;;N;;;;; +19F7;KHMER SYMBOL PRAM-PII ROC;So;0;ON;;;;;N;;;;; +19F8;KHMER SYMBOL PRAM-BEI ROC;So;0;ON;;;;;N;;;;; +19F9;KHMER SYMBOL PRAM-BUON ROC;So;0;ON;;;;;N;;;;; +19FA;KHMER SYMBOL DAP ROC;So;0;ON;;;;;N;;;;; +19FB;KHMER SYMBOL DAP-MUOY ROC;So;0;ON;;;;;N;;;;; +19FC;KHMER SYMBOL DAP-PII ROC;So;0;ON;;;;;N;;;;; +19FD;KHMER SYMBOL DAP-BEI ROC;So;0;ON;;;;;N;;;;; +19FE;KHMER SYMBOL DAP-BUON ROC;So;0;ON;;;;;N;;;;; +19FF;KHMER SYMBOL DAP-PRAM ROC;So;0;ON;;;;;N;;;;; +1A00;BUGINESE LETTER KA;Lo;0;L;;;;;N;;;;; +1A01;BUGINESE LETTER GA;Lo;0;L;;;;;N;;;;; +1A02;BUGINESE LETTER NGA;Lo;0;L;;;;;N;;;;; +1A03;BUGINESE LETTER NGKA;Lo;0;L;;;;;N;;;;; +1A04;BUGINESE LETTER PA;Lo;0;L;;;;;N;;;;; +1A05;BUGINESE LETTER BA;Lo;0;L;;;;;N;;;;; +1A06;BUGINESE LETTER MA;Lo;0;L;;;;;N;;;;; +1A07;BUGINESE LETTER MPA;Lo;0;L;;;;;N;;;;; +1A08;BUGINESE LETTER TA;Lo;0;L;;;;;N;;;;; +1A09;BUGINESE LETTER DA;Lo;0;L;;;;;N;;;;; +1A0A;BUGINESE LETTER NA;Lo;0;L;;;;;N;;;;; +1A0B;BUGINESE LETTER NRA;Lo;0;L;;;;;N;;;;; +1A0C;BUGINESE LETTER CA;Lo;0;L;;;;;N;;;;; +1A0D;BUGINESE LETTER JA;Lo;0;L;;;;;N;;;;; +1A0E;BUGINESE LETTER NYA;Lo;0;L;;;;;N;;;;; +1A0F;BUGINESE LETTER NYCA;Lo;0;L;;;;;N;;;;; +1A10;BUGINESE LETTER YA;Lo;0;L;;;;;N;;;;; +1A11;BUGINESE LETTER RA;Lo;0;L;;;;;N;;;;; +1A12;BUGINESE LETTER LA;Lo;0;L;;;;;N;;;;; +1A13;BUGINESE LETTER VA;Lo;0;L;;;;;N;;;;; +1A14;BUGINESE LETTER SA;Lo;0;L;;;;;N;;;;; +1A15;BUGINESE LETTER A;Lo;0;L;;;;;N;;;;; +1A16;BUGINESE LETTER HA;Lo;0;L;;;;;N;;;;; +1A17;BUGINESE VOWEL SIGN I;Mn;230;NSM;;;;;N;;;;; +1A18;BUGINESE VOWEL SIGN U;Mn;220;NSM;;;;;N;;;;; +1A19;BUGINESE VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +1A1A;BUGINESE VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +1A1B;BUGINESE VOWEL SIGN AE;Mc;0;L;;;;;N;;;;; +1A1E;BUGINESE PALLAWA;Po;0;L;;;;;N;;;;; +1A1F;BUGINESE END OF SECTION;Po;0;L;;;;;N;;;;; +1A20;TAI THAM LETTER HIGH KA;Lo;0;L;;;;;N;;;;; +1A21;TAI THAM LETTER HIGH KHA;Lo;0;L;;;;;N;;;;; +1A22;TAI THAM LETTER HIGH KXA;Lo;0;L;;;;;N;;;;; +1A23;TAI THAM LETTER LOW KA;Lo;0;L;;;;;N;;;;; +1A24;TAI THAM LETTER LOW KXA;Lo;0;L;;;;;N;;;;; +1A25;TAI THAM LETTER LOW KHA;Lo;0;L;;;;;N;;;;; +1A26;TAI THAM LETTER NGA;Lo;0;L;;;;;N;;;;; +1A27;TAI THAM LETTER HIGH CA;Lo;0;L;;;;;N;;;;; +1A28;TAI THAM LETTER HIGH CHA;Lo;0;L;;;;;N;;;;; +1A29;TAI THAM LETTER LOW CA;Lo;0;L;;;;;N;;;;; +1A2A;TAI THAM LETTER LOW SA;Lo;0;L;;;;;N;;;;; +1A2B;TAI THAM LETTER LOW CHA;Lo;0;L;;;;;N;;;;; +1A2C;TAI THAM LETTER NYA;Lo;0;L;;;;;N;;;;; +1A2D;TAI THAM LETTER RATA;Lo;0;L;;;;;N;;;;; +1A2E;TAI THAM LETTER HIGH RATHA;Lo;0;L;;;;;N;;;;; +1A2F;TAI THAM LETTER DA;Lo;0;L;;;;;N;;;;; +1A30;TAI THAM LETTER LOW RATHA;Lo;0;L;;;;;N;;;;; +1A31;TAI THAM LETTER RANA;Lo;0;L;;;;;N;;;;; +1A32;TAI THAM LETTER HIGH TA;Lo;0;L;;;;;N;;;;; +1A33;TAI THAM LETTER HIGH THA;Lo;0;L;;;;;N;;;;; +1A34;TAI THAM LETTER LOW TA;Lo;0;L;;;;;N;;;;; +1A35;TAI THAM LETTER LOW THA;Lo;0;L;;;;;N;;;;; +1A36;TAI THAM LETTER NA;Lo;0;L;;;;;N;;;;; +1A37;TAI THAM LETTER BA;Lo;0;L;;;;;N;;;;; +1A38;TAI THAM LETTER HIGH PA;Lo;0;L;;;;;N;;;;; +1A39;TAI THAM LETTER HIGH PHA;Lo;0;L;;;;;N;;;;; +1A3A;TAI THAM LETTER HIGH FA;Lo;0;L;;;;;N;;;;; +1A3B;TAI THAM LETTER LOW PA;Lo;0;L;;;;;N;;;;; +1A3C;TAI THAM LETTER LOW FA;Lo;0;L;;;;;N;;;;; +1A3D;TAI THAM LETTER LOW PHA;Lo;0;L;;;;;N;;;;; +1A3E;TAI THAM LETTER MA;Lo;0;L;;;;;N;;;;; +1A3F;TAI THAM LETTER LOW YA;Lo;0;L;;;;;N;;;;; +1A40;TAI THAM LETTER HIGH YA;Lo;0;L;;;;;N;;;;; +1A41;TAI THAM LETTER RA;Lo;0;L;;;;;N;;;;; +1A42;TAI THAM LETTER RUE;Lo;0;L;;;;;N;;;;; +1A43;TAI THAM LETTER LA;Lo;0;L;;;;;N;;;;; +1A44;TAI THAM LETTER LUE;Lo;0;L;;;;;N;;;;; +1A45;TAI THAM LETTER WA;Lo;0;L;;;;;N;;;;; +1A46;TAI THAM LETTER HIGH SHA;Lo;0;L;;;;;N;;;;; +1A47;TAI THAM LETTER HIGH SSA;Lo;0;L;;;;;N;;;;; +1A48;TAI THAM LETTER HIGH SA;Lo;0;L;;;;;N;;;;; +1A49;TAI THAM LETTER HIGH HA;Lo;0;L;;;;;N;;;;; +1A4A;TAI THAM LETTER LLA;Lo;0;L;;;;;N;;;;; +1A4B;TAI THAM LETTER A;Lo;0;L;;;;;N;;;;; +1A4C;TAI THAM LETTER LOW HA;Lo;0;L;;;;;N;;;;; +1A4D;TAI THAM LETTER I;Lo;0;L;;;;;N;;;;; +1A4E;TAI THAM LETTER II;Lo;0;L;;;;;N;;;;; +1A4F;TAI THAM LETTER U;Lo;0;L;;;;;N;;;;; +1A50;TAI THAM LETTER UU;Lo;0;L;;;;;N;;;;; +1A51;TAI THAM LETTER EE;Lo;0;L;;;;;N;;;;; +1A52;TAI THAM LETTER OO;Lo;0;L;;;;;N;;;;; +1A53;TAI THAM LETTER LAE;Lo;0;L;;;;;N;;;;; +1A54;TAI THAM LETTER GREAT SA;Lo;0;L;;;;;N;;;;; +1A55;TAI THAM CONSONANT SIGN MEDIAL RA;Mc;0;L;;;;;N;;;;; +1A56;TAI THAM CONSONANT SIGN MEDIAL LA;Mn;0;NSM;;;;;N;;;;; +1A57;TAI THAM CONSONANT SIGN LA TANG LAI;Mc;0;L;;;;;N;;;;; +1A58;TAI THAM SIGN MAI KANG LAI;Mn;0;NSM;;;;;N;;;;; +1A59;TAI THAM CONSONANT SIGN FINAL NGA;Mn;0;NSM;;;;;N;;;;; +1A5A;TAI THAM CONSONANT SIGN LOW PA;Mn;0;NSM;;;;;N;;;;; +1A5B;TAI THAM CONSONANT SIGN HIGH RATHA OR LOW PA;Mn;0;NSM;;;;;N;;;;; +1A5C;TAI THAM CONSONANT SIGN MA;Mn;0;NSM;;;;;N;;;;; +1A5D;TAI THAM CONSONANT SIGN BA;Mn;0;NSM;;;;;N;;;;; +1A5E;TAI THAM CONSONANT SIGN SA;Mn;0;NSM;;;;;N;;;;; +1A60;TAI THAM SIGN SAKOT;Mn;9;NSM;;;;;N;;;;; +1A61;TAI THAM VOWEL SIGN A;Mc;0;L;;;;;N;;;;; +1A62;TAI THAM VOWEL SIGN MAI SAT;Mn;0;NSM;;;;;N;;;;; +1A63;TAI THAM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +1A64;TAI THAM VOWEL SIGN TALL AA;Mc;0;L;;;;;N;;;;; +1A65;TAI THAM VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1A66;TAI THAM VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +1A67;TAI THAM VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +1A68;TAI THAM VOWEL SIGN UUE;Mn;0;NSM;;;;;N;;;;; +1A69;TAI THAM VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1A6A;TAI THAM VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +1A6B;TAI THAM VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +1A6C;TAI THAM VOWEL SIGN OA BELOW;Mn;0;NSM;;;;;N;;;;; +1A6D;TAI THAM VOWEL SIGN OY;Mc;0;L;;;;;N;;;;; +1A6E;TAI THAM VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +1A6F;TAI THAM VOWEL SIGN AE;Mc;0;L;;;;;N;;;;; +1A70;TAI THAM VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +1A71;TAI THAM VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +1A72;TAI THAM VOWEL SIGN THAM AI;Mc;0;L;;;;;N;;;;; +1A73;TAI THAM VOWEL SIGN OA ABOVE;Mn;0;NSM;;;;;N;;;;; +1A74;TAI THAM SIGN MAI KANG;Mn;0;NSM;;;;;N;;;;; +1A75;TAI THAM SIGN TONE-1;Mn;230;NSM;;;;;N;;;;; +1A76;TAI THAM SIGN TONE-2;Mn;230;NSM;;;;;N;;;;; +1A77;TAI THAM SIGN KHUEN TONE-3;Mn;230;NSM;;;;;N;;;;; +1A78;TAI THAM SIGN KHUEN TONE-4;Mn;230;NSM;;;;;N;;;;; +1A79;TAI THAM SIGN KHUEN TONE-5;Mn;230;NSM;;;;;N;;;;; +1A7A;TAI THAM SIGN RA HAAM;Mn;230;NSM;;;;;N;;;;; +1A7B;TAI THAM SIGN MAI SAM;Mn;230;NSM;;;;;N;;;;; +1A7C;TAI THAM SIGN KHUEN-LUE KARAN;Mn;230;NSM;;;;;N;;;;; +1A7F;TAI THAM COMBINING CRYPTOGRAMMIC DOT;Mn;220;NSM;;;;;N;;;;; +1A80;TAI THAM HORA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1A81;TAI THAM HORA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1A82;TAI THAM HORA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1A83;TAI THAM HORA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1A84;TAI THAM HORA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1A85;TAI THAM HORA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1A86;TAI THAM HORA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1A87;TAI THAM HORA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1A88;TAI THAM HORA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1A89;TAI THAM HORA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1A90;TAI THAM THAM DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1A91;TAI THAM THAM DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1A92;TAI THAM THAM DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1A93;TAI THAM THAM DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1A94;TAI THAM THAM DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1A95;TAI THAM THAM DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1A96;TAI THAM THAM DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1A97;TAI THAM THAM DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1A98;TAI THAM THAM DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1A99;TAI THAM THAM DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1AA0;TAI THAM SIGN WIANG;Po;0;L;;;;;N;;;;; +1AA1;TAI THAM SIGN WIANGWAAK;Po;0;L;;;;;N;;;;; +1AA2;TAI THAM SIGN SAWAN;Po;0;L;;;;;N;;;;; +1AA3;TAI THAM SIGN KEOW;Po;0;L;;;;;N;;;;; +1AA4;TAI THAM SIGN HOY;Po;0;L;;;;;N;;;;; +1AA5;TAI THAM SIGN DOKMAI;Po;0;L;;;;;N;;;;; +1AA6;TAI THAM SIGN REVERSED ROTATED RANA;Po;0;L;;;;;N;;;;; +1AA7;TAI THAM SIGN MAI YAMOK;Lm;0;L;;;;;N;;;;; +1AA8;TAI THAM SIGN KAAN;Po;0;L;;;;;N;;;;; +1AA9;TAI THAM SIGN KAANKUU;Po;0;L;;;;;N;;;;; +1AAA;TAI THAM SIGN SATKAAN;Po;0;L;;;;;N;;;;; +1AAB;TAI THAM SIGN SATKAANKUU;Po;0;L;;;;;N;;;;; +1AAC;TAI THAM SIGN HANG;Po;0;L;;;;;N;;;;; +1AAD;TAI THAM SIGN CAANG;Po;0;L;;;;;N;;;;; +1B00;BALINESE SIGN ULU RICEM;Mn;0;NSM;;;;;N;;;;; +1B01;BALINESE SIGN ULU CANDRA;Mn;0;NSM;;;;;N;;;;; +1B02;BALINESE SIGN CECEK;Mn;0;NSM;;;;;N;;;;; +1B03;BALINESE SIGN SURANG;Mn;0;NSM;;;;;N;;;;; +1B04;BALINESE SIGN BISAH;Mc;0;L;;;;;N;;;;; +1B05;BALINESE LETTER AKARA;Lo;0;L;;;;;N;;;;; +1B06;BALINESE LETTER AKARA TEDUNG;Lo;0;L;1B05 1B35;;;;N;;;;; +1B07;BALINESE LETTER IKARA;Lo;0;L;;;;;N;;;;; +1B08;BALINESE LETTER IKARA TEDUNG;Lo;0;L;1B07 1B35;;;;N;;;;; +1B09;BALINESE LETTER UKARA;Lo;0;L;;;;;N;;;;; +1B0A;BALINESE LETTER UKARA TEDUNG;Lo;0;L;1B09 1B35;;;;N;;;;; +1B0B;BALINESE LETTER RA REPA;Lo;0;L;;;;;N;;;;; +1B0C;BALINESE LETTER RA REPA TEDUNG;Lo;0;L;1B0B 1B35;;;;N;;;;; +1B0D;BALINESE LETTER LA LENGA;Lo;0;L;;;;;N;;;;; +1B0E;BALINESE LETTER LA LENGA TEDUNG;Lo;0;L;1B0D 1B35;;;;N;;;;; +1B0F;BALINESE LETTER EKARA;Lo;0;L;;;;;N;;;;; +1B10;BALINESE LETTER AIKARA;Lo;0;L;;;;;N;;;;; +1B11;BALINESE LETTER OKARA;Lo;0;L;;;;;N;;;;; +1B12;BALINESE LETTER OKARA TEDUNG;Lo;0;L;1B11 1B35;;;;N;;;;; +1B13;BALINESE LETTER KA;Lo;0;L;;;;;N;;;;; +1B14;BALINESE LETTER KA MAHAPRANA;Lo;0;L;;;;;N;;;;; +1B15;BALINESE LETTER GA;Lo;0;L;;;;;N;;;;; +1B16;BALINESE LETTER GA GORA;Lo;0;L;;;;;N;;;;; +1B17;BALINESE LETTER NGA;Lo;0;L;;;;;N;;;;; +1B18;BALINESE LETTER CA;Lo;0;L;;;;;N;;;;; +1B19;BALINESE LETTER CA LACA;Lo;0;L;;;;;N;;;;; +1B1A;BALINESE LETTER JA;Lo;0;L;;;;;N;;;;; +1B1B;BALINESE LETTER JA JERA;Lo;0;L;;;;;N;;;;; +1B1C;BALINESE LETTER NYA;Lo;0;L;;;;;N;;;;; +1B1D;BALINESE LETTER TA LATIK;Lo;0;L;;;;;N;;;;; +1B1E;BALINESE LETTER TA MURDA MAHAPRANA;Lo;0;L;;;;;N;;;;; +1B1F;BALINESE LETTER DA MURDA ALPAPRANA;Lo;0;L;;;;;N;;;;; +1B20;BALINESE LETTER DA MURDA MAHAPRANA;Lo;0;L;;;;;N;;;;; +1B21;BALINESE LETTER NA RAMBAT;Lo;0;L;;;;;N;;;;; +1B22;BALINESE LETTER TA;Lo;0;L;;;;;N;;;;; +1B23;BALINESE LETTER TA TAWA;Lo;0;L;;;;;N;;;;; +1B24;BALINESE LETTER DA;Lo;0;L;;;;;N;;;;; +1B25;BALINESE LETTER DA MADU;Lo;0;L;;;;;N;;;;; +1B26;BALINESE LETTER NA;Lo;0;L;;;;;N;;;;; +1B27;BALINESE LETTER PA;Lo;0;L;;;;;N;;;;; +1B28;BALINESE LETTER PA KAPAL;Lo;0;L;;;;;N;;;;; +1B29;BALINESE LETTER BA;Lo;0;L;;;;;N;;;;; +1B2A;BALINESE LETTER BA KEMBANG;Lo;0;L;;;;;N;;;;; +1B2B;BALINESE LETTER MA;Lo;0;L;;;;;N;;;;; +1B2C;BALINESE LETTER YA;Lo;0;L;;;;;N;;;;; +1B2D;BALINESE LETTER RA;Lo;0;L;;;;;N;;;;; +1B2E;BALINESE LETTER LA;Lo;0;L;;;;;N;;;;; +1B2F;BALINESE LETTER WA;Lo;0;L;;;;;N;;;;; +1B30;BALINESE LETTER SA SAGA;Lo;0;L;;;;;N;;;;; +1B31;BALINESE LETTER SA SAPA;Lo;0;L;;;;;N;;;;; +1B32;BALINESE LETTER SA;Lo;0;L;;;;;N;;;;; +1B33;BALINESE LETTER HA;Lo;0;L;;;;;N;;;;; +1B34;BALINESE SIGN REREKAN;Mn;7;NSM;;;;;N;;;;; +1B35;BALINESE VOWEL SIGN TEDUNG;Mc;0;L;;;;;N;;;;; +1B36;BALINESE VOWEL SIGN ULU;Mn;0;NSM;;;;;N;;;;; +1B37;BALINESE VOWEL SIGN ULU SARI;Mn;0;NSM;;;;;N;;;;; +1B38;BALINESE VOWEL SIGN SUKU;Mn;0;NSM;;;;;N;;;;; +1B39;BALINESE VOWEL SIGN SUKU ILUT;Mn;0;NSM;;;;;N;;;;; +1B3A;BALINESE VOWEL SIGN RA REPA;Mn;0;NSM;;;;;N;;;;; +1B3B;BALINESE VOWEL SIGN RA REPA TEDUNG;Mc;0;L;1B3A 1B35;;;;N;;;;; +1B3C;BALINESE VOWEL SIGN LA LENGA;Mn;0;NSM;;;;;N;;;;; +1B3D;BALINESE VOWEL SIGN LA LENGA TEDUNG;Mc;0;L;1B3C 1B35;;;;N;;;;; +1B3E;BALINESE VOWEL SIGN TALING;Mc;0;L;;;;;N;;;;; +1B3F;BALINESE VOWEL SIGN TALING REPA;Mc;0;L;;;;;N;;;;; +1B40;BALINESE VOWEL SIGN TALING TEDUNG;Mc;0;L;1B3E 1B35;;;;N;;;;; +1B41;BALINESE VOWEL SIGN TALING REPA TEDUNG;Mc;0;L;1B3F 1B35;;;;N;;;;; +1B42;BALINESE VOWEL SIGN PEPET;Mn;0;NSM;;;;;N;;;;; +1B43;BALINESE VOWEL SIGN PEPET TEDUNG;Mc;0;L;1B42 1B35;;;;N;;;;; +1B44;BALINESE ADEG ADEG;Mc;9;L;;;;;N;;;;; +1B45;BALINESE LETTER KAF SASAK;Lo;0;L;;;;;N;;;;; +1B46;BALINESE LETTER KHOT SASAK;Lo;0;L;;;;;N;;;;; +1B47;BALINESE LETTER TZIR SASAK;Lo;0;L;;;;;N;;;;; +1B48;BALINESE LETTER EF SASAK;Lo;0;L;;;;;N;;;;; +1B49;BALINESE LETTER VE SASAK;Lo;0;L;;;;;N;;;;; +1B4A;BALINESE LETTER ZAL SASAK;Lo;0;L;;;;;N;;;;; +1B4B;BALINESE LETTER ASYURA SASAK;Lo;0;L;;;;;N;;;;; +1B50;BALINESE DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1B51;BALINESE DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1B52;BALINESE DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1B53;BALINESE DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1B54;BALINESE DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1B55;BALINESE DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1B56;BALINESE DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1B57;BALINESE DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1B58;BALINESE DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1B59;BALINESE DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1B5A;BALINESE PANTI;Po;0;L;;;;;N;;;;; +1B5B;BALINESE PAMADA;Po;0;L;;;;;N;;;;; +1B5C;BALINESE WINDU;Po;0;L;;;;;N;;;;; +1B5D;BALINESE CARIK PAMUNGKAH;Po;0;L;;;;;N;;;;; +1B5E;BALINESE CARIK SIKI;Po;0;L;;;;;N;;;;; +1B5F;BALINESE CARIK PAREREN;Po;0;L;;;;;N;;;;; +1B60;BALINESE PAMENENG;Po;0;L;;;;;N;;;;; +1B61;BALINESE MUSICAL SYMBOL DONG;So;0;L;;;;;N;;;;; +1B62;BALINESE MUSICAL SYMBOL DENG;So;0;L;;;;;N;;;;; +1B63;BALINESE MUSICAL SYMBOL DUNG;So;0;L;;;;;N;;;;; +1B64;BALINESE MUSICAL SYMBOL DANG;So;0;L;;;;;N;;;;; +1B65;BALINESE MUSICAL SYMBOL DANG SURANG;So;0;L;;;;;N;;;;; +1B66;BALINESE MUSICAL SYMBOL DING;So;0;L;;;;;N;;;;; +1B67;BALINESE MUSICAL SYMBOL DAENG;So;0;L;;;;;N;;;;; +1B68;BALINESE MUSICAL SYMBOL DEUNG;So;0;L;;;;;N;;;;; +1B69;BALINESE MUSICAL SYMBOL DAING;So;0;L;;;;;N;;;;; +1B6A;BALINESE MUSICAL SYMBOL DANG GEDE;So;0;L;;;;;N;;;;; +1B6B;BALINESE MUSICAL SYMBOL COMBINING TEGEH;Mn;230;NSM;;;;;N;;;;; +1B6C;BALINESE MUSICAL SYMBOL COMBINING ENDEP;Mn;220;NSM;;;;;N;;;;; +1B6D;BALINESE MUSICAL SYMBOL COMBINING KEMPUL;Mn;230;NSM;;;;;N;;;;; +1B6E;BALINESE MUSICAL SYMBOL COMBINING KEMPLI;Mn;230;NSM;;;;;N;;;;; +1B6F;BALINESE MUSICAL SYMBOL COMBINING JEGOGAN;Mn;230;NSM;;;;;N;;;;; +1B70;BALINESE MUSICAL SYMBOL COMBINING KEMPUL WITH JEGOGAN;Mn;230;NSM;;;;;N;;;;; +1B71;BALINESE MUSICAL SYMBOL COMBINING KEMPLI WITH JEGOGAN;Mn;230;NSM;;;;;N;;;;; +1B72;BALINESE MUSICAL SYMBOL COMBINING BENDE;Mn;230;NSM;;;;;N;;;;; +1B73;BALINESE MUSICAL SYMBOL COMBINING GONG;Mn;230;NSM;;;;;N;;;;; +1B74;BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG;So;0;L;;;;;N;;;;; +1B75;BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DAG;So;0;L;;;;;N;;;;; +1B76;BALINESE MUSICAL SYMBOL RIGHT-HAND CLOSED TUK;So;0;L;;;;;N;;;;; +1B77;BALINESE MUSICAL SYMBOL RIGHT-HAND CLOSED TAK;So;0;L;;;;;N;;;;; +1B78;BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PANG;So;0;L;;;;;N;;;;; +1B79;BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PUNG;So;0;L;;;;;N;;;;; +1B7A;BALINESE MUSICAL SYMBOL LEFT-HAND CLOSED PLAK;So;0;L;;;;;N;;;;; +1B7B;BALINESE MUSICAL SYMBOL LEFT-HAND CLOSED PLUK;So;0;L;;;;;N;;;;; +1B7C;BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING;So;0;L;;;;;N;;;;; +1B80;SUNDANESE SIGN PANYECEK;Mn;0;NSM;;;;;N;;;;; +1B81;SUNDANESE SIGN PANGLAYAR;Mn;0;NSM;;;;;N;;;;; +1B82;SUNDANESE SIGN PANGWISAD;Mc;0;L;;;;;N;;;;; +1B83;SUNDANESE LETTER A;Lo;0;L;;;;;N;;;;; +1B84;SUNDANESE LETTER I;Lo;0;L;;;;;N;;;;; +1B85;SUNDANESE LETTER U;Lo;0;L;;;;;N;;;;; +1B86;SUNDANESE LETTER AE;Lo;0;L;;;;;N;;;;; +1B87;SUNDANESE LETTER O;Lo;0;L;;;;;N;;;;; +1B88;SUNDANESE LETTER E;Lo;0;L;;;;;N;;;;; +1B89;SUNDANESE LETTER EU;Lo;0;L;;;;;N;;;;; +1B8A;SUNDANESE LETTER KA;Lo;0;L;;;;;N;;;;; +1B8B;SUNDANESE LETTER QA;Lo;0;L;;;;;N;;;;; +1B8C;SUNDANESE LETTER GA;Lo;0;L;;;;;N;;;;; +1B8D;SUNDANESE LETTER NGA;Lo;0;L;;;;;N;;;;; +1B8E;SUNDANESE LETTER CA;Lo;0;L;;;;;N;;;;; +1B8F;SUNDANESE LETTER JA;Lo;0;L;;;;;N;;;;; +1B90;SUNDANESE LETTER ZA;Lo;0;L;;;;;N;;;;; +1B91;SUNDANESE LETTER NYA;Lo;0;L;;;;;N;;;;; +1B92;SUNDANESE LETTER TA;Lo;0;L;;;;;N;;;;; +1B93;SUNDANESE LETTER DA;Lo;0;L;;;;;N;;;;; +1B94;SUNDANESE LETTER NA;Lo;0;L;;;;;N;;;;; +1B95;SUNDANESE LETTER PA;Lo;0;L;;;;;N;;;;; +1B96;SUNDANESE LETTER FA;Lo;0;L;;;;;N;;;;; +1B97;SUNDANESE LETTER VA;Lo;0;L;;;;;N;;;;; +1B98;SUNDANESE LETTER BA;Lo;0;L;;;;;N;;;;; +1B99;SUNDANESE LETTER MA;Lo;0;L;;;;;N;;;;; +1B9A;SUNDANESE LETTER YA;Lo;0;L;;;;;N;;;;; +1B9B;SUNDANESE LETTER RA;Lo;0;L;;;;;N;;;;; +1B9C;SUNDANESE LETTER LA;Lo;0;L;;;;;N;;;;; +1B9D;SUNDANESE LETTER WA;Lo;0;L;;;;;N;;;;; +1B9E;SUNDANESE LETTER SA;Lo;0;L;;;;;N;;;;; +1B9F;SUNDANESE LETTER XA;Lo;0;L;;;;;N;;;;; +1BA0;SUNDANESE LETTER HA;Lo;0;L;;;;;N;;;;; +1BA1;SUNDANESE CONSONANT SIGN PAMINGKAL;Mc;0;L;;;;;N;;;;; +1BA2;SUNDANESE CONSONANT SIGN PANYAKRA;Mn;0;NSM;;;;;N;;;;; +1BA3;SUNDANESE CONSONANT SIGN PANYIKU;Mn;0;NSM;;;;;N;;;;; +1BA4;SUNDANESE VOWEL SIGN PANGHULU;Mn;0;NSM;;;;;N;;;;; +1BA5;SUNDANESE VOWEL SIGN PANYUKU;Mn;0;NSM;;;;;N;;;;; +1BA6;SUNDANESE VOWEL SIGN PANAELAENG;Mc;0;L;;;;;N;;;;; +1BA7;SUNDANESE VOWEL SIGN PANOLONG;Mc;0;L;;;;;N;;;;; +1BA8;SUNDANESE VOWEL SIGN PAMEPET;Mn;0;NSM;;;;;N;;;;; +1BA9;SUNDANESE VOWEL SIGN PANEULEUNG;Mn;0;NSM;;;;;N;;;;; +1BAA;SUNDANESE SIGN PAMAAEH;Mc;9;L;;;;;N;;;;; +1BAB;SUNDANESE SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +1BAC;SUNDANESE CONSONANT SIGN PASANGAN MA;Mc;0;L;;;;;N;;;;; +1BAD;SUNDANESE CONSONANT SIGN PASANGAN WA;Mc;0;L;;;;;N;;;;; +1BAE;SUNDANESE LETTER KHA;Lo;0;L;;;;;N;;;;; +1BAF;SUNDANESE LETTER SYA;Lo;0;L;;;;;N;;;;; +1BB0;SUNDANESE DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1BB1;SUNDANESE DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1BB2;SUNDANESE DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1BB3;SUNDANESE DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1BB4;SUNDANESE DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1BB5;SUNDANESE DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1BB6;SUNDANESE DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1BB7;SUNDANESE DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1BB8;SUNDANESE DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1BB9;SUNDANESE DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1BBA;SUNDANESE AVAGRAHA;Lo;0;L;;;;;N;;;;; +1BBB;SUNDANESE LETTER REU;Lo;0;L;;;;;N;;;;; +1BBC;SUNDANESE LETTER LEU;Lo;0;L;;;;;N;;;;; +1BBD;SUNDANESE LETTER BHA;Lo;0;L;;;;;N;;;;; +1BBE;SUNDANESE LETTER FINAL K;Lo;0;L;;;;;N;;;;; +1BBF;SUNDANESE LETTER FINAL M;Lo;0;L;;;;;N;;;;; +1BC0;BATAK LETTER A;Lo;0;L;;;;;N;;;;; +1BC1;BATAK LETTER SIMALUNGUN A;Lo;0;L;;;;;N;;;;; +1BC2;BATAK LETTER HA;Lo;0;L;;;;;N;;;;; +1BC3;BATAK LETTER SIMALUNGUN HA;Lo;0;L;;;;;N;;;;; +1BC4;BATAK LETTER MANDAILING HA;Lo;0;L;;;;;N;;;;; +1BC5;BATAK LETTER BA;Lo;0;L;;;;;N;;;;; +1BC6;BATAK LETTER KARO BA;Lo;0;L;;;;;N;;;;; +1BC7;BATAK LETTER PA;Lo;0;L;;;;;N;;;;; +1BC8;BATAK LETTER SIMALUNGUN PA;Lo;0;L;;;;;N;;;;; +1BC9;BATAK LETTER NA;Lo;0;L;;;;;N;;;;; +1BCA;BATAK LETTER MANDAILING NA;Lo;0;L;;;;;N;;;;; +1BCB;BATAK LETTER WA;Lo;0;L;;;;;N;;;;; +1BCC;BATAK LETTER SIMALUNGUN WA;Lo;0;L;;;;;N;;;;; +1BCD;BATAK LETTER PAKPAK WA;Lo;0;L;;;;;N;;;;; +1BCE;BATAK LETTER GA;Lo;0;L;;;;;N;;;;; +1BCF;BATAK LETTER SIMALUNGUN GA;Lo;0;L;;;;;N;;;;; +1BD0;BATAK LETTER JA;Lo;0;L;;;;;N;;;;; +1BD1;BATAK LETTER DA;Lo;0;L;;;;;N;;;;; +1BD2;BATAK LETTER RA;Lo;0;L;;;;;N;;;;; +1BD3;BATAK LETTER SIMALUNGUN RA;Lo;0;L;;;;;N;;;;; +1BD4;BATAK LETTER MA;Lo;0;L;;;;;N;;;;; +1BD5;BATAK LETTER SIMALUNGUN MA;Lo;0;L;;;;;N;;;;; +1BD6;BATAK LETTER SOUTHERN TA;Lo;0;L;;;;;N;;;;; +1BD7;BATAK LETTER NORTHERN TA;Lo;0;L;;;;;N;;;;; +1BD8;BATAK LETTER SA;Lo;0;L;;;;;N;;;;; +1BD9;BATAK LETTER SIMALUNGUN SA;Lo;0;L;;;;;N;;;;; +1BDA;BATAK LETTER MANDAILING SA;Lo;0;L;;;;;N;;;;; +1BDB;BATAK LETTER YA;Lo;0;L;;;;;N;;;;; +1BDC;BATAK LETTER SIMALUNGUN YA;Lo;0;L;;;;;N;;;;; +1BDD;BATAK LETTER NGA;Lo;0;L;;;;;N;;;;; +1BDE;BATAK LETTER LA;Lo;0;L;;;;;N;;;;; +1BDF;BATAK LETTER SIMALUNGUN LA;Lo;0;L;;;;;N;;;;; +1BE0;BATAK LETTER NYA;Lo;0;L;;;;;N;;;;; +1BE1;BATAK LETTER CA;Lo;0;L;;;;;N;;;;; +1BE2;BATAK LETTER NDA;Lo;0;L;;;;;N;;;;; +1BE3;BATAK LETTER MBA;Lo;0;L;;;;;N;;;;; +1BE4;BATAK LETTER I;Lo;0;L;;;;;N;;;;; +1BE5;BATAK LETTER U;Lo;0;L;;;;;N;;;;; +1BE6;BATAK SIGN TOMPI;Mn;7;NSM;;;;;N;;;;; +1BE7;BATAK VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +1BE8;BATAK VOWEL SIGN PAKPAK E;Mn;0;NSM;;;;;N;;;;; +1BE9;BATAK VOWEL SIGN EE;Mn;0;NSM;;;;;N;;;;; +1BEA;BATAK VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +1BEB;BATAK VOWEL SIGN KARO I;Mc;0;L;;;;;N;;;;; +1BEC;BATAK VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +1BED;BATAK VOWEL SIGN KARO O;Mn;0;NSM;;;;;N;;;;; +1BEE;BATAK VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +1BEF;BATAK VOWEL SIGN U FOR SIMALUNGUN SA;Mn;0;NSM;;;;;N;;;;; +1BF0;BATAK CONSONANT SIGN NG;Mn;0;NSM;;;;;N;;;;; +1BF1;BATAK CONSONANT SIGN H;Mn;0;NSM;;;;;N;;;;; +1BF2;BATAK PANGOLAT;Mc;9;L;;;;;N;;;;; +1BF3;BATAK PANONGONAN;Mc;9;L;;;;;N;;;;; +1BFC;BATAK SYMBOL BINDU NA METEK;Po;0;L;;;;;N;;;;; +1BFD;BATAK SYMBOL BINDU PINARBORAS;Po;0;L;;;;;N;;;;; +1BFE;BATAK SYMBOL BINDU JUDUL;Po;0;L;;;;;N;;;;; +1BFF;BATAK SYMBOL BINDU PANGOLAT;Po;0;L;;;;;N;;;;; +1C00;LEPCHA LETTER KA;Lo;0;L;;;;;N;;;;; +1C01;LEPCHA LETTER KLA;Lo;0;L;;;;;N;;;;; +1C02;LEPCHA LETTER KHA;Lo;0;L;;;;;N;;;;; +1C03;LEPCHA LETTER GA;Lo;0;L;;;;;N;;;;; +1C04;LEPCHA LETTER GLA;Lo;0;L;;;;;N;;;;; +1C05;LEPCHA LETTER NGA;Lo;0;L;;;;;N;;;;; +1C06;LEPCHA LETTER CA;Lo;0;L;;;;;N;;;;; +1C07;LEPCHA LETTER CHA;Lo;0;L;;;;;N;;;;; +1C08;LEPCHA LETTER JA;Lo;0;L;;;;;N;;;;; +1C09;LEPCHA LETTER NYA;Lo;0;L;;;;;N;;;;; +1C0A;LEPCHA LETTER TA;Lo;0;L;;;;;N;;;;; +1C0B;LEPCHA LETTER THA;Lo;0;L;;;;;N;;;;; +1C0C;LEPCHA LETTER DA;Lo;0;L;;;;;N;;;;; +1C0D;LEPCHA LETTER NA;Lo;0;L;;;;;N;;;;; +1C0E;LEPCHA LETTER PA;Lo;0;L;;;;;N;;;;; +1C0F;LEPCHA LETTER PLA;Lo;0;L;;;;;N;;;;; +1C10;LEPCHA LETTER PHA;Lo;0;L;;;;;N;;;;; +1C11;LEPCHA LETTER FA;Lo;0;L;;;;;N;;;;; +1C12;LEPCHA LETTER FLA;Lo;0;L;;;;;N;;;;; +1C13;LEPCHA LETTER BA;Lo;0;L;;;;;N;;;;; +1C14;LEPCHA LETTER BLA;Lo;0;L;;;;;N;;;;; +1C15;LEPCHA LETTER MA;Lo;0;L;;;;;N;;;;; +1C16;LEPCHA LETTER MLA;Lo;0;L;;;;;N;;;;; +1C17;LEPCHA LETTER TSA;Lo;0;L;;;;;N;;;;; +1C18;LEPCHA LETTER TSHA;Lo;0;L;;;;;N;;;;; +1C19;LEPCHA LETTER DZA;Lo;0;L;;;;;N;;;;; +1C1A;LEPCHA LETTER YA;Lo;0;L;;;;;N;;;;; +1C1B;LEPCHA LETTER RA;Lo;0;L;;;;;N;;;;; +1C1C;LEPCHA LETTER LA;Lo;0;L;;;;;N;;;;; +1C1D;LEPCHA LETTER HA;Lo;0;L;;;;;N;;;;; +1C1E;LEPCHA LETTER HLA;Lo;0;L;;;;;N;;;;; +1C1F;LEPCHA LETTER VA;Lo;0;L;;;;;N;;;;; +1C20;LEPCHA LETTER SA;Lo;0;L;;;;;N;;;;; +1C21;LEPCHA LETTER SHA;Lo;0;L;;;;;N;;;;; +1C22;LEPCHA LETTER WA;Lo;0;L;;;;;N;;;;; +1C23;LEPCHA LETTER A;Lo;0;L;;;;;N;;;;; +1C24;LEPCHA SUBJOINED LETTER YA;Mc;0;L;;;;;N;;;;; +1C25;LEPCHA SUBJOINED LETTER RA;Mc;0;L;;;;;N;;;;; +1C26;LEPCHA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +1C27;LEPCHA VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +1C28;LEPCHA VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +1C29;LEPCHA VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +1C2A;LEPCHA VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +1C2B;LEPCHA VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +1C2C;LEPCHA VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +1C2D;LEPCHA CONSONANT SIGN K;Mn;0;NSM;;;;;N;;;;; +1C2E;LEPCHA CONSONANT SIGN M;Mn;0;NSM;;;;;N;;;;; +1C2F;LEPCHA CONSONANT SIGN L;Mn;0;NSM;;;;;N;;;;; +1C30;LEPCHA CONSONANT SIGN N;Mn;0;NSM;;;;;N;;;;; +1C31;LEPCHA CONSONANT SIGN P;Mn;0;NSM;;;;;N;;;;; +1C32;LEPCHA CONSONANT SIGN R;Mn;0;NSM;;;;;N;;;;; +1C33;LEPCHA CONSONANT SIGN T;Mn;0;NSM;;;;;N;;;;; +1C34;LEPCHA CONSONANT SIGN NYIN-DO;Mc;0;L;;;;;N;;;;; +1C35;LEPCHA CONSONANT SIGN KANG;Mc;0;L;;;;;N;;;;; +1C36;LEPCHA SIGN RAN;Mn;0;NSM;;;;;N;;;;; +1C37;LEPCHA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +1C3B;LEPCHA PUNCTUATION TA-ROL;Po;0;L;;;;;N;;;;; +1C3C;LEPCHA PUNCTUATION NYET THYOOM TA-ROL;Po;0;L;;;;;N;;;;; +1C3D;LEPCHA PUNCTUATION CER-WA;Po;0;L;;;;;N;;;;; +1C3E;LEPCHA PUNCTUATION TSHOOK CER-WA;Po;0;L;;;;;N;;;;; +1C3F;LEPCHA PUNCTUATION TSHOOK;Po;0;L;;;;;N;;;;; +1C40;LEPCHA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1C41;LEPCHA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1C42;LEPCHA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1C43;LEPCHA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1C44;LEPCHA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1C45;LEPCHA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1C46;LEPCHA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1C47;LEPCHA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1C48;LEPCHA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1C49;LEPCHA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1C4D;LEPCHA LETTER TTA;Lo;0;L;;;;;N;;;;; +1C4E;LEPCHA LETTER TTHA;Lo;0;L;;;;;N;;;;; +1C4F;LEPCHA LETTER DDA;Lo;0;L;;;;;N;;;;; +1C50;OL CHIKI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1C51;OL CHIKI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1C52;OL CHIKI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1C53;OL CHIKI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1C54;OL CHIKI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1C55;OL CHIKI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1C56;OL CHIKI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1C57;OL CHIKI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1C58;OL CHIKI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1C59;OL CHIKI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1C5A;OL CHIKI LETTER LA;Lo;0;L;;;;;N;;;;; +1C5B;OL CHIKI LETTER AT;Lo;0;L;;;;;N;;;;; +1C5C;OL CHIKI LETTER AG;Lo;0;L;;;;;N;;;;; +1C5D;OL CHIKI LETTER ANG;Lo;0;L;;;;;N;;;;; +1C5E;OL CHIKI LETTER AL;Lo;0;L;;;;;N;;;;; +1C5F;OL CHIKI LETTER LAA;Lo;0;L;;;;;N;;;;; +1C60;OL CHIKI LETTER AAK;Lo;0;L;;;;;N;;;;; +1C61;OL CHIKI LETTER AAJ;Lo;0;L;;;;;N;;;;; +1C62;OL CHIKI LETTER AAM;Lo;0;L;;;;;N;;;;; +1C63;OL CHIKI LETTER AAW;Lo;0;L;;;;;N;;;;; +1C64;OL CHIKI LETTER LI;Lo;0;L;;;;;N;;;;; +1C65;OL CHIKI LETTER IS;Lo;0;L;;;;;N;;;;; +1C66;OL CHIKI LETTER IH;Lo;0;L;;;;;N;;;;; +1C67;OL CHIKI LETTER INY;Lo;0;L;;;;;N;;;;; +1C68;OL CHIKI LETTER IR;Lo;0;L;;;;;N;;;;; +1C69;OL CHIKI LETTER LU;Lo;0;L;;;;;N;;;;; +1C6A;OL CHIKI LETTER UC;Lo;0;L;;;;;N;;;;; +1C6B;OL CHIKI LETTER UD;Lo;0;L;;;;;N;;;;; +1C6C;OL CHIKI LETTER UNN;Lo;0;L;;;;;N;;;;; +1C6D;OL CHIKI LETTER UY;Lo;0;L;;;;;N;;;;; +1C6E;OL CHIKI LETTER LE;Lo;0;L;;;;;N;;;;; +1C6F;OL CHIKI LETTER EP;Lo;0;L;;;;;N;;;;; +1C70;OL CHIKI LETTER EDD;Lo;0;L;;;;;N;;;;; +1C71;OL CHIKI LETTER EN;Lo;0;L;;;;;N;;;;; +1C72;OL CHIKI LETTER ERR;Lo;0;L;;;;;N;;;;; +1C73;OL CHIKI LETTER LO;Lo;0;L;;;;;N;;;;; +1C74;OL CHIKI LETTER OTT;Lo;0;L;;;;;N;;;;; +1C75;OL CHIKI LETTER OB;Lo;0;L;;;;;N;;;;; +1C76;OL CHIKI LETTER OV;Lo;0;L;;;;;N;;;;; +1C77;OL CHIKI LETTER OH;Lo;0;L;;;;;N;;;;; +1C78;OL CHIKI MU TTUDDAG;Lm;0;L;;;;;N;;;;; +1C79;OL CHIKI GAAHLAA TTUDDAAG;Lm;0;L;;;;;N;;;;; +1C7A;OL CHIKI MU-GAAHLAA TTUDDAAG;Lm;0;L;;;;;N;;;;; +1C7B;OL CHIKI RELAA;Lm;0;L;;;;;N;;;;; +1C7C;OL CHIKI PHAARKAA;Lm;0;L;;;;;N;;;;; +1C7D;OL CHIKI AHAD;Lm;0;L;;;;;N;;;;; +1C7E;OL CHIKI PUNCTUATION MUCAAD;Po;0;L;;;;;N;;;;; +1C7F;OL CHIKI PUNCTUATION DOUBLE MUCAAD;Po;0;L;;;;;N;;;;; +1CC0;SUNDANESE PUNCTUATION BINDU SURYA;Po;0;L;;;;;N;;;;; +1CC1;SUNDANESE PUNCTUATION BINDU PANGLONG;Po;0;L;;;;;N;;;;; +1CC2;SUNDANESE PUNCTUATION BINDU PURNAMA;Po;0;L;;;;;N;;;;; +1CC3;SUNDANESE PUNCTUATION BINDU CAKRA;Po;0;L;;;;;N;;;;; +1CC4;SUNDANESE PUNCTUATION BINDU LEU SATANGA;Po;0;L;;;;;N;;;;; +1CC5;SUNDANESE PUNCTUATION BINDU KA SATANGA;Po;0;L;;;;;N;;;;; +1CC6;SUNDANESE PUNCTUATION BINDU DA SATANGA;Po;0;L;;;;;N;;;;; +1CC7;SUNDANESE PUNCTUATION BINDU BA SATANGA;Po;0;L;;;;;N;;;;; +1CD0;VEDIC TONE KARSHANA;Mn;230;NSM;;;;;N;;;;; +1CD1;VEDIC TONE SHARA;Mn;230;NSM;;;;;N;;;;; +1CD2;VEDIC TONE PRENKHA;Mn;230;NSM;;;;;N;;;;; +1CD3;VEDIC SIGN NIHSHVASA;Po;0;L;;;;;N;;;;; +1CD4;VEDIC SIGN YAJURVEDIC MIDLINE SVARITA;Mn;1;NSM;;;;;N;;;;; +1CD5;VEDIC TONE YAJURVEDIC AGGRAVATED INDEPENDENT SVARITA;Mn;220;NSM;;;;;N;;;;; +1CD6;VEDIC TONE YAJURVEDIC INDEPENDENT SVARITA;Mn;220;NSM;;;;;N;;;;; +1CD7;VEDIC TONE YAJURVEDIC KATHAKA INDEPENDENT SVARITA;Mn;220;NSM;;;;;N;;;;; +1CD8;VEDIC TONE CANDRA BELOW;Mn;220;NSM;;;;;N;;;;; +1CD9;VEDIC TONE YAJURVEDIC KATHAKA INDEPENDENT SVARITA SCHROEDER;Mn;220;NSM;;;;;N;;;;; +1CDA;VEDIC TONE DOUBLE SVARITA;Mn;230;NSM;;;;;N;;;;; +1CDB;VEDIC TONE TRIPLE SVARITA;Mn;230;NSM;;;;;N;;;;; +1CDC;VEDIC TONE KATHAKA ANUDATTA;Mn;220;NSM;;;;;N;;;;; +1CDD;VEDIC TONE DOT BELOW;Mn;220;NSM;;;;;N;;;;; +1CDE;VEDIC TONE TWO DOTS BELOW;Mn;220;NSM;;;;;N;;;;; +1CDF;VEDIC TONE THREE DOTS BELOW;Mn;220;NSM;;;;;N;;;;; +1CE0;VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA;Mn;230;NSM;;;;;N;;;;; +1CE1;VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA;Mc;0;L;;;;;N;;;;; +1CE2;VEDIC SIGN VISARGA SVARITA;Mn;1;NSM;;;;;N;;;;; +1CE3;VEDIC SIGN VISARGA UDATTA;Mn;1;NSM;;;;;N;;;;; +1CE4;VEDIC SIGN REVERSED VISARGA UDATTA;Mn;1;NSM;;;;;N;;;;; +1CE5;VEDIC SIGN VISARGA ANUDATTA;Mn;1;NSM;;;;;N;;;;; +1CE6;VEDIC SIGN REVERSED VISARGA ANUDATTA;Mn;1;NSM;;;;;N;;;;; +1CE7;VEDIC SIGN VISARGA UDATTA WITH TAIL;Mn;1;NSM;;;;;N;;;;; +1CE8;VEDIC SIGN VISARGA ANUDATTA WITH TAIL;Mn;1;NSM;;;;;N;;;;; +1CE9;VEDIC SIGN ANUSVARA ANTARGOMUKHA;Lo;0;L;;;;;N;;;;; +1CEA;VEDIC SIGN ANUSVARA BAHIRGOMUKHA;Lo;0;L;;;;;N;;;;; +1CEB;VEDIC SIGN ANUSVARA VAMAGOMUKHA;Lo;0;L;;;;;N;;;;; +1CEC;VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL;Lo;0;L;;;;;N;;;;; +1CED;VEDIC SIGN TIRYAK;Mn;220;NSM;;;;;N;;;;; +1CEE;VEDIC SIGN HEXIFORM LONG ANUSVARA;Lo;0;L;;;;;N;;;;; +1CEF;VEDIC SIGN LONG ANUSVARA;Lo;0;L;;;;;N;;;;; +1CF0;VEDIC SIGN RTHANG LONG ANUSVARA;Lo;0;L;;;;;N;;;;; +1CF1;VEDIC SIGN ANUSVARA UBHAYATO MUKHA;Lo;0;L;;;;;N;;;;; +1CF2;VEDIC SIGN ARDHAVISARGA;Mc;0;L;;;;;N;;;;; +1CF3;VEDIC SIGN ROTATED ARDHAVISARGA;Mc;0;L;;;;;N;;;;; +1CF4;VEDIC TONE CANDRA ABOVE;Mn;230;NSM;;;;;N;;;;; +1CF5;VEDIC SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; +1CF6;VEDIC SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +1D00;LATIN LETTER SMALL CAPITAL A;Ll;0;L;;;;;N;;;;; +1D01;LATIN LETTER SMALL CAPITAL AE;Ll;0;L;;;;;N;;;;; +1D02;LATIN SMALL LETTER TURNED AE;Ll;0;L;;;;;N;;;;; +1D03;LATIN LETTER SMALL CAPITAL BARRED B;Ll;0;L;;;;;N;;;;; +1D04;LATIN LETTER SMALL CAPITAL C;Ll;0;L;;;;;N;;;;; +1D05;LATIN LETTER SMALL CAPITAL D;Ll;0;L;;;;;N;;;;; +1D06;LATIN LETTER SMALL CAPITAL ETH;Ll;0;L;;;;;N;;;;; +1D07;LATIN LETTER SMALL CAPITAL E;Ll;0;L;;;;;N;;;;; +1D08;LATIN SMALL LETTER TURNED OPEN E;Ll;0;L;;;;;N;;;;; +1D09;LATIN SMALL LETTER TURNED I;Ll;0;L;;;;;N;;;;; +1D0A;LATIN LETTER SMALL CAPITAL J;Ll;0;L;;;;;N;;;;; +1D0B;LATIN LETTER SMALL CAPITAL K;Ll;0;L;;;;;N;;;;; +1D0C;LATIN LETTER SMALL CAPITAL L WITH STROKE;Ll;0;L;;;;;N;;;;; +1D0D;LATIN LETTER SMALL CAPITAL M;Ll;0;L;;;;;N;;;;; +1D0E;LATIN LETTER SMALL CAPITAL REVERSED N;Ll;0;L;;;;;N;;;;; +1D0F;LATIN LETTER SMALL CAPITAL O;Ll;0;L;;;;;N;;;;; +1D10;LATIN LETTER SMALL CAPITAL OPEN O;Ll;0;L;;;;;N;;;;; +1D11;LATIN SMALL LETTER SIDEWAYS O;Ll;0;L;;;;;N;;;;; +1D12;LATIN SMALL LETTER SIDEWAYS OPEN O;Ll;0;L;;;;;N;;;;; +1D13;LATIN SMALL LETTER SIDEWAYS O WITH STROKE;Ll;0;L;;;;;N;;;;; +1D14;LATIN SMALL LETTER TURNED OE;Ll;0;L;;;;;N;;;;; +1D15;LATIN LETTER SMALL CAPITAL OU;Ll;0;L;;;;;N;;;;; +1D16;LATIN SMALL LETTER TOP HALF O;Ll;0;L;;;;;N;;;;; +1D17;LATIN SMALL LETTER BOTTOM HALF O;Ll;0;L;;;;;N;;;;; +1D18;LATIN LETTER SMALL CAPITAL P;Ll;0;L;;;;;N;;;;; +1D19;LATIN LETTER SMALL CAPITAL REVERSED R;Ll;0;L;;;;;N;;;;; +1D1A;LATIN LETTER SMALL CAPITAL TURNED R;Ll;0;L;;;;;N;;;;; +1D1B;LATIN LETTER SMALL CAPITAL T;Ll;0;L;;;;;N;;;;; +1D1C;LATIN LETTER SMALL CAPITAL U;Ll;0;L;;;;;N;;;;; +1D1D;LATIN SMALL LETTER SIDEWAYS U;Ll;0;L;;;;;N;;;;; +1D1E;LATIN SMALL LETTER SIDEWAYS DIAERESIZED U;Ll;0;L;;;;;N;;;;; +1D1F;LATIN SMALL LETTER SIDEWAYS TURNED M;Ll;0;L;;;;;N;;;;; +1D20;LATIN LETTER SMALL CAPITAL V;Ll;0;L;;;;;N;;;;; +1D21;LATIN LETTER SMALL CAPITAL W;Ll;0;L;;;;;N;;;;; +1D22;LATIN LETTER SMALL CAPITAL Z;Ll;0;L;;;;;N;;;;; +1D23;LATIN LETTER SMALL CAPITAL EZH;Ll;0;L;;;;;N;;;;; +1D24;LATIN LETTER VOICED LARYNGEAL SPIRANT;Ll;0;L;;;;;N;;;;; +1D25;LATIN LETTER AIN;Ll;0;L;;;;;N;;;;; +1D26;GREEK LETTER SMALL CAPITAL GAMMA;Ll;0;L;;;;;N;;;;; +1D27;GREEK LETTER SMALL CAPITAL LAMDA;Ll;0;L;;;;;N;;;;; +1D28;GREEK LETTER SMALL CAPITAL PI;Ll;0;L;;;;;N;;;;; +1D29;GREEK LETTER SMALL CAPITAL RHO;Ll;0;L;;;;;N;;;;; +1D2A;GREEK LETTER SMALL CAPITAL PSI;Ll;0;L;;;;;N;;;;; +1D2B;CYRILLIC LETTER SMALL CAPITAL EL;Ll;0;L;;;;;N;;;;; +1D2C;MODIFIER LETTER CAPITAL A;Lm;0;L; 0041;;;;N;;;;; +1D2D;MODIFIER LETTER CAPITAL AE;Lm;0;L; 00C6;;;;N;;;;; +1D2E;MODIFIER LETTER CAPITAL B;Lm;0;L; 0042;;;;N;;;;; +1D2F;MODIFIER LETTER CAPITAL BARRED B;Lm;0;L;;;;;N;;;;; +1D30;MODIFIER LETTER CAPITAL D;Lm;0;L; 0044;;;;N;;;;; +1D31;MODIFIER LETTER CAPITAL E;Lm;0;L; 0045;;;;N;;;;; +1D32;MODIFIER LETTER CAPITAL REVERSED E;Lm;0;L; 018E;;;;N;;;;; +1D33;MODIFIER LETTER CAPITAL G;Lm;0;L; 0047;;;;N;;;;; +1D34;MODIFIER LETTER CAPITAL H;Lm;0;L; 0048;;;;N;;;;; +1D35;MODIFIER LETTER CAPITAL I;Lm;0;L; 0049;;;;N;;;;; +1D36;MODIFIER LETTER CAPITAL J;Lm;0;L; 004A;;;;N;;;;; +1D37;MODIFIER LETTER CAPITAL K;Lm;0;L; 004B;;;;N;;;;; +1D38;MODIFIER LETTER CAPITAL L;Lm;0;L; 004C;;;;N;;;;; +1D39;MODIFIER LETTER CAPITAL M;Lm;0;L; 004D;;;;N;;;;; +1D3A;MODIFIER LETTER CAPITAL N;Lm;0;L; 004E;;;;N;;;;; +1D3B;MODIFIER LETTER CAPITAL REVERSED N;Lm;0;L;;;;;N;;;;; +1D3C;MODIFIER LETTER CAPITAL O;Lm;0;L; 004F;;;;N;;;;; +1D3D;MODIFIER LETTER CAPITAL OU;Lm;0;L; 0222;;;;N;;;;; +1D3E;MODIFIER LETTER CAPITAL P;Lm;0;L; 0050;;;;N;;;;; +1D3F;MODIFIER LETTER CAPITAL R;Lm;0;L; 0052;;;;N;;;;; +1D40;MODIFIER LETTER CAPITAL T;Lm;0;L; 0054;;;;N;;;;; +1D41;MODIFIER LETTER CAPITAL U;Lm;0;L; 0055;;;;N;;;;; +1D42;MODIFIER LETTER CAPITAL W;Lm;0;L; 0057;;;;N;;;;; +1D43;MODIFIER LETTER SMALL A;Lm;0;L; 0061;;;;N;;;;; +1D44;MODIFIER LETTER SMALL TURNED A;Lm;0;L; 0250;;;;N;;;;; +1D45;MODIFIER LETTER SMALL ALPHA;Lm;0;L; 0251;;;;N;;;;; +1D46;MODIFIER LETTER SMALL TURNED AE;Lm;0;L; 1D02;;;;N;;;;; +1D47;MODIFIER LETTER SMALL B;Lm;0;L; 0062;;;;N;;;;; +1D48;MODIFIER LETTER SMALL D;Lm;0;L; 0064;;;;N;;;;; +1D49;MODIFIER LETTER SMALL E;Lm;0;L; 0065;;;;N;;;;; +1D4A;MODIFIER LETTER SMALL SCHWA;Lm;0;L; 0259;;;;N;;;;; +1D4B;MODIFIER LETTER SMALL OPEN E;Lm;0;L; 025B;;;;N;;;;; +1D4C;MODIFIER LETTER SMALL TURNED OPEN E;Lm;0;L; 025C;;;;N;;;;; +1D4D;MODIFIER LETTER SMALL G;Lm;0;L; 0067;;;;N;;;;; +1D4E;MODIFIER LETTER SMALL TURNED I;Lm;0;L;;;;;N;;;;; +1D4F;MODIFIER LETTER SMALL K;Lm;0;L; 006B;;;;N;;;;; +1D50;MODIFIER LETTER SMALL M;Lm;0;L; 006D;;;;N;;;;; +1D51;MODIFIER LETTER SMALL ENG;Lm;0;L; 014B;;;;N;;;;; +1D52;MODIFIER LETTER SMALL O;Lm;0;L; 006F;;;;N;;;;; +1D53;MODIFIER LETTER SMALL OPEN O;Lm;0;L; 0254;;;;N;;;;; +1D54;MODIFIER LETTER SMALL TOP HALF O;Lm;0;L; 1D16;;;;N;;;;; +1D55;MODIFIER LETTER SMALL BOTTOM HALF O;Lm;0;L; 1D17;;;;N;;;;; +1D56;MODIFIER LETTER SMALL P;Lm;0;L; 0070;;;;N;;;;; +1D57;MODIFIER LETTER SMALL T;Lm;0;L; 0074;;;;N;;;;; +1D58;MODIFIER LETTER SMALL U;Lm;0;L; 0075;;;;N;;;;; +1D59;MODIFIER LETTER SMALL SIDEWAYS U;Lm;0;L; 1D1D;;;;N;;;;; +1D5A;MODIFIER LETTER SMALL TURNED M;Lm;0;L; 026F;;;;N;;;;; +1D5B;MODIFIER LETTER SMALL V;Lm;0;L; 0076;;;;N;;;;; +1D5C;MODIFIER LETTER SMALL AIN;Lm;0;L; 1D25;;;;N;;;;; +1D5D;MODIFIER LETTER SMALL BETA;Lm;0;L; 03B2;;;;N;;;;; +1D5E;MODIFIER LETTER SMALL GREEK GAMMA;Lm;0;L; 03B3;;;;N;;;;; +1D5F;MODIFIER LETTER SMALL DELTA;Lm;0;L; 03B4;;;;N;;;;; +1D60;MODIFIER LETTER SMALL GREEK PHI;Lm;0;L; 03C6;;;;N;;;;; +1D61;MODIFIER LETTER SMALL CHI;Lm;0;L; 03C7;;;;N;;;;; +1D62;LATIN SUBSCRIPT SMALL LETTER I;Lm;0;L; 0069;;;;N;;;;; +1D63;LATIN SUBSCRIPT SMALL LETTER R;Lm;0;L; 0072;;;;N;;;;; +1D64;LATIN SUBSCRIPT SMALL LETTER U;Lm;0;L; 0075;;;;N;;;;; +1D65;LATIN SUBSCRIPT SMALL LETTER V;Lm;0;L; 0076;;;;N;;;;; +1D66;GREEK SUBSCRIPT SMALL LETTER BETA;Lm;0;L; 03B2;;;;N;;;;; +1D67;GREEK SUBSCRIPT SMALL LETTER GAMMA;Lm;0;L; 03B3;;;;N;;;;; +1D68;GREEK SUBSCRIPT SMALL LETTER RHO;Lm;0;L; 03C1;;;;N;;;;; +1D69;GREEK SUBSCRIPT SMALL LETTER PHI;Lm;0;L; 03C6;;;;N;;;;; +1D6A;GREEK SUBSCRIPT SMALL LETTER CHI;Lm;0;L; 03C7;;;;N;;;;; +1D6B;LATIN SMALL LETTER UE;Ll;0;L;;;;;N;;;;; +1D6C;LATIN SMALL LETTER B WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D6D;LATIN SMALL LETTER D WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D6E;LATIN SMALL LETTER F WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D6F;LATIN SMALL LETTER M WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D70;LATIN SMALL LETTER N WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D71;LATIN SMALL LETTER P WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D72;LATIN SMALL LETTER R WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D73;LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D74;LATIN SMALL LETTER S WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D75;LATIN SMALL LETTER T WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D76;LATIN SMALL LETTER Z WITH MIDDLE TILDE;Ll;0;L;;;;;N;;;;; +1D77;LATIN SMALL LETTER TURNED G;Ll;0;L;;;;;N;;;;; +1D78;MODIFIER LETTER CYRILLIC EN;Lm;0;L; 043D;;;;N;;;;; +1D79;LATIN SMALL LETTER INSULAR G;Ll;0;L;;;;;N;;;A77D;;A77D +1D7A;LATIN SMALL LETTER TH WITH STRIKETHROUGH;Ll;0;L;;;;;N;;;;; +1D7B;LATIN SMALL CAPITAL LETTER I WITH STROKE;Ll;0;L;;;;;N;;;;; +1D7C;LATIN SMALL LETTER IOTA WITH STROKE;Ll;0;L;;;;;N;;;;; +1D7D;LATIN SMALL LETTER P WITH STROKE;Ll;0;L;;;;;N;;;2C63;;2C63 +1D7E;LATIN SMALL CAPITAL LETTER U WITH STROKE;Ll;0;L;;;;;N;;;;; +1D7F;LATIN SMALL LETTER UPSILON WITH STROKE;Ll;0;L;;;;;N;;;;; +1D80;LATIN SMALL LETTER B WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D81;LATIN SMALL LETTER D WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D82;LATIN SMALL LETTER F WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D83;LATIN SMALL LETTER G WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D84;LATIN SMALL LETTER K WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D85;LATIN SMALL LETTER L WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D86;LATIN SMALL LETTER M WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D87;LATIN SMALL LETTER N WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D88;LATIN SMALL LETTER P WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D89;LATIN SMALL LETTER R WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8A;LATIN SMALL LETTER S WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8B;LATIN SMALL LETTER ESH WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8C;LATIN SMALL LETTER V WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8D;LATIN SMALL LETTER X WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8E;LATIN SMALL LETTER Z WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8F;LATIN SMALL LETTER A WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D90;LATIN SMALL LETTER ALPHA WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D91;LATIN SMALL LETTER D WITH HOOK AND TAIL;Ll;0;L;;;;;N;;;;; +1D92;LATIN SMALL LETTER E WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D93;LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D94;LATIN SMALL LETTER REVERSED OPEN E WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D95;LATIN SMALL LETTER SCHWA WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D96;LATIN SMALL LETTER I WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D97;LATIN SMALL LETTER OPEN O WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D98;LATIN SMALL LETTER ESH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D99;LATIN SMALL LETTER U WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D9A;LATIN SMALL LETTER EZH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +1D9B;MODIFIER LETTER SMALL TURNED ALPHA;Lm;0;L; 0252;;;;N;;;;; +1D9C;MODIFIER LETTER SMALL C;Lm;0;L; 0063;;;;N;;;;; +1D9D;MODIFIER LETTER SMALL C WITH CURL;Lm;0;L; 0255;;;;N;;;;; +1D9E;MODIFIER LETTER SMALL ETH;Lm;0;L; 00F0;;;;N;;;;; +1D9F;MODIFIER LETTER SMALL REVERSED OPEN E;Lm;0;L; 025C;;;;N;;;;; +1DA0;MODIFIER LETTER SMALL F;Lm;0;L; 0066;;;;N;;;;; +1DA1;MODIFIER LETTER SMALL DOTLESS J WITH STROKE;Lm;0;L; 025F;;;;N;;;;; +1DA2;MODIFIER LETTER SMALL SCRIPT G;Lm;0;L; 0261;;;;N;;;;; +1DA3;MODIFIER LETTER SMALL TURNED H;Lm;0;L; 0265;;;;N;;;;; +1DA4;MODIFIER LETTER SMALL I WITH STROKE;Lm;0;L; 0268;;;;N;;;;; +1DA5;MODIFIER LETTER SMALL IOTA;Lm;0;L; 0269;;;;N;;;;; +1DA6;MODIFIER LETTER SMALL CAPITAL I;Lm;0;L; 026A;;;;N;;;;; +1DA7;MODIFIER LETTER SMALL CAPITAL I WITH STROKE;Lm;0;L; 1D7B;;;;N;;;;; +1DA8;MODIFIER LETTER SMALL J WITH CROSSED-TAIL;Lm;0;L; 029D;;;;N;;;;; +1DA9;MODIFIER LETTER SMALL L WITH RETROFLEX HOOK;Lm;0;L; 026D;;;;N;;;;; +1DAA;MODIFIER LETTER SMALL L WITH PALATAL HOOK;Lm;0;L; 1D85;;;;N;;;;; +1DAB;MODIFIER LETTER SMALL CAPITAL L;Lm;0;L; 029F;;;;N;;;;; +1DAC;MODIFIER LETTER SMALL M WITH HOOK;Lm;0;L; 0271;;;;N;;;;; +1DAD;MODIFIER LETTER SMALL TURNED M WITH LONG LEG;Lm;0;L; 0270;;;;N;;;;; +1DAE;MODIFIER LETTER SMALL N WITH LEFT HOOK;Lm;0;L; 0272;;;;N;;;;; +1DAF;MODIFIER LETTER SMALL N WITH RETROFLEX HOOK;Lm;0;L; 0273;;;;N;;;;; +1DB0;MODIFIER LETTER SMALL CAPITAL N;Lm;0;L; 0274;;;;N;;;;; +1DB1;MODIFIER LETTER SMALL BARRED O;Lm;0;L; 0275;;;;N;;;;; +1DB2;MODIFIER LETTER SMALL PHI;Lm;0;L; 0278;;;;N;;;;; +1DB3;MODIFIER LETTER SMALL S WITH HOOK;Lm;0;L; 0282;;;;N;;;;; +1DB4;MODIFIER LETTER SMALL ESH;Lm;0;L; 0283;;;;N;;;;; +1DB5;MODIFIER LETTER SMALL T WITH PALATAL HOOK;Lm;0;L; 01AB;;;;N;;;;; +1DB6;MODIFIER LETTER SMALL U BAR;Lm;0;L; 0289;;;;N;;;;; +1DB7;MODIFIER LETTER SMALL UPSILON;Lm;0;L; 028A;;;;N;;;;; +1DB8;MODIFIER LETTER SMALL CAPITAL U;Lm;0;L; 1D1C;;;;N;;;;; +1DB9;MODIFIER LETTER SMALL V WITH HOOK;Lm;0;L; 028B;;;;N;;;;; +1DBA;MODIFIER LETTER SMALL TURNED V;Lm;0;L; 028C;;;;N;;;;; +1DBB;MODIFIER LETTER SMALL Z;Lm;0;L; 007A;;;;N;;;;; +1DBC;MODIFIER LETTER SMALL Z WITH RETROFLEX HOOK;Lm;0;L; 0290;;;;N;;;;; +1DBD;MODIFIER LETTER SMALL Z WITH CURL;Lm;0;L; 0291;;;;N;;;;; +1DBE;MODIFIER LETTER SMALL EZH;Lm;0;L; 0292;;;;N;;;;; +1DBF;MODIFIER LETTER SMALL THETA;Lm;0;L; 03B8;;;;N;;;;; +1DC0;COMBINING DOTTED GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;; +1DC1;COMBINING DOTTED ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;; +1DC2;COMBINING SNAKE BELOW;Mn;220;NSM;;;;;N;;;;; +1DC3;COMBINING SUSPENSION MARK;Mn;230;NSM;;;;;N;;;;; +1DC4;COMBINING MACRON-ACUTE;Mn;230;NSM;;;;;N;;;;; +1DC5;COMBINING GRAVE-MACRON;Mn;230;NSM;;;;;N;;;;; +1DC6;COMBINING MACRON-GRAVE;Mn;230;NSM;;;;;N;;;;; +1DC7;COMBINING ACUTE-MACRON;Mn;230;NSM;;;;;N;;;;; +1DC8;COMBINING GRAVE-ACUTE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1DC9;COMBINING ACUTE-GRAVE-ACUTE;Mn;230;NSM;;;;;N;;;;; +1DCA;COMBINING LATIN SMALL LETTER R BELOW;Mn;220;NSM;;;;;N;;;;; +1DCB;COMBINING BREVE-MACRON;Mn;230;NSM;;;;;N;;;;; +1DCC;COMBINING MACRON-BREVE;Mn;230;NSM;;;;;N;;;;; +1DCD;COMBINING DOUBLE CIRCUMFLEX ABOVE;Mn;234;NSM;;;;;N;;;;; +1DCE;COMBINING OGONEK ABOVE;Mn;214;NSM;;;;;N;;;;; +1DCF;COMBINING ZIGZAG BELOW;Mn;220;NSM;;;;;N;;;;; +1DD0;COMBINING IS BELOW;Mn;202;NSM;;;;;N;;;;; +1DD1;COMBINING UR ABOVE;Mn;230;NSM;;;;;N;;;;; +1DD2;COMBINING US ABOVE;Mn;230;NSM;;;;;N;;;;; +1DD3;COMBINING LATIN SMALL LETTER FLATTENED OPEN A ABOVE;Mn;230;NSM;;;;;N;;;;; +1DD4;COMBINING LATIN SMALL LETTER AE;Mn;230;NSM;;;;;N;;;;; +1DD5;COMBINING LATIN SMALL LETTER AO;Mn;230;NSM;;;;;N;;;;; +1DD6;COMBINING LATIN SMALL LETTER AV;Mn;230;NSM;;;;;N;;;;; +1DD7;COMBINING LATIN SMALL LETTER C CEDILLA;Mn;230;NSM;;;;;N;;;;; +1DD8;COMBINING LATIN SMALL LETTER INSULAR D;Mn;230;NSM;;;;;N;;;;; +1DD9;COMBINING LATIN SMALL LETTER ETH;Mn;230;NSM;;;;;N;;;;; +1DDA;COMBINING LATIN SMALL LETTER G;Mn;230;NSM;;;;;N;;;;; +1DDB;COMBINING LATIN LETTER SMALL CAPITAL G;Mn;230;NSM;;;;;N;;;;; +1DDC;COMBINING LATIN SMALL LETTER K;Mn;230;NSM;;;;;N;;;;; +1DDD;COMBINING LATIN SMALL LETTER L;Mn;230;NSM;;;;;N;;;;; +1DDE;COMBINING LATIN LETTER SMALL CAPITAL L;Mn;230;NSM;;;;;N;;;;; +1DDF;COMBINING LATIN LETTER SMALL CAPITAL M;Mn;230;NSM;;;;;N;;;;; +1DE0;COMBINING LATIN SMALL LETTER N;Mn;230;NSM;;;;;N;;;;; +1DE1;COMBINING LATIN LETTER SMALL CAPITAL N;Mn;230;NSM;;;;;N;;;;; +1DE2;COMBINING LATIN LETTER SMALL CAPITAL R;Mn;230;NSM;;;;;N;;;;; +1DE3;COMBINING LATIN SMALL LETTER R ROTUNDA;Mn;230;NSM;;;;;N;;;;; +1DE4;COMBINING LATIN SMALL LETTER S;Mn;230;NSM;;;;;N;;;;; +1DE5;COMBINING LATIN SMALL LETTER LONG S;Mn;230;NSM;;;;;N;;;;; +1DE6;COMBINING LATIN SMALL LETTER Z;Mn;230;NSM;;;;;N;;;;; +1DFC;COMBINING DOUBLE INVERTED BREVE BELOW;Mn;233;NSM;;;;;N;;;;; +1DFD;COMBINING ALMOST EQUAL TO BELOW;Mn;220;NSM;;;;;N;;;;; +1DFE;COMBINING LEFT ARROWHEAD ABOVE;Mn;230;NSM;;;;;N;;;;; +1DFF;COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW;Mn;220;NSM;;;;;N;;;;; +1E00;LATIN CAPITAL LETTER A WITH RING BELOW;Lu;0;L;0041 0325;;;;N;;;;1E01; +1E01;LATIN SMALL LETTER A WITH RING BELOW;Ll;0;L;0061 0325;;;;N;;;1E00;;1E00 +1E02;LATIN CAPITAL LETTER B WITH DOT ABOVE;Lu;0;L;0042 0307;;;;N;;;;1E03; +1E03;LATIN SMALL LETTER B WITH DOT ABOVE;Ll;0;L;0062 0307;;;;N;;;1E02;;1E02 +1E04;LATIN CAPITAL LETTER B WITH DOT BELOW;Lu;0;L;0042 0323;;;;N;;;;1E05; +1E05;LATIN SMALL LETTER B WITH DOT BELOW;Ll;0;L;0062 0323;;;;N;;;1E04;;1E04 +1E06;LATIN CAPITAL LETTER B WITH LINE BELOW;Lu;0;L;0042 0331;;;;N;;;;1E07; +1E07;LATIN SMALL LETTER B WITH LINE BELOW;Ll;0;L;0062 0331;;;;N;;;1E06;;1E06 +1E08;LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE;Lu;0;L;00C7 0301;;;;N;;;;1E09; +1E09;LATIN SMALL LETTER C WITH CEDILLA AND ACUTE;Ll;0;L;00E7 0301;;;;N;;;1E08;;1E08 +1E0A;LATIN CAPITAL LETTER D WITH DOT ABOVE;Lu;0;L;0044 0307;;;;N;;;;1E0B; +1E0B;LATIN SMALL LETTER D WITH DOT ABOVE;Ll;0;L;0064 0307;;;;N;;;1E0A;;1E0A +1E0C;LATIN CAPITAL LETTER D WITH DOT BELOW;Lu;0;L;0044 0323;;;;N;;;;1E0D; +1E0D;LATIN SMALL LETTER D WITH DOT BELOW;Ll;0;L;0064 0323;;;;N;;;1E0C;;1E0C +1E0E;LATIN CAPITAL LETTER D WITH LINE BELOW;Lu;0;L;0044 0331;;;;N;;;;1E0F; +1E0F;LATIN SMALL LETTER D WITH LINE BELOW;Ll;0;L;0064 0331;;;;N;;;1E0E;;1E0E +1E10;LATIN CAPITAL LETTER D WITH CEDILLA;Lu;0;L;0044 0327;;;;N;;;;1E11; +1E11;LATIN SMALL LETTER D WITH CEDILLA;Ll;0;L;0064 0327;;;;N;;;1E10;;1E10 +1E12;LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW;Lu;0;L;0044 032D;;;;N;;;;1E13; +1E13;LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW;Ll;0;L;0064 032D;;;;N;;;1E12;;1E12 +1E14;LATIN CAPITAL LETTER E WITH MACRON AND GRAVE;Lu;0;L;0112 0300;;;;N;;;;1E15; +1E15;LATIN SMALL LETTER E WITH MACRON AND GRAVE;Ll;0;L;0113 0300;;;;N;;;1E14;;1E14 +1E16;LATIN CAPITAL LETTER E WITH MACRON AND ACUTE;Lu;0;L;0112 0301;;;;N;;;;1E17; +1E17;LATIN SMALL LETTER E WITH MACRON AND ACUTE;Ll;0;L;0113 0301;;;;N;;;1E16;;1E16 +1E18;LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW;Lu;0;L;0045 032D;;;;N;;;;1E19; +1E19;LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW;Ll;0;L;0065 032D;;;;N;;;1E18;;1E18 +1E1A;LATIN CAPITAL LETTER E WITH TILDE BELOW;Lu;0;L;0045 0330;;;;N;;;;1E1B; +1E1B;LATIN SMALL LETTER E WITH TILDE BELOW;Ll;0;L;0065 0330;;;;N;;;1E1A;;1E1A +1E1C;LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE;Lu;0;L;0228 0306;;;;N;;;;1E1D; +1E1D;LATIN SMALL LETTER E WITH CEDILLA AND BREVE;Ll;0;L;0229 0306;;;;N;;;1E1C;;1E1C +1E1E;LATIN CAPITAL LETTER F WITH DOT ABOVE;Lu;0;L;0046 0307;;;;N;;;;1E1F; +1E1F;LATIN SMALL LETTER F WITH DOT ABOVE;Ll;0;L;0066 0307;;;;N;;;1E1E;;1E1E +1E20;LATIN CAPITAL LETTER G WITH MACRON;Lu;0;L;0047 0304;;;;N;;;;1E21; +1E21;LATIN SMALL LETTER G WITH MACRON;Ll;0;L;0067 0304;;;;N;;;1E20;;1E20 +1E22;LATIN CAPITAL LETTER H WITH DOT ABOVE;Lu;0;L;0048 0307;;;;N;;;;1E23; +1E23;LATIN SMALL LETTER H WITH DOT ABOVE;Ll;0;L;0068 0307;;;;N;;;1E22;;1E22 +1E24;LATIN CAPITAL LETTER H WITH DOT BELOW;Lu;0;L;0048 0323;;;;N;;;;1E25; +1E25;LATIN SMALL LETTER H WITH DOT BELOW;Ll;0;L;0068 0323;;;;N;;;1E24;;1E24 +1E26;LATIN CAPITAL LETTER H WITH DIAERESIS;Lu;0;L;0048 0308;;;;N;;;;1E27; +1E27;LATIN SMALL LETTER H WITH DIAERESIS;Ll;0;L;0068 0308;;;;N;;;1E26;;1E26 +1E28;LATIN CAPITAL LETTER H WITH CEDILLA;Lu;0;L;0048 0327;;;;N;;;;1E29; +1E29;LATIN SMALL LETTER H WITH CEDILLA;Ll;0;L;0068 0327;;;;N;;;1E28;;1E28 +1E2A;LATIN CAPITAL LETTER H WITH BREVE BELOW;Lu;0;L;0048 032E;;;;N;;;;1E2B; +1E2B;LATIN SMALL LETTER H WITH BREVE BELOW;Ll;0;L;0068 032E;;;;N;;;1E2A;;1E2A +1E2C;LATIN CAPITAL LETTER I WITH TILDE BELOW;Lu;0;L;0049 0330;;;;N;;;;1E2D; +1E2D;LATIN SMALL LETTER I WITH TILDE BELOW;Ll;0;L;0069 0330;;;;N;;;1E2C;;1E2C +1E2E;LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE;Lu;0;L;00CF 0301;;;;N;;;;1E2F; +1E2F;LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE;Ll;0;L;00EF 0301;;;;N;;;1E2E;;1E2E +1E30;LATIN CAPITAL LETTER K WITH ACUTE;Lu;0;L;004B 0301;;;;N;;;;1E31; +1E31;LATIN SMALL LETTER K WITH ACUTE;Ll;0;L;006B 0301;;;;N;;;1E30;;1E30 +1E32;LATIN CAPITAL LETTER K WITH DOT BELOW;Lu;0;L;004B 0323;;;;N;;;;1E33; +1E33;LATIN SMALL LETTER K WITH DOT BELOW;Ll;0;L;006B 0323;;;;N;;;1E32;;1E32 +1E34;LATIN CAPITAL LETTER K WITH LINE BELOW;Lu;0;L;004B 0331;;;;N;;;;1E35; +1E35;LATIN SMALL LETTER K WITH LINE BELOW;Ll;0;L;006B 0331;;;;N;;;1E34;;1E34 +1E36;LATIN CAPITAL LETTER L WITH DOT BELOW;Lu;0;L;004C 0323;;;;N;;;;1E37; +1E37;LATIN SMALL LETTER L WITH DOT BELOW;Ll;0;L;006C 0323;;;;N;;;1E36;;1E36 +1E38;LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON;Lu;0;L;1E36 0304;;;;N;;;;1E39; +1E39;LATIN SMALL LETTER L WITH DOT BELOW AND MACRON;Ll;0;L;1E37 0304;;;;N;;;1E38;;1E38 +1E3A;LATIN CAPITAL LETTER L WITH LINE BELOW;Lu;0;L;004C 0331;;;;N;;;;1E3B; +1E3B;LATIN SMALL LETTER L WITH LINE BELOW;Ll;0;L;006C 0331;;;;N;;;1E3A;;1E3A +1E3C;LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW;Lu;0;L;004C 032D;;;;N;;;;1E3D; +1E3D;LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW;Ll;0;L;006C 032D;;;;N;;;1E3C;;1E3C +1E3E;LATIN CAPITAL LETTER M WITH ACUTE;Lu;0;L;004D 0301;;;;N;;;;1E3F; +1E3F;LATIN SMALL LETTER M WITH ACUTE;Ll;0;L;006D 0301;;;;N;;;1E3E;;1E3E +1E40;LATIN CAPITAL LETTER M WITH DOT ABOVE;Lu;0;L;004D 0307;;;;N;;;;1E41; +1E41;LATIN SMALL LETTER M WITH DOT ABOVE;Ll;0;L;006D 0307;;;;N;;;1E40;;1E40 +1E42;LATIN CAPITAL LETTER M WITH DOT BELOW;Lu;0;L;004D 0323;;;;N;;;;1E43; +1E43;LATIN SMALL LETTER M WITH DOT BELOW;Ll;0;L;006D 0323;;;;N;;;1E42;;1E42 +1E44;LATIN CAPITAL LETTER N WITH DOT ABOVE;Lu;0;L;004E 0307;;;;N;;;;1E45; +1E45;LATIN SMALL LETTER N WITH DOT ABOVE;Ll;0;L;006E 0307;;;;N;;;1E44;;1E44 +1E46;LATIN CAPITAL LETTER N WITH DOT BELOW;Lu;0;L;004E 0323;;;;N;;;;1E47; +1E47;LATIN SMALL LETTER N WITH DOT BELOW;Ll;0;L;006E 0323;;;;N;;;1E46;;1E46 +1E48;LATIN CAPITAL LETTER N WITH LINE BELOW;Lu;0;L;004E 0331;;;;N;;;;1E49; +1E49;LATIN SMALL LETTER N WITH LINE BELOW;Ll;0;L;006E 0331;;;;N;;;1E48;;1E48 +1E4A;LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW;Lu;0;L;004E 032D;;;;N;;;;1E4B; +1E4B;LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW;Ll;0;L;006E 032D;;;;N;;;1E4A;;1E4A +1E4C;LATIN CAPITAL LETTER O WITH TILDE AND ACUTE;Lu;0;L;00D5 0301;;;;N;;;;1E4D; +1E4D;LATIN SMALL LETTER O WITH TILDE AND ACUTE;Ll;0;L;00F5 0301;;;;N;;;1E4C;;1E4C +1E4E;LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS;Lu;0;L;00D5 0308;;;;N;;;;1E4F; +1E4F;LATIN SMALL LETTER O WITH TILDE AND DIAERESIS;Ll;0;L;00F5 0308;;;;N;;;1E4E;;1E4E +1E50;LATIN CAPITAL LETTER O WITH MACRON AND GRAVE;Lu;0;L;014C 0300;;;;N;;;;1E51; +1E51;LATIN SMALL LETTER O WITH MACRON AND GRAVE;Ll;0;L;014D 0300;;;;N;;;1E50;;1E50 +1E52;LATIN CAPITAL LETTER O WITH MACRON AND ACUTE;Lu;0;L;014C 0301;;;;N;;;;1E53; +1E53;LATIN SMALL LETTER O WITH MACRON AND ACUTE;Ll;0;L;014D 0301;;;;N;;;1E52;;1E52 +1E54;LATIN CAPITAL LETTER P WITH ACUTE;Lu;0;L;0050 0301;;;;N;;;;1E55; +1E55;LATIN SMALL LETTER P WITH ACUTE;Ll;0;L;0070 0301;;;;N;;;1E54;;1E54 +1E56;LATIN CAPITAL LETTER P WITH DOT ABOVE;Lu;0;L;0050 0307;;;;N;;;;1E57; +1E57;LATIN SMALL LETTER P WITH DOT ABOVE;Ll;0;L;0070 0307;;;;N;;;1E56;;1E56 +1E58;LATIN CAPITAL LETTER R WITH DOT ABOVE;Lu;0;L;0052 0307;;;;N;;;;1E59; +1E59;LATIN SMALL LETTER R WITH DOT ABOVE;Ll;0;L;0072 0307;;;;N;;;1E58;;1E58 +1E5A;LATIN CAPITAL LETTER R WITH DOT BELOW;Lu;0;L;0052 0323;;;;N;;;;1E5B; +1E5B;LATIN SMALL LETTER R WITH DOT BELOW;Ll;0;L;0072 0323;;;;N;;;1E5A;;1E5A +1E5C;LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON;Lu;0;L;1E5A 0304;;;;N;;;;1E5D; +1E5D;LATIN SMALL LETTER R WITH DOT BELOW AND MACRON;Ll;0;L;1E5B 0304;;;;N;;;1E5C;;1E5C +1E5E;LATIN CAPITAL LETTER R WITH LINE BELOW;Lu;0;L;0052 0331;;;;N;;;;1E5F; +1E5F;LATIN SMALL LETTER R WITH LINE BELOW;Ll;0;L;0072 0331;;;;N;;;1E5E;;1E5E +1E60;LATIN CAPITAL LETTER S WITH DOT ABOVE;Lu;0;L;0053 0307;;;;N;;;;1E61; +1E61;LATIN SMALL LETTER S WITH DOT ABOVE;Ll;0;L;0073 0307;;;;N;;;1E60;;1E60 +1E62;LATIN CAPITAL LETTER S WITH DOT BELOW;Lu;0;L;0053 0323;;;;N;;;;1E63; +1E63;LATIN SMALL LETTER S WITH DOT BELOW;Ll;0;L;0073 0323;;;;N;;;1E62;;1E62 +1E64;LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE;Lu;0;L;015A 0307;;;;N;;;;1E65; +1E65;LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE;Ll;0;L;015B 0307;;;;N;;;1E64;;1E64 +1E66;LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE;Lu;0;L;0160 0307;;;;N;;;;1E67; +1E67;LATIN SMALL LETTER S WITH CARON AND DOT ABOVE;Ll;0;L;0161 0307;;;;N;;;1E66;;1E66 +1E68;LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE;Lu;0;L;1E62 0307;;;;N;;;;1E69; +1E69;LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE;Ll;0;L;1E63 0307;;;;N;;;1E68;;1E68 +1E6A;LATIN CAPITAL LETTER T WITH DOT ABOVE;Lu;0;L;0054 0307;;;;N;;;;1E6B; +1E6B;LATIN SMALL LETTER T WITH DOT ABOVE;Ll;0;L;0074 0307;;;;N;;;1E6A;;1E6A +1E6C;LATIN CAPITAL LETTER T WITH DOT BELOW;Lu;0;L;0054 0323;;;;N;;;;1E6D; +1E6D;LATIN SMALL LETTER T WITH DOT BELOW;Ll;0;L;0074 0323;;;;N;;;1E6C;;1E6C +1E6E;LATIN CAPITAL LETTER T WITH LINE BELOW;Lu;0;L;0054 0331;;;;N;;;;1E6F; +1E6F;LATIN SMALL LETTER T WITH LINE BELOW;Ll;0;L;0074 0331;;;;N;;;1E6E;;1E6E +1E70;LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW;Lu;0;L;0054 032D;;;;N;;;;1E71; +1E71;LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW;Ll;0;L;0074 032D;;;;N;;;1E70;;1E70 +1E72;LATIN CAPITAL LETTER U WITH DIAERESIS BELOW;Lu;0;L;0055 0324;;;;N;;;;1E73; +1E73;LATIN SMALL LETTER U WITH DIAERESIS BELOW;Ll;0;L;0075 0324;;;;N;;;1E72;;1E72 +1E74;LATIN CAPITAL LETTER U WITH TILDE BELOW;Lu;0;L;0055 0330;;;;N;;;;1E75; +1E75;LATIN SMALL LETTER U WITH TILDE BELOW;Ll;0;L;0075 0330;;;;N;;;1E74;;1E74 +1E76;LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW;Lu;0;L;0055 032D;;;;N;;;;1E77; +1E77;LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW;Ll;0;L;0075 032D;;;;N;;;1E76;;1E76 +1E78;LATIN CAPITAL LETTER U WITH TILDE AND ACUTE;Lu;0;L;0168 0301;;;;N;;;;1E79; +1E79;LATIN SMALL LETTER U WITH TILDE AND ACUTE;Ll;0;L;0169 0301;;;;N;;;1E78;;1E78 +1E7A;LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS;Lu;0;L;016A 0308;;;;N;;;;1E7B; +1E7B;LATIN SMALL LETTER U WITH MACRON AND DIAERESIS;Ll;0;L;016B 0308;;;;N;;;1E7A;;1E7A +1E7C;LATIN CAPITAL LETTER V WITH TILDE;Lu;0;L;0056 0303;;;;N;;;;1E7D; +1E7D;LATIN SMALL LETTER V WITH TILDE;Ll;0;L;0076 0303;;;;N;;;1E7C;;1E7C +1E7E;LATIN CAPITAL LETTER V WITH DOT BELOW;Lu;0;L;0056 0323;;;;N;;;;1E7F; +1E7F;LATIN SMALL LETTER V WITH DOT BELOW;Ll;0;L;0076 0323;;;;N;;;1E7E;;1E7E +1E80;LATIN CAPITAL LETTER W WITH GRAVE;Lu;0;L;0057 0300;;;;N;;;;1E81; +1E81;LATIN SMALL LETTER W WITH GRAVE;Ll;0;L;0077 0300;;;;N;;;1E80;;1E80 +1E82;LATIN CAPITAL LETTER W WITH ACUTE;Lu;0;L;0057 0301;;;;N;;;;1E83; +1E83;LATIN SMALL LETTER W WITH ACUTE;Ll;0;L;0077 0301;;;;N;;;1E82;;1E82 +1E84;LATIN CAPITAL LETTER W WITH DIAERESIS;Lu;0;L;0057 0308;;;;N;;;;1E85; +1E85;LATIN SMALL LETTER W WITH DIAERESIS;Ll;0;L;0077 0308;;;;N;;;1E84;;1E84 +1E86;LATIN CAPITAL LETTER W WITH DOT ABOVE;Lu;0;L;0057 0307;;;;N;;;;1E87; +1E87;LATIN SMALL LETTER W WITH DOT ABOVE;Ll;0;L;0077 0307;;;;N;;;1E86;;1E86 +1E88;LATIN CAPITAL LETTER W WITH DOT BELOW;Lu;0;L;0057 0323;;;;N;;;;1E89; +1E89;LATIN SMALL LETTER W WITH DOT BELOW;Ll;0;L;0077 0323;;;;N;;;1E88;;1E88 +1E8A;LATIN CAPITAL LETTER X WITH DOT ABOVE;Lu;0;L;0058 0307;;;;N;;;;1E8B; +1E8B;LATIN SMALL LETTER X WITH DOT ABOVE;Ll;0;L;0078 0307;;;;N;;;1E8A;;1E8A +1E8C;LATIN CAPITAL LETTER X WITH DIAERESIS;Lu;0;L;0058 0308;;;;N;;;;1E8D; +1E8D;LATIN SMALL LETTER X WITH DIAERESIS;Ll;0;L;0078 0308;;;;N;;;1E8C;;1E8C +1E8E;LATIN CAPITAL LETTER Y WITH DOT ABOVE;Lu;0;L;0059 0307;;;;N;;;;1E8F; +1E8F;LATIN SMALL LETTER Y WITH DOT ABOVE;Ll;0;L;0079 0307;;;;N;;;1E8E;;1E8E +1E90;LATIN CAPITAL LETTER Z WITH CIRCUMFLEX;Lu;0;L;005A 0302;;;;N;;;;1E91; +1E91;LATIN SMALL LETTER Z WITH CIRCUMFLEX;Ll;0;L;007A 0302;;;;N;;;1E90;;1E90 +1E92;LATIN CAPITAL LETTER Z WITH DOT BELOW;Lu;0;L;005A 0323;;;;N;;;;1E93; +1E93;LATIN SMALL LETTER Z WITH DOT BELOW;Ll;0;L;007A 0323;;;;N;;;1E92;;1E92 +1E94;LATIN CAPITAL LETTER Z WITH LINE BELOW;Lu;0;L;005A 0331;;;;N;;;;1E95; +1E95;LATIN SMALL LETTER Z WITH LINE BELOW;Ll;0;L;007A 0331;;;;N;;;1E94;;1E94 +1E96;LATIN SMALL LETTER H WITH LINE BELOW;Ll;0;L;0068 0331;;;;N;;;;; +1E97;LATIN SMALL LETTER T WITH DIAERESIS;Ll;0;L;0074 0308;;;;N;;;;; +1E98;LATIN SMALL LETTER W WITH RING ABOVE;Ll;0;L;0077 030A;;;;N;;;;; +1E99;LATIN SMALL LETTER Y WITH RING ABOVE;Ll;0;L;0079 030A;;;;N;;;;; +1E9A;LATIN SMALL LETTER A WITH RIGHT HALF RING;Ll;0;L; 0061 02BE;;;;N;;;;; +1E9B;LATIN SMALL LETTER LONG S WITH DOT ABOVE;Ll;0;L;017F 0307;;;;N;;;1E60;;1E60 +1E9C;LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;;; +1E9D;LATIN SMALL LETTER LONG S WITH HIGH STROKE;Ll;0;L;;;;;N;;;;; +1E9E;LATIN CAPITAL LETTER SHARP S;Lu;0;L;;;;;N;;;;00DF; +1E9F;LATIN SMALL LETTER DELTA;Ll;0;L;;;;;N;;;;; +1EA0;LATIN CAPITAL LETTER A WITH DOT BELOW;Lu;0;L;0041 0323;;;;N;;;;1EA1; +1EA1;LATIN SMALL LETTER A WITH DOT BELOW;Ll;0;L;0061 0323;;;;N;;;1EA0;;1EA0 +1EA2;LATIN CAPITAL LETTER A WITH HOOK ABOVE;Lu;0;L;0041 0309;;;;N;;;;1EA3; +1EA3;LATIN SMALL LETTER A WITH HOOK ABOVE;Ll;0;L;0061 0309;;;;N;;;1EA2;;1EA2 +1EA4;LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE;Lu;0;L;00C2 0301;;;;N;;;;1EA5; +1EA5;LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE;Ll;0;L;00E2 0301;;;;N;;;1EA4;;1EA4 +1EA6;LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE;Lu;0;L;00C2 0300;;;;N;;;;1EA7; +1EA7;LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE;Ll;0;L;00E2 0300;;;;N;;;1EA6;;1EA6 +1EA8;LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE;Lu;0;L;00C2 0309;;;;N;;;;1EA9; +1EA9;LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE;Ll;0;L;00E2 0309;;;;N;;;1EA8;;1EA8 +1EAA;LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE;Lu;0;L;00C2 0303;;;;N;;;;1EAB; +1EAB;LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE;Ll;0;L;00E2 0303;;;;N;;;1EAA;;1EAA +1EAC;LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW;Lu;0;L;1EA0 0302;;;;N;;;;1EAD; +1EAD;LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1EA1 0302;;;;N;;;1EAC;;1EAC +1EAE;LATIN CAPITAL LETTER A WITH BREVE AND ACUTE;Lu;0;L;0102 0301;;;;N;;;;1EAF; +1EAF;LATIN SMALL LETTER A WITH BREVE AND ACUTE;Ll;0;L;0103 0301;;;;N;;;1EAE;;1EAE +1EB0;LATIN CAPITAL LETTER A WITH BREVE AND GRAVE;Lu;0;L;0102 0300;;;;N;;;;1EB1; +1EB1;LATIN SMALL LETTER A WITH BREVE AND GRAVE;Ll;0;L;0103 0300;;;;N;;;1EB0;;1EB0 +1EB2;LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE;Lu;0;L;0102 0309;;;;N;;;;1EB3; +1EB3;LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE;Ll;0;L;0103 0309;;;;N;;;1EB2;;1EB2 +1EB4;LATIN CAPITAL LETTER A WITH BREVE AND TILDE;Lu;0;L;0102 0303;;;;N;;;;1EB5; +1EB5;LATIN SMALL LETTER A WITH BREVE AND TILDE;Ll;0;L;0103 0303;;;;N;;;1EB4;;1EB4 +1EB6;LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW;Lu;0;L;1EA0 0306;;;;N;;;;1EB7; +1EB7;LATIN SMALL LETTER A WITH BREVE AND DOT BELOW;Ll;0;L;1EA1 0306;;;;N;;;1EB6;;1EB6 +1EB8;LATIN CAPITAL LETTER E WITH DOT BELOW;Lu;0;L;0045 0323;;;;N;;;;1EB9; +1EB9;LATIN SMALL LETTER E WITH DOT BELOW;Ll;0;L;0065 0323;;;;N;;;1EB8;;1EB8 +1EBA;LATIN CAPITAL LETTER E WITH HOOK ABOVE;Lu;0;L;0045 0309;;;;N;;;;1EBB; +1EBB;LATIN SMALL LETTER E WITH HOOK ABOVE;Ll;0;L;0065 0309;;;;N;;;1EBA;;1EBA +1EBC;LATIN CAPITAL LETTER E WITH TILDE;Lu;0;L;0045 0303;;;;N;;;;1EBD; +1EBD;LATIN SMALL LETTER E WITH TILDE;Ll;0;L;0065 0303;;;;N;;;1EBC;;1EBC +1EBE;LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE;Lu;0;L;00CA 0301;;;;N;;;;1EBF; +1EBF;LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE;Ll;0;L;00EA 0301;;;;N;;;1EBE;;1EBE +1EC0;LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE;Lu;0;L;00CA 0300;;;;N;;;;1EC1; +1EC1;LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE;Ll;0;L;00EA 0300;;;;N;;;1EC0;;1EC0 +1EC2;LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE;Lu;0;L;00CA 0309;;;;N;;;;1EC3; +1EC3;LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE;Ll;0;L;00EA 0309;;;;N;;;1EC2;;1EC2 +1EC4;LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE;Lu;0;L;00CA 0303;;;;N;;;;1EC5; +1EC5;LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE;Ll;0;L;00EA 0303;;;;N;;;1EC4;;1EC4 +1EC6;LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW;Lu;0;L;1EB8 0302;;;;N;;;;1EC7; +1EC7;LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1EB9 0302;;;;N;;;1EC6;;1EC6 +1EC8;LATIN CAPITAL LETTER I WITH HOOK ABOVE;Lu;0;L;0049 0309;;;;N;;;;1EC9; +1EC9;LATIN SMALL LETTER I WITH HOOK ABOVE;Ll;0;L;0069 0309;;;;N;;;1EC8;;1EC8 +1ECA;LATIN CAPITAL LETTER I WITH DOT BELOW;Lu;0;L;0049 0323;;;;N;;;;1ECB; +1ECB;LATIN SMALL LETTER I WITH DOT BELOW;Ll;0;L;0069 0323;;;;N;;;1ECA;;1ECA +1ECC;LATIN CAPITAL LETTER O WITH DOT BELOW;Lu;0;L;004F 0323;;;;N;;;;1ECD; +1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC +1ECE;LATIN CAPITAL LETTER O WITH HOOK ABOVE;Lu;0;L;004F 0309;;;;N;;;;1ECF; +1ECF;LATIN SMALL LETTER O WITH HOOK ABOVE;Ll;0;L;006F 0309;;;;N;;;1ECE;;1ECE +1ED0;LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE;Lu;0;L;00D4 0301;;;;N;;;;1ED1; +1ED1;LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE;Ll;0;L;00F4 0301;;;;N;;;1ED0;;1ED0 +1ED2;LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE;Lu;0;L;00D4 0300;;;;N;;;;1ED3; +1ED3;LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE;Ll;0;L;00F4 0300;;;;N;;;1ED2;;1ED2 +1ED4;LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE;Lu;0;L;00D4 0309;;;;N;;;;1ED5; +1ED5;LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE;Ll;0;L;00F4 0309;;;;N;;;1ED4;;1ED4 +1ED6;LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE;Lu;0;L;00D4 0303;;;;N;;;;1ED7; +1ED7;LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE;Ll;0;L;00F4 0303;;;;N;;;1ED6;;1ED6 +1ED8;LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Lu;0;L;1ECC 0302;;;;N;;;;1ED9; +1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8 +1EDA;LATIN CAPITAL LETTER O WITH HORN AND ACUTE;Lu;0;L;01A0 0301;;;;N;;;;1EDB; +1EDB;LATIN SMALL LETTER O WITH HORN AND ACUTE;Ll;0;L;01A1 0301;;;;N;;;1EDA;;1EDA +1EDC;LATIN CAPITAL LETTER O WITH HORN AND GRAVE;Lu;0;L;01A0 0300;;;;N;;;;1EDD; +1EDD;LATIN SMALL LETTER O WITH HORN AND GRAVE;Ll;0;L;01A1 0300;;;;N;;;1EDC;;1EDC +1EDE;LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE;Lu;0;L;01A0 0309;;;;N;;;;1EDF; +1EDF;LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE;Ll;0;L;01A1 0309;;;;N;;;1EDE;;1EDE +1EE0;LATIN CAPITAL LETTER O WITH HORN AND TILDE;Lu;0;L;01A0 0303;;;;N;;;;1EE1; +1EE1;LATIN SMALL LETTER O WITH HORN AND TILDE;Ll;0;L;01A1 0303;;;;N;;;1EE0;;1EE0 +1EE2;LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW;Lu;0;L;01A0 0323;;;;N;;;;1EE3; +1EE3;LATIN SMALL LETTER O WITH HORN AND DOT BELOW;Ll;0;L;01A1 0323;;;;N;;;1EE2;;1EE2 +1EE4;LATIN CAPITAL LETTER U WITH DOT BELOW;Lu;0;L;0055 0323;;;;N;;;;1EE5; +1EE5;LATIN SMALL LETTER U WITH DOT BELOW;Ll;0;L;0075 0323;;;;N;;;1EE4;;1EE4 +1EE6;LATIN CAPITAL LETTER U WITH HOOK ABOVE;Lu;0;L;0055 0309;;;;N;;;;1EE7; +1EE7;LATIN SMALL LETTER U WITH HOOK ABOVE;Ll;0;L;0075 0309;;;;N;;;1EE6;;1EE6 +1EE8;LATIN CAPITAL LETTER U WITH HORN AND ACUTE;Lu;0;L;01AF 0301;;;;N;;;;1EE9; +1EE9;LATIN SMALL LETTER U WITH HORN AND ACUTE;Ll;0;L;01B0 0301;;;;N;;;1EE8;;1EE8 +1EEA;LATIN CAPITAL LETTER U WITH HORN AND GRAVE;Lu;0;L;01AF 0300;;;;N;;;;1EEB; +1EEB;LATIN SMALL LETTER U WITH HORN AND GRAVE;Ll;0;L;01B0 0300;;;;N;;;1EEA;;1EEA +1EEC;LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE;Lu;0;L;01AF 0309;;;;N;;;;1EED; +1EED;LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE;Ll;0;L;01B0 0309;;;;N;;;1EEC;;1EEC +1EEE;LATIN CAPITAL LETTER U WITH HORN AND TILDE;Lu;0;L;01AF 0303;;;;N;;;;1EEF; +1EEF;LATIN SMALL LETTER U WITH HORN AND TILDE;Ll;0;L;01B0 0303;;;;N;;;1EEE;;1EEE +1EF0;LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW;Lu;0;L;01AF 0323;;;;N;;;;1EF1; +1EF1;LATIN SMALL LETTER U WITH HORN AND DOT BELOW;Ll;0;L;01B0 0323;;;;N;;;1EF0;;1EF0 +1EF2;LATIN CAPITAL LETTER Y WITH GRAVE;Lu;0;L;0059 0300;;;;N;;;;1EF3; +1EF3;LATIN SMALL LETTER Y WITH GRAVE;Ll;0;L;0079 0300;;;;N;;;1EF2;;1EF2 +1EF4;LATIN CAPITAL LETTER Y WITH DOT BELOW;Lu;0;L;0059 0323;;;;N;;;;1EF5; +1EF5;LATIN SMALL LETTER Y WITH DOT BELOW;Ll;0;L;0079 0323;;;;N;;;1EF4;;1EF4 +1EF6;LATIN CAPITAL LETTER Y WITH HOOK ABOVE;Lu;0;L;0059 0309;;;;N;;;;1EF7; +1EF7;LATIN SMALL LETTER Y WITH HOOK ABOVE;Ll;0;L;0079 0309;;;;N;;;1EF6;;1EF6 +1EF8;LATIN CAPITAL LETTER Y WITH TILDE;Lu;0;L;0059 0303;;;;N;;;;1EF9; +1EF9;LATIN SMALL LETTER Y WITH TILDE;Ll;0;L;0079 0303;;;;N;;;1EF8;;1EF8 +1EFA;LATIN CAPITAL LETTER MIDDLE-WELSH LL;Lu;0;L;;;;;N;;;;1EFB; +1EFB;LATIN SMALL LETTER MIDDLE-WELSH LL;Ll;0;L;;;;;N;;;1EFA;;1EFA +1EFC;LATIN CAPITAL LETTER MIDDLE-WELSH V;Lu;0;L;;;;;N;;;;1EFD; +1EFD;LATIN SMALL LETTER MIDDLE-WELSH V;Ll;0;L;;;;;N;;;1EFC;;1EFC +1EFE;LATIN CAPITAL LETTER Y WITH LOOP;Lu;0;L;;;;;N;;;;1EFF; +1EFF;LATIN SMALL LETTER Y WITH LOOP;Ll;0;L;;;;;N;;;1EFE;;1EFE +1F00;GREEK SMALL LETTER ALPHA WITH PSILI;Ll;0;L;03B1 0313;;;;N;;;1F08;;1F08 +1F01;GREEK SMALL LETTER ALPHA WITH DASIA;Ll;0;L;03B1 0314;;;;N;;;1F09;;1F09 +1F02;GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA;Ll;0;L;1F00 0300;;;;N;;;1F0A;;1F0A +1F03;GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA;Ll;0;L;1F01 0300;;;;N;;;1F0B;;1F0B +1F04;GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA;Ll;0;L;1F00 0301;;;;N;;;1F0C;;1F0C +1F05;GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA;Ll;0;L;1F01 0301;;;;N;;;1F0D;;1F0D +1F06;GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI;Ll;0;L;1F00 0342;;;;N;;;1F0E;;1F0E +1F07;GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI;Ll;0;L;1F01 0342;;;;N;;;1F0F;;1F0F +1F08;GREEK CAPITAL LETTER ALPHA WITH PSILI;Lu;0;L;0391 0313;;;;N;;;;1F00; +1F09;GREEK CAPITAL LETTER ALPHA WITH DASIA;Lu;0;L;0391 0314;;;;N;;;;1F01; +1F0A;GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA;Lu;0;L;1F08 0300;;;;N;;;;1F02; +1F0B;GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA;Lu;0;L;1F09 0300;;;;N;;;;1F03; +1F0C;GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA;Lu;0;L;1F08 0301;;;;N;;;;1F04; +1F0D;GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA;Lu;0;L;1F09 0301;;;;N;;;;1F05; +1F0E;GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI;Lu;0;L;1F08 0342;;;;N;;;;1F06; +1F0F;GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI;Lu;0;L;1F09 0342;;;;N;;;;1F07; +1F10;GREEK SMALL LETTER EPSILON WITH PSILI;Ll;0;L;03B5 0313;;;;N;;;1F18;;1F18 +1F11;GREEK SMALL LETTER EPSILON WITH DASIA;Ll;0;L;03B5 0314;;;;N;;;1F19;;1F19 +1F12;GREEK SMALL LETTER EPSILON WITH PSILI AND VARIA;Ll;0;L;1F10 0300;;;;N;;;1F1A;;1F1A +1F13;GREEK SMALL LETTER EPSILON WITH DASIA AND VARIA;Ll;0;L;1F11 0300;;;;N;;;1F1B;;1F1B +1F14;GREEK SMALL LETTER EPSILON WITH PSILI AND OXIA;Ll;0;L;1F10 0301;;;;N;;;1F1C;;1F1C +1F15;GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA;Ll;0;L;1F11 0301;;;;N;;;1F1D;;1F1D +1F18;GREEK CAPITAL LETTER EPSILON WITH PSILI;Lu;0;L;0395 0313;;;;N;;;;1F10; +1F19;GREEK CAPITAL LETTER EPSILON WITH DASIA;Lu;0;L;0395 0314;;;;N;;;;1F11; +1F1A;GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA;Lu;0;L;1F18 0300;;;;N;;;;1F12; +1F1B;GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA;Lu;0;L;1F19 0300;;;;N;;;;1F13; +1F1C;GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA;Lu;0;L;1F18 0301;;;;N;;;;1F14; +1F1D;GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA;Lu;0;L;1F19 0301;;;;N;;;;1F15; +1F20;GREEK SMALL LETTER ETA WITH PSILI;Ll;0;L;03B7 0313;;;;N;;;1F28;;1F28 +1F21;GREEK SMALL LETTER ETA WITH DASIA;Ll;0;L;03B7 0314;;;;N;;;1F29;;1F29 +1F22;GREEK SMALL LETTER ETA WITH PSILI AND VARIA;Ll;0;L;1F20 0300;;;;N;;;1F2A;;1F2A +1F23;GREEK SMALL LETTER ETA WITH DASIA AND VARIA;Ll;0;L;1F21 0300;;;;N;;;1F2B;;1F2B +1F24;GREEK SMALL LETTER ETA WITH PSILI AND OXIA;Ll;0;L;1F20 0301;;;;N;;;1F2C;;1F2C +1F25;GREEK SMALL LETTER ETA WITH DASIA AND OXIA;Ll;0;L;1F21 0301;;;;N;;;1F2D;;1F2D +1F26;GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI;Ll;0;L;1F20 0342;;;;N;;;1F2E;;1F2E +1F27;GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI;Ll;0;L;1F21 0342;;;;N;;;1F2F;;1F2F +1F28;GREEK CAPITAL LETTER ETA WITH PSILI;Lu;0;L;0397 0313;;;;N;;;;1F20; +1F29;GREEK CAPITAL LETTER ETA WITH DASIA;Lu;0;L;0397 0314;;;;N;;;;1F21; +1F2A;GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA;Lu;0;L;1F28 0300;;;;N;;;;1F22; +1F2B;GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA;Lu;0;L;1F29 0300;;;;N;;;;1F23; +1F2C;GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA;Lu;0;L;1F28 0301;;;;N;;;;1F24; +1F2D;GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA;Lu;0;L;1F29 0301;;;;N;;;;1F25; +1F2E;GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI;Lu;0;L;1F28 0342;;;;N;;;;1F26; +1F2F;GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI;Lu;0;L;1F29 0342;;;;N;;;;1F27; +1F30;GREEK SMALL LETTER IOTA WITH PSILI;Ll;0;L;03B9 0313;;;;N;;;1F38;;1F38 +1F31;GREEK SMALL LETTER IOTA WITH DASIA;Ll;0;L;03B9 0314;;;;N;;;1F39;;1F39 +1F32;GREEK SMALL LETTER IOTA WITH PSILI AND VARIA;Ll;0;L;1F30 0300;;;;N;;;1F3A;;1F3A +1F33;GREEK SMALL LETTER IOTA WITH DASIA AND VARIA;Ll;0;L;1F31 0300;;;;N;;;1F3B;;1F3B +1F34;GREEK SMALL LETTER IOTA WITH PSILI AND OXIA;Ll;0;L;1F30 0301;;;;N;;;1F3C;;1F3C +1F35;GREEK SMALL LETTER IOTA WITH DASIA AND OXIA;Ll;0;L;1F31 0301;;;;N;;;1F3D;;1F3D +1F36;GREEK SMALL LETTER IOTA WITH PSILI AND PERISPOMENI;Ll;0;L;1F30 0342;;;;N;;;1F3E;;1F3E +1F37;GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI;Ll;0;L;1F31 0342;;;;N;;;1F3F;;1F3F +1F38;GREEK CAPITAL LETTER IOTA WITH PSILI;Lu;0;L;0399 0313;;;;N;;;;1F30; +1F39;GREEK CAPITAL LETTER IOTA WITH DASIA;Lu;0;L;0399 0314;;;;N;;;;1F31; +1F3A;GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA;Lu;0;L;1F38 0300;;;;N;;;;1F32; +1F3B;GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA;Lu;0;L;1F39 0300;;;;N;;;;1F33; +1F3C;GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA;Lu;0;L;1F38 0301;;;;N;;;;1F34; +1F3D;GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA;Lu;0;L;1F39 0301;;;;N;;;;1F35; +1F3E;GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI;Lu;0;L;1F38 0342;;;;N;;;;1F36; +1F3F;GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI;Lu;0;L;1F39 0342;;;;N;;;;1F37; +1F40;GREEK SMALL LETTER OMICRON WITH PSILI;Ll;0;L;03BF 0313;;;;N;;;1F48;;1F48 +1F41;GREEK SMALL LETTER OMICRON WITH DASIA;Ll;0;L;03BF 0314;;;;N;;;1F49;;1F49 +1F42;GREEK SMALL LETTER OMICRON WITH PSILI AND VARIA;Ll;0;L;1F40 0300;;;;N;;;1F4A;;1F4A +1F43;GREEK SMALL LETTER OMICRON WITH DASIA AND VARIA;Ll;0;L;1F41 0300;;;;N;;;1F4B;;1F4B +1F44;GREEK SMALL LETTER OMICRON WITH PSILI AND OXIA;Ll;0;L;1F40 0301;;;;N;;;1F4C;;1F4C +1F45;GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA;Ll;0;L;1F41 0301;;;;N;;;1F4D;;1F4D +1F48;GREEK CAPITAL LETTER OMICRON WITH PSILI;Lu;0;L;039F 0313;;;;N;;;;1F40; +1F49;GREEK CAPITAL LETTER OMICRON WITH DASIA;Lu;0;L;039F 0314;;;;N;;;;1F41; +1F4A;GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA;Lu;0;L;1F48 0300;;;;N;;;;1F42; +1F4B;GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA;Lu;0;L;1F49 0300;;;;N;;;;1F43; +1F4C;GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA;Lu;0;L;1F48 0301;;;;N;;;;1F44; +1F4D;GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA;Lu;0;L;1F49 0301;;;;N;;;;1F45; +1F50;GREEK SMALL LETTER UPSILON WITH PSILI;Ll;0;L;03C5 0313;;;;N;;;;; +1F51;GREEK SMALL LETTER UPSILON WITH DASIA;Ll;0;L;03C5 0314;;;;N;;;1F59;;1F59 +1F52;GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA;Ll;0;L;1F50 0300;;;;N;;;;; +1F53;GREEK SMALL LETTER UPSILON WITH DASIA AND VARIA;Ll;0;L;1F51 0300;;;;N;;;1F5B;;1F5B +1F54;GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA;Ll;0;L;1F50 0301;;;;N;;;;; +1F55;GREEK SMALL LETTER UPSILON WITH DASIA AND OXIA;Ll;0;L;1F51 0301;;;;N;;;1F5D;;1F5D +1F56;GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI;Ll;0;L;1F50 0342;;;;N;;;;; +1F57;GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI;Ll;0;L;1F51 0342;;;;N;;;1F5F;;1F5F +1F59;GREEK CAPITAL LETTER UPSILON WITH DASIA;Lu;0;L;03A5 0314;;;;N;;;;1F51; +1F5B;GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA;Lu;0;L;1F59 0300;;;;N;;;;1F53; +1F5D;GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA;Lu;0;L;1F59 0301;;;;N;;;;1F55; +1F5F;GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI;Lu;0;L;1F59 0342;;;;N;;;;1F57; +1F60;GREEK SMALL LETTER OMEGA WITH PSILI;Ll;0;L;03C9 0313;;;;N;;;1F68;;1F68 +1F61;GREEK SMALL LETTER OMEGA WITH DASIA;Ll;0;L;03C9 0314;;;;N;;;1F69;;1F69 +1F62;GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA;Ll;0;L;1F60 0300;;;;N;;;1F6A;;1F6A +1F63;GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA;Ll;0;L;1F61 0300;;;;N;;;1F6B;;1F6B +1F64;GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA;Ll;0;L;1F60 0301;;;;N;;;1F6C;;1F6C +1F65;GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA;Ll;0;L;1F61 0301;;;;N;;;1F6D;;1F6D +1F66;GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI;Ll;0;L;1F60 0342;;;;N;;;1F6E;;1F6E +1F67;GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI;Ll;0;L;1F61 0342;;;;N;;;1F6F;;1F6F +1F68;GREEK CAPITAL LETTER OMEGA WITH PSILI;Lu;0;L;03A9 0313;;;;N;;;;1F60; +1F69;GREEK CAPITAL LETTER OMEGA WITH DASIA;Lu;0;L;03A9 0314;;;;N;;;;1F61; +1F6A;GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA;Lu;0;L;1F68 0300;;;;N;;;;1F62; +1F6B;GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA;Lu;0;L;1F69 0300;;;;N;;;;1F63; +1F6C;GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA;Lu;0;L;1F68 0301;;;;N;;;;1F64; +1F6D;GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA;Lu;0;L;1F69 0301;;;;N;;;;1F65; +1F6E;GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI;Lu;0;L;1F68 0342;;;;N;;;;1F66; +1F6F;GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI;Lu;0;L;1F69 0342;;;;N;;;;1F67; +1F70;GREEK SMALL LETTER ALPHA WITH VARIA;Ll;0;L;03B1 0300;;;;N;;;1FBA;;1FBA +1F71;GREEK SMALL LETTER ALPHA WITH OXIA;Ll;0;L;03AC;;;;N;;;1FBB;;1FBB +1F72;GREEK SMALL LETTER EPSILON WITH VARIA;Ll;0;L;03B5 0300;;;;N;;;1FC8;;1FC8 +1F73;GREEK SMALL LETTER EPSILON WITH OXIA;Ll;0;L;03AD;;;;N;;;1FC9;;1FC9 +1F74;GREEK SMALL LETTER ETA WITH VARIA;Ll;0;L;03B7 0300;;;;N;;;1FCA;;1FCA +1F75;GREEK SMALL LETTER ETA WITH OXIA;Ll;0;L;03AE;;;;N;;;1FCB;;1FCB +1F76;GREEK SMALL LETTER IOTA WITH VARIA;Ll;0;L;03B9 0300;;;;N;;;1FDA;;1FDA +1F77;GREEK SMALL LETTER IOTA WITH OXIA;Ll;0;L;03AF;;;;N;;;1FDB;;1FDB +1F78;GREEK SMALL LETTER OMICRON WITH VARIA;Ll;0;L;03BF 0300;;;;N;;;1FF8;;1FF8 +1F79;GREEK SMALL LETTER OMICRON WITH OXIA;Ll;0;L;03CC;;;;N;;;1FF9;;1FF9 +1F7A;GREEK SMALL LETTER UPSILON WITH VARIA;Ll;0;L;03C5 0300;;;;N;;;1FEA;;1FEA +1F7B;GREEK SMALL LETTER UPSILON WITH OXIA;Ll;0;L;03CD;;;;N;;;1FEB;;1FEB +1F7C;GREEK SMALL LETTER OMEGA WITH VARIA;Ll;0;L;03C9 0300;;;;N;;;1FFA;;1FFA +1F7D;GREEK SMALL LETTER OMEGA WITH OXIA;Ll;0;L;03CE;;;;N;;;1FFB;;1FFB +1F80;GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI;Ll;0;L;1F00 0345;;;;N;;;1F88;;1F88 +1F81;GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI;Ll;0;L;1F01 0345;;;;N;;;1F89;;1F89 +1F82;GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI;Ll;0;L;1F02 0345;;;;N;;;1F8A;;1F8A +1F83;GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI;Ll;0;L;1F03 0345;;;;N;;;1F8B;;1F8B +1F84;GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI;Ll;0;L;1F04 0345;;;;N;;;1F8C;;1F8C +1F85;GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI;Ll;0;L;1F05 0345;;;;N;;;1F8D;;1F8D +1F86;GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1F06 0345;;;;N;;;1F8E;;1F8E +1F87;GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1F07 0345;;;;N;;;1F8F;;1F8F +1F88;GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI;Lt;0;L;1F08 0345;;;;N;;;;1F80; +1F89;GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI;Lt;0;L;1F09 0345;;;;N;;;;1F81; +1F8A;GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI;Lt;0;L;1F0A 0345;;;;N;;;;1F82; +1F8B;GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI;Lt;0;L;1F0B 0345;;;;N;;;;1F83; +1F8C;GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI;Lt;0;L;1F0C 0345;;;;N;;;;1F84; +1F8D;GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI;Lt;0;L;1F0D 0345;;;;N;;;;1F85; +1F8E;GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI;Lt;0;L;1F0E 0345;;;;N;;;;1F86; +1F8F;GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI;Lt;0;L;1F0F 0345;;;;N;;;;1F87; +1F90;GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI;Ll;0;L;1F20 0345;;;;N;;;1F98;;1F98 +1F91;GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI;Ll;0;L;1F21 0345;;;;N;;;1F99;;1F99 +1F92;GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI;Ll;0;L;1F22 0345;;;;N;;;1F9A;;1F9A +1F93;GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI;Ll;0;L;1F23 0345;;;;N;;;1F9B;;1F9B +1F94;GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI;Ll;0;L;1F24 0345;;;;N;;;1F9C;;1F9C +1F95;GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI;Ll;0;L;1F25 0345;;;;N;;;1F9D;;1F9D +1F96;GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1F26 0345;;;;N;;;1F9E;;1F9E +1F97;GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1F27 0345;;;;N;;;1F9F;;1F9F +1F98;GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI;Lt;0;L;1F28 0345;;;;N;;;;1F90; +1F99;GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI;Lt;0;L;1F29 0345;;;;N;;;;1F91; +1F9A;GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI;Lt;0;L;1F2A 0345;;;;N;;;;1F92; +1F9B;GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI;Lt;0;L;1F2B 0345;;;;N;;;;1F93; +1F9C;GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI;Lt;0;L;1F2C 0345;;;;N;;;;1F94; +1F9D;GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI;Lt;0;L;1F2D 0345;;;;N;;;;1F95; +1F9E;GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI;Lt;0;L;1F2E 0345;;;;N;;;;1F96; +1F9F;GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI;Lt;0;L;1F2F 0345;;;;N;;;;1F97; +1FA0;GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI;Ll;0;L;1F60 0345;;;;N;;;1FA8;;1FA8 +1FA1;GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI;Ll;0;L;1F61 0345;;;;N;;;1FA9;;1FA9 +1FA2;GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI;Ll;0;L;1F62 0345;;;;N;;;1FAA;;1FAA +1FA3;GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI;Ll;0;L;1F63 0345;;;;N;;;1FAB;;1FAB +1FA4;GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI;Ll;0;L;1F64 0345;;;;N;;;1FAC;;1FAC +1FA5;GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI;Ll;0;L;1F65 0345;;;;N;;;1FAD;;1FAD +1FA6;GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1F66 0345;;;;N;;;1FAE;;1FAE +1FA7;GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1F67 0345;;;;N;;;1FAF;;1FAF +1FA8;GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI;Lt;0;L;1F68 0345;;;;N;;;;1FA0; +1FA9;GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI;Lt;0;L;1F69 0345;;;;N;;;;1FA1; +1FAA;GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI;Lt;0;L;1F6A 0345;;;;N;;;;1FA2; +1FAB;GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI;Lt;0;L;1F6B 0345;;;;N;;;;1FA3; +1FAC;GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI;Lt;0;L;1F6C 0345;;;;N;;;;1FA4; +1FAD;GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI;Lt;0;L;1F6D 0345;;;;N;;;;1FA5; +1FAE;GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI;Lt;0;L;1F6E 0345;;;;N;;;;1FA6; +1FAF;GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI;Lt;0;L;1F6F 0345;;;;N;;;;1FA7; +1FB0;GREEK SMALL LETTER ALPHA WITH VRACHY;Ll;0;L;03B1 0306;;;;N;;;1FB8;;1FB8 +1FB1;GREEK SMALL LETTER ALPHA WITH MACRON;Ll;0;L;03B1 0304;;;;N;;;1FB9;;1FB9 +1FB2;GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI;Ll;0;L;1F70 0345;;;;N;;;;; +1FB3;GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI;Ll;0;L;03B1 0345;;;;N;;;1FBC;;1FBC +1FB4;GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI;Ll;0;L;03AC 0345;;;;N;;;;; +1FB6;GREEK SMALL LETTER ALPHA WITH PERISPOMENI;Ll;0;L;03B1 0342;;;;N;;;;; +1FB7;GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1FB6 0345;;;;N;;;;; +1FB8;GREEK CAPITAL LETTER ALPHA WITH VRACHY;Lu;0;L;0391 0306;;;;N;;;;1FB0; +1FB9;GREEK CAPITAL LETTER ALPHA WITH MACRON;Lu;0;L;0391 0304;;;;N;;;;1FB1; +1FBA;GREEK CAPITAL LETTER ALPHA WITH VARIA;Lu;0;L;0391 0300;;;;N;;;;1F70; +1FBB;GREEK CAPITAL LETTER ALPHA WITH OXIA;Lu;0;L;0386;;;;N;;;;1F71; +1FBC;GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI;Lt;0;L;0391 0345;;;;N;;;;1FB3; +1FBD;GREEK KORONIS;Sk;0;ON; 0020 0313;;;;N;;;;; +1FBE;GREEK PROSGEGRAMMENI;Ll;0;L;03B9;;;;N;;;0399;;0399 +1FBF;GREEK PSILI;Sk;0;ON; 0020 0313;;;;N;;;;; +1FC0;GREEK PERISPOMENI;Sk;0;ON; 0020 0342;;;;N;;;;; +1FC1;GREEK DIALYTIKA AND PERISPOMENI;Sk;0;ON;00A8 0342;;;;N;;;;; +1FC2;GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI;Ll;0;L;1F74 0345;;;;N;;;;; +1FC3;GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI;Ll;0;L;03B7 0345;;;;N;;;1FCC;;1FCC +1FC4;GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI;Ll;0;L;03AE 0345;;;;N;;;;; +1FC6;GREEK SMALL LETTER ETA WITH PERISPOMENI;Ll;0;L;03B7 0342;;;;N;;;;; +1FC7;GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1FC6 0345;;;;N;;;;; +1FC8;GREEK CAPITAL LETTER EPSILON WITH VARIA;Lu;0;L;0395 0300;;;;N;;;;1F72; +1FC9;GREEK CAPITAL LETTER EPSILON WITH OXIA;Lu;0;L;0388;;;;N;;;;1F73; +1FCA;GREEK CAPITAL LETTER ETA WITH VARIA;Lu;0;L;0397 0300;;;;N;;;;1F74; +1FCB;GREEK CAPITAL LETTER ETA WITH OXIA;Lu;0;L;0389;;;;N;;;;1F75; +1FCC;GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI;Lt;0;L;0397 0345;;;;N;;;;1FC3; +1FCD;GREEK PSILI AND VARIA;Sk;0;ON;1FBF 0300;;;;N;;;;; +1FCE;GREEK PSILI AND OXIA;Sk;0;ON;1FBF 0301;;;;N;;;;; +1FCF;GREEK PSILI AND PERISPOMENI;Sk;0;ON;1FBF 0342;;;;N;;;;; +1FD0;GREEK SMALL LETTER IOTA WITH VRACHY;Ll;0;L;03B9 0306;;;;N;;;1FD8;;1FD8 +1FD1;GREEK SMALL LETTER IOTA WITH MACRON;Ll;0;L;03B9 0304;;;;N;;;1FD9;;1FD9 +1FD2;GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA;Ll;0;L;03CA 0300;;;;N;;;;; +1FD3;GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA;Ll;0;L;0390;;;;N;;;;; +1FD6;GREEK SMALL LETTER IOTA WITH PERISPOMENI;Ll;0;L;03B9 0342;;;;N;;;;; +1FD7;GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI;Ll;0;L;03CA 0342;;;;N;;;;; +1FD8;GREEK CAPITAL LETTER IOTA WITH VRACHY;Lu;0;L;0399 0306;;;;N;;;;1FD0; +1FD9;GREEK CAPITAL LETTER IOTA WITH MACRON;Lu;0;L;0399 0304;;;;N;;;;1FD1; +1FDA;GREEK CAPITAL LETTER IOTA WITH VARIA;Lu;0;L;0399 0300;;;;N;;;;1F76; +1FDB;GREEK CAPITAL LETTER IOTA WITH OXIA;Lu;0;L;038A;;;;N;;;;1F77; +1FDD;GREEK DASIA AND VARIA;Sk;0;ON;1FFE 0300;;;;N;;;;; +1FDE;GREEK DASIA AND OXIA;Sk;0;ON;1FFE 0301;;;;N;;;;; +1FDF;GREEK DASIA AND PERISPOMENI;Sk;0;ON;1FFE 0342;;;;N;;;;; +1FE0;GREEK SMALL LETTER UPSILON WITH VRACHY;Ll;0;L;03C5 0306;;;;N;;;1FE8;;1FE8 +1FE1;GREEK SMALL LETTER UPSILON WITH MACRON;Ll;0;L;03C5 0304;;;;N;;;1FE9;;1FE9 +1FE2;GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA;Ll;0;L;03CB 0300;;;;N;;;;; +1FE3;GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA;Ll;0;L;03B0;;;;N;;;;; +1FE4;GREEK SMALL LETTER RHO WITH PSILI;Ll;0;L;03C1 0313;;;;N;;;;; +1FE5;GREEK SMALL LETTER RHO WITH DASIA;Ll;0;L;03C1 0314;;;;N;;;1FEC;;1FEC +1FE6;GREEK SMALL LETTER UPSILON WITH PERISPOMENI;Ll;0;L;03C5 0342;;;;N;;;;; +1FE7;GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI;Ll;0;L;03CB 0342;;;;N;;;;; +1FE8;GREEK CAPITAL LETTER UPSILON WITH VRACHY;Lu;0;L;03A5 0306;;;;N;;;;1FE0; +1FE9;GREEK CAPITAL LETTER UPSILON WITH MACRON;Lu;0;L;03A5 0304;;;;N;;;;1FE1; +1FEA;GREEK CAPITAL LETTER UPSILON WITH VARIA;Lu;0;L;03A5 0300;;;;N;;;;1F7A; +1FEB;GREEK CAPITAL LETTER UPSILON WITH OXIA;Lu;0;L;038E;;;;N;;;;1F7B; +1FEC;GREEK CAPITAL LETTER RHO WITH DASIA;Lu;0;L;03A1 0314;;;;N;;;;1FE5; +1FED;GREEK DIALYTIKA AND VARIA;Sk;0;ON;00A8 0300;;;;N;;;;; +1FEE;GREEK DIALYTIKA AND OXIA;Sk;0;ON;0385;;;;N;;;;; +1FEF;GREEK VARIA;Sk;0;ON;0060;;;;N;;;;; +1FF2;GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI;Ll;0;L;1F7C 0345;;;;N;;;;; +1FF3;GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI;Ll;0;L;03C9 0345;;;;N;;;1FFC;;1FFC +1FF4;GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI;Ll;0;L;03CE 0345;;;;N;;;;; +1FF6;GREEK SMALL LETTER OMEGA WITH PERISPOMENI;Ll;0;L;03C9 0342;;;;N;;;;; +1FF7;GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI;Ll;0;L;1FF6 0345;;;;N;;;;; +1FF8;GREEK CAPITAL LETTER OMICRON WITH VARIA;Lu;0;L;039F 0300;;;;N;;;;1F78; +1FF9;GREEK CAPITAL LETTER OMICRON WITH OXIA;Lu;0;L;038C;;;;N;;;;1F79; +1FFA;GREEK CAPITAL LETTER OMEGA WITH VARIA;Lu;0;L;03A9 0300;;;;N;;;;1F7C; +1FFB;GREEK CAPITAL LETTER OMEGA WITH OXIA;Lu;0;L;038F;;;;N;;;;1F7D; +1FFC;GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI;Lt;0;L;03A9 0345;;;;N;;;;1FF3; +1FFD;GREEK OXIA;Sk;0;ON;00B4;;;;N;;;;; +1FFE;GREEK DASIA;Sk;0;ON; 0020 0314;;;;N;;;;; +2000;EN QUAD;Zs;0;WS;2002;;;;N;;;;; +2001;EM QUAD;Zs;0;WS;2003;;;;N;;;;; +2002;EN SPACE;Zs;0;WS; 0020;;;;N;;;;; +2003;EM SPACE;Zs;0;WS; 0020;;;;N;;;;; +2004;THREE-PER-EM SPACE;Zs;0;WS; 0020;;;;N;;;;; +2005;FOUR-PER-EM SPACE;Zs;0;WS; 0020;;;;N;;;;; +2006;SIX-PER-EM SPACE;Zs;0;WS; 0020;;;;N;;;;; +2007;FIGURE SPACE;Zs;0;WS; 0020;;;;N;;;;; +2008;PUNCTUATION SPACE;Zs;0;WS; 0020;;;;N;;;;; +2009;THIN SPACE;Zs;0;WS; 0020;;;;N;;;;; +200A;HAIR SPACE;Zs;0;WS; 0020;;;;N;;;;; +200B;ZERO WIDTH SPACE;Cf;0;BN;;;;;N;;;;; +200C;ZERO WIDTH NON-JOINER;Cf;0;BN;;;;;N;;;;; +200D;ZERO WIDTH JOINER;Cf;0;BN;;;;;N;;;;; +200E;LEFT-TO-RIGHT MARK;Cf;0;L;;;;;N;;;;; +200F;RIGHT-TO-LEFT MARK;Cf;0;R;;;;;N;;;;; +2010;HYPHEN;Pd;0;ON;;;;;N;;;;; +2011;NON-BREAKING HYPHEN;Pd;0;ON; 2010;;;;N;;;;; +2012;FIGURE DASH;Pd;0;ON;;;;;N;;;;; +2013;EN DASH;Pd;0;ON;;;;;N;;;;; +2014;EM DASH;Pd;0;ON;;;;;N;;;;; +2015;HORIZONTAL BAR;Pd;0;ON;;;;;N;QUOTATION DASH;;;; +2016;DOUBLE VERTICAL LINE;Po;0;ON;;;;;N;DOUBLE VERTICAL BAR;;;; +2017;DOUBLE LOW LINE;Po;0;ON; 0020 0333;;;;N;SPACING DOUBLE UNDERSCORE;;;; +2018;LEFT SINGLE QUOTATION MARK;Pi;0;ON;;;;;N;SINGLE TURNED COMMA QUOTATION MARK;;;; +2019;RIGHT SINGLE QUOTATION MARK;Pf;0;ON;;;;;N;SINGLE COMMA QUOTATION MARK;;;; +201A;SINGLE LOW-9 QUOTATION MARK;Ps;0;ON;;;;;N;LOW SINGLE COMMA QUOTATION MARK;;;; +201B;SINGLE HIGH-REVERSED-9 QUOTATION MARK;Pi;0;ON;;;;;N;SINGLE REVERSED COMMA QUOTATION MARK;;;; +201C;LEFT DOUBLE QUOTATION MARK;Pi;0;ON;;;;;N;DOUBLE TURNED COMMA QUOTATION MARK;;;; +201D;RIGHT DOUBLE QUOTATION MARK;Pf;0;ON;;;;;N;DOUBLE COMMA QUOTATION MARK;;;; +201E;DOUBLE LOW-9 QUOTATION MARK;Ps;0;ON;;;;;N;LOW DOUBLE COMMA QUOTATION MARK;;;; +201F;DOUBLE HIGH-REVERSED-9 QUOTATION MARK;Pi;0;ON;;;;;N;DOUBLE REVERSED COMMA QUOTATION MARK;;;; +2020;DAGGER;Po;0;ON;;;;;N;;;;; +2021;DOUBLE DAGGER;Po;0;ON;;;;;N;;;;; +2022;BULLET;Po;0;ON;;;;;N;;;;; +2023;TRIANGULAR BULLET;Po;0;ON;;;;;N;;;;; +2024;ONE DOT LEADER;Po;0;ON; 002E;;;;N;;;;; +2025;TWO DOT LEADER;Po;0;ON; 002E 002E;;;;N;;;;; +2026;HORIZONTAL ELLIPSIS;Po;0;ON; 002E 002E 002E;;;;N;;;;; +2027;HYPHENATION POINT;Po;0;ON;;;;;N;;;;; +2028;LINE SEPARATOR;Zl;0;WS;;;;;N;;;;; +2029;PARAGRAPH SEPARATOR;Zp;0;B;;;;;N;;;;; +202A;LEFT-TO-RIGHT EMBEDDING;Cf;0;LRE;;;;;N;;;;; +202B;RIGHT-TO-LEFT EMBEDDING;Cf;0;RLE;;;;;N;;;;; +202C;POP DIRECTIONAL FORMATTING;Cf;0;PDF;;;;;N;;;;; +202D;LEFT-TO-RIGHT OVERRIDE;Cf;0;LRO;;;;;N;;;;; +202E;RIGHT-TO-LEFT OVERRIDE;Cf;0;RLO;;;;;N;;;;; +202F;NARROW NO-BREAK SPACE;Zs;0;CS; 0020;;;;N;;;;; +2030;PER MILLE SIGN;Po;0;ET;;;;;N;;;;; +2031;PER TEN THOUSAND SIGN;Po;0;ET;;;;;N;;;;; +2032;PRIME;Po;0;ET;;;;;N;;;;; +2033;DOUBLE PRIME;Po;0;ET; 2032 2032;;;;N;;;;; +2034;TRIPLE PRIME;Po;0;ET; 2032 2032 2032;;;;N;;;;; +2035;REVERSED PRIME;Po;0;ON;;;;;N;;;;; +2036;REVERSED DOUBLE PRIME;Po;0;ON; 2035 2035;;;;N;;;;; +2037;REVERSED TRIPLE PRIME;Po;0;ON; 2035 2035 2035;;;;N;;;;; +2038;CARET;Po;0;ON;;;;;N;;;;; +2039;SINGLE LEFT-POINTING ANGLE QUOTATION MARK;Pi;0;ON;;;;;Y;LEFT POINTING SINGLE GUILLEMET;;;; +203A;SINGLE RIGHT-POINTING ANGLE QUOTATION MARK;Pf;0;ON;;;;;Y;RIGHT POINTING SINGLE GUILLEMET;;;; +203B;REFERENCE MARK;Po;0;ON;;;;;N;;;;; +203C;DOUBLE EXCLAMATION MARK;Po;0;ON; 0021 0021;;;;N;;;;; +203D;INTERROBANG;Po;0;ON;;;;;N;;;;; +203E;OVERLINE;Po;0;ON; 0020 0305;;;;N;SPACING OVERSCORE;;;; +203F;UNDERTIE;Pc;0;ON;;;;;N;;;;; +2040;CHARACTER TIE;Pc;0;ON;;;;;N;;;;; +2041;CARET INSERTION POINT;Po;0;ON;;;;;N;;;;; +2042;ASTERISM;Po;0;ON;;;;;N;;;;; +2043;HYPHEN BULLET;Po;0;ON;;;;;N;;;;; +2044;FRACTION SLASH;Sm;0;CS;;;;;N;;;;; +2045;LEFT SQUARE BRACKET WITH QUILL;Ps;0;ON;;;;;Y;;;;; +2046;RIGHT SQUARE BRACKET WITH QUILL;Pe;0;ON;;;;;Y;;;;; +2047;DOUBLE QUESTION MARK;Po;0;ON; 003F 003F;;;;N;;;;; +2048;QUESTION EXCLAMATION MARK;Po;0;ON; 003F 0021;;;;N;;;;; +2049;EXCLAMATION QUESTION MARK;Po;0;ON; 0021 003F;;;;N;;;;; +204A;TIRONIAN SIGN ET;Po;0;ON;;;;;N;;;;; +204B;REVERSED PILCROW SIGN;Po;0;ON;;;;;N;;;;; +204C;BLACK LEFTWARDS BULLET;Po;0;ON;;;;;N;;;;; +204D;BLACK RIGHTWARDS BULLET;Po;0;ON;;;;;N;;;;; +204E;LOW ASTERISK;Po;0;ON;;;;;N;;;;; +204F;REVERSED SEMICOLON;Po;0;ON;;;;;N;;;;; +2050;CLOSE UP;Po;0;ON;;;;;N;;;;; +2051;TWO ASTERISKS ALIGNED VERTICALLY;Po;0;ON;;;;;N;;;;; +2052;COMMERCIAL MINUS SIGN;Sm;0;ON;;;;;N;;;;; +2053;SWUNG DASH;Po;0;ON;;;;;N;;;;; +2054;INVERTED UNDERTIE;Pc;0;ON;;;;;N;;;;; +2055;FLOWER PUNCTUATION MARK;Po;0;ON;;;;;N;;;;; +2056;THREE DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +2057;QUADRUPLE PRIME;Po;0;ON; 2032 2032 2032 2032;;;;N;;;;; +2058;FOUR DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +2059;FIVE DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +205A;TWO DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +205B;FOUR DOT MARK;Po;0;ON;;;;;N;;;;; +205C;DOTTED CROSS;Po;0;ON;;;;;N;;;;; +205D;TRICOLON;Po;0;ON;;;;;N;;;;; +205E;VERTICAL FOUR DOTS;Po;0;ON;;;;;N;;;;; +205F;MEDIUM MATHEMATICAL SPACE;Zs;0;WS; 0020;;;;N;;;;; +2060;WORD JOINER;Cf;0;BN;;;;;N;;;;; +2061;FUNCTION APPLICATION;Cf;0;BN;;;;;N;;;;; +2062;INVISIBLE TIMES;Cf;0;BN;;;;;N;;;;; +2063;INVISIBLE SEPARATOR;Cf;0;BN;;;;;N;;;;; +2064;INVISIBLE PLUS;Cf;0;BN;;;;;N;;;;; +206A;INHIBIT SYMMETRIC SWAPPING;Cf;0;BN;;;;;N;;;;; +206B;ACTIVATE SYMMETRIC SWAPPING;Cf;0;BN;;;;;N;;;;; +206C;INHIBIT ARABIC FORM SHAPING;Cf;0;BN;;;;;N;;;;; +206D;ACTIVATE ARABIC FORM SHAPING;Cf;0;BN;;;;;N;;;;; +206E;NATIONAL DIGIT SHAPES;Cf;0;BN;;;;;N;;;;; +206F;NOMINAL DIGIT SHAPES;Cf;0;BN;;;;;N;;;;; +2070;SUPERSCRIPT ZERO;No;0;EN; 0030;;0;0;N;SUPERSCRIPT DIGIT ZERO;;;; +2071;SUPERSCRIPT LATIN SMALL LETTER I;Lm;0;L; 0069;;;;N;;;;; +2074;SUPERSCRIPT FOUR;No;0;EN; 0034;;4;4;N;SUPERSCRIPT DIGIT FOUR;;;; +2075;SUPERSCRIPT FIVE;No;0;EN; 0035;;5;5;N;SUPERSCRIPT DIGIT FIVE;;;; +2076;SUPERSCRIPT SIX;No;0;EN; 0036;;6;6;N;SUPERSCRIPT DIGIT SIX;;;; +2077;SUPERSCRIPT SEVEN;No;0;EN; 0037;;7;7;N;SUPERSCRIPT DIGIT SEVEN;;;; +2078;SUPERSCRIPT EIGHT;No;0;EN; 0038;;8;8;N;SUPERSCRIPT DIGIT EIGHT;;;; +2079;SUPERSCRIPT NINE;No;0;EN; 0039;;9;9;N;SUPERSCRIPT DIGIT NINE;;;; +207A;SUPERSCRIPT PLUS SIGN;Sm;0;ES; 002B;;;;N;;;;; +207B;SUPERSCRIPT MINUS;Sm;0;ES; 2212;;;;N;SUPERSCRIPT HYPHEN-MINUS;;;; +207C;SUPERSCRIPT EQUALS SIGN;Sm;0;ON; 003D;;;;N;;;;; +207D;SUPERSCRIPT LEFT PARENTHESIS;Ps;0;ON; 0028;;;;Y;SUPERSCRIPT OPENING PARENTHESIS;;;; +207E;SUPERSCRIPT RIGHT PARENTHESIS;Pe;0;ON; 0029;;;;Y;SUPERSCRIPT CLOSING PARENTHESIS;;;; +207F;SUPERSCRIPT LATIN SMALL LETTER N;Lm;0;L; 006E;;;;N;;;;; +2080;SUBSCRIPT ZERO;No;0;EN; 0030;;0;0;N;SUBSCRIPT DIGIT ZERO;;;; +2081;SUBSCRIPT ONE;No;0;EN; 0031;;1;1;N;SUBSCRIPT DIGIT ONE;;;; +2082;SUBSCRIPT TWO;No;0;EN; 0032;;2;2;N;SUBSCRIPT DIGIT TWO;;;; +2083;SUBSCRIPT THREE;No;0;EN; 0033;;3;3;N;SUBSCRIPT DIGIT THREE;;;; +2084;SUBSCRIPT FOUR;No;0;EN; 0034;;4;4;N;SUBSCRIPT DIGIT FOUR;;;; +2085;SUBSCRIPT FIVE;No;0;EN; 0035;;5;5;N;SUBSCRIPT DIGIT FIVE;;;; +2086;SUBSCRIPT SIX;No;0;EN; 0036;;6;6;N;SUBSCRIPT DIGIT SIX;;;; +2087;SUBSCRIPT SEVEN;No;0;EN; 0037;;7;7;N;SUBSCRIPT DIGIT SEVEN;;;; +2088;SUBSCRIPT EIGHT;No;0;EN; 0038;;8;8;N;SUBSCRIPT DIGIT EIGHT;;;; +2089;SUBSCRIPT NINE;No;0;EN; 0039;;9;9;N;SUBSCRIPT DIGIT NINE;;;; +208A;SUBSCRIPT PLUS SIGN;Sm;0;ES; 002B;;;;N;;;;; +208B;SUBSCRIPT MINUS;Sm;0;ES; 2212;;;;N;SUBSCRIPT HYPHEN-MINUS;;;; +208C;SUBSCRIPT EQUALS SIGN;Sm;0;ON; 003D;;;;N;;;;; +208D;SUBSCRIPT LEFT PARENTHESIS;Ps;0;ON; 0028;;;;Y;SUBSCRIPT OPENING PARENTHESIS;;;; +208E;SUBSCRIPT RIGHT PARENTHESIS;Pe;0;ON; 0029;;;;Y;SUBSCRIPT CLOSING PARENTHESIS;;;; +2090;LATIN SUBSCRIPT SMALL LETTER A;Lm;0;L; 0061;;;;N;;;;; +2091;LATIN SUBSCRIPT SMALL LETTER E;Lm;0;L; 0065;;;;N;;;;; +2092;LATIN SUBSCRIPT SMALL LETTER O;Lm;0;L; 006F;;;;N;;;;; +2093;LATIN SUBSCRIPT SMALL LETTER X;Lm;0;L; 0078;;;;N;;;;; +2094;LATIN SUBSCRIPT SMALL LETTER SCHWA;Lm;0;L; 0259;;;;N;;;;; +2095;LATIN SUBSCRIPT SMALL LETTER H;Lm;0;L; 0068;;;;N;;;;; +2096;LATIN SUBSCRIPT SMALL LETTER K;Lm;0;L; 006B;;;;N;;;;; +2097;LATIN SUBSCRIPT SMALL LETTER L;Lm;0;L; 006C;;;;N;;;;; +2098;LATIN SUBSCRIPT SMALL LETTER M;Lm;0;L; 006D;;;;N;;;;; +2099;LATIN SUBSCRIPT SMALL LETTER N;Lm;0;L; 006E;;;;N;;;;; +209A;LATIN SUBSCRIPT SMALL LETTER P;Lm;0;L; 0070;;;;N;;;;; +209B;LATIN SUBSCRIPT SMALL LETTER S;Lm;0;L; 0073;;;;N;;;;; +209C;LATIN SUBSCRIPT SMALL LETTER T;Lm;0;L; 0074;;;;N;;;;; +20A0;EURO-CURRENCY SIGN;Sc;0;ET;;;;;N;;;;; +20A1;COLON SIGN;Sc;0;ET;;;;;N;;;;; +20A2;CRUZEIRO SIGN;Sc;0;ET;;;;;N;;;;; +20A3;FRENCH FRANC SIGN;Sc;0;ET;;;;;N;;;;; +20A4;LIRA SIGN;Sc;0;ET;;;;;N;;;;; +20A5;MILL SIGN;Sc;0;ET;;;;;N;;;;; +20A6;NAIRA SIGN;Sc;0;ET;;;;;N;;;;; +20A7;PESETA SIGN;Sc;0;ET;;;;;N;;;;; +20A8;RUPEE SIGN;Sc;0;ET; 0052 0073;;;;N;;;;; +20A9;WON SIGN;Sc;0;ET;;;;;N;;;;; +20AA;NEW SHEQEL SIGN;Sc;0;ET;;;;;N;;;;; +20AB;DONG SIGN;Sc;0;ET;;;;;N;;;;; +20AC;EURO SIGN;Sc;0;ET;;;;;N;;;;; +20AD;KIP SIGN;Sc;0;ET;;;;;N;;;;; +20AE;TUGRIK SIGN;Sc;0;ET;;;;;N;;;;; +20AF;DRACHMA SIGN;Sc;0;ET;;;;;N;;;;; +20B0;GERMAN PENNY SIGN;Sc;0;ET;;;;;N;;;;; +20B1;PESO SIGN;Sc;0;ET;;;;;N;;;;; +20B2;GUARANI SIGN;Sc;0;ET;;;;;N;;;;; +20B3;AUSTRAL SIGN;Sc;0;ET;;;;;N;;;;; +20B4;HRYVNIA SIGN;Sc;0;ET;;;;;N;;;;; +20B5;CEDI SIGN;Sc;0;ET;;;;;N;;;;; +20B6;LIVRE TOURNOIS SIGN;Sc;0;ET;;;;;N;;;;; +20B7;SPESMILO SIGN;Sc;0;ET;;;;;N;;;;; +20B8;TENGE SIGN;Sc;0;ET;;;;;N;;;;; +20B9;INDIAN RUPEE SIGN;Sc;0;ET;;;;;N;;;;; +20D0;COMBINING LEFT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT HARPOON ABOVE;;;; +20D1;COMBINING RIGHT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RIGHT HARPOON ABOVE;;;; +20D2;COMBINING LONG VERTICAL LINE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG VERTICAL BAR OVERLAY;;;; +20D3;COMBINING SHORT VERTICAL LINE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING SHORT VERTICAL BAR OVERLAY;;;; +20D4;COMBINING ANTICLOCKWISE ARROW ABOVE;Mn;230;NSM;;;;;N;NON-SPACING ANTICLOCKWISE ARROW ABOVE;;;; +20D5;COMBINING CLOCKWISE ARROW ABOVE;Mn;230;NSM;;;;;N;NON-SPACING CLOCKWISE ARROW ABOVE;;;; +20D6;COMBINING LEFT ARROW ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT ARROW ABOVE;;;; +20D7;COMBINING RIGHT ARROW ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RIGHT ARROW ABOVE;;;; +20D8;COMBINING RING OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING RING OVERLAY;;;; +20D9;COMBINING CLOCKWISE RING OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING CLOCKWISE RING OVERLAY;;;; +20DA;COMBINING ANTICLOCKWISE RING OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING ANTICLOCKWISE RING OVERLAY;;;; +20DB;COMBINING THREE DOTS ABOVE;Mn;230;NSM;;;;;N;NON-SPACING THREE DOTS ABOVE;;;; +20DC;COMBINING FOUR DOTS ABOVE;Mn;230;NSM;;;;;N;NON-SPACING FOUR DOTS ABOVE;;;; +20DD;COMBINING ENCLOSING CIRCLE;Me;0;NSM;;;;;N;ENCLOSING CIRCLE;;;; +20DE;COMBINING ENCLOSING SQUARE;Me;0;NSM;;;;;N;ENCLOSING SQUARE;;;; +20DF;COMBINING ENCLOSING DIAMOND;Me;0;NSM;;;;;N;ENCLOSING DIAMOND;;;; +20E0;COMBINING ENCLOSING CIRCLE BACKSLASH;Me;0;NSM;;;;;N;ENCLOSING CIRCLE SLASH;;;; +20E1;COMBINING LEFT RIGHT ARROW ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT RIGHT ARROW ABOVE;;;; +20E2;COMBINING ENCLOSING SCREEN;Me;0;NSM;;;;;N;;;;; +20E3;COMBINING ENCLOSING KEYCAP;Me;0;NSM;;;;;N;;;;; +20E4;COMBINING ENCLOSING UPWARD POINTING TRIANGLE;Me;0;NSM;;;;;N;;;;; +20E5;COMBINING REVERSE SOLIDUS OVERLAY;Mn;1;NSM;;;;;N;;;;; +20E6;COMBINING DOUBLE VERTICAL STROKE OVERLAY;Mn;1;NSM;;;;;N;;;;; +20E7;COMBINING ANNUITY SYMBOL;Mn;230;NSM;;;;;N;;;;; +20E8;COMBINING TRIPLE UNDERDOT;Mn;220;NSM;;;;;N;;;;; +20E9;COMBINING WIDE BRIDGE ABOVE;Mn;230;NSM;;;;;N;;;;; +20EA;COMBINING LEFTWARDS ARROW OVERLAY;Mn;1;NSM;;;;;N;;;;; +20EB;COMBINING LONG DOUBLE SOLIDUS OVERLAY;Mn;1;NSM;;;;;N;;;;; +20EC;COMBINING RIGHTWARDS HARPOON WITH BARB DOWNWARDS;Mn;220;NSM;;;;;N;;;;; +20ED;COMBINING LEFTWARDS HARPOON WITH BARB DOWNWARDS;Mn;220;NSM;;;;;N;;;;; +20EE;COMBINING LEFT ARROW BELOW;Mn;220;NSM;;;;;N;;;;; +20EF;COMBINING RIGHT ARROW BELOW;Mn;220;NSM;;;;;N;;;;; +20F0;COMBINING ASTERISK ABOVE;Mn;230;NSM;;;;;N;;;;; +2100;ACCOUNT OF;So;0;ON; 0061 002F 0063;;;;N;;;;; +2101;ADDRESSED TO THE SUBJECT;So;0;ON; 0061 002F 0073;;;;N;;;;; +2102;DOUBLE-STRUCK CAPITAL C;Lu;0;L; 0043;;;;N;DOUBLE-STRUCK C;;;; +2103;DEGREE CELSIUS;So;0;ON; 00B0 0043;;;;N;DEGREES CENTIGRADE;;;; +2104;CENTRE LINE SYMBOL;So;0;ON;;;;;N;C L SYMBOL;;;; +2105;CARE OF;So;0;ON; 0063 002F 006F;;;;N;;;;; +2106;CADA UNA;So;0;ON; 0063 002F 0075;;;;N;;;;; +2107;EULER CONSTANT;Lu;0;L; 0190;;;;N;EULERS;;;; +2108;SCRUPLE;So;0;ON;;;;;N;;;;; +2109;DEGREE FAHRENHEIT;So;0;ON; 00B0 0046;;;;N;DEGREES FAHRENHEIT;;;; +210A;SCRIPT SMALL G;Ll;0;L; 0067;;;;N;;;;; +210B;SCRIPT CAPITAL H;Lu;0;L; 0048;;;;N;SCRIPT H;;;; +210C;BLACK-LETTER CAPITAL H;Lu;0;L; 0048;;;;N;BLACK-LETTER H;;;; +210D;DOUBLE-STRUCK CAPITAL H;Lu;0;L; 0048;;;;N;DOUBLE-STRUCK H;;;; +210E;PLANCK CONSTANT;Ll;0;L; 0068;;;;N;;;;; +210F;PLANCK CONSTANT OVER TWO PI;Ll;0;L; 0127;;;;N;PLANCK CONSTANT OVER 2 PI;;;; +2110;SCRIPT CAPITAL I;Lu;0;L; 0049;;;;N;SCRIPT I;;;; +2111;BLACK-LETTER CAPITAL I;Lu;0;L; 0049;;;;N;BLACK-LETTER I;;;; +2112;SCRIPT CAPITAL L;Lu;0;L; 004C;;;;N;SCRIPT L;;;; +2113;SCRIPT SMALL L;Ll;0;L; 006C;;;;N;;;;; +2114;L B BAR SYMBOL;So;0;ON;;;;;N;;;;; +2115;DOUBLE-STRUCK CAPITAL N;Lu;0;L; 004E;;;;N;DOUBLE-STRUCK N;;;; +2116;NUMERO SIGN;So;0;ON; 004E 006F;;;;N;NUMERO;;;; +2117;SOUND RECORDING COPYRIGHT;So;0;ON;;;;;N;;;;; +2118;SCRIPT CAPITAL P;Sm;0;ON;;;;;N;SCRIPT P;;;; +2119;DOUBLE-STRUCK CAPITAL P;Lu;0;L; 0050;;;;N;DOUBLE-STRUCK P;;;; +211A;DOUBLE-STRUCK CAPITAL Q;Lu;0;L; 0051;;;;N;DOUBLE-STRUCK Q;;;; +211B;SCRIPT CAPITAL R;Lu;0;L; 0052;;;;N;SCRIPT R;;;; +211C;BLACK-LETTER CAPITAL R;Lu;0;L; 0052;;;;N;BLACK-LETTER R;;;; +211D;DOUBLE-STRUCK CAPITAL R;Lu;0;L; 0052;;;;N;DOUBLE-STRUCK R;;;; +211E;PRESCRIPTION TAKE;So;0;ON;;;;;N;;;;; +211F;RESPONSE;So;0;ON;;;;;N;;;;; +2120;SERVICE MARK;So;0;ON; 0053 004D;;;;N;;;;; +2121;TELEPHONE SIGN;So;0;ON; 0054 0045 004C;;;;N;T E L SYMBOL;;;; +2122;TRADE MARK SIGN;So;0;ON; 0054 004D;;;;N;TRADEMARK;;;; +2123;VERSICLE;So;0;ON;;;;;N;;;;; +2124;DOUBLE-STRUCK CAPITAL Z;Lu;0;L; 005A;;;;N;DOUBLE-STRUCK Z;;;; +2125;OUNCE SIGN;So;0;ON;;;;;N;OUNCE;;;; +2126;OHM SIGN;Lu;0;L;03A9;;;;N;OHM;;;03C9; +2127;INVERTED OHM SIGN;So;0;ON;;;;;N;MHO;;;; +2128;BLACK-LETTER CAPITAL Z;Lu;0;L; 005A;;;;N;BLACK-LETTER Z;;;; +2129;TURNED GREEK SMALL LETTER IOTA;So;0;ON;;;;;N;;;;; +212A;KELVIN SIGN;Lu;0;L;004B;;;;N;DEGREES KELVIN;;;006B; +212B;ANGSTROM SIGN;Lu;0;L;00C5;;;;N;ANGSTROM UNIT;;;00E5; +212C;SCRIPT CAPITAL B;Lu;0;L; 0042;;;;N;SCRIPT B;;;; +212D;BLACK-LETTER CAPITAL C;Lu;0;L; 0043;;;;N;BLACK-LETTER C;;;; +212E;ESTIMATED SYMBOL;So;0;ET;;;;;N;;;;; +212F;SCRIPT SMALL E;Ll;0;L; 0065;;;;N;;;;; +2130;SCRIPT CAPITAL E;Lu;0;L; 0045;;;;N;SCRIPT E;;;; +2131;SCRIPT CAPITAL F;Lu;0;L; 0046;;;;N;SCRIPT F;;;; +2132;TURNED CAPITAL F;Lu;0;L;;;;;N;TURNED F;;;214E; +2133;SCRIPT CAPITAL M;Lu;0;L; 004D;;;;N;SCRIPT M;;;; +2134;SCRIPT SMALL O;Ll;0;L; 006F;;;;N;;;;; +2135;ALEF SYMBOL;Lo;0;L; 05D0;;;;N;FIRST TRANSFINITE CARDINAL;;;; +2136;BET SYMBOL;Lo;0;L; 05D1;;;;N;SECOND TRANSFINITE CARDINAL;;;; +2137;GIMEL SYMBOL;Lo;0;L; 05D2;;;;N;THIRD TRANSFINITE CARDINAL;;;; +2138;DALET SYMBOL;Lo;0;L; 05D3;;;;N;FOURTH TRANSFINITE CARDINAL;;;; +2139;INFORMATION SOURCE;Ll;0;L; 0069;;;;N;;;;; +213A;ROTATED CAPITAL Q;So;0;ON;;;;;N;;;;; +213B;FACSIMILE SIGN;So;0;ON; 0046 0041 0058;;;;N;;;;; +213C;DOUBLE-STRUCK SMALL PI;Ll;0;L; 03C0;;;;N;;;;; +213D;DOUBLE-STRUCK SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; +213E;DOUBLE-STRUCK CAPITAL GAMMA;Lu;0;L; 0393;;;;N;;;;; +213F;DOUBLE-STRUCK CAPITAL PI;Lu;0;L; 03A0;;;;N;;;;; +2140;DOUBLE-STRUCK N-ARY SUMMATION;Sm;0;ON; 2211;;;;Y;;;;; +2141;TURNED SANS-SERIF CAPITAL G;Sm;0;ON;;;;;N;;;;; +2142;TURNED SANS-SERIF CAPITAL L;Sm;0;ON;;;;;N;;;;; +2143;REVERSED SANS-SERIF CAPITAL L;Sm;0;ON;;;;;N;;;;; +2144;TURNED SANS-SERIF CAPITAL Y;Sm;0;ON;;;;;N;;;;; +2145;DOUBLE-STRUCK ITALIC CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +2146;DOUBLE-STRUCK ITALIC SMALL D;Ll;0;L; 0064;;;;N;;;;; +2147;DOUBLE-STRUCK ITALIC SMALL E;Ll;0;L; 0065;;;;N;;;;; +2148;DOUBLE-STRUCK ITALIC SMALL I;Ll;0;L; 0069;;;;N;;;;; +2149;DOUBLE-STRUCK ITALIC SMALL J;Ll;0;L; 006A;;;;N;;;;; +214A;PROPERTY LINE;So;0;ON;;;;;N;;;;; +214B;TURNED AMPERSAND;Sm;0;ON;;;;;N;;;;; +214C;PER SIGN;So;0;ON;;;;;N;;;;; +214D;AKTIESELSKAB;So;0;ON;;;;;N;;;;; +214E;TURNED SMALL F;Ll;0;L;;;;;N;;;2132;;2132 +214F;SYMBOL FOR SAMARITAN SOURCE;So;0;L;;;;;N;;;;; +2150;VULGAR FRACTION ONE SEVENTH;No;0;ON; 0031 2044 0037;;;1/7;N;;;;; +2151;VULGAR FRACTION ONE NINTH;No;0;ON; 0031 2044 0039;;;1/9;N;;;;; +2152;VULGAR FRACTION ONE TENTH;No;0;ON; 0031 2044 0031 0030;;;1/10;N;;;;; +2153;VULGAR FRACTION ONE THIRD;No;0;ON; 0031 2044 0033;;;1/3;N;FRACTION ONE THIRD;;;; +2154;VULGAR FRACTION TWO THIRDS;No;0;ON; 0032 2044 0033;;;2/3;N;FRACTION TWO THIRDS;;;; +2155;VULGAR FRACTION ONE FIFTH;No;0;ON; 0031 2044 0035;;;1/5;N;FRACTION ONE FIFTH;;;; +2156;VULGAR FRACTION TWO FIFTHS;No;0;ON; 0032 2044 0035;;;2/5;N;FRACTION TWO FIFTHS;;;; +2157;VULGAR FRACTION THREE FIFTHS;No;0;ON; 0033 2044 0035;;;3/5;N;FRACTION THREE FIFTHS;;;; +2158;VULGAR FRACTION FOUR FIFTHS;No;0;ON; 0034 2044 0035;;;4/5;N;FRACTION FOUR FIFTHS;;;; +2159;VULGAR FRACTION ONE SIXTH;No;0;ON; 0031 2044 0036;;;1/6;N;FRACTION ONE SIXTH;;;; +215A;VULGAR FRACTION FIVE SIXTHS;No;0;ON; 0035 2044 0036;;;5/6;N;FRACTION FIVE SIXTHS;;;; +215B;VULGAR FRACTION ONE EIGHTH;No;0;ON; 0031 2044 0038;;;1/8;N;FRACTION ONE EIGHTH;;;; +215C;VULGAR FRACTION THREE EIGHTHS;No;0;ON; 0033 2044 0038;;;3/8;N;FRACTION THREE EIGHTHS;;;; +215D;VULGAR FRACTION FIVE EIGHTHS;No;0;ON; 0035 2044 0038;;;5/8;N;FRACTION FIVE EIGHTHS;;;; +215E;VULGAR FRACTION SEVEN EIGHTHS;No;0;ON; 0037 2044 0038;;;7/8;N;FRACTION SEVEN EIGHTHS;;;; +215F;FRACTION NUMERATOR ONE;No;0;ON; 0031 2044;;;1;N;;;;; +2160;ROMAN NUMERAL ONE;Nl;0;L; 0049;;;1;N;;;;2170; +2161;ROMAN NUMERAL TWO;Nl;0;L; 0049 0049;;;2;N;;;;2171; +2162;ROMAN NUMERAL THREE;Nl;0;L; 0049 0049 0049;;;3;N;;;;2172; +2163;ROMAN NUMERAL FOUR;Nl;0;L; 0049 0056;;;4;N;;;;2173; +2164;ROMAN NUMERAL FIVE;Nl;0;L; 0056;;;5;N;;;;2174; +2165;ROMAN NUMERAL SIX;Nl;0;L; 0056 0049;;;6;N;;;;2175; +2166;ROMAN NUMERAL SEVEN;Nl;0;L; 0056 0049 0049;;;7;N;;;;2176; +2167;ROMAN NUMERAL EIGHT;Nl;0;L; 0056 0049 0049 0049;;;8;N;;;;2177; +2168;ROMAN NUMERAL NINE;Nl;0;L; 0049 0058;;;9;N;;;;2178; +2169;ROMAN NUMERAL TEN;Nl;0;L; 0058;;;10;N;;;;2179; +216A;ROMAN NUMERAL ELEVEN;Nl;0;L; 0058 0049;;;11;N;;;;217A; +216B;ROMAN NUMERAL TWELVE;Nl;0;L; 0058 0049 0049;;;12;N;;;;217B; +216C;ROMAN NUMERAL FIFTY;Nl;0;L; 004C;;;50;N;;;;217C; +216D;ROMAN NUMERAL ONE HUNDRED;Nl;0;L; 0043;;;100;N;;;;217D; +216E;ROMAN NUMERAL FIVE HUNDRED;Nl;0;L; 0044;;;500;N;;;;217E; +216F;ROMAN NUMERAL ONE THOUSAND;Nl;0;L; 004D;;;1000;N;;;;217F; +2170;SMALL ROMAN NUMERAL ONE;Nl;0;L; 0069;;;1;N;;;2160;;2160 +2171;SMALL ROMAN NUMERAL TWO;Nl;0;L; 0069 0069;;;2;N;;;2161;;2161 +2172;SMALL ROMAN NUMERAL THREE;Nl;0;L; 0069 0069 0069;;;3;N;;;2162;;2162 +2173;SMALL ROMAN NUMERAL FOUR;Nl;0;L; 0069 0076;;;4;N;;;2163;;2163 +2174;SMALL ROMAN NUMERAL FIVE;Nl;0;L; 0076;;;5;N;;;2164;;2164 +2175;SMALL ROMAN NUMERAL SIX;Nl;0;L; 0076 0069;;;6;N;;;2165;;2165 +2176;SMALL ROMAN NUMERAL SEVEN;Nl;0;L; 0076 0069 0069;;;7;N;;;2166;;2166 +2177;SMALL ROMAN NUMERAL EIGHT;Nl;0;L; 0076 0069 0069 0069;;;8;N;;;2167;;2167 +2178;SMALL ROMAN NUMERAL NINE;Nl;0;L; 0069 0078;;;9;N;;;2168;;2168 +2179;SMALL ROMAN NUMERAL TEN;Nl;0;L; 0078;;;10;N;;;2169;;2169 +217A;SMALL ROMAN NUMERAL ELEVEN;Nl;0;L; 0078 0069;;;11;N;;;216A;;216A +217B;SMALL ROMAN NUMERAL TWELVE;Nl;0;L; 0078 0069 0069;;;12;N;;;216B;;216B +217C;SMALL ROMAN NUMERAL FIFTY;Nl;0;L; 006C;;;50;N;;;216C;;216C +217D;SMALL ROMAN NUMERAL ONE HUNDRED;Nl;0;L; 0063;;;100;N;;;216D;;216D +217E;SMALL ROMAN NUMERAL FIVE HUNDRED;Nl;0;L; 0064;;;500;N;;;216E;;216E +217F;SMALL ROMAN NUMERAL ONE THOUSAND;Nl;0;L; 006D;;;1000;N;;;216F;;216F +2180;ROMAN NUMERAL ONE THOUSAND C D;Nl;0;L;;;;1000;N;;;;; +2181;ROMAN NUMERAL FIVE THOUSAND;Nl;0;L;;;;5000;N;;;;; +2182;ROMAN NUMERAL TEN THOUSAND;Nl;0;L;;;;10000;N;;;;; +2183;ROMAN NUMERAL REVERSED ONE HUNDRED;Lu;0;L;;;;;N;;;;2184; +2184;LATIN SMALL LETTER REVERSED C;Ll;0;L;;;;;N;;;2183;;2183 +2185;ROMAN NUMERAL SIX LATE FORM;Nl;0;L;;;;6;N;;;;; +2186;ROMAN NUMERAL FIFTY EARLY FORM;Nl;0;L;;;;50;N;;;;; +2187;ROMAN NUMERAL FIFTY THOUSAND;Nl;0;L;;;;50000;N;;;;; +2188;ROMAN NUMERAL ONE HUNDRED THOUSAND;Nl;0;L;;;;100000;N;;;;; +2189;VULGAR FRACTION ZERO THIRDS;No;0;ON; 0030 2044 0033;;;0;N;;;;; +2190;LEFTWARDS ARROW;Sm;0;ON;;;;;N;LEFT ARROW;;;; +2191;UPWARDS ARROW;Sm;0;ON;;;;;N;UP ARROW;;;; +2192;RIGHTWARDS ARROW;Sm;0;ON;;;;;N;RIGHT ARROW;;;; +2193;DOWNWARDS ARROW;Sm;0;ON;;;;;N;DOWN ARROW;;;; +2194;LEFT RIGHT ARROW;Sm;0;ON;;;;;N;;;;; +2195;UP DOWN ARROW;So;0;ON;;;;;N;;;;; +2196;NORTH WEST ARROW;So;0;ON;;;;;N;UPPER LEFT ARROW;;;; +2197;NORTH EAST ARROW;So;0;ON;;;;;N;UPPER RIGHT ARROW;;;; +2198;SOUTH EAST ARROW;So;0;ON;;;;;N;LOWER RIGHT ARROW;;;; +2199;SOUTH WEST ARROW;So;0;ON;;;;;N;LOWER LEFT ARROW;;;; +219A;LEFTWARDS ARROW WITH STROKE;Sm;0;ON;2190 0338;;;;N;LEFT ARROW WITH STROKE;;;; +219B;RIGHTWARDS ARROW WITH STROKE;Sm;0;ON;2192 0338;;;;N;RIGHT ARROW WITH STROKE;;;; +219C;LEFTWARDS WAVE ARROW;So;0;ON;;;;;N;LEFT WAVE ARROW;;;; +219D;RIGHTWARDS WAVE ARROW;So;0;ON;;;;;N;RIGHT WAVE ARROW;;;; +219E;LEFTWARDS TWO HEADED ARROW;So;0;ON;;;;;N;LEFT TWO HEADED ARROW;;;; +219F;UPWARDS TWO HEADED ARROW;So;0;ON;;;;;N;UP TWO HEADED ARROW;;;; +21A0;RIGHTWARDS TWO HEADED ARROW;Sm;0;ON;;;;;N;RIGHT TWO HEADED ARROW;;;; +21A1;DOWNWARDS TWO HEADED ARROW;So;0;ON;;;;;N;DOWN TWO HEADED ARROW;;;; +21A2;LEFTWARDS ARROW WITH TAIL;So;0;ON;;;;;N;LEFT ARROW WITH TAIL;;;; +21A3;RIGHTWARDS ARROW WITH TAIL;Sm;0;ON;;;;;N;RIGHT ARROW WITH TAIL;;;; +21A4;LEFTWARDS ARROW FROM BAR;So;0;ON;;;;;N;LEFT ARROW FROM BAR;;;; +21A5;UPWARDS ARROW FROM BAR;So;0;ON;;;;;N;UP ARROW FROM BAR;;;; +21A6;RIGHTWARDS ARROW FROM BAR;Sm;0;ON;;;;;N;RIGHT ARROW FROM BAR;;;; +21A7;DOWNWARDS ARROW FROM BAR;So;0;ON;;;;;N;DOWN ARROW FROM BAR;;;; +21A8;UP DOWN ARROW WITH BASE;So;0;ON;;;;;N;;;;; +21A9;LEFTWARDS ARROW WITH HOOK;So;0;ON;;;;;N;LEFT ARROW WITH HOOK;;;; +21AA;RIGHTWARDS ARROW WITH HOOK;So;0;ON;;;;;N;RIGHT ARROW WITH HOOK;;;; +21AB;LEFTWARDS ARROW WITH LOOP;So;0;ON;;;;;N;LEFT ARROW WITH LOOP;;;; +21AC;RIGHTWARDS ARROW WITH LOOP;So;0;ON;;;;;N;RIGHT ARROW WITH LOOP;;;; +21AD;LEFT RIGHT WAVE ARROW;So;0;ON;;;;;N;;;;; +21AE;LEFT RIGHT ARROW WITH STROKE;Sm;0;ON;2194 0338;;;;N;;;;; +21AF;DOWNWARDS ZIGZAG ARROW;So;0;ON;;;;;N;DOWN ZIGZAG ARROW;;;; +21B0;UPWARDS ARROW WITH TIP LEFTWARDS;So;0;ON;;;;;N;UP ARROW WITH TIP LEFT;;;; +21B1;UPWARDS ARROW WITH TIP RIGHTWARDS;So;0;ON;;;;;N;UP ARROW WITH TIP RIGHT;;;; +21B2;DOWNWARDS ARROW WITH TIP LEFTWARDS;So;0;ON;;;;;N;DOWN ARROW WITH TIP LEFT;;;; +21B3;DOWNWARDS ARROW WITH TIP RIGHTWARDS;So;0;ON;;;;;N;DOWN ARROW WITH TIP RIGHT;;;; +21B4;RIGHTWARDS ARROW WITH CORNER DOWNWARDS;So;0;ON;;;;;N;RIGHT ARROW WITH CORNER DOWN;;;; +21B5;DOWNWARDS ARROW WITH CORNER LEFTWARDS;So;0;ON;;;;;N;DOWN ARROW WITH CORNER LEFT;;;; +21B6;ANTICLOCKWISE TOP SEMICIRCLE ARROW;So;0;ON;;;;;N;;;;; +21B7;CLOCKWISE TOP SEMICIRCLE ARROW;So;0;ON;;;;;N;;;;; +21B8;NORTH WEST ARROW TO LONG BAR;So;0;ON;;;;;N;UPPER LEFT ARROW TO LONG BAR;;;; +21B9;LEFTWARDS ARROW TO BAR OVER RIGHTWARDS ARROW TO BAR;So;0;ON;;;;;N;LEFT ARROW TO BAR OVER RIGHT ARROW TO BAR;;;; +21BA;ANTICLOCKWISE OPEN CIRCLE ARROW;So;0;ON;;;;;N;;;;; +21BB;CLOCKWISE OPEN CIRCLE ARROW;So;0;ON;;;;;N;;;;; +21BC;LEFTWARDS HARPOON WITH BARB UPWARDS;So;0;ON;;;;;N;LEFT HARPOON WITH BARB UP;;;; +21BD;LEFTWARDS HARPOON WITH BARB DOWNWARDS;So;0;ON;;;;;N;LEFT HARPOON WITH BARB DOWN;;;; +21BE;UPWARDS HARPOON WITH BARB RIGHTWARDS;So;0;ON;;;;;N;UP HARPOON WITH BARB RIGHT;;;; +21BF;UPWARDS HARPOON WITH BARB LEFTWARDS;So;0;ON;;;;;N;UP HARPOON WITH BARB LEFT;;;; +21C0;RIGHTWARDS HARPOON WITH BARB UPWARDS;So;0;ON;;;;;N;RIGHT HARPOON WITH BARB UP;;;; +21C1;RIGHTWARDS HARPOON WITH BARB DOWNWARDS;So;0;ON;;;;;N;RIGHT HARPOON WITH BARB DOWN;;;; +21C2;DOWNWARDS HARPOON WITH BARB RIGHTWARDS;So;0;ON;;;;;N;DOWN HARPOON WITH BARB RIGHT;;;; +21C3;DOWNWARDS HARPOON WITH BARB LEFTWARDS;So;0;ON;;;;;N;DOWN HARPOON WITH BARB LEFT;;;; +21C4;RIGHTWARDS ARROW OVER LEFTWARDS ARROW;So;0;ON;;;;;N;RIGHT ARROW OVER LEFT ARROW;;;; +21C5;UPWARDS ARROW LEFTWARDS OF DOWNWARDS ARROW;So;0;ON;;;;;N;UP ARROW LEFT OF DOWN ARROW;;;; +21C6;LEFTWARDS ARROW OVER RIGHTWARDS ARROW;So;0;ON;;;;;N;LEFT ARROW OVER RIGHT ARROW;;;; +21C7;LEFTWARDS PAIRED ARROWS;So;0;ON;;;;;N;LEFT PAIRED ARROWS;;;; +21C8;UPWARDS PAIRED ARROWS;So;0;ON;;;;;N;UP PAIRED ARROWS;;;; +21C9;RIGHTWARDS PAIRED ARROWS;So;0;ON;;;;;N;RIGHT PAIRED ARROWS;;;; +21CA;DOWNWARDS PAIRED ARROWS;So;0;ON;;;;;N;DOWN PAIRED ARROWS;;;; +21CB;LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON;So;0;ON;;;;;N;LEFT HARPOON OVER RIGHT HARPOON;;;; +21CC;RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON;So;0;ON;;;;;N;RIGHT HARPOON OVER LEFT HARPOON;;;; +21CD;LEFTWARDS DOUBLE ARROW WITH STROKE;So;0;ON;21D0 0338;;;;N;LEFT DOUBLE ARROW WITH STROKE;;;; +21CE;LEFT RIGHT DOUBLE ARROW WITH STROKE;Sm;0;ON;21D4 0338;;;;N;;;;; +21CF;RIGHTWARDS DOUBLE ARROW WITH STROKE;Sm;0;ON;21D2 0338;;;;N;RIGHT DOUBLE ARROW WITH STROKE;;;; +21D0;LEFTWARDS DOUBLE ARROW;So;0;ON;;;;;N;LEFT DOUBLE ARROW;;;; +21D1;UPWARDS DOUBLE ARROW;So;0;ON;;;;;N;UP DOUBLE ARROW;;;; +21D2;RIGHTWARDS DOUBLE ARROW;Sm;0;ON;;;;;N;RIGHT DOUBLE ARROW;;;; +21D3;DOWNWARDS DOUBLE ARROW;So;0;ON;;;;;N;DOWN DOUBLE ARROW;;;; +21D4;LEFT RIGHT DOUBLE ARROW;Sm;0;ON;;;;;N;;;;; +21D5;UP DOWN DOUBLE ARROW;So;0;ON;;;;;N;;;;; +21D6;NORTH WEST DOUBLE ARROW;So;0;ON;;;;;N;UPPER LEFT DOUBLE ARROW;;;; +21D7;NORTH EAST DOUBLE ARROW;So;0;ON;;;;;N;UPPER RIGHT DOUBLE ARROW;;;; +21D8;SOUTH EAST DOUBLE ARROW;So;0;ON;;;;;N;LOWER RIGHT DOUBLE ARROW;;;; +21D9;SOUTH WEST DOUBLE ARROW;So;0;ON;;;;;N;LOWER LEFT DOUBLE ARROW;;;; +21DA;LEFTWARDS TRIPLE ARROW;So;0;ON;;;;;N;LEFT TRIPLE ARROW;;;; +21DB;RIGHTWARDS TRIPLE ARROW;So;0;ON;;;;;N;RIGHT TRIPLE ARROW;;;; +21DC;LEFTWARDS SQUIGGLE ARROW;So;0;ON;;;;;N;LEFT SQUIGGLE ARROW;;;; +21DD;RIGHTWARDS SQUIGGLE ARROW;So;0;ON;;;;;N;RIGHT SQUIGGLE ARROW;;;; +21DE;UPWARDS ARROW WITH DOUBLE STROKE;So;0;ON;;;;;N;UP ARROW WITH DOUBLE STROKE;;;; +21DF;DOWNWARDS ARROW WITH DOUBLE STROKE;So;0;ON;;;;;N;DOWN ARROW WITH DOUBLE STROKE;;;; +21E0;LEFTWARDS DASHED ARROW;So;0;ON;;;;;N;LEFT DASHED ARROW;;;; +21E1;UPWARDS DASHED ARROW;So;0;ON;;;;;N;UP DASHED ARROW;;;; +21E2;RIGHTWARDS DASHED ARROW;So;0;ON;;;;;N;RIGHT DASHED ARROW;;;; +21E3;DOWNWARDS DASHED ARROW;So;0;ON;;;;;N;DOWN DASHED ARROW;;;; +21E4;LEFTWARDS ARROW TO BAR;So;0;ON;;;;;N;LEFT ARROW TO BAR;;;; +21E5;RIGHTWARDS ARROW TO BAR;So;0;ON;;;;;N;RIGHT ARROW TO BAR;;;; +21E6;LEFTWARDS WHITE ARROW;So;0;ON;;;;;N;WHITE LEFT ARROW;;;; +21E7;UPWARDS WHITE ARROW;So;0;ON;;;;;N;WHITE UP ARROW;;;; +21E8;RIGHTWARDS WHITE ARROW;So;0;ON;;;;;N;WHITE RIGHT ARROW;;;; +21E9;DOWNWARDS WHITE ARROW;So;0;ON;;;;;N;WHITE DOWN ARROW;;;; +21EA;UPWARDS WHITE ARROW FROM BAR;So;0;ON;;;;;N;WHITE UP ARROW FROM BAR;;;; +21EB;UPWARDS WHITE ARROW ON PEDESTAL;So;0;ON;;;;;N;;;;; +21EC;UPWARDS WHITE ARROW ON PEDESTAL WITH HORIZONTAL BAR;So;0;ON;;;;;N;;;;; +21ED;UPWARDS WHITE ARROW ON PEDESTAL WITH VERTICAL BAR;So;0;ON;;;;;N;;;;; +21EE;UPWARDS WHITE DOUBLE ARROW;So;0;ON;;;;;N;;;;; +21EF;UPWARDS WHITE DOUBLE ARROW ON PEDESTAL;So;0;ON;;;;;N;;;;; +21F0;RIGHTWARDS WHITE ARROW FROM WALL;So;0;ON;;;;;N;;;;; +21F1;NORTH WEST ARROW TO CORNER;So;0;ON;;;;;N;;;;; +21F2;SOUTH EAST ARROW TO CORNER;So;0;ON;;;;;N;;;;; +21F3;UP DOWN WHITE ARROW;So;0;ON;;;;;N;;;;; +21F4;RIGHT ARROW WITH SMALL CIRCLE;Sm;0;ON;;;;;N;;;;; +21F5;DOWNWARDS ARROW LEFTWARDS OF UPWARDS ARROW;Sm;0;ON;;;;;N;;;;; +21F6;THREE RIGHTWARDS ARROWS;Sm;0;ON;;;;;N;;;;; +21F7;LEFTWARDS ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +21F8;RIGHTWARDS ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +21F9;LEFT RIGHT ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +21FA;LEFTWARDS ARROW WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +21FB;RIGHTWARDS ARROW WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +21FC;LEFT RIGHT ARROW WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +21FD;LEFTWARDS OPEN-HEADED ARROW;Sm;0;ON;;;;;N;;;;; +21FE;RIGHTWARDS OPEN-HEADED ARROW;Sm;0;ON;;;;;N;;;;; +21FF;LEFT RIGHT OPEN-HEADED ARROW;Sm;0;ON;;;;;N;;;;; +2200;FOR ALL;Sm;0;ON;;;;;N;;;;; +2201;COMPLEMENT;Sm;0;ON;;;;;Y;;;;; +2202;PARTIAL DIFFERENTIAL;Sm;0;ON;;;;;Y;;;;; +2203;THERE EXISTS;Sm;0;ON;;;;;Y;;;;; +2204;THERE DOES NOT EXIST;Sm;0;ON;2203 0338;;;;Y;;;;; +2205;EMPTY SET;Sm;0;ON;;;;;N;;;;; +2206;INCREMENT;Sm;0;ON;;;;;N;;;;; +2207;NABLA;Sm;0;ON;;;;;N;;;;; +2208;ELEMENT OF;Sm;0;ON;;;;;Y;;;;; +2209;NOT AN ELEMENT OF;Sm;0;ON;2208 0338;;;;Y;;;;; +220A;SMALL ELEMENT OF;Sm;0;ON;;;;;Y;;;;; +220B;CONTAINS AS MEMBER;Sm;0;ON;;;;;Y;;;;; +220C;DOES NOT CONTAIN AS MEMBER;Sm;0;ON;220B 0338;;;;Y;;;;; +220D;SMALL CONTAINS AS MEMBER;Sm;0;ON;;;;;Y;;;;; +220E;END OF PROOF;Sm;0;ON;;;;;N;;;;; +220F;N-ARY PRODUCT;Sm;0;ON;;;;;N;;;;; +2210;N-ARY COPRODUCT;Sm;0;ON;;;;;N;;;;; +2211;N-ARY SUMMATION;Sm;0;ON;;;;;Y;;;;; +2212;MINUS SIGN;Sm;0;ES;;;;;N;;;;; +2213;MINUS-OR-PLUS SIGN;Sm;0;ET;;;;;N;;;;; +2214;DOT PLUS;Sm;0;ON;;;;;N;;;;; +2215;DIVISION SLASH;Sm;0;ON;;;;;Y;;;;; +2216;SET MINUS;Sm;0;ON;;;;;Y;;;;; +2217;ASTERISK OPERATOR;Sm;0;ON;;;;;N;;;;; +2218;RING OPERATOR;Sm;0;ON;;;;;N;;;;; +2219;BULLET OPERATOR;Sm;0;ON;;;;;N;;;;; +221A;SQUARE ROOT;Sm;0;ON;;;;;Y;;;;; +221B;CUBE ROOT;Sm;0;ON;;;;;Y;;;;; +221C;FOURTH ROOT;Sm;0;ON;;;;;Y;;;;; +221D;PROPORTIONAL TO;Sm;0;ON;;;;;Y;;;;; +221E;INFINITY;Sm;0;ON;;;;;N;;;;; +221F;RIGHT ANGLE;Sm;0;ON;;;;;Y;;;;; +2220;ANGLE;Sm;0;ON;;;;;Y;;;;; +2221;MEASURED ANGLE;Sm;0;ON;;;;;Y;;;;; +2222;SPHERICAL ANGLE;Sm;0;ON;;;;;Y;;;;; +2223;DIVIDES;Sm;0;ON;;;;;N;;;;; +2224;DOES NOT DIVIDE;Sm;0;ON;2223 0338;;;;Y;;;;; +2225;PARALLEL TO;Sm;0;ON;;;;;N;;;;; +2226;NOT PARALLEL TO;Sm;0;ON;2225 0338;;;;Y;;;;; +2227;LOGICAL AND;Sm;0;ON;;;;;N;;;;; +2228;LOGICAL OR;Sm;0;ON;;;;;N;;;;; +2229;INTERSECTION;Sm;0;ON;;;;;N;;;;; +222A;UNION;Sm;0;ON;;;;;N;;;;; +222B;INTEGRAL;Sm;0;ON;;;;;Y;;;;; +222C;DOUBLE INTEGRAL;Sm;0;ON; 222B 222B;;;;Y;;;;; +222D;TRIPLE INTEGRAL;Sm;0;ON; 222B 222B 222B;;;;Y;;;;; +222E;CONTOUR INTEGRAL;Sm;0;ON;;;;;Y;;;;; +222F;SURFACE INTEGRAL;Sm;0;ON; 222E 222E;;;;Y;;;;; +2230;VOLUME INTEGRAL;Sm;0;ON; 222E 222E 222E;;;;Y;;;;; +2231;CLOCKWISE INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2232;CLOCKWISE CONTOUR INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2233;ANTICLOCKWISE CONTOUR INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2234;THEREFORE;Sm;0;ON;;;;;N;;;;; +2235;BECAUSE;Sm;0;ON;;;;;N;;;;; +2236;RATIO;Sm;0;ON;;;;;N;;;;; +2237;PROPORTION;Sm;0;ON;;;;;N;;;;; +2238;DOT MINUS;Sm;0;ON;;;;;N;;;;; +2239;EXCESS;Sm;0;ON;;;;;Y;;;;; +223A;GEOMETRIC PROPORTION;Sm;0;ON;;;;;N;;;;; +223B;HOMOTHETIC;Sm;0;ON;;;;;Y;;;;; +223C;TILDE OPERATOR;Sm;0;ON;;;;;Y;;;;; +223D;REVERSED TILDE;Sm;0;ON;;;;;Y;;;;; +223E;INVERTED LAZY S;Sm;0;ON;;;;;Y;;;;; +223F;SINE WAVE;Sm;0;ON;;;;;Y;;;;; +2240;WREATH PRODUCT;Sm;0;ON;;;;;Y;;;;; +2241;NOT TILDE;Sm;0;ON;223C 0338;;;;Y;;;;; +2242;MINUS TILDE;Sm;0;ON;;;;;Y;;;;; +2243;ASYMPTOTICALLY EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2244;NOT ASYMPTOTICALLY EQUAL TO;Sm;0;ON;2243 0338;;;;Y;;;;; +2245;APPROXIMATELY EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2246;APPROXIMATELY BUT NOT ACTUALLY EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2247;NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO;Sm;0;ON;2245 0338;;;;Y;;;;; +2248;ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2249;NOT ALMOST EQUAL TO;Sm;0;ON;2248 0338;;;;Y;;;;; +224A;ALMOST EQUAL OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +224B;TRIPLE TILDE;Sm;0;ON;;;;;Y;;;;; +224C;ALL EQUAL TO;Sm;0;ON;;;;;Y;;;;; +224D;EQUIVALENT TO;Sm;0;ON;;;;;N;;;;; +224E;GEOMETRICALLY EQUIVALENT TO;Sm;0;ON;;;;;N;;;;; +224F;DIFFERENCE BETWEEN;Sm;0;ON;;;;;N;;;;; +2250;APPROACHES THE LIMIT;Sm;0;ON;;;;;N;;;;; +2251;GEOMETRICALLY EQUAL TO;Sm;0;ON;;;;;N;;;;; +2252;APPROXIMATELY EQUAL TO OR THE IMAGE OF;Sm;0;ON;;;;;Y;;;;; +2253;IMAGE OF OR APPROXIMATELY EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2254;COLON EQUALS;Sm;0;ON;;;;;Y;COLON EQUAL;;;; +2255;EQUALS COLON;Sm;0;ON;;;;;Y;EQUAL COLON;;;; +2256;RING IN EQUAL TO;Sm;0;ON;;;;;N;;;;; +2257;RING EQUAL TO;Sm;0;ON;;;;;N;;;;; +2258;CORRESPONDS TO;Sm;0;ON;;;;;N;;;;; +2259;ESTIMATES;Sm;0;ON;;;;;N;;;;; +225A;EQUIANGULAR TO;Sm;0;ON;;;;;N;;;;; +225B;STAR EQUALS;Sm;0;ON;;;;;N;;;;; +225C;DELTA EQUAL TO;Sm;0;ON;;;;;N;;;;; +225D;EQUAL TO BY DEFINITION;Sm;0;ON;;;;;N;;;;; +225E;MEASURED BY;Sm;0;ON;;;;;N;;;;; +225F;QUESTIONED EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2260;NOT EQUAL TO;Sm;0;ON;003D 0338;;;;Y;;;;; +2261;IDENTICAL TO;Sm;0;ON;;;;;N;;;;; +2262;NOT IDENTICAL TO;Sm;0;ON;2261 0338;;;;Y;;;;; +2263;STRICTLY EQUIVALENT TO;Sm;0;ON;;;;;N;;;;; +2264;LESS-THAN OR EQUAL TO;Sm;0;ON;;;;;Y;LESS THAN OR EQUAL TO;;;; +2265;GREATER-THAN OR EQUAL TO;Sm;0;ON;;;;;Y;GREATER THAN OR EQUAL TO;;;; +2266;LESS-THAN OVER EQUAL TO;Sm;0;ON;;;;;Y;LESS THAN OVER EQUAL TO;;;; +2267;GREATER-THAN OVER EQUAL TO;Sm;0;ON;;;;;Y;GREATER THAN OVER EQUAL TO;;;; +2268;LESS-THAN BUT NOT EQUAL TO;Sm;0;ON;;;;;Y;LESS THAN BUT NOT EQUAL TO;;;; +2269;GREATER-THAN BUT NOT EQUAL TO;Sm;0;ON;;;;;Y;GREATER THAN BUT NOT EQUAL TO;;;; +226A;MUCH LESS-THAN;Sm;0;ON;;;;;Y;MUCH LESS THAN;;;; +226B;MUCH GREATER-THAN;Sm;0;ON;;;;;Y;MUCH GREATER THAN;;;; +226C;BETWEEN;Sm;0;ON;;;;;N;;;;; +226D;NOT EQUIVALENT TO;Sm;0;ON;224D 0338;;;;N;;;;; +226E;NOT LESS-THAN;Sm;0;ON;003C 0338;;;;Y;NOT LESS THAN;;;; +226F;NOT GREATER-THAN;Sm;0;ON;003E 0338;;;;Y;NOT GREATER THAN;;;; +2270;NEITHER LESS-THAN NOR EQUAL TO;Sm;0;ON;2264 0338;;;;Y;NEITHER LESS THAN NOR EQUAL TO;;;; +2271;NEITHER GREATER-THAN NOR EQUAL TO;Sm;0;ON;2265 0338;;;;Y;NEITHER GREATER THAN NOR EQUAL TO;;;; +2272;LESS-THAN OR EQUIVALENT TO;Sm;0;ON;;;;;Y;LESS THAN OR EQUIVALENT TO;;;; +2273;GREATER-THAN OR EQUIVALENT TO;Sm;0;ON;;;;;Y;GREATER THAN OR EQUIVALENT TO;;;; +2274;NEITHER LESS-THAN NOR EQUIVALENT TO;Sm;0;ON;2272 0338;;;;Y;NEITHER LESS THAN NOR EQUIVALENT TO;;;; +2275;NEITHER GREATER-THAN NOR EQUIVALENT TO;Sm;0;ON;2273 0338;;;;Y;NEITHER GREATER THAN NOR EQUIVALENT TO;;;; +2276;LESS-THAN OR GREATER-THAN;Sm;0;ON;;;;;Y;LESS THAN OR GREATER THAN;;;; +2277;GREATER-THAN OR LESS-THAN;Sm;0;ON;;;;;Y;GREATER THAN OR LESS THAN;;;; +2278;NEITHER LESS-THAN NOR GREATER-THAN;Sm;0;ON;2276 0338;;;;Y;NEITHER LESS THAN NOR GREATER THAN;;;; +2279;NEITHER GREATER-THAN NOR LESS-THAN;Sm;0;ON;2277 0338;;;;Y;NEITHER GREATER THAN NOR LESS THAN;;;; +227A;PRECEDES;Sm;0;ON;;;;;Y;;;;; +227B;SUCCEEDS;Sm;0;ON;;;;;Y;;;;; +227C;PRECEDES OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +227D;SUCCEEDS OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +227E;PRECEDES OR EQUIVALENT TO;Sm;0;ON;;;;;Y;;;;; +227F;SUCCEEDS OR EQUIVALENT TO;Sm;0;ON;;;;;Y;;;;; +2280;DOES NOT PRECEDE;Sm;0;ON;227A 0338;;;;Y;;;;; +2281;DOES NOT SUCCEED;Sm;0;ON;227B 0338;;;;Y;;;;; +2282;SUBSET OF;Sm;0;ON;;;;;Y;;;;; +2283;SUPERSET OF;Sm;0;ON;;;;;Y;;;;; +2284;NOT A SUBSET OF;Sm;0;ON;2282 0338;;;;Y;;;;; +2285;NOT A SUPERSET OF;Sm;0;ON;2283 0338;;;;Y;;;;; +2286;SUBSET OF OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2287;SUPERSET OF OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2288;NEITHER A SUBSET OF NOR EQUAL TO;Sm;0;ON;2286 0338;;;;Y;;;;; +2289;NEITHER A SUPERSET OF NOR EQUAL TO;Sm;0;ON;2287 0338;;;;Y;;;;; +228A;SUBSET OF WITH NOT EQUAL TO;Sm;0;ON;;;;;Y;SUBSET OF OR NOT EQUAL TO;;;; +228B;SUPERSET OF WITH NOT EQUAL TO;Sm;0;ON;;;;;Y;SUPERSET OF OR NOT EQUAL TO;;;; +228C;MULTISET;Sm;0;ON;;;;;Y;;;;; +228D;MULTISET MULTIPLICATION;Sm;0;ON;;;;;N;;;;; +228E;MULTISET UNION;Sm;0;ON;;;;;N;;;;; +228F;SQUARE IMAGE OF;Sm;0;ON;;;;;Y;;;;; +2290;SQUARE ORIGINAL OF;Sm;0;ON;;;;;Y;;;;; +2291;SQUARE IMAGE OF OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2292;SQUARE ORIGINAL OF OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2293;SQUARE CAP;Sm;0;ON;;;;;N;;;;; +2294;SQUARE CUP;Sm;0;ON;;;;;N;;;;; +2295;CIRCLED PLUS;Sm;0;ON;;;;;N;;;;; +2296;CIRCLED MINUS;Sm;0;ON;;;;;N;;;;; +2297;CIRCLED TIMES;Sm;0;ON;;;;;N;;;;; +2298;CIRCLED DIVISION SLASH;Sm;0;ON;;;;;Y;;;;; +2299;CIRCLED DOT OPERATOR;Sm;0;ON;;;;;N;;;;; +229A;CIRCLED RING OPERATOR;Sm;0;ON;;;;;N;;;;; +229B;CIRCLED ASTERISK OPERATOR;Sm;0;ON;;;;;N;;;;; +229C;CIRCLED EQUALS;Sm;0;ON;;;;;N;;;;; +229D;CIRCLED DASH;Sm;0;ON;;;;;N;;;;; +229E;SQUARED PLUS;Sm;0;ON;;;;;N;;;;; +229F;SQUARED MINUS;Sm;0;ON;;;;;N;;;;; +22A0;SQUARED TIMES;Sm;0;ON;;;;;N;;;;; +22A1;SQUARED DOT OPERATOR;Sm;0;ON;;;;;N;;;;; +22A2;RIGHT TACK;Sm;0;ON;;;;;Y;;;;; +22A3;LEFT TACK;Sm;0;ON;;;;;Y;;;;; +22A4;DOWN TACK;Sm;0;ON;;;;;N;;;;; +22A5;UP TACK;Sm;0;ON;;;;;N;;;;; +22A6;ASSERTION;Sm;0;ON;;;;;Y;;;;; +22A7;MODELS;Sm;0;ON;;;;;Y;;;;; +22A8;TRUE;Sm;0;ON;;;;;Y;;;;; +22A9;FORCES;Sm;0;ON;;;;;Y;;;;; +22AA;TRIPLE VERTICAL BAR RIGHT TURNSTILE;Sm;0;ON;;;;;Y;;;;; +22AB;DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE;Sm;0;ON;;;;;Y;;;;; +22AC;DOES NOT PROVE;Sm;0;ON;22A2 0338;;;;Y;;;;; +22AD;NOT TRUE;Sm;0;ON;22A8 0338;;;;Y;;;;; +22AE;DOES NOT FORCE;Sm;0;ON;22A9 0338;;;;Y;;;;; +22AF;NEGATED DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE;Sm;0;ON;22AB 0338;;;;Y;;;;; +22B0;PRECEDES UNDER RELATION;Sm;0;ON;;;;;Y;;;;; +22B1;SUCCEEDS UNDER RELATION;Sm;0;ON;;;;;Y;;;;; +22B2;NORMAL SUBGROUP OF;Sm;0;ON;;;;;Y;;;;; +22B3;CONTAINS AS NORMAL SUBGROUP;Sm;0;ON;;;;;Y;;;;; +22B4;NORMAL SUBGROUP OF OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +22B5;CONTAINS AS NORMAL SUBGROUP OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +22B6;ORIGINAL OF;Sm;0;ON;;;;;Y;;;;; +22B7;IMAGE OF;Sm;0;ON;;;;;Y;;;;; +22B8;MULTIMAP;Sm;0;ON;;;;;Y;;;;; +22B9;HERMITIAN CONJUGATE MATRIX;Sm;0;ON;;;;;N;;;;; +22BA;INTERCALATE;Sm;0;ON;;;;;N;;;;; +22BB;XOR;Sm;0;ON;;;;;N;;;;; +22BC;NAND;Sm;0;ON;;;;;N;;;;; +22BD;NOR;Sm;0;ON;;;;;N;;;;; +22BE;RIGHT ANGLE WITH ARC;Sm;0;ON;;;;;Y;;;;; +22BF;RIGHT TRIANGLE;Sm;0;ON;;;;;Y;;;;; +22C0;N-ARY LOGICAL AND;Sm;0;ON;;;;;N;;;;; +22C1;N-ARY LOGICAL OR;Sm;0;ON;;;;;N;;;;; +22C2;N-ARY INTERSECTION;Sm;0;ON;;;;;N;;;;; +22C3;N-ARY UNION;Sm;0;ON;;;;;N;;;;; +22C4;DIAMOND OPERATOR;Sm;0;ON;;;;;N;;;;; +22C5;DOT OPERATOR;Sm;0;ON;;;;;N;;;;; +22C6;STAR OPERATOR;Sm;0;ON;;;;;N;;;;; +22C7;DIVISION TIMES;Sm;0;ON;;;;;N;;;;; +22C8;BOWTIE;Sm;0;ON;;;;;N;;;;; +22C9;LEFT NORMAL FACTOR SEMIDIRECT PRODUCT;Sm;0;ON;;;;;Y;;;;; +22CA;RIGHT NORMAL FACTOR SEMIDIRECT PRODUCT;Sm;0;ON;;;;;Y;;;;; +22CB;LEFT SEMIDIRECT PRODUCT;Sm;0;ON;;;;;Y;;;;; +22CC;RIGHT SEMIDIRECT PRODUCT;Sm;0;ON;;;;;Y;;;;; +22CD;REVERSED TILDE EQUALS;Sm;0;ON;;;;;Y;;;;; +22CE;CURLY LOGICAL OR;Sm;0;ON;;;;;N;;;;; +22CF;CURLY LOGICAL AND;Sm;0;ON;;;;;N;;;;; +22D0;DOUBLE SUBSET;Sm;0;ON;;;;;Y;;;;; +22D1;DOUBLE SUPERSET;Sm;0;ON;;;;;Y;;;;; +22D2;DOUBLE INTERSECTION;Sm;0;ON;;;;;N;;;;; +22D3;DOUBLE UNION;Sm;0;ON;;;;;N;;;;; +22D4;PITCHFORK;Sm;0;ON;;;;;N;;;;; +22D5;EQUAL AND PARALLEL TO;Sm;0;ON;;;;;N;;;;; +22D6;LESS-THAN WITH DOT;Sm;0;ON;;;;;Y;LESS THAN WITH DOT;;;; +22D7;GREATER-THAN WITH DOT;Sm;0;ON;;;;;Y;GREATER THAN WITH DOT;;;; +22D8;VERY MUCH LESS-THAN;Sm;0;ON;;;;;Y;VERY MUCH LESS THAN;;;; +22D9;VERY MUCH GREATER-THAN;Sm;0;ON;;;;;Y;VERY MUCH GREATER THAN;;;; +22DA;LESS-THAN EQUAL TO OR GREATER-THAN;Sm;0;ON;;;;;Y;LESS THAN EQUAL TO OR GREATER THAN;;;; +22DB;GREATER-THAN EQUAL TO OR LESS-THAN;Sm;0;ON;;;;;Y;GREATER THAN EQUAL TO OR LESS THAN;;;; +22DC;EQUAL TO OR LESS-THAN;Sm;0;ON;;;;;Y;EQUAL TO OR LESS THAN;;;; +22DD;EQUAL TO OR GREATER-THAN;Sm;0;ON;;;;;Y;EQUAL TO OR GREATER THAN;;;; +22DE;EQUAL TO OR PRECEDES;Sm;0;ON;;;;;Y;;;;; +22DF;EQUAL TO OR SUCCEEDS;Sm;0;ON;;;;;Y;;;;; +22E0;DOES NOT PRECEDE OR EQUAL;Sm;0;ON;227C 0338;;;;Y;;;;; +22E1;DOES NOT SUCCEED OR EQUAL;Sm;0;ON;227D 0338;;;;Y;;;;; +22E2;NOT SQUARE IMAGE OF OR EQUAL TO;Sm;0;ON;2291 0338;;;;Y;;;;; +22E3;NOT SQUARE ORIGINAL OF OR EQUAL TO;Sm;0;ON;2292 0338;;;;Y;;;;; +22E4;SQUARE IMAGE OF OR NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +22E5;SQUARE ORIGINAL OF OR NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +22E6;LESS-THAN BUT NOT EQUIVALENT TO;Sm;0;ON;;;;;Y;LESS THAN BUT NOT EQUIVALENT TO;;;; +22E7;GREATER-THAN BUT NOT EQUIVALENT TO;Sm;0;ON;;;;;Y;GREATER THAN BUT NOT EQUIVALENT TO;;;; +22E8;PRECEDES BUT NOT EQUIVALENT TO;Sm;0;ON;;;;;Y;;;;; +22E9;SUCCEEDS BUT NOT EQUIVALENT TO;Sm;0;ON;;;;;Y;;;;; +22EA;NOT NORMAL SUBGROUP OF;Sm;0;ON;22B2 0338;;;;Y;;;;; +22EB;DOES NOT CONTAIN AS NORMAL SUBGROUP;Sm;0;ON;22B3 0338;;;;Y;;;;; +22EC;NOT NORMAL SUBGROUP OF OR EQUAL TO;Sm;0;ON;22B4 0338;;;;Y;;;;; +22ED;DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL;Sm;0;ON;22B5 0338;;;;Y;;;;; +22EE;VERTICAL ELLIPSIS;Sm;0;ON;;;;;N;;;;; +22EF;MIDLINE HORIZONTAL ELLIPSIS;Sm;0;ON;;;;;N;;;;; +22F0;UP RIGHT DIAGONAL ELLIPSIS;Sm;0;ON;;;;;Y;;;;; +22F1;DOWN RIGHT DIAGONAL ELLIPSIS;Sm;0;ON;;;;;Y;;;;; +22F2;ELEMENT OF WITH LONG HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +22F3;ELEMENT OF WITH VERTICAL BAR AT END OF HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +22F4;SMALL ELEMENT OF WITH VERTICAL BAR AT END OF HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +22F5;ELEMENT OF WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +22F6;ELEMENT OF WITH OVERBAR;Sm;0;ON;;;;;Y;;;;; +22F7;SMALL ELEMENT OF WITH OVERBAR;Sm;0;ON;;;;;Y;;;;; +22F8;ELEMENT OF WITH UNDERBAR;Sm;0;ON;;;;;Y;;;;; +22F9;ELEMENT OF WITH TWO HORIZONTAL STROKES;Sm;0;ON;;;;;Y;;;;; +22FA;CONTAINS WITH LONG HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +22FB;CONTAINS WITH VERTICAL BAR AT END OF HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +22FC;SMALL CONTAINS WITH VERTICAL BAR AT END OF HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +22FD;CONTAINS WITH OVERBAR;Sm;0;ON;;;;;Y;;;;; +22FE;SMALL CONTAINS WITH OVERBAR;Sm;0;ON;;;;;Y;;;;; +22FF;Z NOTATION BAG MEMBERSHIP;Sm;0;ON;;;;;Y;;;;; +2300;DIAMETER SIGN;So;0;ON;;;;;N;;;;; +2301;ELECTRIC ARROW;So;0;ON;;;;;N;;;;; +2302;HOUSE;So;0;ON;;;;;N;;;;; +2303;UP ARROWHEAD;So;0;ON;;;;;N;;;;; +2304;DOWN ARROWHEAD;So;0;ON;;;;;N;;;;; +2305;PROJECTIVE;So;0;ON;;;;;N;;;;; +2306;PERSPECTIVE;So;0;ON;;;;;N;;;;; +2307;WAVY LINE;So;0;ON;;;;;N;;;;; +2308;LEFT CEILING;Sm;0;ON;;;;;Y;;;;; +2309;RIGHT CEILING;Sm;0;ON;;;;;Y;;;;; +230A;LEFT FLOOR;Sm;0;ON;;;;;Y;;;;; +230B;RIGHT FLOOR;Sm;0;ON;;;;;Y;;;;; +230C;BOTTOM RIGHT CROP;So;0;ON;;;;;N;;;;; +230D;BOTTOM LEFT CROP;So;0;ON;;;;;N;;;;; +230E;TOP RIGHT CROP;So;0;ON;;;;;N;;;;; +230F;TOP LEFT CROP;So;0;ON;;;;;N;;;;; +2310;REVERSED NOT SIGN;So;0;ON;;;;;N;;;;; +2311;SQUARE LOZENGE;So;0;ON;;;;;N;;;;; +2312;ARC;So;0;ON;;;;;N;;;;; +2313;SEGMENT;So;0;ON;;;;;N;;;;; +2314;SECTOR;So;0;ON;;;;;N;;;;; +2315;TELEPHONE RECORDER;So;0;ON;;;;;N;;;;; +2316;POSITION INDICATOR;So;0;ON;;;;;N;;;;; +2317;VIEWDATA SQUARE;So;0;ON;;;;;N;;;;; +2318;PLACE OF INTEREST SIGN;So;0;ON;;;;;N;COMMAND KEY;;;; +2319;TURNED NOT SIGN;So;0;ON;;;;;N;;;;; +231A;WATCH;So;0;ON;;;;;N;;;;; +231B;HOURGLASS;So;0;ON;;;;;N;;;;; +231C;TOP LEFT CORNER;So;0;ON;;;;;N;;;;; +231D;TOP RIGHT CORNER;So;0;ON;;;;;N;;;;; +231E;BOTTOM LEFT CORNER;So;0;ON;;;;;N;;;;; +231F;BOTTOM RIGHT CORNER;So;0;ON;;;;;N;;;;; +2320;TOP HALF INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2321;BOTTOM HALF INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2322;FROWN;So;0;ON;;;;;N;;;;; +2323;SMILE;So;0;ON;;;;;N;;;;; +2324;UP ARROWHEAD BETWEEN TWO HORIZONTAL BARS;So;0;ON;;;;;N;ENTER KEY;;;; +2325;OPTION KEY;So;0;ON;;;;;N;;;;; +2326;ERASE TO THE RIGHT;So;0;ON;;;;;N;DELETE TO THE RIGHT KEY;;;; +2327;X IN A RECTANGLE BOX;So;0;ON;;;;;N;CLEAR KEY;;;; +2328;KEYBOARD;So;0;ON;;;;;N;;;;; +2329;LEFT-POINTING ANGLE BRACKET;Ps;0;ON;3008;;;;Y;BRA;;;; +232A;RIGHT-POINTING ANGLE BRACKET;Pe;0;ON;3009;;;;Y;KET;;;; +232B;ERASE TO THE LEFT;So;0;ON;;;;;N;DELETE TO THE LEFT KEY;;;; +232C;BENZENE RING;So;0;ON;;;;;N;;;;; +232D;CYLINDRICITY;So;0;ON;;;;;N;;;;; +232E;ALL AROUND-PROFILE;So;0;ON;;;;;N;;;;; +232F;SYMMETRY;So;0;ON;;;;;N;;;;; +2330;TOTAL RUNOUT;So;0;ON;;;;;N;;;;; +2331;DIMENSION ORIGIN;So;0;ON;;;;;N;;;;; +2332;CONICAL TAPER;So;0;ON;;;;;N;;;;; +2333;SLOPE;So;0;ON;;;;;N;;;;; +2334;COUNTERBORE;So;0;ON;;;;;N;;;;; +2335;COUNTERSINK;So;0;ON;;;;;N;;;;; +2336;APL FUNCTIONAL SYMBOL I-BEAM;So;0;L;;;;;N;;;;; +2337;APL FUNCTIONAL SYMBOL SQUISH QUAD;So;0;L;;;;;N;;;;; +2338;APL FUNCTIONAL SYMBOL QUAD EQUAL;So;0;L;;;;;N;;;;; +2339;APL FUNCTIONAL SYMBOL QUAD DIVIDE;So;0;L;;;;;N;;;;; +233A;APL FUNCTIONAL SYMBOL QUAD DIAMOND;So;0;L;;;;;N;;;;; +233B;APL FUNCTIONAL SYMBOL QUAD JOT;So;0;L;;;;;N;;;;; +233C;APL FUNCTIONAL SYMBOL QUAD CIRCLE;So;0;L;;;;;N;;;;; +233D;APL FUNCTIONAL SYMBOL CIRCLE STILE;So;0;L;;;;;N;;;;; +233E;APL FUNCTIONAL SYMBOL CIRCLE JOT;So;0;L;;;;;N;;;;; +233F;APL FUNCTIONAL SYMBOL SLASH BAR;So;0;L;;;;;N;;;;; +2340;APL FUNCTIONAL SYMBOL BACKSLASH BAR;So;0;L;;;;;N;;;;; +2341;APL FUNCTIONAL SYMBOL QUAD SLASH;So;0;L;;;;;N;;;;; +2342;APL FUNCTIONAL SYMBOL QUAD BACKSLASH;So;0;L;;;;;N;;;;; +2343;APL FUNCTIONAL SYMBOL QUAD LESS-THAN;So;0;L;;;;;N;;;;; +2344;APL FUNCTIONAL SYMBOL QUAD GREATER-THAN;So;0;L;;;;;N;;;;; +2345;APL FUNCTIONAL SYMBOL LEFTWARDS VANE;So;0;L;;;;;N;;;;; +2346;APL FUNCTIONAL SYMBOL RIGHTWARDS VANE;So;0;L;;;;;N;;;;; +2347;APL FUNCTIONAL SYMBOL QUAD LEFTWARDS ARROW;So;0;L;;;;;N;;;;; +2348;APL FUNCTIONAL SYMBOL QUAD RIGHTWARDS ARROW;So;0;L;;;;;N;;;;; +2349;APL FUNCTIONAL SYMBOL CIRCLE BACKSLASH;So;0;L;;;;;N;;;;; +234A;APL FUNCTIONAL SYMBOL DOWN TACK UNDERBAR;So;0;L;;;;;N;;;;; +234B;APL FUNCTIONAL SYMBOL DELTA STILE;So;0;L;;;;;N;;;;; +234C;APL FUNCTIONAL SYMBOL QUAD DOWN CARET;So;0;L;;;;;N;;;;; +234D;APL FUNCTIONAL SYMBOL QUAD DELTA;So;0;L;;;;;N;;;;; +234E;APL FUNCTIONAL SYMBOL DOWN TACK JOT;So;0;L;;;;;N;;;;; +234F;APL FUNCTIONAL SYMBOL UPWARDS VANE;So;0;L;;;;;N;;;;; +2350;APL FUNCTIONAL SYMBOL QUAD UPWARDS ARROW;So;0;L;;;;;N;;;;; +2351;APL FUNCTIONAL SYMBOL UP TACK OVERBAR;So;0;L;;;;;N;;;;; +2352;APL FUNCTIONAL SYMBOL DEL STILE;So;0;L;;;;;N;;;;; +2353;APL FUNCTIONAL SYMBOL QUAD UP CARET;So;0;L;;;;;N;;;;; +2354;APL FUNCTIONAL SYMBOL QUAD DEL;So;0;L;;;;;N;;;;; +2355;APL FUNCTIONAL SYMBOL UP TACK JOT;So;0;L;;;;;N;;;;; +2356;APL FUNCTIONAL SYMBOL DOWNWARDS VANE;So;0;L;;;;;N;;;;; +2357;APL FUNCTIONAL SYMBOL QUAD DOWNWARDS ARROW;So;0;L;;;;;N;;;;; +2358;APL FUNCTIONAL SYMBOL QUOTE UNDERBAR;So;0;L;;;;;N;;;;; +2359;APL FUNCTIONAL SYMBOL DELTA UNDERBAR;So;0;L;;;;;N;;;;; +235A;APL FUNCTIONAL SYMBOL DIAMOND UNDERBAR;So;0;L;;;;;N;;;;; +235B;APL FUNCTIONAL SYMBOL JOT UNDERBAR;So;0;L;;;;;N;;;;; +235C;APL FUNCTIONAL SYMBOL CIRCLE UNDERBAR;So;0;L;;;;;N;;;;; +235D;APL FUNCTIONAL SYMBOL UP SHOE JOT;So;0;L;;;;;N;;;;; +235E;APL FUNCTIONAL SYMBOL QUOTE QUAD;So;0;L;;;;;N;;;;; +235F;APL FUNCTIONAL SYMBOL CIRCLE STAR;So;0;L;;;;;N;;;;; +2360;APL FUNCTIONAL SYMBOL QUAD COLON;So;0;L;;;;;N;;;;; +2361;APL FUNCTIONAL SYMBOL UP TACK DIAERESIS;So;0;L;;;;;N;;;;; +2362;APL FUNCTIONAL SYMBOL DEL DIAERESIS;So;0;L;;;;;N;;;;; +2363;APL FUNCTIONAL SYMBOL STAR DIAERESIS;So;0;L;;;;;N;;;;; +2364;APL FUNCTIONAL SYMBOL JOT DIAERESIS;So;0;L;;;;;N;;;;; +2365;APL FUNCTIONAL SYMBOL CIRCLE DIAERESIS;So;0;L;;;;;N;;;;; +2366;APL FUNCTIONAL SYMBOL DOWN SHOE STILE;So;0;L;;;;;N;;;;; +2367;APL FUNCTIONAL SYMBOL LEFT SHOE STILE;So;0;L;;;;;N;;;;; +2368;APL FUNCTIONAL SYMBOL TILDE DIAERESIS;So;0;L;;;;;N;;;;; +2369;APL FUNCTIONAL SYMBOL GREATER-THAN DIAERESIS;So;0;L;;;;;N;;;;; +236A;APL FUNCTIONAL SYMBOL COMMA BAR;So;0;L;;;;;N;;;;; +236B;APL FUNCTIONAL SYMBOL DEL TILDE;So;0;L;;;;;N;;;;; +236C;APL FUNCTIONAL SYMBOL ZILDE;So;0;L;;;;;N;;;;; +236D;APL FUNCTIONAL SYMBOL STILE TILDE;So;0;L;;;;;N;;;;; +236E;APL FUNCTIONAL SYMBOL SEMICOLON UNDERBAR;So;0;L;;;;;N;;;;; +236F;APL FUNCTIONAL SYMBOL QUAD NOT EQUAL;So;0;L;;;;;N;;;;; +2370;APL FUNCTIONAL SYMBOL QUAD QUESTION;So;0;L;;;;;N;;;;; +2371;APL FUNCTIONAL SYMBOL DOWN CARET TILDE;So;0;L;;;;;N;;;;; +2372;APL FUNCTIONAL SYMBOL UP CARET TILDE;So;0;L;;;;;N;;;;; +2373;APL FUNCTIONAL SYMBOL IOTA;So;0;L;;;;;N;;;;; +2374;APL FUNCTIONAL SYMBOL RHO;So;0;L;;;;;N;;;;; +2375;APL FUNCTIONAL SYMBOL OMEGA;So;0;L;;;;;N;;;;; +2376;APL FUNCTIONAL SYMBOL ALPHA UNDERBAR;So;0;L;;;;;N;;;;; +2377;APL FUNCTIONAL SYMBOL EPSILON UNDERBAR;So;0;L;;;;;N;;;;; +2378;APL FUNCTIONAL SYMBOL IOTA UNDERBAR;So;0;L;;;;;N;;;;; +2379;APL FUNCTIONAL SYMBOL OMEGA UNDERBAR;So;0;L;;;;;N;;;;; +237A;APL FUNCTIONAL SYMBOL ALPHA;So;0;L;;;;;N;;;;; +237B;NOT CHECK MARK;So;0;ON;;;;;N;;;;; +237C;RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW;Sm;0;ON;;;;;N;;;;; +237D;SHOULDERED OPEN BOX;So;0;ON;;;;;N;;;;; +237E;BELL SYMBOL;So;0;ON;;;;;N;;;;; +237F;VERTICAL LINE WITH MIDDLE DOT;So;0;ON;;;;;N;;;;; +2380;INSERTION SYMBOL;So;0;ON;;;;;N;;;;; +2381;CONTINUOUS UNDERLINE SYMBOL;So;0;ON;;;;;N;;;;; +2382;DISCONTINUOUS UNDERLINE SYMBOL;So;0;ON;;;;;N;;;;; +2383;EMPHASIS SYMBOL;So;0;ON;;;;;N;;;;; +2384;COMPOSITION SYMBOL;So;0;ON;;;;;N;;;;; +2385;WHITE SQUARE WITH CENTRE VERTICAL LINE;So;0;ON;;;;;N;;;;; +2386;ENTER SYMBOL;So;0;ON;;;;;N;;;;; +2387;ALTERNATIVE KEY SYMBOL;So;0;ON;;;;;N;;;;; +2388;HELM SYMBOL;So;0;ON;;;;;N;;;;; +2389;CIRCLED HORIZONTAL BAR WITH NOTCH;So;0;ON;;;;;N;;;;; +238A;CIRCLED TRIANGLE DOWN;So;0;ON;;;;;N;;;;; +238B;BROKEN CIRCLE WITH NORTHWEST ARROW;So;0;ON;;;;;N;;;;; +238C;UNDO SYMBOL;So;0;ON;;;;;N;;;;; +238D;MONOSTABLE SYMBOL;So;0;ON;;;;;N;;;;; +238E;HYSTERESIS SYMBOL;So;0;ON;;;;;N;;;;; +238F;OPEN-CIRCUIT-OUTPUT H-TYPE SYMBOL;So;0;ON;;;;;N;;;;; +2390;OPEN-CIRCUIT-OUTPUT L-TYPE SYMBOL;So;0;ON;;;;;N;;;;; +2391;PASSIVE-PULL-DOWN-OUTPUT SYMBOL;So;0;ON;;;;;N;;;;; +2392;PASSIVE-PULL-UP-OUTPUT SYMBOL;So;0;ON;;;;;N;;;;; +2393;DIRECT CURRENT SYMBOL FORM TWO;So;0;ON;;;;;N;;;;; +2394;SOFTWARE-FUNCTION SYMBOL;So;0;ON;;;;;N;;;;; +2395;APL FUNCTIONAL SYMBOL QUAD;So;0;L;;;;;N;;;;; +2396;DECIMAL SEPARATOR KEY SYMBOL;So;0;ON;;;;;N;;;;; +2397;PREVIOUS PAGE;So;0;ON;;;;;N;;;;; +2398;NEXT PAGE;So;0;ON;;;;;N;;;;; +2399;PRINT SCREEN SYMBOL;So;0;ON;;;;;N;;;;; +239A;CLEAR SCREEN SYMBOL;So;0;ON;;;;;N;;;;; +239B;LEFT PARENTHESIS UPPER HOOK;Sm;0;ON;;;;;N;;;;; +239C;LEFT PARENTHESIS EXTENSION;Sm;0;ON;;;;;N;;;;; +239D;LEFT PARENTHESIS LOWER HOOK;Sm;0;ON;;;;;N;;;;; +239E;RIGHT PARENTHESIS UPPER HOOK;Sm;0;ON;;;;;N;;;;; +239F;RIGHT PARENTHESIS EXTENSION;Sm;0;ON;;;;;N;;;;; +23A0;RIGHT PARENTHESIS LOWER HOOK;Sm;0;ON;;;;;N;;;;; +23A1;LEFT SQUARE BRACKET UPPER CORNER;Sm;0;ON;;;;;N;;;;; +23A2;LEFT SQUARE BRACKET EXTENSION;Sm;0;ON;;;;;N;;;;; +23A3;LEFT SQUARE BRACKET LOWER CORNER;Sm;0;ON;;;;;N;;;;; +23A4;RIGHT SQUARE BRACKET UPPER CORNER;Sm;0;ON;;;;;N;;;;; +23A5;RIGHT SQUARE BRACKET EXTENSION;Sm;0;ON;;;;;N;;;;; +23A6;RIGHT SQUARE BRACKET LOWER CORNER;Sm;0;ON;;;;;N;;;;; +23A7;LEFT CURLY BRACKET UPPER HOOK;Sm;0;ON;;;;;N;;;;; +23A8;LEFT CURLY BRACKET MIDDLE PIECE;Sm;0;ON;;;;;N;;;;; +23A9;LEFT CURLY BRACKET LOWER HOOK;Sm;0;ON;;;;;N;;;;; +23AA;CURLY BRACKET EXTENSION;Sm;0;ON;;;;;N;;;;; +23AB;RIGHT CURLY BRACKET UPPER HOOK;Sm;0;ON;;;;;N;;;;; +23AC;RIGHT CURLY BRACKET MIDDLE PIECE;Sm;0;ON;;;;;N;;;;; +23AD;RIGHT CURLY BRACKET LOWER HOOK;Sm;0;ON;;;;;N;;;;; +23AE;INTEGRAL EXTENSION;Sm;0;ON;;;;;N;;;;; +23AF;HORIZONTAL LINE EXTENSION;Sm;0;ON;;;;;N;;;;; +23B0;UPPER LEFT OR LOWER RIGHT CURLY BRACKET SECTION;Sm;0;ON;;;;;N;;;;; +23B1;UPPER RIGHT OR LOWER LEFT CURLY BRACKET SECTION;Sm;0;ON;;;;;N;;;;; +23B2;SUMMATION TOP;Sm;0;ON;;;;;N;;;;; +23B3;SUMMATION BOTTOM;Sm;0;ON;;;;;N;;;;; +23B4;TOP SQUARE BRACKET;So;0;ON;;;;;N;;;;; +23B5;BOTTOM SQUARE BRACKET;So;0;ON;;;;;N;;;;; +23B6;BOTTOM SQUARE BRACKET OVER TOP SQUARE BRACKET;So;0;ON;;;;;N;;;;; +23B7;RADICAL SYMBOL BOTTOM;So;0;ON;;;;;N;;;;; +23B8;LEFT VERTICAL BOX LINE;So;0;ON;;;;;N;;;;; +23B9;RIGHT VERTICAL BOX LINE;So;0;ON;;;;;N;;;;; +23BA;HORIZONTAL SCAN LINE-1;So;0;ON;;;;;N;;;;; +23BB;HORIZONTAL SCAN LINE-3;So;0;ON;;;;;N;;;;; +23BC;HORIZONTAL SCAN LINE-7;So;0;ON;;;;;N;;;;; +23BD;HORIZONTAL SCAN LINE-9;So;0;ON;;;;;N;;;;; +23BE;DENTISTRY SYMBOL LIGHT VERTICAL AND TOP RIGHT;So;0;ON;;;;;N;;;;; +23BF;DENTISTRY SYMBOL LIGHT VERTICAL AND BOTTOM RIGHT;So;0;ON;;;;;N;;;;; +23C0;DENTISTRY SYMBOL LIGHT VERTICAL WITH CIRCLE;So;0;ON;;;;;N;;;;; +23C1;DENTISTRY SYMBOL LIGHT DOWN AND HORIZONTAL WITH CIRCLE;So;0;ON;;;;;N;;;;; +23C2;DENTISTRY SYMBOL LIGHT UP AND HORIZONTAL WITH CIRCLE;So;0;ON;;;;;N;;;;; +23C3;DENTISTRY SYMBOL LIGHT VERTICAL WITH TRIANGLE;So;0;ON;;;;;N;;;;; +23C4;DENTISTRY SYMBOL LIGHT DOWN AND HORIZONTAL WITH TRIANGLE;So;0;ON;;;;;N;;;;; +23C5;DENTISTRY SYMBOL LIGHT UP AND HORIZONTAL WITH TRIANGLE;So;0;ON;;;;;N;;;;; +23C6;DENTISTRY SYMBOL LIGHT VERTICAL AND WAVE;So;0;ON;;;;;N;;;;; +23C7;DENTISTRY SYMBOL LIGHT DOWN AND HORIZONTAL WITH WAVE;So;0;ON;;;;;N;;;;; +23C8;DENTISTRY SYMBOL LIGHT UP AND HORIZONTAL WITH WAVE;So;0;ON;;;;;N;;;;; +23C9;DENTISTRY SYMBOL LIGHT DOWN AND HORIZONTAL;So;0;ON;;;;;N;;;;; +23CA;DENTISTRY SYMBOL LIGHT UP AND HORIZONTAL;So;0;ON;;;;;N;;;;; +23CB;DENTISTRY SYMBOL LIGHT VERTICAL AND TOP LEFT;So;0;ON;;;;;N;;;;; +23CC;DENTISTRY SYMBOL LIGHT VERTICAL AND BOTTOM LEFT;So;0;ON;;;;;N;;;;; +23CD;SQUARE FOOT;So;0;ON;;;;;N;;;;; +23CE;RETURN SYMBOL;So;0;ON;;;;;N;;;;; +23CF;EJECT SYMBOL;So;0;ON;;;;;N;;;;; +23D0;VERTICAL LINE EXTENSION;So;0;ON;;;;;N;;;;; +23D1;METRICAL BREVE;So;0;ON;;;;;N;;;;; +23D2;METRICAL LONG OVER SHORT;So;0;ON;;;;;N;;;;; +23D3;METRICAL SHORT OVER LONG;So;0;ON;;;;;N;;;;; +23D4;METRICAL LONG OVER TWO SHORTS;So;0;ON;;;;;N;;;;; +23D5;METRICAL TWO SHORTS OVER LONG;So;0;ON;;;;;N;;;;; +23D6;METRICAL TWO SHORTS JOINED;So;0;ON;;;;;N;;;;; +23D7;METRICAL TRISEME;So;0;ON;;;;;N;;;;; +23D8;METRICAL TETRASEME;So;0;ON;;;;;N;;;;; +23D9;METRICAL PENTASEME;So;0;ON;;;;;N;;;;; +23DA;EARTH GROUND;So;0;ON;;;;;N;;;;; +23DB;FUSE;So;0;ON;;;;;N;;;;; +23DC;TOP PARENTHESIS;Sm;0;ON;;;;;N;;;;; +23DD;BOTTOM PARENTHESIS;Sm;0;ON;;;;;N;;;;; +23DE;TOP CURLY BRACKET;Sm;0;ON;;;;;N;;;;; +23DF;BOTTOM CURLY BRACKET;Sm;0;ON;;;;;N;;;;; +23E0;TOP TORTOISE SHELL BRACKET;Sm;0;ON;;;;;N;;;;; +23E1;BOTTOM TORTOISE SHELL BRACKET;Sm;0;ON;;;;;N;;;;; +23E2;WHITE TRAPEZIUM;So;0;ON;;;;;N;;;;; +23E3;BENZENE RING WITH CIRCLE;So;0;ON;;;;;N;;;;; +23E4;STRAIGHTNESS;So;0;ON;;;;;N;;;;; +23E5;FLATNESS;So;0;ON;;;;;N;;;;; +23E6;AC CURRENT;So;0;ON;;;;;N;;;;; +23E7;ELECTRICAL INTERSECTION;So;0;ON;;;;;N;;;;; +23E8;DECIMAL EXPONENT SYMBOL;So;0;ON;;;;;N;;;;; +23E9;BLACK RIGHT-POINTING DOUBLE TRIANGLE;So;0;ON;;;;;N;;;;; +23EA;BLACK LEFT-POINTING DOUBLE TRIANGLE;So;0;ON;;;;;N;;;;; +23EB;BLACK UP-POINTING DOUBLE TRIANGLE;So;0;ON;;;;;N;;;;; +23EC;BLACK DOWN-POINTING DOUBLE TRIANGLE;So;0;ON;;;;;N;;;;; +23ED;BLACK RIGHT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR;So;0;ON;;;;;N;;;;; +23EE;BLACK LEFT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR;So;0;ON;;;;;N;;;;; +23EF;BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR;So;0;ON;;;;;N;;;;; +23F0;ALARM CLOCK;So;0;ON;;;;;N;;;;; +23F1;STOPWATCH;So;0;ON;;;;;N;;;;; +23F2;TIMER CLOCK;So;0;ON;;;;;N;;;;; +23F3;HOURGLASS WITH FLOWING SAND;So;0;ON;;;;;N;;;;; +2400;SYMBOL FOR NULL;So;0;ON;;;;;N;GRAPHIC FOR NULL;;;; +2401;SYMBOL FOR START OF HEADING;So;0;ON;;;;;N;GRAPHIC FOR START OF HEADING;;;; +2402;SYMBOL FOR START OF TEXT;So;0;ON;;;;;N;GRAPHIC FOR START OF TEXT;;;; +2403;SYMBOL FOR END OF TEXT;So;0;ON;;;;;N;GRAPHIC FOR END OF TEXT;;;; +2404;SYMBOL FOR END OF TRANSMISSION;So;0;ON;;;;;N;GRAPHIC FOR END OF TRANSMISSION;;;; +2405;SYMBOL FOR ENQUIRY;So;0;ON;;;;;N;GRAPHIC FOR ENQUIRY;;;; +2406;SYMBOL FOR ACKNOWLEDGE;So;0;ON;;;;;N;GRAPHIC FOR ACKNOWLEDGE;;;; +2407;SYMBOL FOR BELL;So;0;ON;;;;;N;GRAPHIC FOR BELL;;;; +2408;SYMBOL FOR BACKSPACE;So;0;ON;;;;;N;GRAPHIC FOR BACKSPACE;;;; +2409;SYMBOL FOR HORIZONTAL TABULATION;So;0;ON;;;;;N;GRAPHIC FOR HORIZONTAL TABULATION;;;; +240A;SYMBOL FOR LINE FEED;So;0;ON;;;;;N;GRAPHIC FOR LINE FEED;;;; +240B;SYMBOL FOR VERTICAL TABULATION;So;0;ON;;;;;N;GRAPHIC FOR VERTICAL TABULATION;;;; +240C;SYMBOL FOR FORM FEED;So;0;ON;;;;;N;GRAPHIC FOR FORM FEED;;;; +240D;SYMBOL FOR CARRIAGE RETURN;So;0;ON;;;;;N;GRAPHIC FOR CARRIAGE RETURN;;;; +240E;SYMBOL FOR SHIFT OUT;So;0;ON;;;;;N;GRAPHIC FOR SHIFT OUT;;;; +240F;SYMBOL FOR SHIFT IN;So;0;ON;;;;;N;GRAPHIC FOR SHIFT IN;;;; +2410;SYMBOL FOR DATA LINK ESCAPE;So;0;ON;;;;;N;GRAPHIC FOR DATA LINK ESCAPE;;;; +2411;SYMBOL FOR DEVICE CONTROL ONE;So;0;ON;;;;;N;GRAPHIC FOR DEVICE CONTROL ONE;;;; +2412;SYMBOL FOR DEVICE CONTROL TWO;So;0;ON;;;;;N;GRAPHIC FOR DEVICE CONTROL TWO;;;; +2413;SYMBOL FOR DEVICE CONTROL THREE;So;0;ON;;;;;N;GRAPHIC FOR DEVICE CONTROL THREE;;;; +2414;SYMBOL FOR DEVICE CONTROL FOUR;So;0;ON;;;;;N;GRAPHIC FOR DEVICE CONTROL FOUR;;;; +2415;SYMBOL FOR NEGATIVE ACKNOWLEDGE;So;0;ON;;;;;N;GRAPHIC FOR NEGATIVE ACKNOWLEDGE;;;; +2416;SYMBOL FOR SYNCHRONOUS IDLE;So;0;ON;;;;;N;GRAPHIC FOR SYNCHRONOUS IDLE;;;; +2417;SYMBOL FOR END OF TRANSMISSION BLOCK;So;0;ON;;;;;N;GRAPHIC FOR END OF TRANSMISSION BLOCK;;;; +2418;SYMBOL FOR CANCEL;So;0;ON;;;;;N;GRAPHIC FOR CANCEL;;;; +2419;SYMBOL FOR END OF MEDIUM;So;0;ON;;;;;N;GRAPHIC FOR END OF MEDIUM;;;; +241A;SYMBOL FOR SUBSTITUTE;So;0;ON;;;;;N;GRAPHIC FOR SUBSTITUTE;;;; +241B;SYMBOL FOR ESCAPE;So;0;ON;;;;;N;GRAPHIC FOR ESCAPE;;;; +241C;SYMBOL FOR FILE SEPARATOR;So;0;ON;;;;;N;GRAPHIC FOR FILE SEPARATOR;;;; +241D;SYMBOL FOR GROUP SEPARATOR;So;0;ON;;;;;N;GRAPHIC FOR GROUP SEPARATOR;;;; +241E;SYMBOL FOR RECORD SEPARATOR;So;0;ON;;;;;N;GRAPHIC FOR RECORD SEPARATOR;;;; +241F;SYMBOL FOR UNIT SEPARATOR;So;0;ON;;;;;N;GRAPHIC FOR UNIT SEPARATOR;;;; +2420;SYMBOL FOR SPACE;So;0;ON;;;;;N;GRAPHIC FOR SPACE;;;; +2421;SYMBOL FOR DELETE;So;0;ON;;;;;N;GRAPHIC FOR DELETE;;;; +2422;BLANK SYMBOL;So;0;ON;;;;;N;BLANK;;;; +2423;OPEN BOX;So;0;ON;;;;;N;;;;; +2424;SYMBOL FOR NEWLINE;So;0;ON;;;;;N;GRAPHIC FOR NEWLINE;;;; +2425;SYMBOL FOR DELETE FORM TWO;So;0;ON;;;;;N;;;;; +2426;SYMBOL FOR SUBSTITUTE FORM TWO;So;0;ON;;;;;N;;;;; +2440;OCR HOOK;So;0;ON;;;;;N;;;;; +2441;OCR CHAIR;So;0;ON;;;;;N;;;;; +2442;OCR FORK;So;0;ON;;;;;N;;;;; +2443;OCR INVERTED FORK;So;0;ON;;;;;N;;;;; +2444;OCR BELT BUCKLE;So;0;ON;;;;;N;;;;; +2445;OCR BOW TIE;So;0;ON;;;;;N;;;;; +2446;OCR BRANCH BANK IDENTIFICATION;So;0;ON;;;;;N;;;;; +2447;OCR AMOUNT OF CHECK;So;0;ON;;;;;N;;;;; +2448;OCR DASH;So;0;ON;;;;;N;;;;; +2449;OCR CUSTOMER ACCOUNT NUMBER;So;0;ON;;;;;N;;;;; +244A;OCR DOUBLE BACKSLASH;So;0;ON;;;;;N;;;;; +2460;CIRCLED DIGIT ONE;No;0;ON; 0031;;1;1;N;;;;; +2461;CIRCLED DIGIT TWO;No;0;ON; 0032;;2;2;N;;;;; +2462;CIRCLED DIGIT THREE;No;0;ON; 0033;;3;3;N;;;;; +2463;CIRCLED DIGIT FOUR;No;0;ON; 0034;;4;4;N;;;;; +2464;CIRCLED DIGIT FIVE;No;0;ON; 0035;;5;5;N;;;;; +2465;CIRCLED DIGIT SIX;No;0;ON; 0036;;6;6;N;;;;; +2466;CIRCLED DIGIT SEVEN;No;0;ON; 0037;;7;7;N;;;;; +2467;CIRCLED DIGIT EIGHT;No;0;ON; 0038;;8;8;N;;;;; +2468;CIRCLED DIGIT NINE;No;0;ON; 0039;;9;9;N;;;;; +2469;CIRCLED NUMBER TEN;No;0;ON; 0031 0030;;;10;N;;;;; +246A;CIRCLED NUMBER ELEVEN;No;0;ON; 0031 0031;;;11;N;;;;; +246B;CIRCLED NUMBER TWELVE;No;0;ON; 0031 0032;;;12;N;;;;; +246C;CIRCLED NUMBER THIRTEEN;No;0;ON; 0031 0033;;;13;N;;;;; +246D;CIRCLED NUMBER FOURTEEN;No;0;ON; 0031 0034;;;14;N;;;;; +246E;CIRCLED NUMBER FIFTEEN;No;0;ON; 0031 0035;;;15;N;;;;; +246F;CIRCLED NUMBER SIXTEEN;No;0;ON; 0031 0036;;;16;N;;;;; +2470;CIRCLED NUMBER SEVENTEEN;No;0;ON; 0031 0037;;;17;N;;;;; +2471;CIRCLED NUMBER EIGHTEEN;No;0;ON; 0031 0038;;;18;N;;;;; +2472;CIRCLED NUMBER NINETEEN;No;0;ON; 0031 0039;;;19;N;;;;; +2473;CIRCLED NUMBER TWENTY;No;0;ON; 0032 0030;;;20;N;;;;; +2474;PARENTHESIZED DIGIT ONE;No;0;ON; 0028 0031 0029;;1;1;N;;;;; +2475;PARENTHESIZED DIGIT TWO;No;0;ON; 0028 0032 0029;;2;2;N;;;;; +2476;PARENTHESIZED DIGIT THREE;No;0;ON; 0028 0033 0029;;3;3;N;;;;; +2477;PARENTHESIZED DIGIT FOUR;No;0;ON; 0028 0034 0029;;4;4;N;;;;; +2478;PARENTHESIZED DIGIT FIVE;No;0;ON; 0028 0035 0029;;5;5;N;;;;; +2479;PARENTHESIZED DIGIT SIX;No;0;ON; 0028 0036 0029;;6;6;N;;;;; +247A;PARENTHESIZED DIGIT SEVEN;No;0;ON; 0028 0037 0029;;7;7;N;;;;; +247B;PARENTHESIZED DIGIT EIGHT;No;0;ON; 0028 0038 0029;;8;8;N;;;;; +247C;PARENTHESIZED DIGIT NINE;No;0;ON; 0028 0039 0029;;9;9;N;;;;; +247D;PARENTHESIZED NUMBER TEN;No;0;ON; 0028 0031 0030 0029;;;10;N;;;;; +247E;PARENTHESIZED NUMBER ELEVEN;No;0;ON; 0028 0031 0031 0029;;;11;N;;;;; +247F;PARENTHESIZED NUMBER TWELVE;No;0;ON; 0028 0031 0032 0029;;;12;N;;;;; +2480;PARENTHESIZED NUMBER THIRTEEN;No;0;ON; 0028 0031 0033 0029;;;13;N;;;;; +2481;PARENTHESIZED NUMBER FOURTEEN;No;0;ON; 0028 0031 0034 0029;;;14;N;;;;; +2482;PARENTHESIZED NUMBER FIFTEEN;No;0;ON; 0028 0031 0035 0029;;;15;N;;;;; +2483;PARENTHESIZED NUMBER SIXTEEN;No;0;ON; 0028 0031 0036 0029;;;16;N;;;;; +2484;PARENTHESIZED NUMBER SEVENTEEN;No;0;ON; 0028 0031 0037 0029;;;17;N;;;;; +2485;PARENTHESIZED NUMBER EIGHTEEN;No;0;ON; 0028 0031 0038 0029;;;18;N;;;;; +2486;PARENTHESIZED NUMBER NINETEEN;No;0;ON; 0028 0031 0039 0029;;;19;N;;;;; +2487;PARENTHESIZED NUMBER TWENTY;No;0;ON; 0028 0032 0030 0029;;;20;N;;;;; +2488;DIGIT ONE FULL STOP;No;0;EN; 0031 002E;;1;1;N;DIGIT ONE PERIOD;;;; +2489;DIGIT TWO FULL STOP;No;0;EN; 0032 002E;;2;2;N;DIGIT TWO PERIOD;;;; +248A;DIGIT THREE FULL STOP;No;0;EN; 0033 002E;;3;3;N;DIGIT THREE PERIOD;;;; +248B;DIGIT FOUR FULL STOP;No;0;EN; 0034 002E;;4;4;N;DIGIT FOUR PERIOD;;;; +248C;DIGIT FIVE FULL STOP;No;0;EN; 0035 002E;;5;5;N;DIGIT FIVE PERIOD;;;; +248D;DIGIT SIX FULL STOP;No;0;EN; 0036 002E;;6;6;N;DIGIT SIX PERIOD;;;; +248E;DIGIT SEVEN FULL STOP;No;0;EN; 0037 002E;;7;7;N;DIGIT SEVEN PERIOD;;;; +248F;DIGIT EIGHT FULL STOP;No;0;EN; 0038 002E;;8;8;N;DIGIT EIGHT PERIOD;;;; +2490;DIGIT NINE FULL STOP;No;0;EN; 0039 002E;;9;9;N;DIGIT NINE PERIOD;;;; +2491;NUMBER TEN FULL STOP;No;0;EN; 0031 0030 002E;;;10;N;NUMBER TEN PERIOD;;;; +2492;NUMBER ELEVEN FULL STOP;No;0;EN; 0031 0031 002E;;;11;N;NUMBER ELEVEN PERIOD;;;; +2493;NUMBER TWELVE FULL STOP;No;0;EN; 0031 0032 002E;;;12;N;NUMBER TWELVE PERIOD;;;; +2494;NUMBER THIRTEEN FULL STOP;No;0;EN; 0031 0033 002E;;;13;N;NUMBER THIRTEEN PERIOD;;;; +2495;NUMBER FOURTEEN FULL STOP;No;0;EN; 0031 0034 002E;;;14;N;NUMBER FOURTEEN PERIOD;;;; +2496;NUMBER FIFTEEN FULL STOP;No;0;EN; 0031 0035 002E;;;15;N;NUMBER FIFTEEN PERIOD;;;; +2497;NUMBER SIXTEEN FULL STOP;No;0;EN; 0031 0036 002E;;;16;N;NUMBER SIXTEEN PERIOD;;;; +2498;NUMBER SEVENTEEN FULL STOP;No;0;EN; 0031 0037 002E;;;17;N;NUMBER SEVENTEEN PERIOD;;;; +2499;NUMBER EIGHTEEN FULL STOP;No;0;EN; 0031 0038 002E;;;18;N;NUMBER EIGHTEEN PERIOD;;;; +249A;NUMBER NINETEEN FULL STOP;No;0;EN; 0031 0039 002E;;;19;N;NUMBER NINETEEN PERIOD;;;; +249B;NUMBER TWENTY FULL STOP;No;0;EN; 0032 0030 002E;;;20;N;NUMBER TWENTY PERIOD;;;; +249C;PARENTHESIZED LATIN SMALL LETTER A;So;0;L; 0028 0061 0029;;;;N;;;;; +249D;PARENTHESIZED LATIN SMALL LETTER B;So;0;L; 0028 0062 0029;;;;N;;;;; +249E;PARENTHESIZED LATIN SMALL LETTER C;So;0;L; 0028 0063 0029;;;;N;;;;; +249F;PARENTHESIZED LATIN SMALL LETTER D;So;0;L; 0028 0064 0029;;;;N;;;;; +24A0;PARENTHESIZED LATIN SMALL LETTER E;So;0;L; 0028 0065 0029;;;;N;;;;; +24A1;PARENTHESIZED LATIN SMALL LETTER F;So;0;L; 0028 0066 0029;;;;N;;;;; +24A2;PARENTHESIZED LATIN SMALL LETTER G;So;0;L; 0028 0067 0029;;;;N;;;;; +24A3;PARENTHESIZED LATIN SMALL LETTER H;So;0;L; 0028 0068 0029;;;;N;;;;; +24A4;PARENTHESIZED LATIN SMALL LETTER I;So;0;L; 0028 0069 0029;;;;N;;;;; +24A5;PARENTHESIZED LATIN SMALL LETTER J;So;0;L; 0028 006A 0029;;;;N;;;;; +24A6;PARENTHESIZED LATIN SMALL LETTER K;So;0;L; 0028 006B 0029;;;;N;;;;; +24A7;PARENTHESIZED LATIN SMALL LETTER L;So;0;L; 0028 006C 0029;;;;N;;;;; +24A8;PARENTHESIZED LATIN SMALL LETTER M;So;0;L; 0028 006D 0029;;;;N;;;;; +24A9;PARENTHESIZED LATIN SMALL LETTER N;So;0;L; 0028 006E 0029;;;;N;;;;; +24AA;PARENTHESIZED LATIN SMALL LETTER O;So;0;L; 0028 006F 0029;;;;N;;;;; +24AB;PARENTHESIZED LATIN SMALL LETTER P;So;0;L; 0028 0070 0029;;;;N;;;;; +24AC;PARENTHESIZED LATIN SMALL LETTER Q;So;0;L; 0028 0071 0029;;;;N;;;;; +24AD;PARENTHESIZED LATIN SMALL LETTER R;So;0;L; 0028 0072 0029;;;;N;;;;; +24AE;PARENTHESIZED LATIN SMALL LETTER S;So;0;L; 0028 0073 0029;;;;N;;;;; +24AF;PARENTHESIZED LATIN SMALL LETTER T;So;0;L; 0028 0074 0029;;;;N;;;;; +24B0;PARENTHESIZED LATIN SMALL LETTER U;So;0;L; 0028 0075 0029;;;;N;;;;; +24B1;PARENTHESIZED LATIN SMALL LETTER V;So;0;L; 0028 0076 0029;;;;N;;;;; +24B2;PARENTHESIZED LATIN SMALL LETTER W;So;0;L; 0028 0077 0029;;;;N;;;;; +24B3;PARENTHESIZED LATIN SMALL LETTER X;So;0;L; 0028 0078 0029;;;;N;;;;; +24B4;PARENTHESIZED LATIN SMALL LETTER Y;So;0;L; 0028 0079 0029;;;;N;;;;; +24B5;PARENTHESIZED LATIN SMALL LETTER Z;So;0;L; 0028 007A 0029;;;;N;;;;; +24B6;CIRCLED LATIN CAPITAL LETTER A;So;0;L; 0041;;;;N;;;;24D0; +24B7;CIRCLED LATIN CAPITAL LETTER B;So;0;L; 0042;;;;N;;;;24D1; +24B8;CIRCLED LATIN CAPITAL LETTER C;So;0;L; 0043;;;;N;;;;24D2; +24B9;CIRCLED LATIN CAPITAL LETTER D;So;0;L; 0044;;;;N;;;;24D3; +24BA;CIRCLED LATIN CAPITAL LETTER E;So;0;L; 0045;;;;N;;;;24D4; +24BB;CIRCLED LATIN CAPITAL LETTER F;So;0;L; 0046;;;;N;;;;24D5; +24BC;CIRCLED LATIN CAPITAL LETTER G;So;0;L; 0047;;;;N;;;;24D6; +24BD;CIRCLED LATIN CAPITAL LETTER H;So;0;L; 0048;;;;N;;;;24D7; +24BE;CIRCLED LATIN CAPITAL LETTER I;So;0;L; 0049;;;;N;;;;24D8; +24BF;CIRCLED LATIN CAPITAL LETTER J;So;0;L; 004A;;;;N;;;;24D9; +24C0;CIRCLED LATIN CAPITAL LETTER K;So;0;L; 004B;;;;N;;;;24DA; +24C1;CIRCLED LATIN CAPITAL LETTER L;So;0;L; 004C;;;;N;;;;24DB; +24C2;CIRCLED LATIN CAPITAL LETTER M;So;0;L; 004D;;;;N;;;;24DC; +24C3;CIRCLED LATIN CAPITAL LETTER N;So;0;L; 004E;;;;N;;;;24DD; +24C4;CIRCLED LATIN CAPITAL LETTER O;So;0;L; 004F;;;;N;;;;24DE; +24C5;CIRCLED LATIN CAPITAL LETTER P;So;0;L; 0050;;;;N;;;;24DF; +24C6;CIRCLED LATIN CAPITAL LETTER Q;So;0;L; 0051;;;;N;;;;24E0; +24C7;CIRCLED LATIN CAPITAL LETTER R;So;0;L; 0052;;;;N;;;;24E1; +24C8;CIRCLED LATIN CAPITAL LETTER S;So;0;L; 0053;;;;N;;;;24E2; +24C9;CIRCLED LATIN CAPITAL LETTER T;So;0;L; 0054;;;;N;;;;24E3; +24CA;CIRCLED LATIN CAPITAL LETTER U;So;0;L; 0055;;;;N;;;;24E4; +24CB;CIRCLED LATIN CAPITAL LETTER V;So;0;L; 0056;;;;N;;;;24E5; +24CC;CIRCLED LATIN CAPITAL LETTER W;So;0;L; 0057;;;;N;;;;24E6; +24CD;CIRCLED LATIN CAPITAL LETTER X;So;0;L; 0058;;;;N;;;;24E7; +24CE;CIRCLED LATIN CAPITAL LETTER Y;So;0;L; 0059;;;;N;;;;24E8; +24CF;CIRCLED LATIN CAPITAL LETTER Z;So;0;L; 005A;;;;N;;;;24E9; +24D0;CIRCLED LATIN SMALL LETTER A;So;0;L; 0061;;;;N;;;24B6;;24B6 +24D1;CIRCLED LATIN SMALL LETTER B;So;0;L; 0062;;;;N;;;24B7;;24B7 +24D2;CIRCLED LATIN SMALL LETTER C;So;0;L; 0063;;;;N;;;24B8;;24B8 +24D3;CIRCLED LATIN SMALL LETTER D;So;0;L; 0064;;;;N;;;24B9;;24B9 +24D4;CIRCLED LATIN SMALL LETTER E;So;0;L; 0065;;;;N;;;24BA;;24BA +24D5;CIRCLED LATIN SMALL LETTER F;So;0;L; 0066;;;;N;;;24BB;;24BB +24D6;CIRCLED LATIN SMALL LETTER G;So;0;L; 0067;;;;N;;;24BC;;24BC +24D7;CIRCLED LATIN SMALL LETTER H;So;0;L; 0068;;;;N;;;24BD;;24BD +24D8;CIRCLED LATIN SMALL LETTER I;So;0;L; 0069;;;;N;;;24BE;;24BE +24D9;CIRCLED LATIN SMALL LETTER J;So;0;L; 006A;;;;N;;;24BF;;24BF +24DA;CIRCLED LATIN SMALL LETTER K;So;0;L; 006B;;;;N;;;24C0;;24C0 +24DB;CIRCLED LATIN SMALL LETTER L;So;0;L; 006C;;;;N;;;24C1;;24C1 +24DC;CIRCLED LATIN SMALL LETTER M;So;0;L; 006D;;;;N;;;24C2;;24C2 +24DD;CIRCLED LATIN SMALL LETTER N;So;0;L; 006E;;;;N;;;24C3;;24C3 +24DE;CIRCLED LATIN SMALL LETTER O;So;0;L; 006F;;;;N;;;24C4;;24C4 +24DF;CIRCLED LATIN SMALL LETTER P;So;0;L; 0070;;;;N;;;24C5;;24C5 +24E0;CIRCLED LATIN SMALL LETTER Q;So;0;L; 0071;;;;N;;;24C6;;24C6 +24E1;CIRCLED LATIN SMALL LETTER R;So;0;L; 0072;;;;N;;;24C7;;24C7 +24E2;CIRCLED LATIN SMALL LETTER S;So;0;L; 0073;;;;N;;;24C8;;24C8 +24E3;CIRCLED LATIN SMALL LETTER T;So;0;L; 0074;;;;N;;;24C9;;24C9 +24E4;CIRCLED LATIN SMALL LETTER U;So;0;L; 0075;;;;N;;;24CA;;24CA +24E5;CIRCLED LATIN SMALL LETTER V;So;0;L; 0076;;;;N;;;24CB;;24CB +24E6;CIRCLED LATIN SMALL LETTER W;So;0;L; 0077;;;;N;;;24CC;;24CC +24E7;CIRCLED LATIN SMALL LETTER X;So;0;L; 0078;;;;N;;;24CD;;24CD +24E8;CIRCLED LATIN SMALL LETTER Y;So;0;L; 0079;;;;N;;;24CE;;24CE +24E9;CIRCLED LATIN SMALL LETTER Z;So;0;L; 007A;;;;N;;;24CF;;24CF +24EA;CIRCLED DIGIT ZERO;No;0;ON; 0030;;0;0;N;;;;; +24EB;NEGATIVE CIRCLED NUMBER ELEVEN;No;0;ON;;;;11;N;;;;; +24EC;NEGATIVE CIRCLED NUMBER TWELVE;No;0;ON;;;;12;N;;;;; +24ED;NEGATIVE CIRCLED NUMBER THIRTEEN;No;0;ON;;;;13;N;;;;; +24EE;NEGATIVE CIRCLED NUMBER FOURTEEN;No;0;ON;;;;14;N;;;;; +24EF;NEGATIVE CIRCLED NUMBER FIFTEEN;No;0;ON;;;;15;N;;;;; +24F0;NEGATIVE CIRCLED NUMBER SIXTEEN;No;0;ON;;;;16;N;;;;; +24F1;NEGATIVE CIRCLED NUMBER SEVENTEEN;No;0;ON;;;;17;N;;;;; +24F2;NEGATIVE CIRCLED NUMBER EIGHTEEN;No;0;ON;;;;18;N;;;;; +24F3;NEGATIVE CIRCLED NUMBER NINETEEN;No;0;ON;;;;19;N;;;;; +24F4;NEGATIVE CIRCLED NUMBER TWENTY;No;0;ON;;;;20;N;;;;; +24F5;DOUBLE CIRCLED DIGIT ONE;No;0;ON;;;1;1;N;;;;; +24F6;DOUBLE CIRCLED DIGIT TWO;No;0;ON;;;2;2;N;;;;; +24F7;DOUBLE CIRCLED DIGIT THREE;No;0;ON;;;3;3;N;;;;; +24F8;DOUBLE CIRCLED DIGIT FOUR;No;0;ON;;;4;4;N;;;;; +24F9;DOUBLE CIRCLED DIGIT FIVE;No;0;ON;;;5;5;N;;;;; +24FA;DOUBLE CIRCLED DIGIT SIX;No;0;ON;;;6;6;N;;;;; +24FB;DOUBLE CIRCLED DIGIT SEVEN;No;0;ON;;;7;7;N;;;;; +24FC;DOUBLE CIRCLED DIGIT EIGHT;No;0;ON;;;8;8;N;;;;; +24FD;DOUBLE CIRCLED DIGIT NINE;No;0;ON;;;9;9;N;;;;; +24FE;DOUBLE CIRCLED NUMBER TEN;No;0;ON;;;;10;N;;;;; +24FF;NEGATIVE CIRCLED DIGIT ZERO;No;0;ON;;;0;0;N;;;;; +2500;BOX DRAWINGS LIGHT HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT HORIZONTAL;;;; +2501;BOX DRAWINGS HEAVY HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY HORIZONTAL;;;; +2502;BOX DRAWINGS LIGHT VERTICAL;So;0;ON;;;;;N;FORMS LIGHT VERTICAL;;;; +2503;BOX DRAWINGS HEAVY VERTICAL;So;0;ON;;;;;N;FORMS HEAVY VERTICAL;;;; +2504;BOX DRAWINGS LIGHT TRIPLE DASH HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT TRIPLE DASH HORIZONTAL;;;; +2505;BOX DRAWINGS HEAVY TRIPLE DASH HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY TRIPLE DASH HORIZONTAL;;;; +2506;BOX DRAWINGS LIGHT TRIPLE DASH VERTICAL;So;0;ON;;;;;N;FORMS LIGHT TRIPLE DASH VERTICAL;;;; +2507;BOX DRAWINGS HEAVY TRIPLE DASH VERTICAL;So;0;ON;;;;;N;FORMS HEAVY TRIPLE DASH VERTICAL;;;; +2508;BOX DRAWINGS LIGHT QUADRUPLE DASH HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT QUADRUPLE DASH HORIZONTAL;;;; +2509;BOX DRAWINGS HEAVY QUADRUPLE DASH HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY QUADRUPLE DASH HORIZONTAL;;;; +250A;BOX DRAWINGS LIGHT QUADRUPLE DASH VERTICAL;So;0;ON;;;;;N;FORMS LIGHT QUADRUPLE DASH VERTICAL;;;; +250B;BOX DRAWINGS HEAVY QUADRUPLE DASH VERTICAL;So;0;ON;;;;;N;FORMS HEAVY QUADRUPLE DASH VERTICAL;;;; +250C;BOX DRAWINGS LIGHT DOWN AND RIGHT;So;0;ON;;;;;N;FORMS LIGHT DOWN AND RIGHT;;;; +250D;BOX DRAWINGS DOWN LIGHT AND RIGHT HEAVY;So;0;ON;;;;;N;FORMS DOWN LIGHT AND RIGHT HEAVY;;;; +250E;BOX DRAWINGS DOWN HEAVY AND RIGHT LIGHT;So;0;ON;;;;;N;FORMS DOWN HEAVY AND RIGHT LIGHT;;;; +250F;BOX DRAWINGS HEAVY DOWN AND RIGHT;So;0;ON;;;;;N;FORMS HEAVY DOWN AND RIGHT;;;; +2510;BOX DRAWINGS LIGHT DOWN AND LEFT;So;0;ON;;;;;N;FORMS LIGHT DOWN AND LEFT;;;; +2511;BOX DRAWINGS DOWN LIGHT AND LEFT HEAVY;So;0;ON;;;;;N;FORMS DOWN LIGHT AND LEFT HEAVY;;;; +2512;BOX DRAWINGS DOWN HEAVY AND LEFT LIGHT;So;0;ON;;;;;N;FORMS DOWN HEAVY AND LEFT LIGHT;;;; +2513;BOX DRAWINGS HEAVY DOWN AND LEFT;So;0;ON;;;;;N;FORMS HEAVY DOWN AND LEFT;;;; +2514;BOX DRAWINGS LIGHT UP AND RIGHT;So;0;ON;;;;;N;FORMS LIGHT UP AND RIGHT;;;; +2515;BOX DRAWINGS UP LIGHT AND RIGHT HEAVY;So;0;ON;;;;;N;FORMS UP LIGHT AND RIGHT HEAVY;;;; +2516;BOX DRAWINGS UP HEAVY AND RIGHT LIGHT;So;0;ON;;;;;N;FORMS UP HEAVY AND RIGHT LIGHT;;;; +2517;BOX DRAWINGS HEAVY UP AND RIGHT;So;0;ON;;;;;N;FORMS HEAVY UP AND RIGHT;;;; +2518;BOX DRAWINGS LIGHT UP AND LEFT;So;0;ON;;;;;N;FORMS LIGHT UP AND LEFT;;;; +2519;BOX DRAWINGS UP LIGHT AND LEFT HEAVY;So;0;ON;;;;;N;FORMS UP LIGHT AND LEFT HEAVY;;;; +251A;BOX DRAWINGS UP HEAVY AND LEFT LIGHT;So;0;ON;;;;;N;FORMS UP HEAVY AND LEFT LIGHT;;;; +251B;BOX DRAWINGS HEAVY UP AND LEFT;So;0;ON;;;;;N;FORMS HEAVY UP AND LEFT;;;; +251C;BOX DRAWINGS LIGHT VERTICAL AND RIGHT;So;0;ON;;;;;N;FORMS LIGHT VERTICAL AND RIGHT;;;; +251D;BOX DRAWINGS VERTICAL LIGHT AND RIGHT HEAVY;So;0;ON;;;;;N;FORMS VERTICAL LIGHT AND RIGHT HEAVY;;;; +251E;BOX DRAWINGS UP HEAVY AND RIGHT DOWN LIGHT;So;0;ON;;;;;N;FORMS UP HEAVY AND RIGHT DOWN LIGHT;;;; +251F;BOX DRAWINGS DOWN HEAVY AND RIGHT UP LIGHT;So;0;ON;;;;;N;FORMS DOWN HEAVY AND RIGHT UP LIGHT;;;; +2520;BOX DRAWINGS VERTICAL HEAVY AND RIGHT LIGHT;So;0;ON;;;;;N;FORMS VERTICAL HEAVY AND RIGHT LIGHT;;;; +2521;BOX DRAWINGS DOWN LIGHT AND RIGHT UP HEAVY;So;0;ON;;;;;N;FORMS DOWN LIGHT AND RIGHT UP HEAVY;;;; +2522;BOX DRAWINGS UP LIGHT AND RIGHT DOWN HEAVY;So;0;ON;;;;;N;FORMS UP LIGHT AND RIGHT DOWN HEAVY;;;; +2523;BOX DRAWINGS HEAVY VERTICAL AND RIGHT;So;0;ON;;;;;N;FORMS HEAVY VERTICAL AND RIGHT;;;; +2524;BOX DRAWINGS LIGHT VERTICAL AND LEFT;So;0;ON;;;;;N;FORMS LIGHT VERTICAL AND LEFT;;;; +2525;BOX DRAWINGS VERTICAL LIGHT AND LEFT HEAVY;So;0;ON;;;;;N;FORMS VERTICAL LIGHT AND LEFT HEAVY;;;; +2526;BOX DRAWINGS UP HEAVY AND LEFT DOWN LIGHT;So;0;ON;;;;;N;FORMS UP HEAVY AND LEFT DOWN LIGHT;;;; +2527;BOX DRAWINGS DOWN HEAVY AND LEFT UP LIGHT;So;0;ON;;;;;N;FORMS DOWN HEAVY AND LEFT UP LIGHT;;;; +2528;BOX DRAWINGS VERTICAL HEAVY AND LEFT LIGHT;So;0;ON;;;;;N;FORMS VERTICAL HEAVY AND LEFT LIGHT;;;; +2529;BOX DRAWINGS DOWN LIGHT AND LEFT UP HEAVY;So;0;ON;;;;;N;FORMS DOWN LIGHT AND LEFT UP HEAVY;;;; +252A;BOX DRAWINGS UP LIGHT AND LEFT DOWN HEAVY;So;0;ON;;;;;N;FORMS UP LIGHT AND LEFT DOWN HEAVY;;;; +252B;BOX DRAWINGS HEAVY VERTICAL AND LEFT;So;0;ON;;;;;N;FORMS HEAVY VERTICAL AND LEFT;;;; +252C;BOX DRAWINGS LIGHT DOWN AND HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT DOWN AND HORIZONTAL;;;; +252D;BOX DRAWINGS LEFT HEAVY AND RIGHT DOWN LIGHT;So;0;ON;;;;;N;FORMS LEFT HEAVY AND RIGHT DOWN LIGHT;;;; +252E;BOX DRAWINGS RIGHT HEAVY AND LEFT DOWN LIGHT;So;0;ON;;;;;N;FORMS RIGHT HEAVY AND LEFT DOWN LIGHT;;;; +252F;BOX DRAWINGS DOWN LIGHT AND HORIZONTAL HEAVY;So;0;ON;;;;;N;FORMS DOWN LIGHT AND HORIZONTAL HEAVY;;;; +2530;BOX DRAWINGS DOWN HEAVY AND HORIZONTAL LIGHT;So;0;ON;;;;;N;FORMS DOWN HEAVY AND HORIZONTAL LIGHT;;;; +2531;BOX DRAWINGS RIGHT LIGHT AND LEFT DOWN HEAVY;So;0;ON;;;;;N;FORMS RIGHT LIGHT AND LEFT DOWN HEAVY;;;; +2532;BOX DRAWINGS LEFT LIGHT AND RIGHT DOWN HEAVY;So;0;ON;;;;;N;FORMS LEFT LIGHT AND RIGHT DOWN HEAVY;;;; +2533;BOX DRAWINGS HEAVY DOWN AND HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY DOWN AND HORIZONTAL;;;; +2534;BOX DRAWINGS LIGHT UP AND HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT UP AND HORIZONTAL;;;; +2535;BOX DRAWINGS LEFT HEAVY AND RIGHT UP LIGHT;So;0;ON;;;;;N;FORMS LEFT HEAVY AND RIGHT UP LIGHT;;;; +2536;BOX DRAWINGS RIGHT HEAVY AND LEFT UP LIGHT;So;0;ON;;;;;N;FORMS RIGHT HEAVY AND LEFT UP LIGHT;;;; +2537;BOX DRAWINGS UP LIGHT AND HORIZONTAL HEAVY;So;0;ON;;;;;N;FORMS UP LIGHT AND HORIZONTAL HEAVY;;;; +2538;BOX DRAWINGS UP HEAVY AND HORIZONTAL LIGHT;So;0;ON;;;;;N;FORMS UP HEAVY AND HORIZONTAL LIGHT;;;; +2539;BOX DRAWINGS RIGHT LIGHT AND LEFT UP HEAVY;So;0;ON;;;;;N;FORMS RIGHT LIGHT AND LEFT UP HEAVY;;;; +253A;BOX DRAWINGS LEFT LIGHT AND RIGHT UP HEAVY;So;0;ON;;;;;N;FORMS LEFT LIGHT AND RIGHT UP HEAVY;;;; +253B;BOX DRAWINGS HEAVY UP AND HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY UP AND HORIZONTAL;;;; +253C;BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT VERTICAL AND HORIZONTAL;;;; +253D;BOX DRAWINGS LEFT HEAVY AND RIGHT VERTICAL LIGHT;So;0;ON;;;;;N;FORMS LEFT HEAVY AND RIGHT VERTICAL LIGHT;;;; +253E;BOX DRAWINGS RIGHT HEAVY AND LEFT VERTICAL LIGHT;So;0;ON;;;;;N;FORMS RIGHT HEAVY AND LEFT VERTICAL LIGHT;;;; +253F;BOX DRAWINGS VERTICAL LIGHT AND HORIZONTAL HEAVY;So;0;ON;;;;;N;FORMS VERTICAL LIGHT AND HORIZONTAL HEAVY;;;; +2540;BOX DRAWINGS UP HEAVY AND DOWN HORIZONTAL LIGHT;So;0;ON;;;;;N;FORMS UP HEAVY AND DOWN HORIZONTAL LIGHT;;;; +2541;BOX DRAWINGS DOWN HEAVY AND UP HORIZONTAL LIGHT;So;0;ON;;;;;N;FORMS DOWN HEAVY AND UP HORIZONTAL LIGHT;;;; +2542;BOX DRAWINGS VERTICAL HEAVY AND HORIZONTAL LIGHT;So;0;ON;;;;;N;FORMS VERTICAL HEAVY AND HORIZONTAL LIGHT;;;; +2543;BOX DRAWINGS LEFT UP HEAVY AND RIGHT DOWN LIGHT;So;0;ON;;;;;N;FORMS LEFT UP HEAVY AND RIGHT DOWN LIGHT;;;; +2544;BOX DRAWINGS RIGHT UP HEAVY AND LEFT DOWN LIGHT;So;0;ON;;;;;N;FORMS RIGHT UP HEAVY AND LEFT DOWN LIGHT;;;; +2545;BOX DRAWINGS LEFT DOWN HEAVY AND RIGHT UP LIGHT;So;0;ON;;;;;N;FORMS LEFT DOWN HEAVY AND RIGHT UP LIGHT;;;; +2546;BOX DRAWINGS RIGHT DOWN HEAVY AND LEFT UP LIGHT;So;0;ON;;;;;N;FORMS RIGHT DOWN HEAVY AND LEFT UP LIGHT;;;; +2547;BOX DRAWINGS DOWN LIGHT AND UP HORIZONTAL HEAVY;So;0;ON;;;;;N;FORMS DOWN LIGHT AND UP HORIZONTAL HEAVY;;;; +2548;BOX DRAWINGS UP LIGHT AND DOWN HORIZONTAL HEAVY;So;0;ON;;;;;N;FORMS UP LIGHT AND DOWN HORIZONTAL HEAVY;;;; +2549;BOX DRAWINGS RIGHT LIGHT AND LEFT VERTICAL HEAVY;So;0;ON;;;;;N;FORMS RIGHT LIGHT AND LEFT VERTICAL HEAVY;;;; +254A;BOX DRAWINGS LEFT LIGHT AND RIGHT VERTICAL HEAVY;So;0;ON;;;;;N;FORMS LEFT LIGHT AND RIGHT VERTICAL HEAVY;;;; +254B;BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY VERTICAL AND HORIZONTAL;;;; +254C;BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL;So;0;ON;;;;;N;FORMS LIGHT DOUBLE DASH HORIZONTAL;;;; +254D;BOX DRAWINGS HEAVY DOUBLE DASH HORIZONTAL;So;0;ON;;;;;N;FORMS HEAVY DOUBLE DASH HORIZONTAL;;;; +254E;BOX DRAWINGS LIGHT DOUBLE DASH VERTICAL;So;0;ON;;;;;N;FORMS LIGHT DOUBLE DASH VERTICAL;;;; +254F;BOX DRAWINGS HEAVY DOUBLE DASH VERTICAL;So;0;ON;;;;;N;FORMS HEAVY DOUBLE DASH VERTICAL;;;; +2550;BOX DRAWINGS DOUBLE HORIZONTAL;So;0;ON;;;;;N;FORMS DOUBLE HORIZONTAL;;;; +2551;BOX DRAWINGS DOUBLE VERTICAL;So;0;ON;;;;;N;FORMS DOUBLE VERTICAL;;;; +2552;BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE;So;0;ON;;;;;N;FORMS DOWN SINGLE AND RIGHT DOUBLE;;;; +2553;BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE;So;0;ON;;;;;N;FORMS DOWN DOUBLE AND RIGHT SINGLE;;;; +2554;BOX DRAWINGS DOUBLE DOWN AND RIGHT;So;0;ON;;;;;N;FORMS DOUBLE DOWN AND RIGHT;;;; +2555;BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE;So;0;ON;;;;;N;FORMS DOWN SINGLE AND LEFT DOUBLE;;;; +2556;BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE;So;0;ON;;;;;N;FORMS DOWN DOUBLE AND LEFT SINGLE;;;; +2557;BOX DRAWINGS DOUBLE DOWN AND LEFT;So;0;ON;;;;;N;FORMS DOUBLE DOWN AND LEFT;;;; +2558;BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE;So;0;ON;;;;;N;FORMS UP SINGLE AND RIGHT DOUBLE;;;; +2559;BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE;So;0;ON;;;;;N;FORMS UP DOUBLE AND RIGHT SINGLE;;;; +255A;BOX DRAWINGS DOUBLE UP AND RIGHT;So;0;ON;;;;;N;FORMS DOUBLE UP AND RIGHT;;;; +255B;BOX DRAWINGS UP SINGLE AND LEFT DOUBLE;So;0;ON;;;;;N;FORMS UP SINGLE AND LEFT DOUBLE;;;; +255C;BOX DRAWINGS UP DOUBLE AND LEFT SINGLE;So;0;ON;;;;;N;FORMS UP DOUBLE AND LEFT SINGLE;;;; +255D;BOX DRAWINGS DOUBLE UP AND LEFT;So;0;ON;;;;;N;FORMS DOUBLE UP AND LEFT;;;; +255E;BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE;So;0;ON;;;;;N;FORMS VERTICAL SINGLE AND RIGHT DOUBLE;;;; +255F;BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE;So;0;ON;;;;;N;FORMS VERTICAL DOUBLE AND RIGHT SINGLE;;;; +2560;BOX DRAWINGS DOUBLE VERTICAL AND RIGHT;So;0;ON;;;;;N;FORMS DOUBLE VERTICAL AND RIGHT;;;; +2561;BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE;So;0;ON;;;;;N;FORMS VERTICAL SINGLE AND LEFT DOUBLE;;;; +2562;BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE;So;0;ON;;;;;N;FORMS VERTICAL DOUBLE AND LEFT SINGLE;;;; +2563;BOX DRAWINGS DOUBLE VERTICAL AND LEFT;So;0;ON;;;;;N;FORMS DOUBLE VERTICAL AND LEFT;;;; +2564;BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE;So;0;ON;;;;;N;FORMS DOWN SINGLE AND HORIZONTAL DOUBLE;;;; +2565;BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE;So;0;ON;;;;;N;FORMS DOWN DOUBLE AND HORIZONTAL SINGLE;;;; +2566;BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL;So;0;ON;;;;;N;FORMS DOUBLE DOWN AND HORIZONTAL;;;; +2567;BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE;So;0;ON;;;;;N;FORMS UP SINGLE AND HORIZONTAL DOUBLE;;;; +2568;BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE;So;0;ON;;;;;N;FORMS UP DOUBLE AND HORIZONTAL SINGLE;;;; +2569;BOX DRAWINGS DOUBLE UP AND HORIZONTAL;So;0;ON;;;;;N;FORMS DOUBLE UP AND HORIZONTAL;;;; +256A;BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE;So;0;ON;;;;;N;FORMS VERTICAL SINGLE AND HORIZONTAL DOUBLE;;;; +256B;BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE;So;0;ON;;;;;N;FORMS VERTICAL DOUBLE AND HORIZONTAL SINGLE;;;; +256C;BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL;So;0;ON;;;;;N;FORMS DOUBLE VERTICAL AND HORIZONTAL;;;; +256D;BOX DRAWINGS LIGHT ARC DOWN AND RIGHT;So;0;ON;;;;;N;FORMS LIGHT ARC DOWN AND RIGHT;;;; +256E;BOX DRAWINGS LIGHT ARC DOWN AND LEFT;So;0;ON;;;;;N;FORMS LIGHT ARC DOWN AND LEFT;;;; +256F;BOX DRAWINGS LIGHT ARC UP AND LEFT;So;0;ON;;;;;N;FORMS LIGHT ARC UP AND LEFT;;;; +2570;BOX DRAWINGS LIGHT ARC UP AND RIGHT;So;0;ON;;;;;N;FORMS LIGHT ARC UP AND RIGHT;;;; +2571;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT;So;0;ON;;;;;N;FORMS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT;;;; +2572;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT;So;0;ON;;;;;N;FORMS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT;;;; +2573;BOX DRAWINGS LIGHT DIAGONAL CROSS;So;0;ON;;;;;N;FORMS LIGHT DIAGONAL CROSS;;;; +2574;BOX DRAWINGS LIGHT LEFT;So;0;ON;;;;;N;FORMS LIGHT LEFT;;;; +2575;BOX DRAWINGS LIGHT UP;So;0;ON;;;;;N;FORMS LIGHT UP;;;; +2576;BOX DRAWINGS LIGHT RIGHT;So;0;ON;;;;;N;FORMS LIGHT RIGHT;;;; +2577;BOX DRAWINGS LIGHT DOWN;So;0;ON;;;;;N;FORMS LIGHT DOWN;;;; +2578;BOX DRAWINGS HEAVY LEFT;So;0;ON;;;;;N;FORMS HEAVY LEFT;;;; +2579;BOX DRAWINGS HEAVY UP;So;0;ON;;;;;N;FORMS HEAVY UP;;;; +257A;BOX DRAWINGS HEAVY RIGHT;So;0;ON;;;;;N;FORMS HEAVY RIGHT;;;; +257B;BOX DRAWINGS HEAVY DOWN;So;0;ON;;;;;N;FORMS HEAVY DOWN;;;; +257C;BOX DRAWINGS LIGHT LEFT AND HEAVY RIGHT;So;0;ON;;;;;N;FORMS LIGHT LEFT AND HEAVY RIGHT;;;; +257D;BOX DRAWINGS LIGHT UP AND HEAVY DOWN;So;0;ON;;;;;N;FORMS LIGHT UP AND HEAVY DOWN;;;; +257E;BOX DRAWINGS HEAVY LEFT AND LIGHT RIGHT;So;0;ON;;;;;N;FORMS HEAVY LEFT AND LIGHT RIGHT;;;; +257F;BOX DRAWINGS HEAVY UP AND LIGHT DOWN;So;0;ON;;;;;N;FORMS HEAVY UP AND LIGHT DOWN;;;; +2580;UPPER HALF BLOCK;So;0;ON;;;;;N;;;;; +2581;LOWER ONE EIGHTH BLOCK;So;0;ON;;;;;N;;;;; +2582;LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +2583;LOWER THREE EIGHTHS BLOCK;So;0;ON;;;;;N;;;;; +2584;LOWER HALF BLOCK;So;0;ON;;;;;N;;;;; +2585;LOWER FIVE EIGHTHS BLOCK;So;0;ON;;;;;N;;;;; +2586;LOWER THREE QUARTERS BLOCK;So;0;ON;;;;;N;LOWER THREE QUARTER BLOCK;;;; +2587;LOWER SEVEN EIGHTHS BLOCK;So;0;ON;;;;;N;;;;; +2588;FULL BLOCK;So;0;ON;;;;;N;;;;; +2589;LEFT SEVEN EIGHTHS BLOCK;So;0;ON;;;;;N;;;;; +258A;LEFT THREE QUARTERS BLOCK;So;0;ON;;;;;N;LEFT THREE QUARTER BLOCK;;;; +258B;LEFT FIVE EIGHTHS BLOCK;So;0;ON;;;;;N;;;;; +258C;LEFT HALF BLOCK;So;0;ON;;;;;N;;;;; +258D;LEFT THREE EIGHTHS BLOCK;So;0;ON;;;;;N;;;;; +258E;LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +258F;LEFT ONE EIGHTH BLOCK;So;0;ON;;;;;N;;;;; +2590;RIGHT HALF BLOCK;So;0;ON;;;;;N;;;;; +2591;LIGHT SHADE;So;0;ON;;;;;N;;;;; +2592;MEDIUM SHADE;So;0;ON;;;;;N;;;;; +2593;DARK SHADE;So;0;ON;;;;;N;;;;; +2594;UPPER ONE EIGHTH BLOCK;So;0;ON;;;;;N;;;;; +2595;RIGHT ONE EIGHTH BLOCK;So;0;ON;;;;;N;;;;; +2596;QUADRANT LOWER LEFT;So;0;ON;;;;;N;;;;; +2597;QUADRANT LOWER RIGHT;So;0;ON;;;;;N;;;;; +2598;QUADRANT UPPER LEFT;So;0;ON;;;;;N;;;;; +2599;QUADRANT UPPER LEFT AND LOWER LEFT AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +259A;QUADRANT UPPER LEFT AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +259B;QUADRANT UPPER LEFT AND UPPER RIGHT AND LOWER LEFT;So;0;ON;;;;;N;;;;; +259C;QUADRANT UPPER LEFT AND UPPER RIGHT AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +259D;QUADRANT UPPER RIGHT;So;0;ON;;;;;N;;;;; +259E;QUADRANT UPPER RIGHT AND LOWER LEFT;So;0;ON;;;;;N;;;;; +259F;QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +25A0;BLACK SQUARE;So;0;ON;;;;;N;;;;; +25A1;WHITE SQUARE;So;0;ON;;;;;N;;;;; +25A2;WHITE SQUARE WITH ROUNDED CORNERS;So;0;ON;;;;;N;;;;; +25A3;WHITE SQUARE CONTAINING BLACK SMALL SQUARE;So;0;ON;;;;;N;;;;; +25A4;SQUARE WITH HORIZONTAL FILL;So;0;ON;;;;;N;;;;; +25A5;SQUARE WITH VERTICAL FILL;So;0;ON;;;;;N;;;;; +25A6;SQUARE WITH ORTHOGONAL CROSSHATCH FILL;So;0;ON;;;;;N;;;;; +25A7;SQUARE WITH UPPER LEFT TO LOWER RIGHT FILL;So;0;ON;;;;;N;;;;; +25A8;SQUARE WITH UPPER RIGHT TO LOWER LEFT FILL;So;0;ON;;;;;N;;;;; +25A9;SQUARE WITH DIAGONAL CROSSHATCH FILL;So;0;ON;;;;;N;;;;; +25AA;BLACK SMALL SQUARE;So;0;ON;;;;;N;;;;; +25AB;WHITE SMALL SQUARE;So;0;ON;;;;;N;;;;; +25AC;BLACK RECTANGLE;So;0;ON;;;;;N;;;;; +25AD;WHITE RECTANGLE;So;0;ON;;;;;N;;;;; +25AE;BLACK VERTICAL RECTANGLE;So;0;ON;;;;;N;;;;; +25AF;WHITE VERTICAL RECTANGLE;So;0;ON;;;;;N;;;;; +25B0;BLACK PARALLELOGRAM;So;0;ON;;;;;N;;;;; +25B1;WHITE PARALLELOGRAM;So;0;ON;;;;;N;;;;; +25B2;BLACK UP-POINTING TRIANGLE;So;0;ON;;;;;N;BLACK UP POINTING TRIANGLE;;;; +25B3;WHITE UP-POINTING TRIANGLE;So;0;ON;;;;;N;WHITE UP POINTING TRIANGLE;;;; +25B4;BLACK UP-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;BLACK UP POINTING SMALL TRIANGLE;;;; +25B5;WHITE UP-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;WHITE UP POINTING SMALL TRIANGLE;;;; +25B6;BLACK RIGHT-POINTING TRIANGLE;So;0;ON;;;;;N;BLACK RIGHT POINTING TRIANGLE;;;; +25B7;WHITE RIGHT-POINTING TRIANGLE;Sm;0;ON;;;;;N;WHITE RIGHT POINTING TRIANGLE;;;; +25B8;BLACK RIGHT-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;BLACK RIGHT POINTING SMALL TRIANGLE;;;; +25B9;WHITE RIGHT-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;WHITE RIGHT POINTING SMALL TRIANGLE;;;; +25BA;BLACK RIGHT-POINTING POINTER;So;0;ON;;;;;N;BLACK RIGHT POINTING POINTER;;;; +25BB;WHITE RIGHT-POINTING POINTER;So;0;ON;;;;;N;WHITE RIGHT POINTING POINTER;;;; +25BC;BLACK DOWN-POINTING TRIANGLE;So;0;ON;;;;;N;BLACK DOWN POINTING TRIANGLE;;;; +25BD;WHITE DOWN-POINTING TRIANGLE;So;0;ON;;;;;N;WHITE DOWN POINTING TRIANGLE;;;; +25BE;BLACK DOWN-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;BLACK DOWN POINTING SMALL TRIANGLE;;;; +25BF;WHITE DOWN-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;WHITE DOWN POINTING SMALL TRIANGLE;;;; +25C0;BLACK LEFT-POINTING TRIANGLE;So;0;ON;;;;;N;BLACK LEFT POINTING TRIANGLE;;;; +25C1;WHITE LEFT-POINTING TRIANGLE;Sm;0;ON;;;;;N;WHITE LEFT POINTING TRIANGLE;;;; +25C2;BLACK LEFT-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;BLACK LEFT POINTING SMALL TRIANGLE;;;; +25C3;WHITE LEFT-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;WHITE LEFT POINTING SMALL TRIANGLE;;;; +25C4;BLACK LEFT-POINTING POINTER;So;0;ON;;;;;N;BLACK LEFT POINTING POINTER;;;; +25C5;WHITE LEFT-POINTING POINTER;So;0;ON;;;;;N;WHITE LEFT POINTING POINTER;;;; +25C6;BLACK DIAMOND;So;0;ON;;;;;N;;;;; +25C7;WHITE DIAMOND;So;0;ON;;;;;N;;;;; +25C8;WHITE DIAMOND CONTAINING BLACK SMALL DIAMOND;So;0;ON;;;;;N;;;;; +25C9;FISHEYE;So;0;ON;;;;;N;;;;; +25CA;LOZENGE;So;0;ON;;;;;N;;;;; +25CB;WHITE CIRCLE;So;0;ON;;;;;N;;;;; +25CC;DOTTED CIRCLE;So;0;ON;;;;;N;;;;; +25CD;CIRCLE WITH VERTICAL FILL;So;0;ON;;;;;N;;;;; +25CE;BULLSEYE;So;0;ON;;;;;N;;;;; +25CF;BLACK CIRCLE;So;0;ON;;;;;N;;;;; +25D0;CIRCLE WITH LEFT HALF BLACK;So;0;ON;;;;;N;;;;; +25D1;CIRCLE WITH RIGHT HALF BLACK;So;0;ON;;;;;N;;;;; +25D2;CIRCLE WITH LOWER HALF BLACK;So;0;ON;;;;;N;;;;; +25D3;CIRCLE WITH UPPER HALF BLACK;So;0;ON;;;;;N;;;;; +25D4;CIRCLE WITH UPPER RIGHT QUADRANT BLACK;So;0;ON;;;;;N;;;;; +25D5;CIRCLE WITH ALL BUT UPPER LEFT QUADRANT BLACK;So;0;ON;;;;;N;;;;; +25D6;LEFT HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +25D7;RIGHT HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +25D8;INVERSE BULLET;So;0;ON;;;;;N;;;;; +25D9;INVERSE WHITE CIRCLE;So;0;ON;;;;;N;;;;; +25DA;UPPER HALF INVERSE WHITE CIRCLE;So;0;ON;;;;;N;;;;; +25DB;LOWER HALF INVERSE WHITE CIRCLE;So;0;ON;;;;;N;;;;; +25DC;UPPER LEFT QUADRANT CIRCULAR ARC;So;0;ON;;;;;N;;;;; +25DD;UPPER RIGHT QUADRANT CIRCULAR ARC;So;0;ON;;;;;N;;;;; +25DE;LOWER RIGHT QUADRANT CIRCULAR ARC;So;0;ON;;;;;N;;;;; +25DF;LOWER LEFT QUADRANT CIRCULAR ARC;So;0;ON;;;;;N;;;;; +25E0;UPPER HALF CIRCLE;So;0;ON;;;;;N;;;;; +25E1;LOWER HALF CIRCLE;So;0;ON;;;;;N;;;;; +25E2;BLACK LOWER RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; +25E3;BLACK LOWER LEFT TRIANGLE;So;0;ON;;;;;N;;;;; +25E4;BLACK UPPER LEFT TRIANGLE;So;0;ON;;;;;N;;;;; +25E5;BLACK UPPER RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; +25E6;WHITE BULLET;So;0;ON;;;;;N;;;;; +25E7;SQUARE WITH LEFT HALF BLACK;So;0;ON;;;;;N;;;;; +25E8;SQUARE WITH RIGHT HALF BLACK;So;0;ON;;;;;N;;;;; +25E9;SQUARE WITH UPPER LEFT DIAGONAL HALF BLACK;So;0;ON;;;;;N;;;;; +25EA;SQUARE WITH LOWER RIGHT DIAGONAL HALF BLACK;So;0;ON;;;;;N;;;;; +25EB;WHITE SQUARE WITH VERTICAL BISECTING LINE;So;0;ON;;;;;N;;;;; +25EC;WHITE UP-POINTING TRIANGLE WITH DOT;So;0;ON;;;;;N;WHITE UP POINTING TRIANGLE WITH DOT;;;; +25ED;UP-POINTING TRIANGLE WITH LEFT HALF BLACK;So;0;ON;;;;;N;UP POINTING TRIANGLE WITH LEFT HALF BLACK;;;; +25EE;UP-POINTING TRIANGLE WITH RIGHT HALF BLACK;So;0;ON;;;;;N;UP POINTING TRIANGLE WITH RIGHT HALF BLACK;;;; +25EF;LARGE CIRCLE;So;0;ON;;;;;N;;;;; +25F0;WHITE SQUARE WITH UPPER LEFT QUADRANT;So;0;ON;;;;;N;;;;; +25F1;WHITE SQUARE WITH LOWER LEFT QUADRANT;So;0;ON;;;;;N;;;;; +25F2;WHITE SQUARE WITH LOWER RIGHT QUADRANT;So;0;ON;;;;;N;;;;; +25F3;WHITE SQUARE WITH UPPER RIGHT QUADRANT;So;0;ON;;;;;N;;;;; +25F4;WHITE CIRCLE WITH UPPER LEFT QUADRANT;So;0;ON;;;;;N;;;;; +25F5;WHITE CIRCLE WITH LOWER LEFT QUADRANT;So;0;ON;;;;;N;;;;; +25F6;WHITE CIRCLE WITH LOWER RIGHT QUADRANT;So;0;ON;;;;;N;;;;; +25F7;WHITE CIRCLE WITH UPPER RIGHT QUADRANT;So;0;ON;;;;;N;;;;; +25F8;UPPER LEFT TRIANGLE;Sm;0;ON;;;;;N;;;;; +25F9;UPPER RIGHT TRIANGLE;Sm;0;ON;;;;;N;;;;; +25FA;LOWER LEFT TRIANGLE;Sm;0;ON;;;;;N;;;;; +25FB;WHITE MEDIUM SQUARE;Sm;0;ON;;;;;N;;;;; +25FC;BLACK MEDIUM SQUARE;Sm;0;ON;;;;;N;;;;; +25FD;WHITE MEDIUM SMALL SQUARE;Sm;0;ON;;;;;N;;;;; +25FE;BLACK MEDIUM SMALL SQUARE;Sm;0;ON;;;;;N;;;;; +25FF;LOWER RIGHT TRIANGLE;Sm;0;ON;;;;;N;;;;; +2600;BLACK SUN WITH RAYS;So;0;ON;;;;;N;;;;; +2601;CLOUD;So;0;ON;;;;;N;;;;; +2602;UMBRELLA;So;0;ON;;;;;N;;;;; +2603;SNOWMAN;So;0;ON;;;;;N;;;;; +2604;COMET;So;0;ON;;;;;N;;;;; +2605;BLACK STAR;So;0;ON;;;;;N;;;;; +2606;WHITE STAR;So;0;ON;;;;;N;;;;; +2607;LIGHTNING;So;0;ON;;;;;N;;;;; +2608;THUNDERSTORM;So;0;ON;;;;;N;;;;; +2609;SUN;So;0;ON;;;;;N;;;;; +260A;ASCENDING NODE;So;0;ON;;;;;N;;;;; +260B;DESCENDING NODE;So;0;ON;;;;;N;;;;; +260C;CONJUNCTION;So;0;ON;;;;;N;;;;; +260D;OPPOSITION;So;0;ON;;;;;N;;;;; +260E;BLACK TELEPHONE;So;0;ON;;;;;N;;;;; +260F;WHITE TELEPHONE;So;0;ON;;;;;N;;;;; +2610;BALLOT BOX;So;0;ON;;;;;N;;;;; +2611;BALLOT BOX WITH CHECK;So;0;ON;;;;;N;;;;; +2612;BALLOT BOX WITH X;So;0;ON;;;;;N;;;;; +2613;SALTIRE;So;0;ON;;;;;N;;;;; +2614;UMBRELLA WITH RAIN DROPS;So;0;ON;;;;;N;;;;; +2615;HOT BEVERAGE;So;0;ON;;;;;N;;;;; +2616;WHITE SHOGI PIECE;So;0;ON;;;;;N;;;;; +2617;BLACK SHOGI PIECE;So;0;ON;;;;;N;;;;; +2618;SHAMROCK;So;0;ON;;;;;N;;;;; +2619;REVERSED ROTATED FLORAL HEART BULLET;So;0;ON;;;;;N;;;;; +261A;BLACK LEFT POINTING INDEX;So;0;ON;;;;;N;;;;; +261B;BLACK RIGHT POINTING INDEX;So;0;ON;;;;;N;;;;; +261C;WHITE LEFT POINTING INDEX;So;0;ON;;;;;N;;;;; +261D;WHITE UP POINTING INDEX;So;0;ON;;;;;N;;;;; +261E;WHITE RIGHT POINTING INDEX;So;0;ON;;;;;N;;;;; +261F;WHITE DOWN POINTING INDEX;So;0;ON;;;;;N;;;;; +2620;SKULL AND CROSSBONES;So;0;ON;;;;;N;;;;; +2621;CAUTION SIGN;So;0;ON;;;;;N;;;;; +2622;RADIOACTIVE SIGN;So;0;ON;;;;;N;;;;; +2623;BIOHAZARD SIGN;So;0;ON;;;;;N;;;;; +2624;CADUCEUS;So;0;ON;;;;;N;;;;; +2625;ANKH;So;0;ON;;;;;N;;;;; +2626;ORTHODOX CROSS;So;0;ON;;;;;N;;;;; +2627;CHI RHO;So;0;ON;;;;;N;;;;; +2628;CROSS OF LORRAINE;So;0;ON;;;;;N;;;;; +2629;CROSS OF JERUSALEM;So;0;ON;;;;;N;;;;; +262A;STAR AND CRESCENT;So;0;ON;;;;;N;;;;; +262B;FARSI SYMBOL;So;0;ON;;;;;N;SYMBOL OF IRAN;;;; +262C;ADI SHAKTI;So;0;ON;;;;;N;;;;; +262D;HAMMER AND SICKLE;So;0;ON;;;;;N;;;;; +262E;PEACE SYMBOL;So;0;ON;;;;;N;;;;; +262F;YIN YANG;So;0;ON;;;;;N;;;;; +2630;TRIGRAM FOR HEAVEN;So;0;ON;;;;;N;;;;; +2631;TRIGRAM FOR LAKE;So;0;ON;;;;;N;;;;; +2632;TRIGRAM FOR FIRE;So;0;ON;;;;;N;;;;; +2633;TRIGRAM FOR THUNDER;So;0;ON;;;;;N;;;;; +2634;TRIGRAM FOR WIND;So;0;ON;;;;;N;;;;; +2635;TRIGRAM FOR WATER;So;0;ON;;;;;N;;;;; +2636;TRIGRAM FOR MOUNTAIN;So;0;ON;;;;;N;;;;; +2637;TRIGRAM FOR EARTH;So;0;ON;;;;;N;;;;; +2638;WHEEL OF DHARMA;So;0;ON;;;;;N;;;;; +2639;WHITE FROWNING FACE;So;0;ON;;;;;N;;;;; +263A;WHITE SMILING FACE;So;0;ON;;;;;N;;;;; +263B;BLACK SMILING FACE;So;0;ON;;;;;N;;;;; +263C;WHITE SUN WITH RAYS;So;0;ON;;;;;N;;;;; +263D;FIRST QUARTER MOON;So;0;ON;;;;;N;;;;; +263E;LAST QUARTER MOON;So;0;ON;;;;;N;;;;; +263F;MERCURY;So;0;ON;;;;;N;;;;; +2640;FEMALE SIGN;So;0;ON;;;;;N;;;;; +2641;EARTH;So;0;ON;;;;;N;;;;; +2642;MALE SIGN;So;0;ON;;;;;N;;;;; +2643;JUPITER;So;0;ON;;;;;N;;;;; +2644;SATURN;So;0;ON;;;;;N;;;;; +2645;URANUS;So;0;ON;;;;;N;;;;; +2646;NEPTUNE;So;0;ON;;;;;N;;;;; +2647;PLUTO;So;0;ON;;;;;N;;;;; +2648;ARIES;So;0;ON;;;;;N;;;;; +2649;TAURUS;So;0;ON;;;;;N;;;;; +264A;GEMINI;So;0;ON;;;;;N;;;;; +264B;CANCER;So;0;ON;;;;;N;;;;; +264C;LEO;So;0;ON;;;;;N;;;;; +264D;VIRGO;So;0;ON;;;;;N;;;;; +264E;LIBRA;So;0;ON;;;;;N;;;;; +264F;SCORPIUS;So;0;ON;;;;;N;;;;; +2650;SAGITTARIUS;So;0;ON;;;;;N;;;;; +2651;CAPRICORN;So;0;ON;;;;;N;;;;; +2652;AQUARIUS;So;0;ON;;;;;N;;;;; +2653;PISCES;So;0;ON;;;;;N;;;;; +2654;WHITE CHESS KING;So;0;ON;;;;;N;;;;; +2655;WHITE CHESS QUEEN;So;0;ON;;;;;N;;;;; +2656;WHITE CHESS ROOK;So;0;ON;;;;;N;;;;; +2657;WHITE CHESS BISHOP;So;0;ON;;;;;N;;;;; +2658;WHITE CHESS KNIGHT;So;0;ON;;;;;N;;;;; +2659;WHITE CHESS PAWN;So;0;ON;;;;;N;;;;; +265A;BLACK CHESS KING;So;0;ON;;;;;N;;;;; +265B;BLACK CHESS QUEEN;So;0;ON;;;;;N;;;;; +265C;BLACK CHESS ROOK;So;0;ON;;;;;N;;;;; +265D;BLACK CHESS BISHOP;So;0;ON;;;;;N;;;;; +265E;BLACK CHESS KNIGHT;So;0;ON;;;;;N;;;;; +265F;BLACK CHESS PAWN;So;0;ON;;;;;N;;;;; +2660;BLACK SPADE SUIT;So;0;ON;;;;;N;;;;; +2661;WHITE HEART SUIT;So;0;ON;;;;;N;;;;; +2662;WHITE DIAMOND SUIT;So;0;ON;;;;;N;;;;; +2663;BLACK CLUB SUIT;So;0;ON;;;;;N;;;;; +2664;WHITE SPADE SUIT;So;0;ON;;;;;N;;;;; +2665;BLACK HEART SUIT;So;0;ON;;;;;N;;;;; +2666;BLACK DIAMOND SUIT;So;0;ON;;;;;N;;;;; +2667;WHITE CLUB SUIT;So;0;ON;;;;;N;;;;; +2668;HOT SPRINGS;So;0;ON;;;;;N;;;;; +2669;QUARTER NOTE;So;0;ON;;;;;N;;;;; +266A;EIGHTH NOTE;So;0;ON;;;;;N;;;;; +266B;BEAMED EIGHTH NOTES;So;0;ON;;;;;N;BARRED EIGHTH NOTES;;;; +266C;BEAMED SIXTEENTH NOTES;So;0;ON;;;;;N;BARRED SIXTEENTH NOTES;;;; +266D;MUSIC FLAT SIGN;So;0;ON;;;;;N;FLAT;;;; +266E;MUSIC NATURAL SIGN;So;0;ON;;;;;N;NATURAL;;;; +266F;MUSIC SHARP SIGN;Sm;0;ON;;;;;N;SHARP;;;; +2670;WEST SYRIAC CROSS;So;0;ON;;;;;N;;;;; +2671;EAST SYRIAC CROSS;So;0;ON;;;;;N;;;;; +2672;UNIVERSAL RECYCLING SYMBOL;So;0;ON;;;;;N;;;;; +2673;RECYCLING SYMBOL FOR TYPE-1 PLASTICS;So;0;ON;;;;;N;;;;; +2674;RECYCLING SYMBOL FOR TYPE-2 PLASTICS;So;0;ON;;;;;N;;;;; +2675;RECYCLING SYMBOL FOR TYPE-3 PLASTICS;So;0;ON;;;;;N;;;;; +2676;RECYCLING SYMBOL FOR TYPE-4 PLASTICS;So;0;ON;;;;;N;;;;; +2677;RECYCLING SYMBOL FOR TYPE-5 PLASTICS;So;0;ON;;;;;N;;;;; +2678;RECYCLING SYMBOL FOR TYPE-6 PLASTICS;So;0;ON;;;;;N;;;;; +2679;RECYCLING SYMBOL FOR TYPE-7 PLASTICS;So;0;ON;;;;;N;;;;; +267A;RECYCLING SYMBOL FOR GENERIC MATERIALS;So;0;ON;;;;;N;;;;; +267B;BLACK UNIVERSAL RECYCLING SYMBOL;So;0;ON;;;;;N;;;;; +267C;RECYCLED PAPER SYMBOL;So;0;ON;;;;;N;;;;; +267D;PARTIALLY-RECYCLED PAPER SYMBOL;So;0;ON;;;;;N;;;;; +267E;PERMANENT PAPER SIGN;So;0;ON;;;;;N;;;;; +267F;WHEELCHAIR SYMBOL;So;0;ON;;;;;N;;;;; +2680;DIE FACE-1;So;0;ON;;;;;N;;;;; +2681;DIE FACE-2;So;0;ON;;;;;N;;;;; +2682;DIE FACE-3;So;0;ON;;;;;N;;;;; +2683;DIE FACE-4;So;0;ON;;;;;N;;;;; +2684;DIE FACE-5;So;0;ON;;;;;N;;;;; +2685;DIE FACE-6;So;0;ON;;;;;N;;;;; +2686;WHITE CIRCLE WITH DOT RIGHT;So;0;ON;;;;;N;;;;; +2687;WHITE CIRCLE WITH TWO DOTS;So;0;ON;;;;;N;;;;; +2688;BLACK CIRCLE WITH WHITE DOT RIGHT;So;0;ON;;;;;N;;;;; +2689;BLACK CIRCLE WITH TWO WHITE DOTS;So;0;ON;;;;;N;;;;; +268A;MONOGRAM FOR YANG;So;0;ON;;;;;N;;;;; +268B;MONOGRAM FOR YIN;So;0;ON;;;;;N;;;;; +268C;DIGRAM FOR GREATER YANG;So;0;ON;;;;;N;;;;; +268D;DIGRAM FOR LESSER YIN;So;0;ON;;;;;N;;;;; +268E;DIGRAM FOR LESSER YANG;So;0;ON;;;;;N;;;;; +268F;DIGRAM FOR GREATER YIN;So;0;ON;;;;;N;;;;; +2690;WHITE FLAG;So;0;ON;;;;;N;;;;; +2691;BLACK FLAG;So;0;ON;;;;;N;;;;; +2692;HAMMER AND PICK;So;0;ON;;;;;N;;;;; +2693;ANCHOR;So;0;ON;;;;;N;;;;; +2694;CROSSED SWORDS;So;0;ON;;;;;N;;;;; +2695;STAFF OF AESCULAPIUS;So;0;ON;;;;;N;;;;; +2696;SCALES;So;0;ON;;;;;N;;;;; +2697;ALEMBIC;So;0;ON;;;;;N;;;;; +2698;FLOWER;So;0;ON;;;;;N;;;;; +2699;GEAR;So;0;ON;;;;;N;;;;; +269A;STAFF OF HERMES;So;0;ON;;;;;N;;;;; +269B;ATOM SYMBOL;So;0;ON;;;;;N;;;;; +269C;FLEUR-DE-LIS;So;0;ON;;;;;N;;;;; +269D;OUTLINED WHITE STAR;So;0;ON;;;;;N;;;;; +269E;THREE LINES CONVERGING RIGHT;So;0;ON;;;;;N;;;;; +269F;THREE LINES CONVERGING LEFT;So;0;ON;;;;;N;;;;; +26A0;WARNING SIGN;So;0;ON;;;;;N;;;;; +26A1;HIGH VOLTAGE SIGN;So;0;ON;;;;;N;;;;; +26A2;DOUBLED FEMALE SIGN;So;0;ON;;;;;N;;;;; +26A3;DOUBLED MALE SIGN;So;0;ON;;;;;N;;;;; +26A4;INTERLOCKED FEMALE AND MALE SIGN;So;0;ON;;;;;N;;;;; +26A5;MALE AND FEMALE SIGN;So;0;ON;;;;;N;;;;; +26A6;MALE WITH STROKE SIGN;So;0;ON;;;;;N;;;;; +26A7;MALE WITH STROKE AND MALE AND FEMALE SIGN;So;0;ON;;;;;N;;;;; +26A8;VERTICAL MALE WITH STROKE SIGN;So;0;ON;;;;;N;;;;; +26A9;HORIZONTAL MALE WITH STROKE SIGN;So;0;ON;;;;;N;;;;; +26AA;MEDIUM WHITE CIRCLE;So;0;ON;;;;;N;;;;; +26AB;MEDIUM BLACK CIRCLE;So;0;ON;;;;;N;;;;; +26AC;MEDIUM SMALL WHITE CIRCLE;So;0;L;;;;;N;;;;; +26AD;MARRIAGE SYMBOL;So;0;ON;;;;;N;;;;; +26AE;DIVORCE SYMBOL;So;0;ON;;;;;N;;;;; +26AF;UNMARRIED PARTNERSHIP SYMBOL;So;0;ON;;;;;N;;;;; +26B0;COFFIN;So;0;ON;;;;;N;;;;; +26B1;FUNERAL URN;So;0;ON;;;;;N;;;;; +26B2;NEUTER;So;0;ON;;;;;N;;;;; +26B3;CERES;So;0;ON;;;;;N;;;;; +26B4;PALLAS;So;0;ON;;;;;N;;;;; +26B5;JUNO;So;0;ON;;;;;N;;;;; +26B6;VESTA;So;0;ON;;;;;N;;;;; +26B7;CHIRON;So;0;ON;;;;;N;;;;; +26B8;BLACK MOON LILITH;So;0;ON;;;;;N;;;;; +26B9;SEXTILE;So;0;ON;;;;;N;;;;; +26BA;SEMISEXTILE;So;0;ON;;;;;N;;;;; +26BB;QUINCUNX;So;0;ON;;;;;N;;;;; +26BC;SESQUIQUADRATE;So;0;ON;;;;;N;;;;; +26BD;SOCCER BALL;So;0;ON;;;;;N;;;;; +26BE;BASEBALL;So;0;ON;;;;;N;;;;; +26BF;SQUARED KEY;So;0;ON;;;;;N;;;;; +26C0;WHITE DRAUGHTS MAN;So;0;ON;;;;;N;;;;; +26C1;WHITE DRAUGHTS KING;So;0;ON;;;;;N;;;;; +26C2;BLACK DRAUGHTS MAN;So;0;ON;;;;;N;;;;; +26C3;BLACK DRAUGHTS KING;So;0;ON;;;;;N;;;;; +26C4;SNOWMAN WITHOUT SNOW;So;0;ON;;;;;N;;;;; +26C5;SUN BEHIND CLOUD;So;0;ON;;;;;N;;;;; +26C6;RAIN;So;0;ON;;;;;N;;;;; +26C7;BLACK SNOWMAN;So;0;ON;;;;;N;;;;; +26C8;THUNDER CLOUD AND RAIN;So;0;ON;;;;;N;;;;; +26C9;TURNED WHITE SHOGI PIECE;So;0;ON;;;;;N;;;;; +26CA;TURNED BLACK SHOGI PIECE;So;0;ON;;;;;N;;;;; +26CB;WHITE DIAMOND IN SQUARE;So;0;ON;;;;;N;;;;; +26CC;CROSSING LANES;So;0;ON;;;;;N;;;;; +26CD;DISABLED CAR;So;0;ON;;;;;N;;;;; +26CE;OPHIUCHUS;So;0;ON;;;;;N;;;;; +26CF;PICK;So;0;ON;;;;;N;;;;; +26D0;CAR SLIDING;So;0;ON;;;;;N;;;;; +26D1;HELMET WITH WHITE CROSS;So;0;ON;;;;;N;;;;; +26D2;CIRCLED CROSSING LANES;So;0;ON;;;;;N;;;;; +26D3;CHAINS;So;0;ON;;;;;N;;;;; +26D4;NO ENTRY;So;0;ON;;;;;N;;;;; +26D5;ALTERNATE ONE-WAY LEFT WAY TRAFFIC;So;0;ON;;;;;N;;;;; +26D6;BLACK TWO-WAY LEFT WAY TRAFFIC;So;0;ON;;;;;N;;;;; +26D7;WHITE TWO-WAY LEFT WAY TRAFFIC;So;0;ON;;;;;N;;;;; +26D8;BLACK LEFT LANE MERGE;So;0;ON;;;;;N;;;;; +26D9;WHITE LEFT LANE MERGE;So;0;ON;;;;;N;;;;; +26DA;DRIVE SLOW SIGN;So;0;ON;;;;;N;;;;; +26DB;HEAVY WHITE DOWN-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +26DC;LEFT CLOSED ENTRY;So;0;ON;;;;;N;;;;; +26DD;SQUARED SALTIRE;So;0;ON;;;;;N;;;;; +26DE;FALLING DIAGONAL IN WHITE CIRCLE IN BLACK SQUARE;So;0;ON;;;;;N;;;;; +26DF;BLACK TRUCK;So;0;ON;;;;;N;;;;; +26E0;RESTRICTED LEFT ENTRY-1;So;0;ON;;;;;N;;;;; +26E1;RESTRICTED LEFT ENTRY-2;So;0;ON;;;;;N;;;;; +26E2;ASTRONOMICAL SYMBOL FOR URANUS;So;0;ON;;;;;N;;;;; +26E3;HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE;So;0;ON;;;;;N;;;;; +26E4;PENTAGRAM;So;0;ON;;;;;N;;;;; +26E5;RIGHT-HANDED INTERLACED PENTAGRAM;So;0;ON;;;;;N;;;;; +26E6;LEFT-HANDED INTERLACED PENTAGRAM;So;0;ON;;;;;N;;;;; +26E7;INVERTED PENTAGRAM;So;0;ON;;;;;N;;;;; +26E8;BLACK CROSS ON SHIELD;So;0;ON;;;;;N;;;;; +26E9;SHINTO SHRINE;So;0;ON;;;;;N;;;;; +26EA;CHURCH;So;0;ON;;;;;N;;;;; +26EB;CASTLE;So;0;ON;;;;;N;;;;; +26EC;HISTORIC SITE;So;0;ON;;;;;N;;;;; +26ED;GEAR WITHOUT HUB;So;0;ON;;;;;N;;;;; +26EE;GEAR WITH HANDLES;So;0;ON;;;;;N;;;;; +26EF;MAP SYMBOL FOR LIGHTHOUSE;So;0;ON;;;;;N;;;;; +26F0;MOUNTAIN;So;0;ON;;;;;N;;;;; +26F1;UMBRELLA ON GROUND;So;0;ON;;;;;N;;;;; +26F2;FOUNTAIN;So;0;ON;;;;;N;;;;; +26F3;FLAG IN HOLE;So;0;ON;;;;;N;;;;; +26F4;FERRY;So;0;ON;;;;;N;;;;; +26F5;SAILBOAT;So;0;ON;;;;;N;;;;; +26F6;SQUARE FOUR CORNERS;So;0;ON;;;;;N;;;;; +26F7;SKIER;So;0;ON;;;;;N;;;;; +26F8;ICE SKATE;So;0;ON;;;;;N;;;;; +26F9;PERSON WITH BALL;So;0;ON;;;;;N;;;;; +26FA;TENT;So;0;ON;;;;;N;;;;; +26FB;JAPANESE BANK SYMBOL;So;0;ON;;;;;N;;;;; +26FC;HEADSTONE GRAVEYARD SYMBOL;So;0;ON;;;;;N;;;;; +26FD;FUEL PUMP;So;0;ON;;;;;N;;;;; +26FE;CUP ON BLACK SQUARE;So;0;ON;;;;;N;;;;; +26FF;WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE;So;0;ON;;;;;N;;;;; +2701;UPPER BLADE SCISSORS;So;0;ON;;;;;N;;;;; +2702;BLACK SCISSORS;So;0;ON;;;;;N;;;;; +2703;LOWER BLADE SCISSORS;So;0;ON;;;;;N;;;;; +2704;WHITE SCISSORS;So;0;ON;;;;;N;;;;; +2705;WHITE HEAVY CHECK MARK;So;0;ON;;;;;N;;;;; +2706;TELEPHONE LOCATION SIGN;So;0;ON;;;;;N;;;;; +2707;TAPE DRIVE;So;0;ON;;;;;N;;;;; +2708;AIRPLANE;So;0;ON;;;;;N;;;;; +2709;ENVELOPE;So;0;ON;;;;;N;;;;; +270A;RAISED FIST;So;0;ON;;;;;N;;;;; +270B;RAISED HAND;So;0;ON;;;;;N;;;;; +270C;VICTORY HAND;So;0;ON;;;;;N;;;;; +270D;WRITING HAND;So;0;ON;;;;;N;;;;; +270E;LOWER RIGHT PENCIL;So;0;ON;;;;;N;;;;; +270F;PENCIL;So;0;ON;;;;;N;;;;; +2710;UPPER RIGHT PENCIL;So;0;ON;;;;;N;;;;; +2711;WHITE NIB;So;0;ON;;;;;N;;;;; +2712;BLACK NIB;So;0;ON;;;;;N;;;;; +2713;CHECK MARK;So;0;ON;;;;;N;;;;; +2714;HEAVY CHECK MARK;So;0;ON;;;;;N;;;;; +2715;MULTIPLICATION X;So;0;ON;;;;;N;;;;; +2716;HEAVY MULTIPLICATION X;So;0;ON;;;;;N;;;;; +2717;BALLOT X;So;0;ON;;;;;N;;;;; +2718;HEAVY BALLOT X;So;0;ON;;;;;N;;;;; +2719;OUTLINED GREEK CROSS;So;0;ON;;;;;N;;;;; +271A;HEAVY GREEK CROSS;So;0;ON;;;;;N;;;;; +271B;OPEN CENTRE CROSS;So;0;ON;;;;;N;OPEN CENTER CROSS;;;; +271C;HEAVY OPEN CENTRE CROSS;So;0;ON;;;;;N;HEAVY OPEN CENTER CROSS;;;; +271D;LATIN CROSS;So;0;ON;;;;;N;;;;; +271E;SHADOWED WHITE LATIN CROSS;So;0;ON;;;;;N;;;;; +271F;OUTLINED LATIN CROSS;So;0;ON;;;;;N;;;;; +2720;MALTESE CROSS;So;0;ON;;;;;N;;;;; +2721;STAR OF DAVID;So;0;ON;;;;;N;;;;; +2722;FOUR TEARDROP-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +2723;FOUR BALLOON-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +2724;HEAVY FOUR BALLOON-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +2725;FOUR CLUB-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +2726;BLACK FOUR POINTED STAR;So;0;ON;;;;;N;;;;; +2727;WHITE FOUR POINTED STAR;So;0;ON;;;;;N;;;;; +2728;SPARKLES;So;0;ON;;;;;N;;;;; +2729;STRESS OUTLINED WHITE STAR;So;0;ON;;;;;N;;;;; +272A;CIRCLED WHITE STAR;So;0;ON;;;;;N;;;;; +272B;OPEN CENTRE BLACK STAR;So;0;ON;;;;;N;OPEN CENTER BLACK STAR;;;; +272C;BLACK CENTRE WHITE STAR;So;0;ON;;;;;N;BLACK CENTER WHITE STAR;;;; +272D;OUTLINED BLACK STAR;So;0;ON;;;;;N;;;;; +272E;HEAVY OUTLINED BLACK STAR;So;0;ON;;;;;N;;;;; +272F;PINWHEEL STAR;So;0;ON;;;;;N;;;;; +2730;SHADOWED WHITE STAR;So;0;ON;;;;;N;;;;; +2731;HEAVY ASTERISK;So;0;ON;;;;;N;;;;; +2732;OPEN CENTRE ASTERISK;So;0;ON;;;;;N;OPEN CENTER ASTERISK;;;; +2733;EIGHT SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +2734;EIGHT POINTED BLACK STAR;So;0;ON;;;;;N;;;;; +2735;EIGHT POINTED PINWHEEL STAR;So;0;ON;;;;;N;;;;; +2736;SIX POINTED BLACK STAR;So;0;ON;;;;;N;;;;; +2737;EIGHT POINTED RECTILINEAR BLACK STAR;So;0;ON;;;;;N;;;;; +2738;HEAVY EIGHT POINTED RECTILINEAR BLACK STAR;So;0;ON;;;;;N;;;;; +2739;TWELVE POINTED BLACK STAR;So;0;ON;;;;;N;;;;; +273A;SIXTEEN POINTED ASTERISK;So;0;ON;;;;;N;;;;; +273B;TEARDROP-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +273C;OPEN CENTRE TEARDROP-SPOKED ASTERISK;So;0;ON;;;;;N;OPEN CENTER TEARDROP-SPOKED ASTERISK;;;; +273D;HEAVY TEARDROP-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +273E;SIX PETALLED BLACK AND WHITE FLORETTE;So;0;ON;;;;;N;;;;; +273F;BLACK FLORETTE;So;0;ON;;;;;N;;;;; +2740;WHITE FLORETTE;So;0;ON;;;;;N;;;;; +2741;EIGHT PETALLED OUTLINED BLACK FLORETTE;So;0;ON;;;;;N;;;;; +2742;CIRCLED OPEN CENTRE EIGHT POINTED STAR;So;0;ON;;;;;N;CIRCLED OPEN CENTER EIGHT POINTED STAR;;;; +2743;HEAVY TEARDROP-SPOKED PINWHEEL ASTERISK;So;0;ON;;;;;N;;;;; +2744;SNOWFLAKE;So;0;ON;;;;;N;;;;; +2745;TIGHT TRIFOLIATE SNOWFLAKE;So;0;ON;;;;;N;;;;; +2746;HEAVY CHEVRON SNOWFLAKE;So;0;ON;;;;;N;;;;; +2747;SPARKLE;So;0;ON;;;;;N;;;;; +2748;HEAVY SPARKLE;So;0;ON;;;;;N;;;;; +2749;BALLOON-SPOKED ASTERISK;So;0;ON;;;;;N;;;;; +274A;EIGHT TEARDROP-SPOKED PROPELLER ASTERISK;So;0;ON;;;;;N;;;;; +274B;HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK;So;0;ON;;;;;N;;;;; +274C;CROSS MARK;So;0;ON;;;;;N;;;;; +274D;SHADOWED WHITE CIRCLE;So;0;ON;;;;;N;;;;; +274E;NEGATIVE SQUARED CROSS MARK;So;0;ON;;;;;N;;;;; +274F;LOWER RIGHT DROP-SHADOWED WHITE SQUARE;So;0;ON;;;;;N;;;;; +2750;UPPER RIGHT DROP-SHADOWED WHITE SQUARE;So;0;ON;;;;;N;;;;; +2751;LOWER RIGHT SHADOWED WHITE SQUARE;So;0;ON;;;;;N;;;;; +2752;UPPER RIGHT SHADOWED WHITE SQUARE;So;0;ON;;;;;N;;;;; +2753;BLACK QUESTION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2754;WHITE QUESTION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2755;WHITE EXCLAMATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2756;BLACK DIAMOND MINUS WHITE X;So;0;ON;;;;;N;;;;; +2757;HEAVY EXCLAMATION MARK SYMBOL;So;0;ON;;;;;N;;;;; +2758;LIGHT VERTICAL BAR;So;0;ON;;;;;N;;;;; +2759;MEDIUM VERTICAL BAR;So;0;ON;;;;;N;;;;; +275A;HEAVY VERTICAL BAR;So;0;ON;;;;;N;;;;; +275B;HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +275C;HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +275D;HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +275E;HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +275F;HEAVY LOW SINGLE COMMA QUOTATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2760;HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2761;CURVED STEM PARAGRAPH SIGN ORNAMENT;So;0;ON;;;;;N;;;;; +2762;HEAVY EXCLAMATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2763;HEAVY HEART EXCLAMATION MARK ORNAMENT;So;0;ON;;;;;N;;;;; +2764;HEAVY BLACK HEART;So;0;ON;;;;;N;;;;; +2765;ROTATED HEAVY BLACK HEART BULLET;So;0;ON;;;;;N;;;;; +2766;FLORAL HEART;So;0;ON;;;;;N;;;;; +2767;ROTATED FLORAL HEART BULLET;So;0;ON;;;;;N;;;;; +2768;MEDIUM LEFT PARENTHESIS ORNAMENT;Ps;0;ON;;;;;Y;;;;; +2769;MEDIUM RIGHT PARENTHESIS ORNAMENT;Pe;0;ON;;;;;Y;;;;; +276A;MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT;Ps;0;ON;;;;;Y;;;;; +276B;MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT;Pe;0;ON;;;;;Y;;;;; +276C;MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT;Ps;0;ON;;;;;Y;;;;; +276D;MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT;Pe;0;ON;;;;;Y;;;;; +276E;HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT;Ps;0;ON;;;;;Y;;;;; +276F;HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT;Pe;0;ON;;;;;Y;;;;; +2770;HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT;Ps;0;ON;;;;;Y;;;;; +2771;HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT;Pe;0;ON;;;;;Y;;;;; +2772;LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT;Ps;0;ON;;;;;Y;;;;; +2773;LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT;Pe;0;ON;;;;;Y;;;;; +2774;MEDIUM LEFT CURLY BRACKET ORNAMENT;Ps;0;ON;;;;;Y;;;;; +2775;MEDIUM RIGHT CURLY BRACKET ORNAMENT;Pe;0;ON;;;;;Y;;;;; +2776;DINGBAT NEGATIVE CIRCLED DIGIT ONE;No;0;ON;;;1;1;N;INVERSE CIRCLED DIGIT ONE;;;; +2777;DINGBAT NEGATIVE CIRCLED DIGIT TWO;No;0;ON;;;2;2;N;INVERSE CIRCLED DIGIT TWO;;;; +2778;DINGBAT NEGATIVE CIRCLED DIGIT THREE;No;0;ON;;;3;3;N;INVERSE CIRCLED DIGIT THREE;;;; +2779;DINGBAT NEGATIVE CIRCLED DIGIT FOUR;No;0;ON;;;4;4;N;INVERSE CIRCLED DIGIT FOUR;;;; +277A;DINGBAT NEGATIVE CIRCLED DIGIT FIVE;No;0;ON;;;5;5;N;INVERSE CIRCLED DIGIT FIVE;;;; +277B;DINGBAT NEGATIVE CIRCLED DIGIT SIX;No;0;ON;;;6;6;N;INVERSE CIRCLED DIGIT SIX;;;; +277C;DINGBAT NEGATIVE CIRCLED DIGIT SEVEN;No;0;ON;;;7;7;N;INVERSE CIRCLED DIGIT SEVEN;;;; +277D;DINGBAT NEGATIVE CIRCLED DIGIT EIGHT;No;0;ON;;;8;8;N;INVERSE CIRCLED DIGIT EIGHT;;;; +277E;DINGBAT NEGATIVE CIRCLED DIGIT NINE;No;0;ON;;;9;9;N;INVERSE CIRCLED DIGIT NINE;;;; +277F;DINGBAT NEGATIVE CIRCLED NUMBER TEN;No;0;ON;;;;10;N;INVERSE CIRCLED NUMBER TEN;;;; +2780;DINGBAT CIRCLED SANS-SERIF DIGIT ONE;No;0;ON;;;1;1;N;CIRCLED SANS-SERIF DIGIT ONE;;;; +2781;DINGBAT CIRCLED SANS-SERIF DIGIT TWO;No;0;ON;;;2;2;N;CIRCLED SANS-SERIF DIGIT TWO;;;; +2782;DINGBAT CIRCLED SANS-SERIF DIGIT THREE;No;0;ON;;;3;3;N;CIRCLED SANS-SERIF DIGIT THREE;;;; +2783;DINGBAT CIRCLED SANS-SERIF DIGIT FOUR;No;0;ON;;;4;4;N;CIRCLED SANS-SERIF DIGIT FOUR;;;; +2784;DINGBAT CIRCLED SANS-SERIF DIGIT FIVE;No;0;ON;;;5;5;N;CIRCLED SANS-SERIF DIGIT FIVE;;;; +2785;DINGBAT CIRCLED SANS-SERIF DIGIT SIX;No;0;ON;;;6;6;N;CIRCLED SANS-SERIF DIGIT SIX;;;; +2786;DINGBAT CIRCLED SANS-SERIF DIGIT SEVEN;No;0;ON;;;7;7;N;CIRCLED SANS-SERIF DIGIT SEVEN;;;; +2787;DINGBAT CIRCLED SANS-SERIF DIGIT EIGHT;No;0;ON;;;8;8;N;CIRCLED SANS-SERIF DIGIT EIGHT;;;; +2788;DINGBAT CIRCLED SANS-SERIF DIGIT NINE;No;0;ON;;;9;9;N;CIRCLED SANS-SERIF DIGIT NINE;;;; +2789;DINGBAT CIRCLED SANS-SERIF NUMBER TEN;No;0;ON;;;;10;N;CIRCLED SANS-SERIF NUMBER TEN;;;; +278A;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ONE;No;0;ON;;;1;1;N;INVERSE CIRCLED SANS-SERIF DIGIT ONE;;;; +278B;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT TWO;No;0;ON;;;2;2;N;INVERSE CIRCLED SANS-SERIF DIGIT TWO;;;; +278C;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT THREE;No;0;ON;;;3;3;N;INVERSE CIRCLED SANS-SERIF DIGIT THREE;;;; +278D;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FOUR;No;0;ON;;;4;4;N;INVERSE CIRCLED SANS-SERIF DIGIT FOUR;;;; +278E;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FIVE;No;0;ON;;;5;5;N;INVERSE CIRCLED SANS-SERIF DIGIT FIVE;;;; +278F;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SIX;No;0;ON;;;6;6;N;INVERSE CIRCLED SANS-SERIF DIGIT SIX;;;; +2790;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SEVEN;No;0;ON;;;7;7;N;INVERSE CIRCLED SANS-SERIF DIGIT SEVEN;;;; +2791;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT EIGHT;No;0;ON;;;8;8;N;INVERSE CIRCLED SANS-SERIF DIGIT EIGHT;;;; +2792;DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT NINE;No;0;ON;;;9;9;N;INVERSE CIRCLED SANS-SERIF DIGIT NINE;;;; +2793;DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN;No;0;ON;;;;10;N;INVERSE CIRCLED SANS-SERIF NUMBER TEN;;;; +2794;HEAVY WIDE-HEADED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY WIDE-HEADED RIGHT ARROW;;;; +2795;HEAVY PLUS SIGN;So;0;ON;;;;;N;;;;; +2796;HEAVY MINUS SIGN;So;0;ON;;;;;N;;;;; +2797;HEAVY DIVISION SIGN;So;0;ON;;;;;N;;;;; +2798;HEAVY SOUTH EAST ARROW;So;0;ON;;;;;N;HEAVY LOWER RIGHT ARROW;;;; +2799;HEAVY RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY RIGHT ARROW;;;; +279A;HEAVY NORTH EAST ARROW;So;0;ON;;;;;N;HEAVY UPPER RIGHT ARROW;;;; +279B;DRAFTING POINT RIGHTWARDS ARROW;So;0;ON;;;;;N;DRAFTING POINT RIGHT ARROW;;;; +279C;HEAVY ROUND-TIPPED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY ROUND-TIPPED RIGHT ARROW;;;; +279D;TRIANGLE-HEADED RIGHTWARDS ARROW;So;0;ON;;;;;N;TRIANGLE-HEADED RIGHT ARROW;;;; +279E;HEAVY TRIANGLE-HEADED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY TRIANGLE-HEADED RIGHT ARROW;;;; +279F;DASHED TRIANGLE-HEADED RIGHTWARDS ARROW;So;0;ON;;;;;N;DASHED TRIANGLE-HEADED RIGHT ARROW;;;; +27A0;HEAVY DASHED TRIANGLE-HEADED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY DASHED TRIANGLE-HEADED RIGHT ARROW;;;; +27A1;BLACK RIGHTWARDS ARROW;So;0;ON;;;;;N;BLACK RIGHT ARROW;;;; +27A2;THREE-D TOP-LIGHTED RIGHTWARDS ARROWHEAD;So;0;ON;;;;;N;THREE-D TOP-LIGHTED RIGHT ARROWHEAD;;;; +27A3;THREE-D BOTTOM-LIGHTED RIGHTWARDS ARROWHEAD;So;0;ON;;;;;N;THREE-D BOTTOM-LIGHTED RIGHT ARROWHEAD;;;; +27A4;BLACK RIGHTWARDS ARROWHEAD;So;0;ON;;;;;N;BLACK RIGHT ARROWHEAD;;;; +27A5;HEAVY BLACK CURVED DOWNWARDS AND RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY BLACK CURVED DOWN AND RIGHT ARROW;;;; +27A6;HEAVY BLACK CURVED UPWARDS AND RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY BLACK CURVED UP AND RIGHT ARROW;;;; +27A7;SQUAT BLACK RIGHTWARDS ARROW;So;0;ON;;;;;N;SQUAT BLACK RIGHT ARROW;;;; +27A8;HEAVY CONCAVE-POINTED BLACK RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY CONCAVE-POINTED BLACK RIGHT ARROW;;;; +27A9;RIGHT-SHADED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;RIGHT-SHADED WHITE RIGHT ARROW;;;; +27AA;LEFT-SHADED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;LEFT-SHADED WHITE RIGHT ARROW;;;; +27AB;BACK-TILTED SHADOWED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;BACK-TILTED SHADOWED WHITE RIGHT ARROW;;;; +27AC;FRONT-TILTED SHADOWED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;FRONT-TILTED SHADOWED WHITE RIGHT ARROW;;;; +27AD;HEAVY LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY LOWER RIGHT-SHADOWED WHITE RIGHT ARROW;;;; +27AE;HEAVY UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY UPPER RIGHT-SHADOWED WHITE RIGHT ARROW;;;; +27AF;NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHT ARROW;;;; +27B0;CURLY LOOP;So;0;ON;;;;;N;;;;; +27B1;NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHT ARROW;;;; +27B2;CIRCLED HEAVY WHITE RIGHTWARDS ARROW;So;0;ON;;;;;N;CIRCLED HEAVY WHITE RIGHT ARROW;;;; +27B3;WHITE-FEATHERED RIGHTWARDS ARROW;So;0;ON;;;;;N;WHITE-FEATHERED RIGHT ARROW;;;; +27B4;BLACK-FEATHERED SOUTH EAST ARROW;So;0;ON;;;;;N;BLACK-FEATHERED LOWER RIGHT ARROW;;;; +27B5;BLACK-FEATHERED RIGHTWARDS ARROW;So;0;ON;;;;;N;BLACK-FEATHERED RIGHT ARROW;;;; +27B6;BLACK-FEATHERED NORTH EAST ARROW;So;0;ON;;;;;N;BLACK-FEATHERED UPPER RIGHT ARROW;;;; +27B7;HEAVY BLACK-FEATHERED SOUTH EAST ARROW;So;0;ON;;;;;N;HEAVY BLACK-FEATHERED LOWER RIGHT ARROW;;;; +27B8;HEAVY BLACK-FEATHERED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY BLACK-FEATHERED RIGHT ARROW;;;; +27B9;HEAVY BLACK-FEATHERED NORTH EAST ARROW;So;0;ON;;;;;N;HEAVY BLACK-FEATHERED UPPER RIGHT ARROW;;;; +27BA;TEARDROP-BARBED RIGHTWARDS ARROW;So;0;ON;;;;;N;TEARDROP-BARBED RIGHT ARROW;;;; +27BB;HEAVY TEARDROP-SHANKED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY TEARDROP-SHANKED RIGHT ARROW;;;; +27BC;WEDGE-TAILED RIGHTWARDS ARROW;So;0;ON;;;;;N;WEDGE-TAILED RIGHT ARROW;;;; +27BD;HEAVY WEDGE-TAILED RIGHTWARDS ARROW;So;0;ON;;;;;N;HEAVY WEDGE-TAILED RIGHT ARROW;;;; +27BE;OPEN-OUTLINED RIGHTWARDS ARROW;So;0;ON;;;;;N;OPEN-OUTLINED RIGHT ARROW;;;; +27BF;DOUBLE CURLY LOOP;So;0;ON;;;;;N;;;;; +27C0;THREE DIMENSIONAL ANGLE;Sm;0;ON;;;;;Y;;;;; +27C1;WHITE TRIANGLE CONTAINING SMALL WHITE TRIANGLE;Sm;0;ON;;;;;N;;;;; +27C2;PERPENDICULAR;Sm;0;ON;;;;;N;;;;; +27C3;OPEN SUBSET;Sm;0;ON;;;;;Y;;;;; +27C4;OPEN SUPERSET;Sm;0;ON;;;;;Y;;;;; +27C5;LEFT S-SHAPED BAG DELIMITER;Ps;0;ON;;;;;Y;;;;; +27C6;RIGHT S-SHAPED BAG DELIMITER;Pe;0;ON;;;;;Y;;;;; +27C7;OR WITH DOT INSIDE;Sm;0;ON;;;;;N;;;;; +27C8;REVERSE SOLIDUS PRECEDING SUBSET;Sm;0;ON;;;;;Y;;;;; +27C9;SUPERSET PRECEDING SOLIDUS;Sm;0;ON;;;;;Y;;;;; +27CA;VERTICAL BAR WITH HORIZONTAL STROKE;Sm;0;ON;;;;;N;;;;; +27CB;MATHEMATICAL RISING DIAGONAL;Sm;0;ON;;;;;Y;;;;; +27CC;LONG DIVISION;Sm;0;ON;;;;;Y;;;;; +27CD;MATHEMATICAL FALLING DIAGONAL;Sm;0;ON;;;;;Y;;;;; +27CE;SQUARED LOGICAL AND;Sm;0;ON;;;;;N;;;;; +27CF;SQUARED LOGICAL OR;Sm;0;ON;;;;;N;;;;; +27D0;WHITE DIAMOND WITH CENTRED DOT;Sm;0;ON;;;;;N;;;;; +27D1;AND WITH DOT;Sm;0;ON;;;;;N;;;;; +27D2;ELEMENT OF OPENING UPWARDS;Sm;0;ON;;;;;N;;;;; +27D3;LOWER RIGHT CORNER WITH DOT;Sm;0;ON;;;;;Y;;;;; +27D4;UPPER LEFT CORNER WITH DOT;Sm;0;ON;;;;;Y;;;;; +27D5;LEFT OUTER JOIN;Sm;0;ON;;;;;Y;;;;; +27D6;RIGHT OUTER JOIN;Sm;0;ON;;;;;Y;;;;; +27D7;FULL OUTER JOIN;Sm;0;ON;;;;;N;;;;; +27D8;LARGE UP TACK;Sm;0;ON;;;;;N;;;;; +27D9;LARGE DOWN TACK;Sm;0;ON;;;;;N;;;;; +27DA;LEFT AND RIGHT DOUBLE TURNSTILE;Sm;0;ON;;;;;N;;;;; +27DB;LEFT AND RIGHT TACK;Sm;0;ON;;;;;N;;;;; +27DC;LEFT MULTIMAP;Sm;0;ON;;;;;Y;;;;; +27DD;LONG RIGHT TACK;Sm;0;ON;;;;;Y;;;;; +27DE;LONG LEFT TACK;Sm;0;ON;;;;;Y;;;;; +27DF;UP TACK WITH CIRCLE ABOVE;Sm;0;ON;;;;;N;;;;; +27E0;LOZENGE DIVIDED BY HORIZONTAL RULE;Sm;0;ON;;;;;N;;;;; +27E1;WHITE CONCAVE-SIDED DIAMOND;Sm;0;ON;;;;;N;;;;; +27E2;WHITE CONCAVE-SIDED DIAMOND WITH LEFTWARDS TICK;Sm;0;ON;;;;;Y;;;;; +27E3;WHITE CONCAVE-SIDED DIAMOND WITH RIGHTWARDS TICK;Sm;0;ON;;;;;Y;;;;; +27E4;WHITE SQUARE WITH LEFTWARDS TICK;Sm;0;ON;;;;;Y;;;;; +27E5;WHITE SQUARE WITH RIGHTWARDS TICK;Sm;0;ON;;;;;Y;;;;; +27E6;MATHEMATICAL LEFT WHITE SQUARE BRACKET;Ps;0;ON;;;;;Y;;;;; +27E7;MATHEMATICAL RIGHT WHITE SQUARE BRACKET;Pe;0;ON;;;;;Y;;;;; +27E8;MATHEMATICAL LEFT ANGLE BRACKET;Ps;0;ON;;;;;Y;;;;; +27E9;MATHEMATICAL RIGHT ANGLE BRACKET;Pe;0;ON;;;;;Y;;;;; +27EA;MATHEMATICAL LEFT DOUBLE ANGLE BRACKET;Ps;0;ON;;;;;Y;;;;; +27EB;MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET;Pe;0;ON;;;;;Y;;;;; +27EC;MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET;Ps;0;ON;;;;;Y;;;;; +27ED;MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET;Pe;0;ON;;;;;Y;;;;; +27EE;MATHEMATICAL LEFT FLATTENED PARENTHESIS;Ps;0;ON;;;;;Y;;;;; +27EF;MATHEMATICAL RIGHT FLATTENED PARENTHESIS;Pe;0;ON;;;;;Y;;;;; +27F0;UPWARDS QUADRUPLE ARROW;Sm;0;ON;;;;;N;;;;; +27F1;DOWNWARDS QUADRUPLE ARROW;Sm;0;ON;;;;;N;;;;; +27F2;ANTICLOCKWISE GAPPED CIRCLE ARROW;Sm;0;ON;;;;;N;;;;; +27F3;CLOCKWISE GAPPED CIRCLE ARROW;Sm;0;ON;;;;;N;;;;; +27F4;RIGHT ARROW WITH CIRCLED PLUS;Sm;0;ON;;;;;N;;;;; +27F5;LONG LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +27F6;LONG RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +27F7;LONG LEFT RIGHT ARROW;Sm;0;ON;;;;;N;;;;; +27F8;LONG LEFTWARDS DOUBLE ARROW;Sm;0;ON;;;;;N;;;;; +27F9;LONG RIGHTWARDS DOUBLE ARROW;Sm;0;ON;;;;;N;;;;; +27FA;LONG LEFT RIGHT DOUBLE ARROW;Sm;0;ON;;;;;N;;;;; +27FB;LONG LEFTWARDS ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +27FC;LONG RIGHTWARDS ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +27FD;LONG LEFTWARDS DOUBLE ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +27FE;LONG RIGHTWARDS DOUBLE ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +27FF;LONG RIGHTWARDS SQUIGGLE ARROW;Sm;0;ON;;;;;N;;;;; +2800;BRAILLE PATTERN BLANK;So;0;L;;;;;N;;;;; +2801;BRAILLE PATTERN DOTS-1;So;0;L;;;;;N;;;;; +2802;BRAILLE PATTERN DOTS-2;So;0;L;;;;;N;;;;; +2803;BRAILLE PATTERN DOTS-12;So;0;L;;;;;N;;;;; +2804;BRAILLE PATTERN DOTS-3;So;0;L;;;;;N;;;;; +2805;BRAILLE PATTERN DOTS-13;So;0;L;;;;;N;;;;; +2806;BRAILLE PATTERN DOTS-23;So;0;L;;;;;N;;;;; +2807;BRAILLE PATTERN DOTS-123;So;0;L;;;;;N;;;;; +2808;BRAILLE PATTERN DOTS-4;So;0;L;;;;;N;;;;; +2809;BRAILLE PATTERN DOTS-14;So;0;L;;;;;N;;;;; +280A;BRAILLE PATTERN DOTS-24;So;0;L;;;;;N;;;;; +280B;BRAILLE PATTERN DOTS-124;So;0;L;;;;;N;;;;; +280C;BRAILLE PATTERN DOTS-34;So;0;L;;;;;N;;;;; +280D;BRAILLE PATTERN DOTS-134;So;0;L;;;;;N;;;;; +280E;BRAILLE PATTERN DOTS-234;So;0;L;;;;;N;;;;; +280F;BRAILLE PATTERN DOTS-1234;So;0;L;;;;;N;;;;; +2810;BRAILLE PATTERN DOTS-5;So;0;L;;;;;N;;;;; +2811;BRAILLE PATTERN DOTS-15;So;0;L;;;;;N;;;;; +2812;BRAILLE PATTERN DOTS-25;So;0;L;;;;;N;;;;; +2813;BRAILLE PATTERN DOTS-125;So;0;L;;;;;N;;;;; +2814;BRAILLE PATTERN DOTS-35;So;0;L;;;;;N;;;;; +2815;BRAILLE PATTERN DOTS-135;So;0;L;;;;;N;;;;; +2816;BRAILLE PATTERN DOTS-235;So;0;L;;;;;N;;;;; +2817;BRAILLE PATTERN DOTS-1235;So;0;L;;;;;N;;;;; +2818;BRAILLE PATTERN DOTS-45;So;0;L;;;;;N;;;;; +2819;BRAILLE PATTERN DOTS-145;So;0;L;;;;;N;;;;; +281A;BRAILLE PATTERN DOTS-245;So;0;L;;;;;N;;;;; +281B;BRAILLE PATTERN DOTS-1245;So;0;L;;;;;N;;;;; +281C;BRAILLE PATTERN DOTS-345;So;0;L;;;;;N;;;;; +281D;BRAILLE PATTERN DOTS-1345;So;0;L;;;;;N;;;;; +281E;BRAILLE PATTERN DOTS-2345;So;0;L;;;;;N;;;;; +281F;BRAILLE PATTERN DOTS-12345;So;0;L;;;;;N;;;;; +2820;BRAILLE PATTERN DOTS-6;So;0;L;;;;;N;;;;; +2821;BRAILLE PATTERN DOTS-16;So;0;L;;;;;N;;;;; +2822;BRAILLE PATTERN DOTS-26;So;0;L;;;;;N;;;;; +2823;BRAILLE PATTERN DOTS-126;So;0;L;;;;;N;;;;; +2824;BRAILLE PATTERN DOTS-36;So;0;L;;;;;N;;;;; +2825;BRAILLE PATTERN DOTS-136;So;0;L;;;;;N;;;;; +2826;BRAILLE PATTERN DOTS-236;So;0;L;;;;;N;;;;; +2827;BRAILLE PATTERN DOTS-1236;So;0;L;;;;;N;;;;; +2828;BRAILLE PATTERN DOTS-46;So;0;L;;;;;N;;;;; +2829;BRAILLE PATTERN DOTS-146;So;0;L;;;;;N;;;;; +282A;BRAILLE PATTERN DOTS-246;So;0;L;;;;;N;;;;; +282B;BRAILLE PATTERN DOTS-1246;So;0;L;;;;;N;;;;; +282C;BRAILLE PATTERN DOTS-346;So;0;L;;;;;N;;;;; +282D;BRAILLE PATTERN DOTS-1346;So;0;L;;;;;N;;;;; +282E;BRAILLE PATTERN DOTS-2346;So;0;L;;;;;N;;;;; +282F;BRAILLE PATTERN DOTS-12346;So;0;L;;;;;N;;;;; +2830;BRAILLE PATTERN DOTS-56;So;0;L;;;;;N;;;;; +2831;BRAILLE PATTERN DOTS-156;So;0;L;;;;;N;;;;; +2832;BRAILLE PATTERN DOTS-256;So;0;L;;;;;N;;;;; +2833;BRAILLE PATTERN DOTS-1256;So;0;L;;;;;N;;;;; +2834;BRAILLE PATTERN DOTS-356;So;0;L;;;;;N;;;;; +2835;BRAILLE PATTERN DOTS-1356;So;0;L;;;;;N;;;;; +2836;BRAILLE PATTERN DOTS-2356;So;0;L;;;;;N;;;;; +2837;BRAILLE PATTERN DOTS-12356;So;0;L;;;;;N;;;;; +2838;BRAILLE PATTERN DOTS-456;So;0;L;;;;;N;;;;; +2839;BRAILLE PATTERN DOTS-1456;So;0;L;;;;;N;;;;; +283A;BRAILLE PATTERN DOTS-2456;So;0;L;;;;;N;;;;; +283B;BRAILLE PATTERN DOTS-12456;So;0;L;;;;;N;;;;; +283C;BRAILLE PATTERN DOTS-3456;So;0;L;;;;;N;;;;; +283D;BRAILLE PATTERN DOTS-13456;So;0;L;;;;;N;;;;; +283E;BRAILLE PATTERN DOTS-23456;So;0;L;;;;;N;;;;; +283F;BRAILLE PATTERN DOTS-123456;So;0;L;;;;;N;;;;; +2840;BRAILLE PATTERN DOTS-7;So;0;L;;;;;N;;;;; +2841;BRAILLE PATTERN DOTS-17;So;0;L;;;;;N;;;;; +2842;BRAILLE PATTERN DOTS-27;So;0;L;;;;;N;;;;; +2843;BRAILLE PATTERN DOTS-127;So;0;L;;;;;N;;;;; +2844;BRAILLE PATTERN DOTS-37;So;0;L;;;;;N;;;;; +2845;BRAILLE PATTERN DOTS-137;So;0;L;;;;;N;;;;; +2846;BRAILLE PATTERN DOTS-237;So;0;L;;;;;N;;;;; +2847;BRAILLE PATTERN DOTS-1237;So;0;L;;;;;N;;;;; +2848;BRAILLE PATTERN DOTS-47;So;0;L;;;;;N;;;;; +2849;BRAILLE PATTERN DOTS-147;So;0;L;;;;;N;;;;; +284A;BRAILLE PATTERN DOTS-247;So;0;L;;;;;N;;;;; +284B;BRAILLE PATTERN DOTS-1247;So;0;L;;;;;N;;;;; +284C;BRAILLE PATTERN DOTS-347;So;0;L;;;;;N;;;;; +284D;BRAILLE PATTERN DOTS-1347;So;0;L;;;;;N;;;;; +284E;BRAILLE PATTERN DOTS-2347;So;0;L;;;;;N;;;;; +284F;BRAILLE PATTERN DOTS-12347;So;0;L;;;;;N;;;;; +2850;BRAILLE PATTERN DOTS-57;So;0;L;;;;;N;;;;; +2851;BRAILLE PATTERN DOTS-157;So;0;L;;;;;N;;;;; +2852;BRAILLE PATTERN DOTS-257;So;0;L;;;;;N;;;;; +2853;BRAILLE PATTERN DOTS-1257;So;0;L;;;;;N;;;;; +2854;BRAILLE PATTERN DOTS-357;So;0;L;;;;;N;;;;; +2855;BRAILLE PATTERN DOTS-1357;So;0;L;;;;;N;;;;; +2856;BRAILLE PATTERN DOTS-2357;So;0;L;;;;;N;;;;; +2857;BRAILLE PATTERN DOTS-12357;So;0;L;;;;;N;;;;; +2858;BRAILLE PATTERN DOTS-457;So;0;L;;;;;N;;;;; +2859;BRAILLE PATTERN DOTS-1457;So;0;L;;;;;N;;;;; +285A;BRAILLE PATTERN DOTS-2457;So;0;L;;;;;N;;;;; +285B;BRAILLE PATTERN DOTS-12457;So;0;L;;;;;N;;;;; +285C;BRAILLE PATTERN DOTS-3457;So;0;L;;;;;N;;;;; +285D;BRAILLE PATTERN DOTS-13457;So;0;L;;;;;N;;;;; +285E;BRAILLE PATTERN DOTS-23457;So;0;L;;;;;N;;;;; +285F;BRAILLE PATTERN DOTS-123457;So;0;L;;;;;N;;;;; +2860;BRAILLE PATTERN DOTS-67;So;0;L;;;;;N;;;;; +2861;BRAILLE PATTERN DOTS-167;So;0;L;;;;;N;;;;; +2862;BRAILLE PATTERN DOTS-267;So;0;L;;;;;N;;;;; +2863;BRAILLE PATTERN DOTS-1267;So;0;L;;;;;N;;;;; +2864;BRAILLE PATTERN DOTS-367;So;0;L;;;;;N;;;;; +2865;BRAILLE PATTERN DOTS-1367;So;0;L;;;;;N;;;;; +2866;BRAILLE PATTERN DOTS-2367;So;0;L;;;;;N;;;;; +2867;BRAILLE PATTERN DOTS-12367;So;0;L;;;;;N;;;;; +2868;BRAILLE PATTERN DOTS-467;So;0;L;;;;;N;;;;; +2869;BRAILLE PATTERN DOTS-1467;So;0;L;;;;;N;;;;; +286A;BRAILLE PATTERN DOTS-2467;So;0;L;;;;;N;;;;; +286B;BRAILLE PATTERN DOTS-12467;So;0;L;;;;;N;;;;; +286C;BRAILLE PATTERN DOTS-3467;So;0;L;;;;;N;;;;; +286D;BRAILLE PATTERN DOTS-13467;So;0;L;;;;;N;;;;; +286E;BRAILLE PATTERN DOTS-23467;So;0;L;;;;;N;;;;; +286F;BRAILLE PATTERN DOTS-123467;So;0;L;;;;;N;;;;; +2870;BRAILLE PATTERN DOTS-567;So;0;L;;;;;N;;;;; +2871;BRAILLE PATTERN DOTS-1567;So;0;L;;;;;N;;;;; +2872;BRAILLE PATTERN DOTS-2567;So;0;L;;;;;N;;;;; +2873;BRAILLE PATTERN DOTS-12567;So;0;L;;;;;N;;;;; +2874;BRAILLE PATTERN DOTS-3567;So;0;L;;;;;N;;;;; +2875;BRAILLE PATTERN DOTS-13567;So;0;L;;;;;N;;;;; +2876;BRAILLE PATTERN DOTS-23567;So;0;L;;;;;N;;;;; +2877;BRAILLE PATTERN DOTS-123567;So;0;L;;;;;N;;;;; +2878;BRAILLE PATTERN DOTS-4567;So;0;L;;;;;N;;;;; +2879;BRAILLE PATTERN DOTS-14567;So;0;L;;;;;N;;;;; +287A;BRAILLE PATTERN DOTS-24567;So;0;L;;;;;N;;;;; +287B;BRAILLE PATTERN DOTS-124567;So;0;L;;;;;N;;;;; +287C;BRAILLE PATTERN DOTS-34567;So;0;L;;;;;N;;;;; +287D;BRAILLE PATTERN DOTS-134567;So;0;L;;;;;N;;;;; +287E;BRAILLE PATTERN DOTS-234567;So;0;L;;;;;N;;;;; +287F;BRAILLE PATTERN DOTS-1234567;So;0;L;;;;;N;;;;; +2880;BRAILLE PATTERN DOTS-8;So;0;L;;;;;N;;;;; +2881;BRAILLE PATTERN DOTS-18;So;0;L;;;;;N;;;;; +2882;BRAILLE PATTERN DOTS-28;So;0;L;;;;;N;;;;; +2883;BRAILLE PATTERN DOTS-128;So;0;L;;;;;N;;;;; +2884;BRAILLE PATTERN DOTS-38;So;0;L;;;;;N;;;;; +2885;BRAILLE PATTERN DOTS-138;So;0;L;;;;;N;;;;; +2886;BRAILLE PATTERN DOTS-238;So;0;L;;;;;N;;;;; +2887;BRAILLE PATTERN DOTS-1238;So;0;L;;;;;N;;;;; +2888;BRAILLE PATTERN DOTS-48;So;0;L;;;;;N;;;;; +2889;BRAILLE PATTERN DOTS-148;So;0;L;;;;;N;;;;; +288A;BRAILLE PATTERN DOTS-248;So;0;L;;;;;N;;;;; +288B;BRAILLE PATTERN DOTS-1248;So;0;L;;;;;N;;;;; +288C;BRAILLE PATTERN DOTS-348;So;0;L;;;;;N;;;;; +288D;BRAILLE PATTERN DOTS-1348;So;0;L;;;;;N;;;;; +288E;BRAILLE PATTERN DOTS-2348;So;0;L;;;;;N;;;;; +288F;BRAILLE PATTERN DOTS-12348;So;0;L;;;;;N;;;;; +2890;BRAILLE PATTERN DOTS-58;So;0;L;;;;;N;;;;; +2891;BRAILLE PATTERN DOTS-158;So;0;L;;;;;N;;;;; +2892;BRAILLE PATTERN DOTS-258;So;0;L;;;;;N;;;;; +2893;BRAILLE PATTERN DOTS-1258;So;0;L;;;;;N;;;;; +2894;BRAILLE PATTERN DOTS-358;So;0;L;;;;;N;;;;; +2895;BRAILLE PATTERN DOTS-1358;So;0;L;;;;;N;;;;; +2896;BRAILLE PATTERN DOTS-2358;So;0;L;;;;;N;;;;; +2897;BRAILLE PATTERN DOTS-12358;So;0;L;;;;;N;;;;; +2898;BRAILLE PATTERN DOTS-458;So;0;L;;;;;N;;;;; +2899;BRAILLE PATTERN DOTS-1458;So;0;L;;;;;N;;;;; +289A;BRAILLE PATTERN DOTS-2458;So;0;L;;;;;N;;;;; +289B;BRAILLE PATTERN DOTS-12458;So;0;L;;;;;N;;;;; +289C;BRAILLE PATTERN DOTS-3458;So;0;L;;;;;N;;;;; +289D;BRAILLE PATTERN DOTS-13458;So;0;L;;;;;N;;;;; +289E;BRAILLE PATTERN DOTS-23458;So;0;L;;;;;N;;;;; +289F;BRAILLE PATTERN DOTS-123458;So;0;L;;;;;N;;;;; +28A0;BRAILLE PATTERN DOTS-68;So;0;L;;;;;N;;;;; +28A1;BRAILLE PATTERN DOTS-168;So;0;L;;;;;N;;;;; +28A2;BRAILLE PATTERN DOTS-268;So;0;L;;;;;N;;;;; +28A3;BRAILLE PATTERN DOTS-1268;So;0;L;;;;;N;;;;; +28A4;BRAILLE PATTERN DOTS-368;So;0;L;;;;;N;;;;; +28A5;BRAILLE PATTERN DOTS-1368;So;0;L;;;;;N;;;;; +28A6;BRAILLE PATTERN DOTS-2368;So;0;L;;;;;N;;;;; +28A7;BRAILLE PATTERN DOTS-12368;So;0;L;;;;;N;;;;; +28A8;BRAILLE PATTERN DOTS-468;So;0;L;;;;;N;;;;; +28A9;BRAILLE PATTERN DOTS-1468;So;0;L;;;;;N;;;;; +28AA;BRAILLE PATTERN DOTS-2468;So;0;L;;;;;N;;;;; +28AB;BRAILLE PATTERN DOTS-12468;So;0;L;;;;;N;;;;; +28AC;BRAILLE PATTERN DOTS-3468;So;0;L;;;;;N;;;;; +28AD;BRAILLE PATTERN DOTS-13468;So;0;L;;;;;N;;;;; +28AE;BRAILLE PATTERN DOTS-23468;So;0;L;;;;;N;;;;; +28AF;BRAILLE PATTERN DOTS-123468;So;0;L;;;;;N;;;;; +28B0;BRAILLE PATTERN DOTS-568;So;0;L;;;;;N;;;;; +28B1;BRAILLE PATTERN DOTS-1568;So;0;L;;;;;N;;;;; +28B2;BRAILLE PATTERN DOTS-2568;So;0;L;;;;;N;;;;; +28B3;BRAILLE PATTERN DOTS-12568;So;0;L;;;;;N;;;;; +28B4;BRAILLE PATTERN DOTS-3568;So;0;L;;;;;N;;;;; +28B5;BRAILLE PATTERN DOTS-13568;So;0;L;;;;;N;;;;; +28B6;BRAILLE PATTERN DOTS-23568;So;0;L;;;;;N;;;;; +28B7;BRAILLE PATTERN DOTS-123568;So;0;L;;;;;N;;;;; +28B8;BRAILLE PATTERN DOTS-4568;So;0;L;;;;;N;;;;; +28B9;BRAILLE PATTERN DOTS-14568;So;0;L;;;;;N;;;;; +28BA;BRAILLE PATTERN DOTS-24568;So;0;L;;;;;N;;;;; +28BB;BRAILLE PATTERN DOTS-124568;So;0;L;;;;;N;;;;; +28BC;BRAILLE PATTERN DOTS-34568;So;0;L;;;;;N;;;;; +28BD;BRAILLE PATTERN DOTS-134568;So;0;L;;;;;N;;;;; +28BE;BRAILLE PATTERN DOTS-234568;So;0;L;;;;;N;;;;; +28BF;BRAILLE PATTERN DOTS-1234568;So;0;L;;;;;N;;;;; +28C0;BRAILLE PATTERN DOTS-78;So;0;L;;;;;N;;;;; +28C1;BRAILLE PATTERN DOTS-178;So;0;L;;;;;N;;;;; +28C2;BRAILLE PATTERN DOTS-278;So;0;L;;;;;N;;;;; +28C3;BRAILLE PATTERN DOTS-1278;So;0;L;;;;;N;;;;; +28C4;BRAILLE PATTERN DOTS-378;So;0;L;;;;;N;;;;; +28C5;BRAILLE PATTERN DOTS-1378;So;0;L;;;;;N;;;;; +28C6;BRAILLE PATTERN DOTS-2378;So;0;L;;;;;N;;;;; +28C7;BRAILLE PATTERN DOTS-12378;So;0;L;;;;;N;;;;; +28C8;BRAILLE PATTERN DOTS-478;So;0;L;;;;;N;;;;; +28C9;BRAILLE PATTERN DOTS-1478;So;0;L;;;;;N;;;;; +28CA;BRAILLE PATTERN DOTS-2478;So;0;L;;;;;N;;;;; +28CB;BRAILLE PATTERN DOTS-12478;So;0;L;;;;;N;;;;; +28CC;BRAILLE PATTERN DOTS-3478;So;0;L;;;;;N;;;;; +28CD;BRAILLE PATTERN DOTS-13478;So;0;L;;;;;N;;;;; +28CE;BRAILLE PATTERN DOTS-23478;So;0;L;;;;;N;;;;; +28CF;BRAILLE PATTERN DOTS-123478;So;0;L;;;;;N;;;;; +28D0;BRAILLE PATTERN DOTS-578;So;0;L;;;;;N;;;;; +28D1;BRAILLE PATTERN DOTS-1578;So;0;L;;;;;N;;;;; +28D2;BRAILLE PATTERN DOTS-2578;So;0;L;;;;;N;;;;; +28D3;BRAILLE PATTERN DOTS-12578;So;0;L;;;;;N;;;;; +28D4;BRAILLE PATTERN DOTS-3578;So;0;L;;;;;N;;;;; +28D5;BRAILLE PATTERN DOTS-13578;So;0;L;;;;;N;;;;; +28D6;BRAILLE PATTERN DOTS-23578;So;0;L;;;;;N;;;;; +28D7;BRAILLE PATTERN DOTS-123578;So;0;L;;;;;N;;;;; +28D8;BRAILLE PATTERN DOTS-4578;So;0;L;;;;;N;;;;; +28D9;BRAILLE PATTERN DOTS-14578;So;0;L;;;;;N;;;;; +28DA;BRAILLE PATTERN DOTS-24578;So;0;L;;;;;N;;;;; +28DB;BRAILLE PATTERN DOTS-124578;So;0;L;;;;;N;;;;; +28DC;BRAILLE PATTERN DOTS-34578;So;0;L;;;;;N;;;;; +28DD;BRAILLE PATTERN DOTS-134578;So;0;L;;;;;N;;;;; +28DE;BRAILLE PATTERN DOTS-234578;So;0;L;;;;;N;;;;; +28DF;BRAILLE PATTERN DOTS-1234578;So;0;L;;;;;N;;;;; +28E0;BRAILLE PATTERN DOTS-678;So;0;L;;;;;N;;;;; +28E1;BRAILLE PATTERN DOTS-1678;So;0;L;;;;;N;;;;; +28E2;BRAILLE PATTERN DOTS-2678;So;0;L;;;;;N;;;;; +28E3;BRAILLE PATTERN DOTS-12678;So;0;L;;;;;N;;;;; +28E4;BRAILLE PATTERN DOTS-3678;So;0;L;;;;;N;;;;; +28E5;BRAILLE PATTERN DOTS-13678;So;0;L;;;;;N;;;;; +28E6;BRAILLE PATTERN DOTS-23678;So;0;L;;;;;N;;;;; +28E7;BRAILLE PATTERN DOTS-123678;So;0;L;;;;;N;;;;; +28E8;BRAILLE PATTERN DOTS-4678;So;0;L;;;;;N;;;;; +28E9;BRAILLE PATTERN DOTS-14678;So;0;L;;;;;N;;;;; +28EA;BRAILLE PATTERN DOTS-24678;So;0;L;;;;;N;;;;; +28EB;BRAILLE PATTERN DOTS-124678;So;0;L;;;;;N;;;;; +28EC;BRAILLE PATTERN DOTS-34678;So;0;L;;;;;N;;;;; +28ED;BRAILLE PATTERN DOTS-134678;So;0;L;;;;;N;;;;; +28EE;BRAILLE PATTERN DOTS-234678;So;0;L;;;;;N;;;;; +28EF;BRAILLE PATTERN DOTS-1234678;So;0;L;;;;;N;;;;; +28F0;BRAILLE PATTERN DOTS-5678;So;0;L;;;;;N;;;;; +28F1;BRAILLE PATTERN DOTS-15678;So;0;L;;;;;N;;;;; +28F2;BRAILLE PATTERN DOTS-25678;So;0;L;;;;;N;;;;; +28F3;BRAILLE PATTERN DOTS-125678;So;0;L;;;;;N;;;;; +28F4;BRAILLE PATTERN DOTS-35678;So;0;L;;;;;N;;;;; +28F5;BRAILLE PATTERN DOTS-135678;So;0;L;;;;;N;;;;; +28F6;BRAILLE PATTERN DOTS-235678;So;0;L;;;;;N;;;;; +28F7;BRAILLE PATTERN DOTS-1235678;So;0;L;;;;;N;;;;; +28F8;BRAILLE PATTERN DOTS-45678;So;0;L;;;;;N;;;;; +28F9;BRAILLE PATTERN DOTS-145678;So;0;L;;;;;N;;;;; +28FA;BRAILLE PATTERN DOTS-245678;So;0;L;;;;;N;;;;; +28FB;BRAILLE PATTERN DOTS-1245678;So;0;L;;;;;N;;;;; +28FC;BRAILLE PATTERN DOTS-345678;So;0;L;;;;;N;;;;; +28FD;BRAILLE PATTERN DOTS-1345678;So;0;L;;;;;N;;;;; +28FE;BRAILLE PATTERN DOTS-2345678;So;0;L;;;;;N;;;;; +28FF;BRAILLE PATTERN DOTS-12345678;So;0;L;;;;;N;;;;; +2900;RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2901;RIGHTWARDS TWO-HEADED ARROW WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2902;LEFTWARDS DOUBLE ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2903;RIGHTWARDS DOUBLE ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2904;LEFT RIGHT DOUBLE ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2905;RIGHTWARDS TWO-HEADED ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +2906;LEFTWARDS DOUBLE ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +2907;RIGHTWARDS DOUBLE ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +2908;DOWNWARDS ARROW WITH HORIZONTAL STROKE;Sm;0;ON;;;;;N;;;;; +2909;UPWARDS ARROW WITH HORIZONTAL STROKE;Sm;0;ON;;;;;N;;;;; +290A;UPWARDS TRIPLE ARROW;Sm;0;ON;;;;;N;;;;; +290B;DOWNWARDS TRIPLE ARROW;Sm;0;ON;;;;;N;;;;; +290C;LEFTWARDS DOUBLE DASH ARROW;Sm;0;ON;;;;;N;;;;; +290D;RIGHTWARDS DOUBLE DASH ARROW;Sm;0;ON;;;;;N;;;;; +290E;LEFTWARDS TRIPLE DASH ARROW;Sm;0;ON;;;;;N;;;;; +290F;RIGHTWARDS TRIPLE DASH ARROW;Sm;0;ON;;;;;N;;;;; +2910;RIGHTWARDS TWO-HEADED TRIPLE DASH ARROW;Sm;0;ON;;;;;N;;;;; +2911;RIGHTWARDS ARROW WITH DOTTED STEM;Sm;0;ON;;;;;N;;;;; +2912;UPWARDS ARROW TO BAR;Sm;0;ON;;;;;N;;;;; +2913;DOWNWARDS ARROW TO BAR;Sm;0;ON;;;;;N;;;;; +2914;RIGHTWARDS ARROW WITH TAIL WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2915;RIGHTWARDS ARROW WITH TAIL WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2916;RIGHTWARDS TWO-HEADED ARROW WITH TAIL;Sm;0;ON;;;;;N;;;;; +2917;RIGHTWARDS TWO-HEADED ARROW WITH TAIL WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2918;RIGHTWARDS TWO-HEADED ARROW WITH TAIL WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2919;LEFTWARDS ARROW-TAIL;Sm;0;ON;;;;;N;;;;; +291A;RIGHTWARDS ARROW-TAIL;Sm;0;ON;;;;;N;;;;; +291B;LEFTWARDS DOUBLE ARROW-TAIL;Sm;0;ON;;;;;N;;;;; +291C;RIGHTWARDS DOUBLE ARROW-TAIL;Sm;0;ON;;;;;N;;;;; +291D;LEFTWARDS ARROW TO BLACK DIAMOND;Sm;0;ON;;;;;N;;;;; +291E;RIGHTWARDS ARROW TO BLACK DIAMOND;Sm;0;ON;;;;;N;;;;; +291F;LEFTWARDS ARROW FROM BAR TO BLACK DIAMOND;Sm;0;ON;;;;;N;;;;; +2920;RIGHTWARDS ARROW FROM BAR TO BLACK DIAMOND;Sm;0;ON;;;;;N;;;;; +2921;NORTH WEST AND SOUTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +2922;NORTH EAST AND SOUTH WEST ARROW;Sm;0;ON;;;;;N;;;;; +2923;NORTH WEST ARROW WITH HOOK;Sm;0;ON;;;;;N;;;;; +2924;NORTH EAST ARROW WITH HOOK;Sm;0;ON;;;;;N;;;;; +2925;SOUTH EAST ARROW WITH HOOK;Sm;0;ON;;;;;N;;;;; +2926;SOUTH WEST ARROW WITH HOOK;Sm;0;ON;;;;;N;;;;; +2927;NORTH WEST ARROW AND NORTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +2928;NORTH EAST ARROW AND SOUTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +2929;SOUTH EAST ARROW AND SOUTH WEST ARROW;Sm;0;ON;;;;;N;;;;; +292A;SOUTH WEST ARROW AND NORTH WEST ARROW;Sm;0;ON;;;;;N;;;;; +292B;RISING DIAGONAL CROSSING FALLING DIAGONAL;Sm;0;ON;;;;;N;;;;; +292C;FALLING DIAGONAL CROSSING RISING DIAGONAL;Sm;0;ON;;;;;N;;;;; +292D;SOUTH EAST ARROW CROSSING NORTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +292E;NORTH EAST ARROW CROSSING SOUTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +292F;FALLING DIAGONAL CROSSING NORTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +2930;RISING DIAGONAL CROSSING SOUTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +2931;NORTH EAST ARROW CROSSING NORTH WEST ARROW;Sm;0;ON;;;;;N;;;;; +2932;NORTH WEST ARROW CROSSING NORTH EAST ARROW;Sm;0;ON;;;;;N;;;;; +2933;WAVE ARROW POINTING DIRECTLY RIGHT;Sm;0;ON;;;;;N;;;;; +2934;ARROW POINTING RIGHTWARDS THEN CURVING UPWARDS;Sm;0;ON;;;;;N;;;;; +2935;ARROW POINTING RIGHTWARDS THEN CURVING DOWNWARDS;Sm;0;ON;;;;;N;;;;; +2936;ARROW POINTING DOWNWARDS THEN CURVING LEFTWARDS;Sm;0;ON;;;;;N;;;;; +2937;ARROW POINTING DOWNWARDS THEN CURVING RIGHTWARDS;Sm;0;ON;;;;;N;;;;; +2938;RIGHT-SIDE ARC CLOCKWISE ARROW;Sm;0;ON;;;;;N;;;;; +2939;LEFT-SIDE ARC ANTICLOCKWISE ARROW;Sm;0;ON;;;;;N;;;;; +293A;TOP ARC ANTICLOCKWISE ARROW;Sm;0;ON;;;;;N;;;;; +293B;BOTTOM ARC ANTICLOCKWISE ARROW;Sm;0;ON;;;;;N;;;;; +293C;TOP ARC CLOCKWISE ARROW WITH MINUS;Sm;0;ON;;;;;N;;;;; +293D;TOP ARC ANTICLOCKWISE ARROW WITH PLUS;Sm;0;ON;;;;;N;;;;; +293E;LOWER RIGHT SEMICIRCULAR CLOCKWISE ARROW;Sm;0;ON;;;;;N;;;;; +293F;LOWER LEFT SEMICIRCULAR ANTICLOCKWISE ARROW;Sm;0;ON;;;;;N;;;;; +2940;ANTICLOCKWISE CLOSED CIRCLE ARROW;Sm;0;ON;;;;;N;;;;; +2941;CLOCKWISE CLOSED CIRCLE ARROW;Sm;0;ON;;;;;N;;;;; +2942;RIGHTWARDS ARROW ABOVE SHORT LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2943;LEFTWARDS ARROW ABOVE SHORT RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2944;SHORT RIGHTWARDS ARROW ABOVE LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2945;RIGHTWARDS ARROW WITH PLUS BELOW;Sm;0;ON;;;;;N;;;;; +2946;LEFTWARDS ARROW WITH PLUS BELOW;Sm;0;ON;;;;;N;;;;; +2947;RIGHTWARDS ARROW THROUGH X;Sm;0;ON;;;;;N;;;;; +2948;LEFT RIGHT ARROW THROUGH SMALL CIRCLE;Sm;0;ON;;;;;N;;;;; +2949;UPWARDS TWO-HEADED ARROW FROM SMALL CIRCLE;Sm;0;ON;;;;;N;;;;; +294A;LEFT BARB UP RIGHT BARB DOWN HARPOON;Sm;0;ON;;;;;N;;;;; +294B;LEFT BARB DOWN RIGHT BARB UP HARPOON;Sm;0;ON;;;;;N;;;;; +294C;UP BARB RIGHT DOWN BARB LEFT HARPOON;Sm;0;ON;;;;;N;;;;; +294D;UP BARB LEFT DOWN BARB RIGHT HARPOON;Sm;0;ON;;;;;N;;;;; +294E;LEFT BARB UP RIGHT BARB UP HARPOON;Sm;0;ON;;;;;N;;;;; +294F;UP BARB RIGHT DOWN BARB RIGHT HARPOON;Sm;0;ON;;;;;N;;;;; +2950;LEFT BARB DOWN RIGHT BARB DOWN HARPOON;Sm;0;ON;;;;;N;;;;; +2951;UP BARB LEFT DOWN BARB LEFT HARPOON;Sm;0;ON;;;;;N;;;;; +2952;LEFTWARDS HARPOON WITH BARB UP TO BAR;Sm;0;ON;;;;;N;;;;; +2953;RIGHTWARDS HARPOON WITH BARB UP TO BAR;Sm;0;ON;;;;;N;;;;; +2954;UPWARDS HARPOON WITH BARB RIGHT TO BAR;Sm;0;ON;;;;;N;;;;; +2955;DOWNWARDS HARPOON WITH BARB RIGHT TO BAR;Sm;0;ON;;;;;N;;;;; +2956;LEFTWARDS HARPOON WITH BARB DOWN TO BAR;Sm;0;ON;;;;;N;;;;; +2957;RIGHTWARDS HARPOON WITH BARB DOWN TO BAR;Sm;0;ON;;;;;N;;;;; +2958;UPWARDS HARPOON WITH BARB LEFT TO BAR;Sm;0;ON;;;;;N;;;;; +2959;DOWNWARDS HARPOON WITH BARB LEFT TO BAR;Sm;0;ON;;;;;N;;;;; +295A;LEFTWARDS HARPOON WITH BARB UP FROM BAR;Sm;0;ON;;;;;N;;;;; +295B;RIGHTWARDS HARPOON WITH BARB UP FROM BAR;Sm;0;ON;;;;;N;;;;; +295C;UPWARDS HARPOON WITH BARB RIGHT FROM BAR;Sm;0;ON;;;;;N;;;;; +295D;DOWNWARDS HARPOON WITH BARB RIGHT FROM BAR;Sm;0;ON;;;;;N;;;;; +295E;LEFTWARDS HARPOON WITH BARB DOWN FROM BAR;Sm;0;ON;;;;;N;;;;; +295F;RIGHTWARDS HARPOON WITH BARB DOWN FROM BAR;Sm;0;ON;;;;;N;;;;; +2960;UPWARDS HARPOON WITH BARB LEFT FROM BAR;Sm;0;ON;;;;;N;;;;; +2961;DOWNWARDS HARPOON WITH BARB LEFT FROM BAR;Sm;0;ON;;;;;N;;;;; +2962;LEFTWARDS HARPOON WITH BARB UP ABOVE LEFTWARDS HARPOON WITH BARB DOWN;Sm;0;ON;;;;;N;;;;; +2963;UPWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT;Sm;0;ON;;;;;N;;;;; +2964;RIGHTWARDS HARPOON WITH BARB UP ABOVE RIGHTWARDS HARPOON WITH BARB DOWN;Sm;0;ON;;;;;N;;;;; +2965;DOWNWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT;Sm;0;ON;;;;;N;;;;; +2966;LEFTWARDS HARPOON WITH BARB UP ABOVE RIGHTWARDS HARPOON WITH BARB UP;Sm;0;ON;;;;;N;;;;; +2967;LEFTWARDS HARPOON WITH BARB DOWN ABOVE RIGHTWARDS HARPOON WITH BARB DOWN;Sm;0;ON;;;;;N;;;;; +2968;RIGHTWARDS HARPOON WITH BARB UP ABOVE LEFTWARDS HARPOON WITH BARB UP;Sm;0;ON;;;;;N;;;;; +2969;RIGHTWARDS HARPOON WITH BARB DOWN ABOVE LEFTWARDS HARPOON WITH BARB DOWN;Sm;0;ON;;;;;N;;;;; +296A;LEFTWARDS HARPOON WITH BARB UP ABOVE LONG DASH;Sm;0;ON;;;;;N;;;;; +296B;LEFTWARDS HARPOON WITH BARB DOWN BELOW LONG DASH;Sm;0;ON;;;;;N;;;;; +296C;RIGHTWARDS HARPOON WITH BARB UP ABOVE LONG DASH;Sm;0;ON;;;;;N;;;;; +296D;RIGHTWARDS HARPOON WITH BARB DOWN BELOW LONG DASH;Sm;0;ON;;;;;N;;;;; +296E;UPWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT;Sm;0;ON;;;;;N;;;;; +296F;DOWNWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT;Sm;0;ON;;;;;N;;;;; +2970;RIGHT DOUBLE ARROW WITH ROUNDED HEAD;Sm;0;ON;;;;;N;;;;; +2971;EQUALS SIGN ABOVE RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2972;TILDE OPERATOR ABOVE RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2973;LEFTWARDS ARROW ABOVE TILDE OPERATOR;Sm;0;ON;;;;;N;;;;; +2974;RIGHTWARDS ARROW ABOVE TILDE OPERATOR;Sm;0;ON;;;;;N;;;;; +2975;RIGHTWARDS ARROW ABOVE ALMOST EQUAL TO;Sm;0;ON;;;;;N;;;;; +2976;LESS-THAN ABOVE LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2977;LEFTWARDS ARROW THROUGH LESS-THAN;Sm;0;ON;;;;;N;;;;; +2978;GREATER-THAN ABOVE RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2979;SUBSET ABOVE RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +297A;LEFTWARDS ARROW THROUGH SUBSET;Sm;0;ON;;;;;N;;;;; +297B;SUPERSET ABOVE LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +297C;LEFT FISH TAIL;Sm;0;ON;;;;;N;;;;; +297D;RIGHT FISH TAIL;Sm;0;ON;;;;;N;;;;; +297E;UP FISH TAIL;Sm;0;ON;;;;;N;;;;; +297F;DOWN FISH TAIL;Sm;0;ON;;;;;N;;;;; +2980;TRIPLE VERTICAL BAR DELIMITER;Sm;0;ON;;;;;N;;;;; +2981;Z NOTATION SPOT;Sm;0;ON;;;;;N;;;;; +2982;Z NOTATION TYPE COLON;Sm;0;ON;;;;;N;;;;; +2983;LEFT WHITE CURLY BRACKET;Ps;0;ON;;;;;Y;;;;; +2984;RIGHT WHITE CURLY BRACKET;Pe;0;ON;;;;;Y;;;;; +2985;LEFT WHITE PARENTHESIS;Ps;0;ON;;;;;Y;;;;; +2986;RIGHT WHITE PARENTHESIS;Pe;0;ON;;;;;Y;;;;; +2987;Z NOTATION LEFT IMAGE BRACKET;Ps;0;ON;;;;;Y;;;;; +2988;Z NOTATION RIGHT IMAGE BRACKET;Pe;0;ON;;;;;Y;;;;; +2989;Z NOTATION LEFT BINDING BRACKET;Ps;0;ON;;;;;Y;;;;; +298A;Z NOTATION RIGHT BINDING BRACKET;Pe;0;ON;;;;;Y;;;;; +298B;LEFT SQUARE BRACKET WITH UNDERBAR;Ps;0;ON;;;;;Y;;;;; +298C;RIGHT SQUARE BRACKET WITH UNDERBAR;Pe;0;ON;;;;;Y;;;;; +298D;LEFT SQUARE BRACKET WITH TICK IN TOP CORNER;Ps;0;ON;;;;;Y;;;;; +298E;RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER;Pe;0;ON;;;;;Y;;;;; +298F;LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER;Ps;0;ON;;;;;Y;;;;; +2990;RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER;Pe;0;ON;;;;;Y;;;;; +2991;LEFT ANGLE BRACKET WITH DOT;Ps;0;ON;;;;;Y;;;;; +2992;RIGHT ANGLE BRACKET WITH DOT;Pe;0;ON;;;;;Y;;;;; +2993;LEFT ARC LESS-THAN BRACKET;Ps;0;ON;;;;;Y;;;;; +2994;RIGHT ARC GREATER-THAN BRACKET;Pe;0;ON;;;;;Y;;;;; +2995;DOUBLE LEFT ARC GREATER-THAN BRACKET;Ps;0;ON;;;;;Y;;;;; +2996;DOUBLE RIGHT ARC LESS-THAN BRACKET;Pe;0;ON;;;;;Y;;;;; +2997;LEFT BLACK TORTOISE SHELL BRACKET;Ps;0;ON;;;;;Y;;;;; +2998;RIGHT BLACK TORTOISE SHELL BRACKET;Pe;0;ON;;;;;Y;;;;; +2999;DOTTED FENCE;Sm;0;ON;;;;;N;;;;; +299A;VERTICAL ZIGZAG LINE;Sm;0;ON;;;;;N;;;;; +299B;MEASURED ANGLE OPENING LEFT;Sm;0;ON;;;;;Y;;;;; +299C;RIGHT ANGLE VARIANT WITH SQUARE;Sm;0;ON;;;;;Y;;;;; +299D;MEASURED RIGHT ANGLE WITH DOT;Sm;0;ON;;;;;Y;;;;; +299E;ANGLE WITH S INSIDE;Sm;0;ON;;;;;Y;;;;; +299F;ACUTE ANGLE;Sm;0;ON;;;;;Y;;;;; +29A0;SPHERICAL ANGLE OPENING LEFT;Sm;0;ON;;;;;Y;;;;; +29A1;SPHERICAL ANGLE OPENING UP;Sm;0;ON;;;;;Y;;;;; +29A2;TURNED ANGLE;Sm;0;ON;;;;;Y;;;;; +29A3;REVERSED ANGLE;Sm;0;ON;;;;;Y;;;;; +29A4;ANGLE WITH UNDERBAR;Sm;0;ON;;;;;Y;;;;; +29A5;REVERSED ANGLE WITH UNDERBAR;Sm;0;ON;;;;;Y;;;;; +29A6;OBLIQUE ANGLE OPENING UP;Sm;0;ON;;;;;Y;;;;; +29A7;OBLIQUE ANGLE OPENING DOWN;Sm;0;ON;;;;;Y;;;;; +29A8;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING UP AND RIGHT;Sm;0;ON;;;;;Y;;;;; +29A9;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING UP AND LEFT;Sm;0;ON;;;;;Y;;;;; +29AA;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING DOWN AND RIGHT;Sm;0;ON;;;;;Y;;;;; +29AB;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING DOWN AND LEFT;Sm;0;ON;;;;;Y;;;;; +29AC;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING RIGHT AND UP;Sm;0;ON;;;;;Y;;;;; +29AD;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING LEFT AND UP;Sm;0;ON;;;;;Y;;;;; +29AE;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING RIGHT AND DOWN;Sm;0;ON;;;;;Y;;;;; +29AF;MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING LEFT AND DOWN;Sm;0;ON;;;;;Y;;;;; +29B0;REVERSED EMPTY SET;Sm;0;ON;;;;;N;;;;; +29B1;EMPTY SET WITH OVERBAR;Sm;0;ON;;;;;N;;;;; +29B2;EMPTY SET WITH SMALL CIRCLE ABOVE;Sm;0;ON;;;;;N;;;;; +29B3;EMPTY SET WITH RIGHT ARROW ABOVE;Sm;0;ON;;;;;N;;;;; +29B4;EMPTY SET WITH LEFT ARROW ABOVE;Sm;0;ON;;;;;N;;;;; +29B5;CIRCLE WITH HORIZONTAL BAR;Sm;0;ON;;;;;N;;;;; +29B6;CIRCLED VERTICAL BAR;Sm;0;ON;;;;;N;;;;; +29B7;CIRCLED PARALLEL;Sm;0;ON;;;;;N;;;;; +29B8;CIRCLED REVERSE SOLIDUS;Sm;0;ON;;;;;Y;;;;; +29B9;CIRCLED PERPENDICULAR;Sm;0;ON;;;;;N;;;;; +29BA;CIRCLE DIVIDED BY HORIZONTAL BAR AND TOP HALF DIVIDED BY VERTICAL BAR;Sm;0;ON;;;;;N;;;;; +29BB;CIRCLE WITH SUPERIMPOSED X;Sm;0;ON;;;;;N;;;;; +29BC;CIRCLED ANTICLOCKWISE-ROTATED DIVISION SIGN;Sm;0;ON;;;;;N;;;;; +29BD;UP ARROW THROUGH CIRCLE;Sm;0;ON;;;;;N;;;;; +29BE;CIRCLED WHITE BULLET;Sm;0;ON;;;;;N;;;;; +29BF;CIRCLED BULLET;Sm;0;ON;;;;;N;;;;; +29C0;CIRCLED LESS-THAN;Sm;0;ON;;;;;Y;;;;; +29C1;CIRCLED GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +29C2;CIRCLE WITH SMALL CIRCLE TO THE RIGHT;Sm;0;ON;;;;;Y;;;;; +29C3;CIRCLE WITH TWO HORIZONTAL STROKES TO THE RIGHT;Sm;0;ON;;;;;Y;;;;; +29C4;SQUARED RISING DIAGONAL SLASH;Sm;0;ON;;;;;Y;;;;; +29C5;SQUARED FALLING DIAGONAL SLASH;Sm;0;ON;;;;;Y;;;;; +29C6;SQUARED ASTERISK;Sm;0;ON;;;;;N;;;;; +29C7;SQUARED SMALL CIRCLE;Sm;0;ON;;;;;N;;;;; +29C8;SQUARED SQUARE;Sm;0;ON;;;;;N;;;;; +29C9;TWO JOINED SQUARES;Sm;0;ON;;;;;Y;;;;; +29CA;TRIANGLE WITH DOT ABOVE;Sm;0;ON;;;;;N;;;;; +29CB;TRIANGLE WITH UNDERBAR;Sm;0;ON;;;;;N;;;;; +29CC;S IN TRIANGLE;Sm;0;ON;;;;;N;;;;; +29CD;TRIANGLE WITH SERIFS AT BOTTOM;Sm;0;ON;;;;;N;;;;; +29CE;RIGHT TRIANGLE ABOVE LEFT TRIANGLE;Sm;0;ON;;;;;Y;;;;; +29CF;LEFT TRIANGLE BESIDE VERTICAL BAR;Sm;0;ON;;;;;Y;;;;; +29D0;VERTICAL BAR BESIDE RIGHT TRIANGLE;Sm;0;ON;;;;;Y;;;;; +29D1;BOWTIE WITH LEFT HALF BLACK;Sm;0;ON;;;;;Y;;;;; +29D2;BOWTIE WITH RIGHT HALF BLACK;Sm;0;ON;;;;;Y;;;;; +29D3;BLACK BOWTIE;Sm;0;ON;;;;;N;;;;; +29D4;TIMES WITH LEFT HALF BLACK;Sm;0;ON;;;;;Y;;;;; +29D5;TIMES WITH RIGHT HALF BLACK;Sm;0;ON;;;;;Y;;;;; +29D6;WHITE HOURGLASS;Sm;0;ON;;;;;N;;;;; +29D7;BLACK HOURGLASS;Sm;0;ON;;;;;N;;;;; +29D8;LEFT WIGGLY FENCE;Ps;0;ON;;;;;Y;;;;; +29D9;RIGHT WIGGLY FENCE;Pe;0;ON;;;;;Y;;;;; +29DA;LEFT DOUBLE WIGGLY FENCE;Ps;0;ON;;;;;Y;;;;; +29DB;RIGHT DOUBLE WIGGLY FENCE;Pe;0;ON;;;;;Y;;;;; +29DC;INCOMPLETE INFINITY;Sm;0;ON;;;;;Y;;;;; +29DD;TIE OVER INFINITY;Sm;0;ON;;;;;N;;;;; +29DE;INFINITY NEGATED WITH VERTICAL BAR;Sm;0;ON;;;;;N;;;;; +29DF;DOUBLE-ENDED MULTIMAP;Sm;0;ON;;;;;N;;;;; +29E0;SQUARE WITH CONTOURED OUTLINE;Sm;0;ON;;;;;N;;;;; +29E1;INCREASES AS;Sm;0;ON;;;;;Y;;;;; +29E2;SHUFFLE PRODUCT;Sm;0;ON;;;;;N;;;;; +29E3;EQUALS SIGN AND SLANTED PARALLEL;Sm;0;ON;;;;;Y;;;;; +29E4;EQUALS SIGN AND SLANTED PARALLEL WITH TILDE ABOVE;Sm;0;ON;;;;;Y;;;;; +29E5;IDENTICAL TO AND SLANTED PARALLEL;Sm;0;ON;;;;;Y;;;;; +29E6;GLEICH STARK;Sm;0;ON;;;;;N;;;;; +29E7;THERMODYNAMIC;Sm;0;ON;;;;;N;;;;; +29E8;DOWN-POINTING TRIANGLE WITH LEFT HALF BLACK;Sm;0;ON;;;;;Y;;;;; +29E9;DOWN-POINTING TRIANGLE WITH RIGHT HALF BLACK;Sm;0;ON;;;;;Y;;;;; +29EA;BLACK DIAMOND WITH DOWN ARROW;Sm;0;ON;;;;;N;;;;; +29EB;BLACK LOZENGE;Sm;0;ON;;;;;N;;;;; +29EC;WHITE CIRCLE WITH DOWN ARROW;Sm;0;ON;;;;;N;;;;; +29ED;BLACK CIRCLE WITH DOWN ARROW;Sm;0;ON;;;;;N;;;;; +29EE;ERROR-BARRED WHITE SQUARE;Sm;0;ON;;;;;N;;;;; +29EF;ERROR-BARRED BLACK SQUARE;Sm;0;ON;;;;;N;;;;; +29F0;ERROR-BARRED WHITE DIAMOND;Sm;0;ON;;;;;N;;;;; +29F1;ERROR-BARRED BLACK DIAMOND;Sm;0;ON;;;;;N;;;;; +29F2;ERROR-BARRED WHITE CIRCLE;Sm;0;ON;;;;;N;;;;; +29F3;ERROR-BARRED BLACK CIRCLE;Sm;0;ON;;;;;N;;;;; +29F4;RULE-DELAYED;Sm;0;ON;;;;;Y;;;;; +29F5;REVERSE SOLIDUS OPERATOR;Sm;0;ON;;;;;Y;;;;; +29F6;SOLIDUS WITH OVERBAR;Sm;0;ON;;;;;Y;;;;; +29F7;REVERSE SOLIDUS WITH HORIZONTAL STROKE;Sm;0;ON;;;;;Y;;;;; +29F8;BIG SOLIDUS;Sm;0;ON;;;;;Y;;;;; +29F9;BIG REVERSE SOLIDUS;Sm;0;ON;;;;;Y;;;;; +29FA;DOUBLE PLUS;Sm;0;ON;;;;;N;;;;; +29FB;TRIPLE PLUS;Sm;0;ON;;;;;N;;;;; +29FC;LEFT-POINTING CURVED ANGLE BRACKET;Ps;0;ON;;;;;Y;;;;; +29FD;RIGHT-POINTING CURVED ANGLE BRACKET;Pe;0;ON;;;;;Y;;;;; +29FE;TINY;Sm;0;ON;;;;;N;;;;; +29FF;MINY;Sm;0;ON;;;;;N;;;;; +2A00;N-ARY CIRCLED DOT OPERATOR;Sm;0;ON;;;;;N;;;;; +2A01;N-ARY CIRCLED PLUS OPERATOR;Sm;0;ON;;;;;N;;;;; +2A02;N-ARY CIRCLED TIMES OPERATOR;Sm;0;ON;;;;;N;;;;; +2A03;N-ARY UNION OPERATOR WITH DOT;Sm;0;ON;;;;;N;;;;; +2A04;N-ARY UNION OPERATOR WITH PLUS;Sm;0;ON;;;;;N;;;;; +2A05;N-ARY SQUARE INTERSECTION OPERATOR;Sm;0;ON;;;;;N;;;;; +2A06;N-ARY SQUARE UNION OPERATOR;Sm;0;ON;;;;;N;;;;; +2A07;TWO LOGICAL AND OPERATOR;Sm;0;ON;;;;;N;;;;; +2A08;TWO LOGICAL OR OPERATOR;Sm;0;ON;;;;;N;;;;; +2A09;N-ARY TIMES OPERATOR;Sm;0;ON;;;;;N;;;;; +2A0A;MODULO TWO SUM;Sm;0;ON;;;;;Y;;;;; +2A0B;SUMMATION WITH INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2A0C;QUADRUPLE INTEGRAL OPERATOR;Sm;0;ON; 222B 222B 222B 222B;;;;Y;;;;; +2A0D;FINITE PART INTEGRAL;Sm;0;ON;;;;;Y;;;;; +2A0E;INTEGRAL WITH DOUBLE STROKE;Sm;0;ON;;;;;Y;;;;; +2A0F;INTEGRAL AVERAGE WITH SLASH;Sm;0;ON;;;;;Y;;;;; +2A10;CIRCULATION FUNCTION;Sm;0;ON;;;;;Y;;;;; +2A11;ANTICLOCKWISE INTEGRATION;Sm;0;ON;;;;;Y;;;;; +2A12;LINE INTEGRATION WITH RECTANGULAR PATH AROUND POLE;Sm;0;ON;;;;;Y;;;;; +2A13;LINE INTEGRATION WITH SEMICIRCULAR PATH AROUND POLE;Sm;0;ON;;;;;Y;;;;; +2A14;LINE INTEGRATION NOT INCLUDING THE POLE;Sm;0;ON;;;;;Y;;;;; +2A15;INTEGRAL AROUND A POINT OPERATOR;Sm;0;ON;;;;;Y;;;;; +2A16;QUATERNION INTEGRAL OPERATOR;Sm;0;ON;;;;;Y;;;;; +2A17;INTEGRAL WITH LEFTWARDS ARROW WITH HOOK;Sm;0;ON;;;;;Y;;;;; +2A18;INTEGRAL WITH TIMES SIGN;Sm;0;ON;;;;;Y;;;;; +2A19;INTEGRAL WITH INTERSECTION;Sm;0;ON;;;;;Y;;;;; +2A1A;INTEGRAL WITH UNION;Sm;0;ON;;;;;Y;;;;; +2A1B;INTEGRAL WITH OVERBAR;Sm;0;ON;;;;;Y;;;;; +2A1C;INTEGRAL WITH UNDERBAR;Sm;0;ON;;;;;Y;;;;; +2A1D;JOIN;Sm;0;ON;;;;;N;;;;; +2A1E;LARGE LEFT TRIANGLE OPERATOR;Sm;0;ON;;;;;Y;;;;; +2A1F;Z NOTATION SCHEMA COMPOSITION;Sm;0;ON;;;;;Y;;;;; +2A20;Z NOTATION SCHEMA PIPING;Sm;0;ON;;;;;Y;;;;; +2A21;Z NOTATION SCHEMA PROJECTION;Sm;0;ON;;;;;Y;;;;; +2A22;PLUS SIGN WITH SMALL CIRCLE ABOVE;Sm;0;ON;;;;;N;;;;; +2A23;PLUS SIGN WITH CIRCUMFLEX ACCENT ABOVE;Sm;0;ON;;;;;N;;;;; +2A24;PLUS SIGN WITH TILDE ABOVE;Sm;0;ON;;;;;Y;;;;; +2A25;PLUS SIGN WITH DOT BELOW;Sm;0;ON;;;;;N;;;;; +2A26;PLUS SIGN WITH TILDE BELOW;Sm;0;ON;;;;;Y;;;;; +2A27;PLUS SIGN WITH SUBSCRIPT TWO;Sm;0;ON;;;;;N;;;;; +2A28;PLUS SIGN WITH BLACK TRIANGLE;Sm;0;ON;;;;;N;;;;; +2A29;MINUS SIGN WITH COMMA ABOVE;Sm;0;ON;;;;;Y;;;;; +2A2A;MINUS SIGN WITH DOT BELOW;Sm;0;ON;;;;;N;;;;; +2A2B;MINUS SIGN WITH FALLING DOTS;Sm;0;ON;;;;;Y;;;;; +2A2C;MINUS SIGN WITH RISING DOTS;Sm;0;ON;;;;;Y;;;;; +2A2D;PLUS SIGN IN LEFT HALF CIRCLE;Sm;0;ON;;;;;Y;;;;; +2A2E;PLUS SIGN IN RIGHT HALF CIRCLE;Sm;0;ON;;;;;Y;;;;; +2A2F;VECTOR OR CROSS PRODUCT;Sm;0;ON;;;;;N;;;;; +2A30;MULTIPLICATION SIGN WITH DOT ABOVE;Sm;0;ON;;;;;N;;;;; +2A31;MULTIPLICATION SIGN WITH UNDERBAR;Sm;0;ON;;;;;N;;;;; +2A32;SEMIDIRECT PRODUCT WITH BOTTOM CLOSED;Sm;0;ON;;;;;N;;;;; +2A33;SMASH PRODUCT;Sm;0;ON;;;;;N;;;;; +2A34;MULTIPLICATION SIGN IN LEFT HALF CIRCLE;Sm;0;ON;;;;;Y;;;;; +2A35;MULTIPLICATION SIGN IN RIGHT HALF CIRCLE;Sm;0;ON;;;;;Y;;;;; +2A36;CIRCLED MULTIPLICATION SIGN WITH CIRCUMFLEX ACCENT;Sm;0;ON;;;;;N;;;;; +2A37;MULTIPLICATION SIGN IN DOUBLE CIRCLE;Sm;0;ON;;;;;N;;;;; +2A38;CIRCLED DIVISION SIGN;Sm;0;ON;;;;;N;;;;; +2A39;PLUS SIGN IN TRIANGLE;Sm;0;ON;;;;;N;;;;; +2A3A;MINUS SIGN IN TRIANGLE;Sm;0;ON;;;;;N;;;;; +2A3B;MULTIPLICATION SIGN IN TRIANGLE;Sm;0;ON;;;;;N;;;;; +2A3C;INTERIOR PRODUCT;Sm;0;ON;;;;;Y;;;;; +2A3D;RIGHTHAND INTERIOR PRODUCT;Sm;0;ON;;;;;Y;;;;; +2A3E;Z NOTATION RELATIONAL COMPOSITION;Sm;0;ON;;;;;Y;;;;; +2A3F;AMALGAMATION OR COPRODUCT;Sm;0;ON;;;;;N;;;;; +2A40;INTERSECTION WITH DOT;Sm;0;ON;;;;;N;;;;; +2A41;UNION WITH MINUS SIGN;Sm;0;ON;;;;;N;;;;; +2A42;UNION WITH OVERBAR;Sm;0;ON;;;;;N;;;;; +2A43;INTERSECTION WITH OVERBAR;Sm;0;ON;;;;;N;;;;; +2A44;INTERSECTION WITH LOGICAL AND;Sm;0;ON;;;;;N;;;;; +2A45;UNION WITH LOGICAL OR;Sm;0;ON;;;;;N;;;;; +2A46;UNION ABOVE INTERSECTION;Sm;0;ON;;;;;N;;;;; +2A47;INTERSECTION ABOVE UNION;Sm;0;ON;;;;;N;;;;; +2A48;UNION ABOVE BAR ABOVE INTERSECTION;Sm;0;ON;;;;;N;;;;; +2A49;INTERSECTION ABOVE BAR ABOVE UNION;Sm;0;ON;;;;;N;;;;; +2A4A;UNION BESIDE AND JOINED WITH UNION;Sm;0;ON;;;;;N;;;;; +2A4B;INTERSECTION BESIDE AND JOINED WITH INTERSECTION;Sm;0;ON;;;;;N;;;;; +2A4C;CLOSED UNION WITH SERIFS;Sm;0;ON;;;;;N;;;;; +2A4D;CLOSED INTERSECTION WITH SERIFS;Sm;0;ON;;;;;N;;;;; +2A4E;DOUBLE SQUARE INTERSECTION;Sm;0;ON;;;;;N;;;;; +2A4F;DOUBLE SQUARE UNION;Sm;0;ON;;;;;N;;;;; +2A50;CLOSED UNION WITH SERIFS AND SMASH PRODUCT;Sm;0;ON;;;;;N;;;;; +2A51;LOGICAL AND WITH DOT ABOVE;Sm;0;ON;;;;;N;;;;; +2A52;LOGICAL OR WITH DOT ABOVE;Sm;0;ON;;;;;N;;;;; +2A53;DOUBLE LOGICAL AND;Sm;0;ON;;;;;N;;;;; +2A54;DOUBLE LOGICAL OR;Sm;0;ON;;;;;N;;;;; +2A55;TWO INTERSECTING LOGICAL AND;Sm;0;ON;;;;;N;;;;; +2A56;TWO INTERSECTING LOGICAL OR;Sm;0;ON;;;;;N;;;;; +2A57;SLOPING LARGE OR;Sm;0;ON;;;;;Y;;;;; +2A58;SLOPING LARGE AND;Sm;0;ON;;;;;Y;;;;; +2A59;LOGICAL OR OVERLAPPING LOGICAL AND;Sm;0;ON;;;;;N;;;;; +2A5A;LOGICAL AND WITH MIDDLE STEM;Sm;0;ON;;;;;N;;;;; +2A5B;LOGICAL OR WITH MIDDLE STEM;Sm;0;ON;;;;;N;;;;; +2A5C;LOGICAL AND WITH HORIZONTAL DASH;Sm;0;ON;;;;;N;;;;; +2A5D;LOGICAL OR WITH HORIZONTAL DASH;Sm;0;ON;;;;;N;;;;; +2A5E;LOGICAL AND WITH DOUBLE OVERBAR;Sm;0;ON;;;;;N;;;;; +2A5F;LOGICAL AND WITH UNDERBAR;Sm;0;ON;;;;;N;;;;; +2A60;LOGICAL AND WITH DOUBLE UNDERBAR;Sm;0;ON;;;;;N;;;;; +2A61;SMALL VEE WITH UNDERBAR;Sm;0;ON;;;;;N;;;;; +2A62;LOGICAL OR WITH DOUBLE OVERBAR;Sm;0;ON;;;;;N;;;;; +2A63;LOGICAL OR WITH DOUBLE UNDERBAR;Sm;0;ON;;;;;N;;;;; +2A64;Z NOTATION DOMAIN ANTIRESTRICTION;Sm;0;ON;;;;;Y;;;;; +2A65;Z NOTATION RANGE ANTIRESTRICTION;Sm;0;ON;;;;;Y;;;;; +2A66;EQUALS SIGN WITH DOT BELOW;Sm;0;ON;;;;;N;;;;; +2A67;IDENTICAL WITH DOT ABOVE;Sm;0;ON;;;;;N;;;;; +2A68;TRIPLE HORIZONTAL BAR WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2A69;TRIPLE HORIZONTAL BAR WITH TRIPLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2A6A;TILDE OPERATOR WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +2A6B;TILDE OPERATOR WITH RISING DOTS;Sm;0;ON;;;;;Y;;;;; +2A6C;SIMILAR MINUS SIMILAR;Sm;0;ON;;;;;Y;;;;; +2A6D;CONGRUENT WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +2A6E;EQUALS WITH ASTERISK;Sm;0;ON;;;;;N;;;;; +2A6F;ALMOST EQUAL TO WITH CIRCUMFLEX ACCENT;Sm;0;ON;;;;;Y;;;;; +2A70;APPROXIMATELY EQUAL OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2A71;EQUALS SIGN ABOVE PLUS SIGN;Sm;0;ON;;;;;N;;;;; +2A72;PLUS SIGN ABOVE EQUALS SIGN;Sm;0;ON;;;;;N;;;;; +2A73;EQUALS SIGN ABOVE TILDE OPERATOR;Sm;0;ON;;;;;Y;;;;; +2A74;DOUBLE COLON EQUAL;Sm;0;ON; 003A 003A 003D;;;;Y;;;;; +2A75;TWO CONSECUTIVE EQUALS SIGNS;Sm;0;ON; 003D 003D;;;;N;;;;; +2A76;THREE CONSECUTIVE EQUALS SIGNS;Sm;0;ON; 003D 003D 003D;;;;N;;;;; +2A77;EQUALS SIGN WITH TWO DOTS ABOVE AND TWO DOTS BELOW;Sm;0;ON;;;;;N;;;;; +2A78;EQUIVALENT WITH FOUR DOTS ABOVE;Sm;0;ON;;;;;N;;;;; +2A79;LESS-THAN WITH CIRCLE INSIDE;Sm;0;ON;;;;;Y;;;;; +2A7A;GREATER-THAN WITH CIRCLE INSIDE;Sm;0;ON;;;;;Y;;;;; +2A7B;LESS-THAN WITH QUESTION MARK ABOVE;Sm;0;ON;;;;;Y;;;;; +2A7C;GREATER-THAN WITH QUESTION MARK ABOVE;Sm;0;ON;;;;;Y;;;;; +2A7D;LESS-THAN OR SLANTED EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2A7E;GREATER-THAN OR SLANTED EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2A7F;LESS-THAN OR SLANTED EQUAL TO WITH DOT INSIDE;Sm;0;ON;;;;;Y;;;;; +2A80;GREATER-THAN OR SLANTED EQUAL TO WITH DOT INSIDE;Sm;0;ON;;;;;Y;;;;; +2A81;LESS-THAN OR SLANTED EQUAL TO WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +2A82;GREATER-THAN OR SLANTED EQUAL TO WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +2A83;LESS-THAN OR SLANTED EQUAL TO WITH DOT ABOVE RIGHT;Sm;0;ON;;;;;Y;;;;; +2A84;GREATER-THAN OR SLANTED EQUAL TO WITH DOT ABOVE LEFT;Sm;0;ON;;;;;Y;;;;; +2A85;LESS-THAN OR APPROXIMATE;Sm;0;ON;;;;;Y;;;;; +2A86;GREATER-THAN OR APPROXIMATE;Sm;0;ON;;;;;Y;;;;; +2A87;LESS-THAN AND SINGLE-LINE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2A88;GREATER-THAN AND SINGLE-LINE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2A89;LESS-THAN AND NOT APPROXIMATE;Sm;0;ON;;;;;Y;;;;; +2A8A;GREATER-THAN AND NOT APPROXIMATE;Sm;0;ON;;;;;Y;;;;; +2A8B;LESS-THAN ABOVE DOUBLE-LINE EQUAL ABOVE GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2A8C;GREATER-THAN ABOVE DOUBLE-LINE EQUAL ABOVE LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2A8D;LESS-THAN ABOVE SIMILAR OR EQUAL;Sm;0;ON;;;;;Y;;;;; +2A8E;GREATER-THAN ABOVE SIMILAR OR EQUAL;Sm;0;ON;;;;;Y;;;;; +2A8F;LESS-THAN ABOVE SIMILAR ABOVE GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2A90;GREATER-THAN ABOVE SIMILAR ABOVE LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2A91;LESS-THAN ABOVE GREATER-THAN ABOVE DOUBLE-LINE EQUAL;Sm;0;ON;;;;;Y;;;;; +2A92;GREATER-THAN ABOVE LESS-THAN ABOVE DOUBLE-LINE EQUAL;Sm;0;ON;;;;;Y;;;;; +2A93;LESS-THAN ABOVE SLANTED EQUAL ABOVE GREATER-THAN ABOVE SLANTED EQUAL;Sm;0;ON;;;;;Y;;;;; +2A94;GREATER-THAN ABOVE SLANTED EQUAL ABOVE LESS-THAN ABOVE SLANTED EQUAL;Sm;0;ON;;;;;Y;;;;; +2A95;SLANTED EQUAL TO OR LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2A96;SLANTED EQUAL TO OR GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2A97;SLANTED EQUAL TO OR LESS-THAN WITH DOT INSIDE;Sm;0;ON;;;;;Y;;;;; +2A98;SLANTED EQUAL TO OR GREATER-THAN WITH DOT INSIDE;Sm;0;ON;;;;;Y;;;;; +2A99;DOUBLE-LINE EQUAL TO OR LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2A9A;DOUBLE-LINE EQUAL TO OR GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2A9B;DOUBLE-LINE SLANTED EQUAL TO OR LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2A9C;DOUBLE-LINE SLANTED EQUAL TO OR GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2A9D;SIMILAR OR LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2A9E;SIMILAR OR GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2A9F;SIMILAR ABOVE LESS-THAN ABOVE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AA0;SIMILAR ABOVE GREATER-THAN ABOVE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AA1;DOUBLE NESTED LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2AA2;DOUBLE NESTED GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2AA3;DOUBLE NESTED LESS-THAN WITH UNDERBAR;Sm;0;ON;;;;;Y;;;;; +2AA4;GREATER-THAN OVERLAPPING LESS-THAN;Sm;0;ON;;;;;N;;;;; +2AA5;GREATER-THAN BESIDE LESS-THAN;Sm;0;ON;;;;;N;;;;; +2AA6;LESS-THAN CLOSED BY CURVE;Sm;0;ON;;;;;Y;;;;; +2AA7;GREATER-THAN CLOSED BY CURVE;Sm;0;ON;;;;;Y;;;;; +2AA8;LESS-THAN CLOSED BY CURVE ABOVE SLANTED EQUAL;Sm;0;ON;;;;;Y;;;;; +2AA9;GREATER-THAN CLOSED BY CURVE ABOVE SLANTED EQUAL;Sm;0;ON;;;;;Y;;;;; +2AAA;SMALLER THAN;Sm;0;ON;;;;;Y;;;;; +2AAB;LARGER THAN;Sm;0;ON;;;;;Y;;;;; +2AAC;SMALLER THAN OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AAD;LARGER THAN OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AAE;EQUALS SIGN WITH BUMPY ABOVE;Sm;0;ON;;;;;N;;;;; +2AAF;PRECEDES ABOVE SINGLE-LINE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AB0;SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AB1;PRECEDES ABOVE SINGLE-LINE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AB2;SUCCEEDS ABOVE SINGLE-LINE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AB3;PRECEDES ABOVE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AB4;SUCCEEDS ABOVE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AB5;PRECEDES ABOVE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AB6;SUCCEEDS ABOVE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AB7;PRECEDES ABOVE ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AB8;SUCCEEDS ABOVE ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AB9;PRECEDES ABOVE NOT ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2ABA;SUCCEEDS ABOVE NOT ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2ABB;DOUBLE PRECEDES;Sm;0;ON;;;;;Y;;;;; +2ABC;DOUBLE SUCCEEDS;Sm;0;ON;;;;;Y;;;;; +2ABD;SUBSET WITH DOT;Sm;0;ON;;;;;Y;;;;; +2ABE;SUPERSET WITH DOT;Sm;0;ON;;;;;Y;;;;; +2ABF;SUBSET WITH PLUS SIGN BELOW;Sm;0;ON;;;;;Y;;;;; +2AC0;SUPERSET WITH PLUS SIGN BELOW;Sm;0;ON;;;;;Y;;;;; +2AC1;SUBSET WITH MULTIPLICATION SIGN BELOW;Sm;0;ON;;;;;Y;;;;; +2AC2;SUPERSET WITH MULTIPLICATION SIGN BELOW;Sm;0;ON;;;;;Y;;;;; +2AC3;SUBSET OF OR EQUAL TO WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +2AC4;SUPERSET OF OR EQUAL TO WITH DOT ABOVE;Sm;0;ON;;;;;Y;;;;; +2AC5;SUBSET OF ABOVE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AC6;SUPERSET OF ABOVE EQUALS SIGN;Sm;0;ON;;;;;Y;;;;; +2AC7;SUBSET OF ABOVE TILDE OPERATOR;Sm;0;ON;;;;;Y;;;;; +2AC8;SUPERSET OF ABOVE TILDE OPERATOR;Sm;0;ON;;;;;Y;;;;; +2AC9;SUBSET OF ABOVE ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2ACA;SUPERSET OF ABOVE ALMOST EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2ACB;SUBSET OF ABOVE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2ACC;SUPERSET OF ABOVE NOT EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2ACD;SQUARE LEFT OPEN BOX OPERATOR;Sm;0;ON;;;;;Y;;;;; +2ACE;SQUARE RIGHT OPEN BOX OPERATOR;Sm;0;ON;;;;;Y;;;;; +2ACF;CLOSED SUBSET;Sm;0;ON;;;;;Y;;;;; +2AD0;CLOSED SUPERSET;Sm;0;ON;;;;;Y;;;;; +2AD1;CLOSED SUBSET OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AD2;CLOSED SUPERSET OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AD3;SUBSET ABOVE SUPERSET;Sm;0;ON;;;;;Y;;;;; +2AD4;SUPERSET ABOVE SUBSET;Sm;0;ON;;;;;Y;;;;; +2AD5;SUBSET ABOVE SUBSET;Sm;0;ON;;;;;Y;;;;; +2AD6;SUPERSET ABOVE SUPERSET;Sm;0;ON;;;;;Y;;;;; +2AD7;SUPERSET BESIDE SUBSET;Sm;0;ON;;;;;N;;;;; +2AD8;SUPERSET BESIDE AND JOINED BY DASH WITH SUBSET;Sm;0;ON;;;;;N;;;;; +2AD9;ELEMENT OF OPENING DOWNWARDS;Sm;0;ON;;;;;N;;;;; +2ADA;PITCHFORK WITH TEE TOP;Sm;0;ON;;;;;N;;;;; +2ADB;TRANSVERSAL INTERSECTION;Sm;0;ON;;;;;N;;;;; +2ADC;FORKING;Sm;0;ON;2ADD 0338;;;;Y;;;;; +2ADD;NONFORKING;Sm;0;ON;;;;;N;;;;; +2ADE;SHORT LEFT TACK;Sm;0;ON;;;;;Y;;;;; +2ADF;SHORT DOWN TACK;Sm;0;ON;;;;;N;;;;; +2AE0;SHORT UP TACK;Sm;0;ON;;;;;N;;;;; +2AE1;PERPENDICULAR WITH S;Sm;0;ON;;;;;N;;;;; +2AE2;VERTICAL BAR TRIPLE RIGHT TURNSTILE;Sm;0;ON;;;;;Y;;;;; +2AE3;DOUBLE VERTICAL BAR LEFT TURNSTILE;Sm;0;ON;;;;;Y;;;;; +2AE4;VERTICAL BAR DOUBLE LEFT TURNSTILE;Sm;0;ON;;;;;Y;;;;; +2AE5;DOUBLE VERTICAL BAR DOUBLE LEFT TURNSTILE;Sm;0;ON;;;;;Y;;;;; +2AE6;LONG DASH FROM LEFT MEMBER OF DOUBLE VERTICAL;Sm;0;ON;;;;;Y;;;;; +2AE7;SHORT DOWN TACK WITH OVERBAR;Sm;0;ON;;;;;N;;;;; +2AE8;SHORT UP TACK WITH UNDERBAR;Sm;0;ON;;;;;N;;;;; +2AE9;SHORT UP TACK ABOVE SHORT DOWN TACK;Sm;0;ON;;;;;N;;;;; +2AEA;DOUBLE DOWN TACK;Sm;0;ON;;;;;N;;;;; +2AEB;DOUBLE UP TACK;Sm;0;ON;;;;;N;;;;; +2AEC;DOUBLE STROKE NOT SIGN;Sm;0;ON;;;;;Y;;;;; +2AED;REVERSED DOUBLE STROKE NOT SIGN;Sm;0;ON;;;;;Y;;;;; +2AEE;DOES NOT DIVIDE WITH REVERSED NEGATION SLASH;Sm;0;ON;;;;;Y;;;;; +2AEF;VERTICAL LINE WITH CIRCLE ABOVE;Sm;0;ON;;;;;N;;;;; +2AF0;VERTICAL LINE WITH CIRCLE BELOW;Sm;0;ON;;;;;N;;;;; +2AF1;DOWN TACK WITH CIRCLE BELOW;Sm;0;ON;;;;;N;;;;; +2AF2;PARALLEL WITH HORIZONTAL STROKE;Sm;0;ON;;;;;N;;;;; +2AF3;PARALLEL WITH TILDE OPERATOR;Sm;0;ON;;;;;Y;;;;; +2AF4;TRIPLE VERTICAL BAR BINARY RELATION;Sm;0;ON;;;;;N;;;;; +2AF5;TRIPLE VERTICAL BAR WITH HORIZONTAL STROKE;Sm;0;ON;;;;;N;;;;; +2AF6;TRIPLE COLON OPERATOR;Sm;0;ON;;;;;N;;;;; +2AF7;TRIPLE NESTED LESS-THAN;Sm;0;ON;;;;;Y;;;;; +2AF8;TRIPLE NESTED GREATER-THAN;Sm;0;ON;;;;;Y;;;;; +2AF9;DOUBLE-LINE SLANTED LESS-THAN OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AFA;DOUBLE-LINE SLANTED GREATER-THAN OR EQUAL TO;Sm;0;ON;;;;;Y;;;;; +2AFB;TRIPLE SOLIDUS BINARY RELATION;Sm;0;ON;;;;;Y;;;;; +2AFC;LARGE TRIPLE VERTICAL BAR OPERATOR;Sm;0;ON;;;;;N;;;;; +2AFD;DOUBLE SOLIDUS OPERATOR;Sm;0;ON;;;;;Y;;;;; +2AFE;WHITE VERTICAL BAR;Sm;0;ON;;;;;N;;;;; +2AFF;N-ARY WHITE VERTICAL BAR;Sm;0;ON;;;;;N;;;;; +2B00;NORTH EAST WHITE ARROW;So;0;ON;;;;;N;;;;; +2B01;NORTH WEST WHITE ARROW;So;0;ON;;;;;N;;;;; +2B02;SOUTH EAST WHITE ARROW;So;0;ON;;;;;N;;;;; +2B03;SOUTH WEST WHITE ARROW;So;0;ON;;;;;N;;;;; +2B04;LEFT RIGHT WHITE ARROW;So;0;ON;;;;;N;;;;; +2B05;LEFTWARDS BLACK ARROW;So;0;ON;;;;;N;;;;; +2B06;UPWARDS BLACK ARROW;So;0;ON;;;;;N;;;;; +2B07;DOWNWARDS BLACK ARROW;So;0;ON;;;;;N;;;;; +2B08;NORTH EAST BLACK ARROW;So;0;ON;;;;;N;;;;; +2B09;NORTH WEST BLACK ARROW;So;0;ON;;;;;N;;;;; +2B0A;SOUTH EAST BLACK ARROW;So;0;ON;;;;;N;;;;; +2B0B;SOUTH WEST BLACK ARROW;So;0;ON;;;;;N;;;;; +2B0C;LEFT RIGHT BLACK ARROW;So;0;ON;;;;;N;;;;; +2B0D;UP DOWN BLACK ARROW;So;0;ON;;;;;N;;;;; +2B0E;RIGHTWARDS ARROW WITH TIP DOWNWARDS;So;0;ON;;;;;N;;;;; +2B0F;RIGHTWARDS ARROW WITH TIP UPWARDS;So;0;ON;;;;;N;;;;; +2B10;LEFTWARDS ARROW WITH TIP DOWNWARDS;So;0;ON;;;;;N;;;;; +2B11;LEFTWARDS ARROW WITH TIP UPWARDS;So;0;ON;;;;;N;;;;; +2B12;SQUARE WITH TOP HALF BLACK;So;0;ON;;;;;N;;;;; +2B13;SQUARE WITH BOTTOM HALF BLACK;So;0;ON;;;;;N;;;;; +2B14;SQUARE WITH UPPER RIGHT DIAGONAL HALF BLACK;So;0;ON;;;;;N;;;;; +2B15;SQUARE WITH LOWER LEFT DIAGONAL HALF BLACK;So;0;ON;;;;;N;;;;; +2B16;DIAMOND WITH LEFT HALF BLACK;So;0;ON;;;;;N;;;;; +2B17;DIAMOND WITH RIGHT HALF BLACK;So;0;ON;;;;;N;;;;; +2B18;DIAMOND WITH TOP HALF BLACK;So;0;ON;;;;;N;;;;; +2B19;DIAMOND WITH BOTTOM HALF BLACK;So;0;ON;;;;;N;;;;; +2B1A;DOTTED SQUARE;So;0;ON;;;;;N;;;;; +2B1B;BLACK LARGE SQUARE;So;0;ON;;;;;N;;;;; +2B1C;WHITE LARGE SQUARE;So;0;ON;;;;;N;;;;; +2B1D;BLACK VERY SMALL SQUARE;So;0;ON;;;;;N;;;;; +2B1E;WHITE VERY SMALL SQUARE;So;0;ON;;;;;N;;;;; +2B1F;BLACK PENTAGON;So;0;ON;;;;;N;;;;; +2B20;WHITE PENTAGON;So;0;ON;;;;;N;;;;; +2B21;WHITE HEXAGON;So;0;ON;;;;;N;;;;; +2B22;BLACK HEXAGON;So;0;ON;;;;;N;;;;; +2B23;HORIZONTAL BLACK HEXAGON;So;0;ON;;;;;N;;;;; +2B24;BLACK LARGE CIRCLE;So;0;ON;;;;;N;;;;; +2B25;BLACK MEDIUM DIAMOND;So;0;ON;;;;;N;;;;; +2B26;WHITE MEDIUM DIAMOND;So;0;ON;;;;;N;;;;; +2B27;BLACK MEDIUM LOZENGE;So;0;ON;;;;;N;;;;; +2B28;WHITE MEDIUM LOZENGE;So;0;ON;;;;;N;;;;; +2B29;BLACK SMALL DIAMOND;So;0;ON;;;;;N;;;;; +2B2A;BLACK SMALL LOZENGE;So;0;ON;;;;;N;;;;; +2B2B;WHITE SMALL LOZENGE;So;0;ON;;;;;N;;;;; +2B2C;BLACK HORIZONTAL ELLIPSE;So;0;ON;;;;;N;;;;; +2B2D;WHITE HORIZONTAL ELLIPSE;So;0;ON;;;;;N;;;;; +2B2E;BLACK VERTICAL ELLIPSE;So;0;ON;;;;;N;;;;; +2B2F;WHITE VERTICAL ELLIPSE;So;0;ON;;;;;N;;;;; +2B30;LEFT ARROW WITH SMALL CIRCLE;Sm;0;ON;;;;;N;;;;; +2B31;THREE LEFTWARDS ARROWS;Sm;0;ON;;;;;N;;;;; +2B32;LEFT ARROW WITH CIRCLED PLUS;Sm;0;ON;;;;;N;;;;; +2B33;LONG LEFTWARDS SQUIGGLE ARROW;Sm;0;ON;;;;;N;;;;; +2B34;LEFTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2B35;LEFTWARDS TWO-HEADED ARROW WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2B36;LEFTWARDS TWO-HEADED ARROW FROM BAR;Sm;0;ON;;;;;N;;;;; +2B37;LEFTWARDS TWO-HEADED TRIPLE DASH ARROW;Sm;0;ON;;;;;N;;;;; +2B38;LEFTWARDS ARROW WITH DOTTED STEM;Sm;0;ON;;;;;N;;;;; +2B39;LEFTWARDS ARROW WITH TAIL WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2B3A;LEFTWARDS ARROW WITH TAIL WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2B3B;LEFTWARDS TWO-HEADED ARROW WITH TAIL;Sm;0;ON;;;;;N;;;;; +2B3C;LEFTWARDS TWO-HEADED ARROW WITH TAIL WITH VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2B3D;LEFTWARDS TWO-HEADED ARROW WITH TAIL WITH DOUBLE VERTICAL STROKE;Sm;0;ON;;;;;N;;;;; +2B3E;LEFTWARDS ARROW THROUGH X;Sm;0;ON;;;;;N;;;;; +2B3F;WAVE ARROW POINTING DIRECTLY LEFT;Sm;0;ON;;;;;N;;;;; +2B40;EQUALS SIGN ABOVE LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2B41;REVERSE TILDE OPERATOR ABOVE LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2B42;LEFTWARDS ARROW ABOVE REVERSE ALMOST EQUAL TO;Sm;0;ON;;;;;N;;;;; +2B43;RIGHTWARDS ARROW THROUGH GREATER-THAN;Sm;0;ON;;;;;N;;;;; +2B44;RIGHTWARDS ARROW THROUGH SUPERSET;Sm;0;ON;;;;;N;;;;; +2B45;LEFTWARDS QUADRUPLE ARROW;So;0;ON;;;;;N;;;;; +2B46;RIGHTWARDS QUADRUPLE ARROW;So;0;ON;;;;;N;;;;; +2B47;REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2B48;RIGHTWARDS ARROW ABOVE REVERSE ALMOST EQUAL TO;Sm;0;ON;;;;;N;;;;; +2B49;TILDE OPERATOR ABOVE LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +2B4A;LEFTWARDS ARROW ABOVE ALMOST EQUAL TO;Sm;0;ON;;;;;N;;;;; +2B4B;LEFTWARDS ARROW ABOVE REVERSE TILDE OPERATOR;Sm;0;ON;;;;;N;;;;; +2B4C;RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR;Sm;0;ON;;;;;N;;;;; +2B50;WHITE MEDIUM STAR;So;0;ON;;;;;N;;;;; +2B51;BLACK SMALL STAR;So;0;ON;;;;;N;;;;; +2B52;WHITE SMALL STAR;So;0;ON;;;;;N;;;;; +2B53;BLACK RIGHT-POINTING PENTAGON;So;0;ON;;;;;N;;;;; +2B54;WHITE RIGHT-POINTING PENTAGON;So;0;ON;;;;;N;;;;; +2B55;HEAVY LARGE CIRCLE;So;0;ON;;;;;N;;;;; +2B56;HEAVY OVAL WITH OVAL INSIDE;So;0;ON;;;;;N;;;;; +2B57;HEAVY CIRCLE WITH CIRCLE INSIDE;So;0;ON;;;;;N;;;;; +2B58;HEAVY CIRCLE;So;0;ON;;;;;N;;;;; +2B59;HEAVY CIRCLED SALTIRE;So;0;ON;;;;;N;;;;; +2C00;GLAGOLITIC CAPITAL LETTER AZU;Lu;0;L;;;;;N;;;;2C30; +2C01;GLAGOLITIC CAPITAL LETTER BUKY;Lu;0;L;;;;;N;;;;2C31; +2C02;GLAGOLITIC CAPITAL LETTER VEDE;Lu;0;L;;;;;N;;;;2C32; +2C03;GLAGOLITIC CAPITAL LETTER GLAGOLI;Lu;0;L;;;;;N;;;;2C33; +2C04;GLAGOLITIC CAPITAL LETTER DOBRO;Lu;0;L;;;;;N;;;;2C34; +2C05;GLAGOLITIC CAPITAL LETTER YESTU;Lu;0;L;;;;;N;;;;2C35; +2C06;GLAGOLITIC CAPITAL LETTER ZHIVETE;Lu;0;L;;;;;N;;;;2C36; +2C07;GLAGOLITIC CAPITAL LETTER DZELO;Lu;0;L;;;;;N;;;;2C37; +2C08;GLAGOLITIC CAPITAL LETTER ZEMLJA;Lu;0;L;;;;;N;;;;2C38; +2C09;GLAGOLITIC CAPITAL LETTER IZHE;Lu;0;L;;;;;N;;;;2C39; +2C0A;GLAGOLITIC CAPITAL LETTER INITIAL IZHE;Lu;0;L;;;;;N;;;;2C3A; +2C0B;GLAGOLITIC CAPITAL LETTER I;Lu;0;L;;;;;N;;;;2C3B; +2C0C;GLAGOLITIC CAPITAL LETTER DJERVI;Lu;0;L;;;;;N;;;;2C3C; +2C0D;GLAGOLITIC CAPITAL LETTER KAKO;Lu;0;L;;;;;N;;;;2C3D; +2C0E;GLAGOLITIC CAPITAL LETTER LJUDIJE;Lu;0;L;;;;;N;;;;2C3E; +2C0F;GLAGOLITIC CAPITAL LETTER MYSLITE;Lu;0;L;;;;;N;;;;2C3F; +2C10;GLAGOLITIC CAPITAL LETTER NASHI;Lu;0;L;;;;;N;;;;2C40; +2C11;GLAGOLITIC CAPITAL LETTER ONU;Lu;0;L;;;;;N;;;;2C41; +2C12;GLAGOLITIC CAPITAL LETTER POKOJI;Lu;0;L;;;;;N;;;;2C42; +2C13;GLAGOLITIC CAPITAL LETTER RITSI;Lu;0;L;;;;;N;;;;2C43; +2C14;GLAGOLITIC CAPITAL LETTER SLOVO;Lu;0;L;;;;;N;;;;2C44; +2C15;GLAGOLITIC CAPITAL LETTER TVRIDO;Lu;0;L;;;;;N;;;;2C45; +2C16;GLAGOLITIC CAPITAL LETTER UKU;Lu;0;L;;;;;N;;;;2C46; +2C17;GLAGOLITIC CAPITAL LETTER FRITU;Lu;0;L;;;;;N;;;;2C47; +2C18;GLAGOLITIC CAPITAL LETTER HERU;Lu;0;L;;;;;N;;;;2C48; +2C19;GLAGOLITIC CAPITAL LETTER OTU;Lu;0;L;;;;;N;;;;2C49; +2C1A;GLAGOLITIC CAPITAL LETTER PE;Lu;0;L;;;;;N;;;;2C4A; +2C1B;GLAGOLITIC CAPITAL LETTER SHTA;Lu;0;L;;;;;N;;;;2C4B; +2C1C;GLAGOLITIC CAPITAL LETTER TSI;Lu;0;L;;;;;N;;;;2C4C; +2C1D;GLAGOLITIC CAPITAL LETTER CHRIVI;Lu;0;L;;;;;N;;;;2C4D; +2C1E;GLAGOLITIC CAPITAL LETTER SHA;Lu;0;L;;;;;N;;;;2C4E; +2C1F;GLAGOLITIC CAPITAL LETTER YERU;Lu;0;L;;;;;N;;;;2C4F; +2C20;GLAGOLITIC CAPITAL LETTER YERI;Lu;0;L;;;;;N;;;;2C50; +2C21;GLAGOLITIC CAPITAL LETTER YATI;Lu;0;L;;;;;N;;;;2C51; +2C22;GLAGOLITIC CAPITAL LETTER SPIDERY HA;Lu;0;L;;;;;N;;;;2C52; +2C23;GLAGOLITIC CAPITAL LETTER YU;Lu;0;L;;;;;N;;;;2C53; +2C24;GLAGOLITIC CAPITAL LETTER SMALL YUS;Lu;0;L;;;;;N;;;;2C54; +2C25;GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL;Lu;0;L;;;;;N;;;;2C55; +2C26;GLAGOLITIC CAPITAL LETTER YO;Lu;0;L;;;;;N;;;;2C56; +2C27;GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS;Lu;0;L;;;;;N;;;;2C57; +2C28;GLAGOLITIC CAPITAL LETTER BIG YUS;Lu;0;L;;;;;N;;;;2C58; +2C29;GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS;Lu;0;L;;;;;N;;;;2C59; +2C2A;GLAGOLITIC CAPITAL LETTER FITA;Lu;0;L;;;;;N;;;;2C5A; +2C2B;GLAGOLITIC CAPITAL LETTER IZHITSA;Lu;0;L;;;;;N;;;;2C5B; +2C2C;GLAGOLITIC CAPITAL LETTER SHTAPIC;Lu;0;L;;;;;N;;;;2C5C; +2C2D;GLAGOLITIC CAPITAL LETTER TROKUTASTI A;Lu;0;L;;;;;N;;;;2C5D; +2C2E;GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE;Lu;0;L;;;;;N;;;;2C5E; +2C30;GLAGOLITIC SMALL LETTER AZU;Ll;0;L;;;;;N;;;2C00;;2C00 +2C31;GLAGOLITIC SMALL LETTER BUKY;Ll;0;L;;;;;N;;;2C01;;2C01 +2C32;GLAGOLITIC SMALL LETTER VEDE;Ll;0;L;;;;;N;;;2C02;;2C02 +2C33;GLAGOLITIC SMALL LETTER GLAGOLI;Ll;0;L;;;;;N;;;2C03;;2C03 +2C34;GLAGOLITIC SMALL LETTER DOBRO;Ll;0;L;;;;;N;;;2C04;;2C04 +2C35;GLAGOLITIC SMALL LETTER YESTU;Ll;0;L;;;;;N;;;2C05;;2C05 +2C36;GLAGOLITIC SMALL LETTER ZHIVETE;Ll;0;L;;;;;N;;;2C06;;2C06 +2C37;GLAGOLITIC SMALL LETTER DZELO;Ll;0;L;;;;;N;;;2C07;;2C07 +2C38;GLAGOLITIC SMALL LETTER ZEMLJA;Ll;0;L;;;;;N;;;2C08;;2C08 +2C39;GLAGOLITIC SMALL LETTER IZHE;Ll;0;L;;;;;N;;;2C09;;2C09 +2C3A;GLAGOLITIC SMALL LETTER INITIAL IZHE;Ll;0;L;;;;;N;;;2C0A;;2C0A +2C3B;GLAGOLITIC SMALL LETTER I;Ll;0;L;;;;;N;;;2C0B;;2C0B +2C3C;GLAGOLITIC SMALL LETTER DJERVI;Ll;0;L;;;;;N;;;2C0C;;2C0C +2C3D;GLAGOLITIC SMALL LETTER KAKO;Ll;0;L;;;;;N;;;2C0D;;2C0D +2C3E;GLAGOLITIC SMALL LETTER LJUDIJE;Ll;0;L;;;;;N;;;2C0E;;2C0E +2C3F;GLAGOLITIC SMALL LETTER MYSLITE;Ll;0;L;;;;;N;;;2C0F;;2C0F +2C40;GLAGOLITIC SMALL LETTER NASHI;Ll;0;L;;;;;N;;;2C10;;2C10 +2C41;GLAGOLITIC SMALL LETTER ONU;Ll;0;L;;;;;N;;;2C11;;2C11 +2C42;GLAGOLITIC SMALL LETTER POKOJI;Ll;0;L;;;;;N;;;2C12;;2C12 +2C43;GLAGOLITIC SMALL LETTER RITSI;Ll;0;L;;;;;N;;;2C13;;2C13 +2C44;GLAGOLITIC SMALL LETTER SLOVO;Ll;0;L;;;;;N;;;2C14;;2C14 +2C45;GLAGOLITIC SMALL LETTER TVRIDO;Ll;0;L;;;;;N;;;2C15;;2C15 +2C46;GLAGOLITIC SMALL LETTER UKU;Ll;0;L;;;;;N;;;2C16;;2C16 +2C47;GLAGOLITIC SMALL LETTER FRITU;Ll;0;L;;;;;N;;;2C17;;2C17 +2C48;GLAGOLITIC SMALL LETTER HERU;Ll;0;L;;;;;N;;;2C18;;2C18 +2C49;GLAGOLITIC SMALL LETTER OTU;Ll;0;L;;;;;N;;;2C19;;2C19 +2C4A;GLAGOLITIC SMALL LETTER PE;Ll;0;L;;;;;N;;;2C1A;;2C1A +2C4B;GLAGOLITIC SMALL LETTER SHTA;Ll;0;L;;;;;N;;;2C1B;;2C1B +2C4C;GLAGOLITIC SMALL LETTER TSI;Ll;0;L;;;;;N;;;2C1C;;2C1C +2C4D;GLAGOLITIC SMALL LETTER CHRIVI;Ll;0;L;;;;;N;;;2C1D;;2C1D +2C4E;GLAGOLITIC SMALL LETTER SHA;Ll;0;L;;;;;N;;;2C1E;;2C1E +2C4F;GLAGOLITIC SMALL LETTER YERU;Ll;0;L;;;;;N;;;2C1F;;2C1F +2C50;GLAGOLITIC SMALL LETTER YERI;Ll;0;L;;;;;N;;;2C20;;2C20 +2C51;GLAGOLITIC SMALL LETTER YATI;Ll;0;L;;;;;N;;;2C21;;2C21 +2C52;GLAGOLITIC SMALL LETTER SPIDERY HA;Ll;0;L;;;;;N;;;2C22;;2C22 +2C53;GLAGOLITIC SMALL LETTER YU;Ll;0;L;;;;;N;;;2C23;;2C23 +2C54;GLAGOLITIC SMALL LETTER SMALL YUS;Ll;0;L;;;;;N;;;2C24;;2C24 +2C55;GLAGOLITIC SMALL LETTER SMALL YUS WITH TAIL;Ll;0;L;;;;;N;;;2C25;;2C25 +2C56;GLAGOLITIC SMALL LETTER YO;Ll;0;L;;;;;N;;;2C26;;2C26 +2C57;GLAGOLITIC SMALL LETTER IOTATED SMALL YUS;Ll;0;L;;;;;N;;;2C27;;2C27 +2C58;GLAGOLITIC SMALL LETTER BIG YUS;Ll;0;L;;;;;N;;;2C28;;2C28 +2C59;GLAGOLITIC SMALL LETTER IOTATED BIG YUS;Ll;0;L;;;;;N;;;2C29;;2C29 +2C5A;GLAGOLITIC SMALL LETTER FITA;Ll;0;L;;;;;N;;;2C2A;;2C2A +2C5B;GLAGOLITIC SMALL LETTER IZHITSA;Ll;0;L;;;;;N;;;2C2B;;2C2B +2C5C;GLAGOLITIC SMALL LETTER SHTAPIC;Ll;0;L;;;;;N;;;2C2C;;2C2C +2C5D;GLAGOLITIC SMALL LETTER TROKUTASTI A;Ll;0;L;;;;;N;;;2C2D;;2C2D +2C5E;GLAGOLITIC SMALL LETTER LATINATE MYSLITE;Ll;0;L;;;;;N;;;2C2E;;2C2E +2C60;LATIN CAPITAL LETTER L WITH DOUBLE BAR;Lu;0;L;;;;;N;;;;2C61; +2C61;LATIN SMALL LETTER L WITH DOUBLE BAR;Ll;0;L;;;;;N;;;2C60;;2C60 +2C62;LATIN CAPITAL LETTER L WITH MIDDLE TILDE;Lu;0;L;;;;;N;;;;026B; +2C63;LATIN CAPITAL LETTER P WITH STROKE;Lu;0;L;;;;;N;;;;1D7D; +2C64;LATIN CAPITAL LETTER R WITH TAIL;Lu;0;L;;;;;N;;;;027D; +2C65;LATIN SMALL LETTER A WITH STROKE;Ll;0;L;;;;;N;;;023A;;023A +2C66;LATIN SMALL LETTER T WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;023E;;023E +2C67;LATIN CAPITAL LETTER H WITH DESCENDER;Lu;0;L;;;;;N;;;;2C68; +2C68;LATIN SMALL LETTER H WITH DESCENDER;Ll;0;L;;;;;N;;;2C67;;2C67 +2C69;LATIN CAPITAL LETTER K WITH DESCENDER;Lu;0;L;;;;;N;;;;2C6A; +2C6A;LATIN SMALL LETTER K WITH DESCENDER;Ll;0;L;;;;;N;;;2C69;;2C69 +2C6B;LATIN CAPITAL LETTER Z WITH DESCENDER;Lu;0;L;;;;;N;;;;2C6C; +2C6C;LATIN SMALL LETTER Z WITH DESCENDER;Ll;0;L;;;;;N;;;2C6B;;2C6B +2C6D;LATIN CAPITAL LETTER ALPHA;Lu;0;L;;;;;N;;;;0251; +2C6E;LATIN CAPITAL LETTER M WITH HOOK;Lu;0;L;;;;;N;;;;0271; +2C6F;LATIN CAPITAL LETTER TURNED A;Lu;0;L;;;;;N;;;;0250; +2C70;LATIN CAPITAL LETTER TURNED ALPHA;Lu;0;L;;;;;N;;;;0252; +2C71;LATIN SMALL LETTER V WITH RIGHT HOOK;Ll;0;L;;;;;N;;;;; +2C72;LATIN CAPITAL LETTER W WITH HOOK;Lu;0;L;;;;;N;;;;2C73; +2C73;LATIN SMALL LETTER W WITH HOOK;Ll;0;L;;;;;N;;;2C72;;2C72 +2C74;LATIN SMALL LETTER V WITH CURL;Ll;0;L;;;;;N;;;;; +2C75;LATIN CAPITAL LETTER HALF H;Lu;0;L;;;;;N;;;;2C76; +2C76;LATIN SMALL LETTER HALF H;Ll;0;L;;;;;N;;;2C75;;2C75 +2C77;LATIN SMALL LETTER TAILLESS PHI;Ll;0;L;;;;;N;;;;; +2C78;LATIN SMALL LETTER E WITH NOTCH;Ll;0;L;;;;;N;;;;; +2C79;LATIN SMALL LETTER TURNED R WITH TAIL;Ll;0;L;;;;;N;;;;; +2C7A;LATIN SMALL LETTER O WITH LOW RING INSIDE;Ll;0;L;;;;;N;;;;; +2C7B;LATIN LETTER SMALL CAPITAL TURNED E;Ll;0;L;;;;;N;;;;; +2C7C;LATIN SUBSCRIPT SMALL LETTER J;Lm;0;L; 006A;;;;N;;;;; +2C7D;MODIFIER LETTER CAPITAL V;Lm;0;L; 0056;;;;N;;;;; +2C7E;LATIN CAPITAL LETTER S WITH SWASH TAIL;Lu;0;L;;;;;N;;;;023F; +2C7F;LATIN CAPITAL LETTER Z WITH SWASH TAIL;Lu;0;L;;;;;N;;;;0240; +2C80;COPTIC CAPITAL LETTER ALFA;Lu;0;L;;;;;N;;;;2C81; +2C81;COPTIC SMALL LETTER ALFA;Ll;0;L;;;;;N;;;2C80;;2C80 +2C82;COPTIC CAPITAL LETTER VIDA;Lu;0;L;;;;;N;;;;2C83; +2C83;COPTIC SMALL LETTER VIDA;Ll;0;L;;;;;N;;;2C82;;2C82 +2C84;COPTIC CAPITAL LETTER GAMMA;Lu;0;L;;;;;N;;;;2C85; +2C85;COPTIC SMALL LETTER GAMMA;Ll;0;L;;;;;N;;;2C84;;2C84 +2C86;COPTIC CAPITAL LETTER DALDA;Lu;0;L;;;;;N;;;;2C87; +2C87;COPTIC SMALL LETTER DALDA;Ll;0;L;;;;;N;;;2C86;;2C86 +2C88;COPTIC CAPITAL LETTER EIE;Lu;0;L;;;;;N;;;;2C89; +2C89;COPTIC SMALL LETTER EIE;Ll;0;L;;;;;N;;;2C88;;2C88 +2C8A;COPTIC CAPITAL LETTER SOU;Lu;0;L;;;;;N;;;;2C8B; +2C8B;COPTIC SMALL LETTER SOU;Ll;0;L;;;;;N;;;2C8A;;2C8A +2C8C;COPTIC CAPITAL LETTER ZATA;Lu;0;L;;;;;N;;;;2C8D; +2C8D;COPTIC SMALL LETTER ZATA;Ll;0;L;;;;;N;;;2C8C;;2C8C +2C8E;COPTIC CAPITAL LETTER HATE;Lu;0;L;;;;;N;;;;2C8F; +2C8F;COPTIC SMALL LETTER HATE;Ll;0;L;;;;;N;;;2C8E;;2C8E +2C90;COPTIC CAPITAL LETTER THETHE;Lu;0;L;;;;;N;;;;2C91; +2C91;COPTIC SMALL LETTER THETHE;Ll;0;L;;;;;N;;;2C90;;2C90 +2C92;COPTIC CAPITAL LETTER IAUDA;Lu;0;L;;;;;N;;;;2C93; +2C93;COPTIC SMALL LETTER IAUDA;Ll;0;L;;;;;N;;;2C92;;2C92 +2C94;COPTIC CAPITAL LETTER KAPA;Lu;0;L;;;;;N;;;;2C95; +2C95;COPTIC SMALL LETTER KAPA;Ll;0;L;;;;;N;;;2C94;;2C94 +2C96;COPTIC CAPITAL LETTER LAULA;Lu;0;L;;;;;N;;;;2C97; +2C97;COPTIC SMALL LETTER LAULA;Ll;0;L;;;;;N;;;2C96;;2C96 +2C98;COPTIC CAPITAL LETTER MI;Lu;0;L;;;;;N;;;;2C99; +2C99;COPTIC SMALL LETTER MI;Ll;0;L;;;;;N;;;2C98;;2C98 +2C9A;COPTIC CAPITAL LETTER NI;Lu;0;L;;;;;N;;;;2C9B; +2C9B;COPTIC SMALL LETTER NI;Ll;0;L;;;;;N;;;2C9A;;2C9A +2C9C;COPTIC CAPITAL LETTER KSI;Lu;0;L;;;;;N;;;;2C9D; +2C9D;COPTIC SMALL LETTER KSI;Ll;0;L;;;;;N;;;2C9C;;2C9C +2C9E;COPTIC CAPITAL LETTER O;Lu;0;L;;;;;N;;;;2C9F; +2C9F;COPTIC SMALL LETTER O;Ll;0;L;;;;;N;;;2C9E;;2C9E +2CA0;COPTIC CAPITAL LETTER PI;Lu;0;L;;;;;N;;;;2CA1; +2CA1;COPTIC SMALL LETTER PI;Ll;0;L;;;;;N;;;2CA0;;2CA0 +2CA2;COPTIC CAPITAL LETTER RO;Lu;0;L;;;;;N;;;;2CA3; +2CA3;COPTIC SMALL LETTER RO;Ll;0;L;;;;;N;;;2CA2;;2CA2 +2CA4;COPTIC CAPITAL LETTER SIMA;Lu;0;L;;;;;N;;;;2CA5; +2CA5;COPTIC SMALL LETTER SIMA;Ll;0;L;;;;;N;;;2CA4;;2CA4 +2CA6;COPTIC CAPITAL LETTER TAU;Lu;0;L;;;;;N;;;;2CA7; +2CA7;COPTIC SMALL LETTER TAU;Ll;0;L;;;;;N;;;2CA6;;2CA6 +2CA8;COPTIC CAPITAL LETTER UA;Lu;0;L;;;;;N;;;;2CA9; +2CA9;COPTIC SMALL LETTER UA;Ll;0;L;;;;;N;;;2CA8;;2CA8 +2CAA;COPTIC CAPITAL LETTER FI;Lu;0;L;;;;;N;;;;2CAB; +2CAB;COPTIC SMALL LETTER FI;Ll;0;L;;;;;N;;;2CAA;;2CAA +2CAC;COPTIC CAPITAL LETTER KHI;Lu;0;L;;;;;N;;;;2CAD; +2CAD;COPTIC SMALL LETTER KHI;Ll;0;L;;;;;N;;;2CAC;;2CAC +2CAE;COPTIC CAPITAL LETTER PSI;Lu;0;L;;;;;N;;;;2CAF; +2CAF;COPTIC SMALL LETTER PSI;Ll;0;L;;;;;N;;;2CAE;;2CAE +2CB0;COPTIC CAPITAL LETTER OOU;Lu;0;L;;;;;N;;;;2CB1; +2CB1;COPTIC SMALL LETTER OOU;Ll;0;L;;;;;N;;;2CB0;;2CB0 +2CB2;COPTIC CAPITAL LETTER DIALECT-P ALEF;Lu;0;L;;;;;N;;;;2CB3; +2CB3;COPTIC SMALL LETTER DIALECT-P ALEF;Ll;0;L;;;;;N;;;2CB2;;2CB2 +2CB4;COPTIC CAPITAL LETTER OLD COPTIC AIN;Lu;0;L;;;;;N;;;;2CB5; +2CB5;COPTIC SMALL LETTER OLD COPTIC AIN;Ll;0;L;;;;;N;;;2CB4;;2CB4 +2CB6;COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE;Lu;0;L;;;;;N;;;;2CB7; +2CB7;COPTIC SMALL LETTER CRYPTOGRAMMIC EIE;Ll;0;L;;;;;N;;;2CB6;;2CB6 +2CB8;COPTIC CAPITAL LETTER DIALECT-P KAPA;Lu;0;L;;;;;N;;;;2CB9; +2CB9;COPTIC SMALL LETTER DIALECT-P KAPA;Ll;0;L;;;;;N;;;2CB8;;2CB8 +2CBA;COPTIC CAPITAL LETTER DIALECT-P NI;Lu;0;L;;;;;N;;;;2CBB; +2CBB;COPTIC SMALL LETTER DIALECT-P NI;Ll;0;L;;;;;N;;;2CBA;;2CBA +2CBC;COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI;Lu;0;L;;;;;N;;;;2CBD; +2CBD;COPTIC SMALL LETTER CRYPTOGRAMMIC NI;Ll;0;L;;;;;N;;;2CBC;;2CBC +2CBE;COPTIC CAPITAL LETTER OLD COPTIC OOU;Lu;0;L;;;;;N;;;;2CBF; +2CBF;COPTIC SMALL LETTER OLD COPTIC OOU;Ll;0;L;;;;;N;;;2CBE;;2CBE +2CC0;COPTIC CAPITAL LETTER SAMPI;Lu;0;L;;;;;N;;;;2CC1; +2CC1;COPTIC SMALL LETTER SAMPI;Ll;0;L;;;;;N;;;2CC0;;2CC0 +2CC2;COPTIC CAPITAL LETTER CROSSED SHEI;Lu;0;L;;;;;N;;;;2CC3; +2CC3;COPTIC SMALL LETTER CROSSED SHEI;Ll;0;L;;;;;N;;;2CC2;;2CC2 +2CC4;COPTIC CAPITAL LETTER OLD COPTIC SHEI;Lu;0;L;;;;;N;;;;2CC5; +2CC5;COPTIC SMALL LETTER OLD COPTIC SHEI;Ll;0;L;;;;;N;;;2CC4;;2CC4 +2CC6;COPTIC CAPITAL LETTER OLD COPTIC ESH;Lu;0;L;;;;;N;;;;2CC7; +2CC7;COPTIC SMALL LETTER OLD COPTIC ESH;Ll;0;L;;;;;N;;;2CC6;;2CC6 +2CC8;COPTIC CAPITAL LETTER AKHMIMIC KHEI;Lu;0;L;;;;;N;;;;2CC9; +2CC9;COPTIC SMALL LETTER AKHMIMIC KHEI;Ll;0;L;;;;;N;;;2CC8;;2CC8 +2CCA;COPTIC CAPITAL LETTER DIALECT-P HORI;Lu;0;L;;;;;N;;;;2CCB; +2CCB;COPTIC SMALL LETTER DIALECT-P HORI;Ll;0;L;;;;;N;;;2CCA;;2CCA +2CCC;COPTIC CAPITAL LETTER OLD COPTIC HORI;Lu;0;L;;;;;N;;;;2CCD; +2CCD;COPTIC SMALL LETTER OLD COPTIC HORI;Ll;0;L;;;;;N;;;2CCC;;2CCC +2CCE;COPTIC CAPITAL LETTER OLD COPTIC HA;Lu;0;L;;;;;N;;;;2CCF; +2CCF;COPTIC SMALL LETTER OLD COPTIC HA;Ll;0;L;;;;;N;;;2CCE;;2CCE +2CD0;COPTIC CAPITAL LETTER L-SHAPED HA;Lu;0;L;;;;;N;;;;2CD1; +2CD1;COPTIC SMALL LETTER L-SHAPED HA;Ll;0;L;;;;;N;;;2CD0;;2CD0 +2CD2;COPTIC CAPITAL LETTER OLD COPTIC HEI;Lu;0;L;;;;;N;;;;2CD3; +2CD3;COPTIC SMALL LETTER OLD COPTIC HEI;Ll;0;L;;;;;N;;;2CD2;;2CD2 +2CD4;COPTIC CAPITAL LETTER OLD COPTIC HAT;Lu;0;L;;;;;N;;;;2CD5; +2CD5;COPTIC SMALL LETTER OLD COPTIC HAT;Ll;0;L;;;;;N;;;2CD4;;2CD4 +2CD6;COPTIC CAPITAL LETTER OLD COPTIC GANGIA;Lu;0;L;;;;;N;;;;2CD7; +2CD7;COPTIC SMALL LETTER OLD COPTIC GANGIA;Ll;0;L;;;;;N;;;2CD6;;2CD6 +2CD8;COPTIC CAPITAL LETTER OLD COPTIC DJA;Lu;0;L;;;;;N;;;;2CD9; +2CD9;COPTIC SMALL LETTER OLD COPTIC DJA;Ll;0;L;;;;;N;;;2CD8;;2CD8 +2CDA;COPTIC CAPITAL LETTER OLD COPTIC SHIMA;Lu;0;L;;;;;N;;;;2CDB; +2CDB;COPTIC SMALL LETTER OLD COPTIC SHIMA;Ll;0;L;;;;;N;;;2CDA;;2CDA +2CDC;COPTIC CAPITAL LETTER OLD NUBIAN SHIMA;Lu;0;L;;;;;N;;;;2CDD; +2CDD;COPTIC SMALL LETTER OLD NUBIAN SHIMA;Ll;0;L;;;;;N;;;2CDC;;2CDC +2CDE;COPTIC CAPITAL LETTER OLD NUBIAN NGI;Lu;0;L;;;;;N;;;;2CDF; +2CDF;COPTIC SMALL LETTER OLD NUBIAN NGI;Ll;0;L;;;;;N;;;2CDE;;2CDE +2CE0;COPTIC CAPITAL LETTER OLD NUBIAN NYI;Lu;0;L;;;;;N;;;;2CE1; +2CE1;COPTIC SMALL LETTER OLD NUBIAN NYI;Ll;0;L;;;;;N;;;2CE0;;2CE0 +2CE2;COPTIC CAPITAL LETTER OLD NUBIAN WAU;Lu;0;L;;;;;N;;;;2CE3; +2CE3;COPTIC SMALL LETTER OLD NUBIAN WAU;Ll;0;L;;;;;N;;;2CE2;;2CE2 +2CE4;COPTIC SYMBOL KAI;Ll;0;L;;;;;N;;;;; +2CE5;COPTIC SYMBOL MI RO;So;0;ON;;;;;N;;;;; +2CE6;COPTIC SYMBOL PI RO;So;0;ON;;;;;N;;;;; +2CE7;COPTIC SYMBOL STAUROS;So;0;ON;;;;;N;;;;; +2CE8;COPTIC SYMBOL TAU RO;So;0;ON;;;;;N;;;;; +2CE9;COPTIC SYMBOL KHI RO;So;0;ON;;;;;N;;;;; +2CEA;COPTIC SYMBOL SHIMA SIMA;So;0;ON;;;;;N;;;;; +2CEB;COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI;Lu;0;L;;;;;N;;;;2CEC; +2CEC;COPTIC SMALL LETTER CRYPTOGRAMMIC SHEI;Ll;0;L;;;;;N;;;2CEB;;2CEB +2CED;COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA;Lu;0;L;;;;;N;;;;2CEE; +2CEE;COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA;Ll;0;L;;;;;N;;;2CED;;2CED +2CEF;COPTIC COMBINING NI ABOVE;Mn;230;NSM;;;;;N;;;;; +2CF0;COPTIC COMBINING SPIRITUS ASPER;Mn;230;NSM;;;;;N;;;;; +2CF1;COPTIC COMBINING SPIRITUS LENIS;Mn;230;NSM;;;;;N;;;;; +2CF2;COPTIC CAPITAL LETTER BOHAIRIC KHEI;Lu;0;L;;;;;N;;;;2CF3; +2CF3;COPTIC SMALL LETTER BOHAIRIC KHEI;Ll;0;L;;;;;N;;;2CF2;;2CF2 +2CF9;COPTIC OLD NUBIAN FULL STOP;Po;0;ON;;;;;N;;;;; +2CFA;COPTIC OLD NUBIAN DIRECT QUESTION MARK;Po;0;ON;;;;;N;;;;; +2CFB;COPTIC OLD NUBIAN INDIRECT QUESTION MARK;Po;0;ON;;;;;N;;;;; +2CFC;COPTIC OLD NUBIAN VERSE DIVIDER;Po;0;ON;;;;;N;;;;; +2CFD;COPTIC FRACTION ONE HALF;No;0;ON;;;;1/2;N;;;;; +2CFE;COPTIC FULL STOP;Po;0;ON;;;;;N;;;;; +2CFF;COPTIC MORPHOLOGICAL DIVIDER;Po;0;ON;;;;;N;;;;; +2D00;GEORGIAN SMALL LETTER AN;Ll;0;L;;;;;N;;;10A0;;10A0 +2D01;GEORGIAN SMALL LETTER BAN;Ll;0;L;;;;;N;;;10A1;;10A1 +2D02;GEORGIAN SMALL LETTER GAN;Ll;0;L;;;;;N;;;10A2;;10A2 +2D03;GEORGIAN SMALL LETTER DON;Ll;0;L;;;;;N;;;10A3;;10A3 +2D04;GEORGIAN SMALL LETTER EN;Ll;0;L;;;;;N;;;10A4;;10A4 +2D05;GEORGIAN SMALL LETTER VIN;Ll;0;L;;;;;N;;;10A5;;10A5 +2D06;GEORGIAN SMALL LETTER ZEN;Ll;0;L;;;;;N;;;10A6;;10A6 +2D07;GEORGIAN SMALL LETTER TAN;Ll;0;L;;;;;N;;;10A7;;10A7 +2D08;GEORGIAN SMALL LETTER IN;Ll;0;L;;;;;N;;;10A8;;10A8 +2D09;GEORGIAN SMALL LETTER KAN;Ll;0;L;;;;;N;;;10A9;;10A9 +2D0A;GEORGIAN SMALL LETTER LAS;Ll;0;L;;;;;N;;;10AA;;10AA +2D0B;GEORGIAN SMALL LETTER MAN;Ll;0;L;;;;;N;;;10AB;;10AB +2D0C;GEORGIAN SMALL LETTER NAR;Ll;0;L;;;;;N;;;10AC;;10AC +2D0D;GEORGIAN SMALL LETTER ON;Ll;0;L;;;;;N;;;10AD;;10AD +2D0E;GEORGIAN SMALL LETTER PAR;Ll;0;L;;;;;N;;;10AE;;10AE +2D0F;GEORGIAN SMALL LETTER ZHAR;Ll;0;L;;;;;N;;;10AF;;10AF +2D10;GEORGIAN SMALL LETTER RAE;Ll;0;L;;;;;N;;;10B0;;10B0 +2D11;GEORGIAN SMALL LETTER SAN;Ll;0;L;;;;;N;;;10B1;;10B1 +2D12;GEORGIAN SMALL LETTER TAR;Ll;0;L;;;;;N;;;10B2;;10B2 +2D13;GEORGIAN SMALL LETTER UN;Ll;0;L;;;;;N;;;10B3;;10B3 +2D14;GEORGIAN SMALL LETTER PHAR;Ll;0;L;;;;;N;;;10B4;;10B4 +2D15;GEORGIAN SMALL LETTER KHAR;Ll;0;L;;;;;N;;;10B5;;10B5 +2D16;GEORGIAN SMALL LETTER GHAN;Ll;0;L;;;;;N;;;10B6;;10B6 +2D17;GEORGIAN SMALL LETTER QAR;Ll;0;L;;;;;N;;;10B7;;10B7 +2D18;GEORGIAN SMALL LETTER SHIN;Ll;0;L;;;;;N;;;10B8;;10B8 +2D19;GEORGIAN SMALL LETTER CHIN;Ll;0;L;;;;;N;;;10B9;;10B9 +2D1A;GEORGIAN SMALL LETTER CAN;Ll;0;L;;;;;N;;;10BA;;10BA +2D1B;GEORGIAN SMALL LETTER JIL;Ll;0;L;;;;;N;;;10BB;;10BB +2D1C;GEORGIAN SMALL LETTER CIL;Ll;0;L;;;;;N;;;10BC;;10BC +2D1D;GEORGIAN SMALL LETTER CHAR;Ll;0;L;;;;;N;;;10BD;;10BD +2D1E;GEORGIAN SMALL LETTER XAN;Ll;0;L;;;;;N;;;10BE;;10BE +2D1F;GEORGIAN SMALL LETTER JHAN;Ll;0;L;;;;;N;;;10BF;;10BF +2D20;GEORGIAN SMALL LETTER HAE;Ll;0;L;;;;;N;;;10C0;;10C0 +2D21;GEORGIAN SMALL LETTER HE;Ll;0;L;;;;;N;;;10C1;;10C1 +2D22;GEORGIAN SMALL LETTER HIE;Ll;0;L;;;;;N;;;10C2;;10C2 +2D23;GEORGIAN SMALL LETTER WE;Ll;0;L;;;;;N;;;10C3;;10C3 +2D24;GEORGIAN SMALL LETTER HAR;Ll;0;L;;;;;N;;;10C4;;10C4 +2D25;GEORGIAN SMALL LETTER HOE;Ll;0;L;;;;;N;;;10C5;;10C5 +2D27;GEORGIAN SMALL LETTER YN;Ll;0;L;;;;;N;;;10C7;;10C7 +2D2D;GEORGIAN SMALL LETTER AEN;Ll;0;L;;;;;N;;;10CD;;10CD +2D30;TIFINAGH LETTER YA;Lo;0;L;;;;;N;;;;; +2D31;TIFINAGH LETTER YAB;Lo;0;L;;;;;N;;;;; +2D32;TIFINAGH LETTER YABH;Lo;0;L;;;;;N;;;;; +2D33;TIFINAGH LETTER YAG;Lo;0;L;;;;;N;;;;; +2D34;TIFINAGH LETTER YAGHH;Lo;0;L;;;;;N;;;;; +2D35;TIFINAGH LETTER BERBER ACADEMY YAJ;Lo;0;L;;;;;N;;;;; +2D36;TIFINAGH LETTER YAJ;Lo;0;L;;;;;N;;;;; +2D37;TIFINAGH LETTER YAD;Lo;0;L;;;;;N;;;;; +2D38;TIFINAGH LETTER YADH;Lo;0;L;;;;;N;;;;; +2D39;TIFINAGH LETTER YADD;Lo;0;L;;;;;N;;;;; +2D3A;TIFINAGH LETTER YADDH;Lo;0;L;;;;;N;;;;; +2D3B;TIFINAGH LETTER YEY;Lo;0;L;;;;;N;;;;; +2D3C;TIFINAGH LETTER YAF;Lo;0;L;;;;;N;;;;; +2D3D;TIFINAGH LETTER YAK;Lo;0;L;;;;;N;;;;; +2D3E;TIFINAGH LETTER TUAREG YAK;Lo;0;L;;;;;N;;;;; +2D3F;TIFINAGH LETTER YAKHH;Lo;0;L;;;;;N;;;;; +2D40;TIFINAGH LETTER YAH;Lo;0;L;;;;;N;;;;; +2D41;TIFINAGH LETTER BERBER ACADEMY YAH;Lo;0;L;;;;;N;;;;; +2D42;TIFINAGH LETTER TUAREG YAH;Lo;0;L;;;;;N;;;;; +2D43;TIFINAGH LETTER YAHH;Lo;0;L;;;;;N;;;;; +2D44;TIFINAGH LETTER YAA;Lo;0;L;;;;;N;;;;; +2D45;TIFINAGH LETTER YAKH;Lo;0;L;;;;;N;;;;; +2D46;TIFINAGH LETTER TUAREG YAKH;Lo;0;L;;;;;N;;;;; +2D47;TIFINAGH LETTER YAQ;Lo;0;L;;;;;N;;;;; +2D48;TIFINAGH LETTER TUAREG YAQ;Lo;0;L;;;;;N;;;;; +2D49;TIFINAGH LETTER YI;Lo;0;L;;;;;N;;;;; +2D4A;TIFINAGH LETTER YAZH;Lo;0;L;;;;;N;;;;; +2D4B;TIFINAGH LETTER AHAGGAR YAZH;Lo;0;L;;;;;N;;;;; +2D4C;TIFINAGH LETTER TUAREG YAZH;Lo;0;L;;;;;N;;;;; +2D4D;TIFINAGH LETTER YAL;Lo;0;L;;;;;N;;;;; +2D4E;TIFINAGH LETTER YAM;Lo;0;L;;;;;N;;;;; +2D4F;TIFINAGH LETTER YAN;Lo;0;L;;;;;N;;;;; +2D50;TIFINAGH LETTER TUAREG YAGN;Lo;0;L;;;;;N;;;;; +2D51;TIFINAGH LETTER TUAREG YANG;Lo;0;L;;;;;N;;;;; +2D52;TIFINAGH LETTER YAP;Lo;0;L;;;;;N;;;;; +2D53;TIFINAGH LETTER YU;Lo;0;L;;;;;N;;;;; +2D54;TIFINAGH LETTER YAR;Lo;0;L;;;;;N;;;;; +2D55;TIFINAGH LETTER YARR;Lo;0;L;;;;;N;;;;; +2D56;TIFINAGH LETTER YAGH;Lo;0;L;;;;;N;;;;; +2D57;TIFINAGH LETTER TUAREG YAGH;Lo;0;L;;;;;N;;;;; +2D58;TIFINAGH LETTER AYER YAGH;Lo;0;L;;;;;N;;;;; +2D59;TIFINAGH LETTER YAS;Lo;0;L;;;;;N;;;;; +2D5A;TIFINAGH LETTER YASS;Lo;0;L;;;;;N;;;;; +2D5B;TIFINAGH LETTER YASH;Lo;0;L;;;;;N;;;;; +2D5C;TIFINAGH LETTER YAT;Lo;0;L;;;;;N;;;;; +2D5D;TIFINAGH LETTER YATH;Lo;0;L;;;;;N;;;;; +2D5E;TIFINAGH LETTER YACH;Lo;0;L;;;;;N;;;;; +2D5F;TIFINAGH LETTER YATT;Lo;0;L;;;;;N;;;;; +2D60;TIFINAGH LETTER YAV;Lo;0;L;;;;;N;;;;; +2D61;TIFINAGH LETTER YAW;Lo;0;L;;;;;N;;;;; +2D62;TIFINAGH LETTER YAY;Lo;0;L;;;;;N;;;;; +2D63;TIFINAGH LETTER YAZ;Lo;0;L;;;;;N;;;;; +2D64;TIFINAGH LETTER TAWELLEMET YAZ;Lo;0;L;;;;;N;;;;; +2D65;TIFINAGH LETTER YAZZ;Lo;0;L;;;;;N;;;;; +2D66;TIFINAGH LETTER YE;Lo;0;L;;;;;N;;;;; +2D67;TIFINAGH LETTER YO;Lo;0;L;;;;;N;;;;; +2D6F;TIFINAGH MODIFIER LETTER LABIALIZATION MARK;Lm;0;L; 2D61;;;;N;;;;; +2D70;TIFINAGH SEPARATOR MARK;Po;0;L;;;;;N;;;;; +2D7F;TIFINAGH CONSONANT JOINER;Mn;9;NSM;;;;;N;;;;; +2D80;ETHIOPIC SYLLABLE LOA;Lo;0;L;;;;;N;;;;; +2D81;ETHIOPIC SYLLABLE MOA;Lo;0;L;;;;;N;;;;; +2D82;ETHIOPIC SYLLABLE ROA;Lo;0;L;;;;;N;;;;; +2D83;ETHIOPIC SYLLABLE SOA;Lo;0;L;;;;;N;;;;; +2D84;ETHIOPIC SYLLABLE SHOA;Lo;0;L;;;;;N;;;;; +2D85;ETHIOPIC SYLLABLE BOA;Lo;0;L;;;;;N;;;;; +2D86;ETHIOPIC SYLLABLE TOA;Lo;0;L;;;;;N;;;;; +2D87;ETHIOPIC SYLLABLE COA;Lo;0;L;;;;;N;;;;; +2D88;ETHIOPIC SYLLABLE NOA;Lo;0;L;;;;;N;;;;; +2D89;ETHIOPIC SYLLABLE NYOA;Lo;0;L;;;;;N;;;;; +2D8A;ETHIOPIC SYLLABLE GLOTTAL OA;Lo;0;L;;;;;N;;;;; +2D8B;ETHIOPIC SYLLABLE ZOA;Lo;0;L;;;;;N;;;;; +2D8C;ETHIOPIC SYLLABLE DOA;Lo;0;L;;;;;N;;;;; +2D8D;ETHIOPIC SYLLABLE DDOA;Lo;0;L;;;;;N;;;;; +2D8E;ETHIOPIC SYLLABLE JOA;Lo;0;L;;;;;N;;;;; +2D8F;ETHIOPIC SYLLABLE THOA;Lo;0;L;;;;;N;;;;; +2D90;ETHIOPIC SYLLABLE CHOA;Lo;0;L;;;;;N;;;;; +2D91;ETHIOPIC SYLLABLE PHOA;Lo;0;L;;;;;N;;;;; +2D92;ETHIOPIC SYLLABLE POA;Lo;0;L;;;;;N;;;;; +2D93;ETHIOPIC SYLLABLE GGWA;Lo;0;L;;;;;N;;;;; +2D94;ETHIOPIC SYLLABLE GGWI;Lo;0;L;;;;;N;;;;; +2D95;ETHIOPIC SYLLABLE GGWEE;Lo;0;L;;;;;N;;;;; +2D96;ETHIOPIC SYLLABLE GGWE;Lo;0;L;;;;;N;;;;; +2DA0;ETHIOPIC SYLLABLE SSA;Lo;0;L;;;;;N;;;;; +2DA1;ETHIOPIC SYLLABLE SSU;Lo;0;L;;;;;N;;;;; +2DA2;ETHIOPIC SYLLABLE SSI;Lo;0;L;;;;;N;;;;; +2DA3;ETHIOPIC SYLLABLE SSAA;Lo;0;L;;;;;N;;;;; +2DA4;ETHIOPIC SYLLABLE SSEE;Lo;0;L;;;;;N;;;;; +2DA5;ETHIOPIC SYLLABLE SSE;Lo;0;L;;;;;N;;;;; +2DA6;ETHIOPIC SYLLABLE SSO;Lo;0;L;;;;;N;;;;; +2DA8;ETHIOPIC SYLLABLE CCA;Lo;0;L;;;;;N;;;;; +2DA9;ETHIOPIC SYLLABLE CCU;Lo;0;L;;;;;N;;;;; +2DAA;ETHIOPIC SYLLABLE CCI;Lo;0;L;;;;;N;;;;; +2DAB;ETHIOPIC SYLLABLE CCAA;Lo;0;L;;;;;N;;;;; +2DAC;ETHIOPIC SYLLABLE CCEE;Lo;0;L;;;;;N;;;;; +2DAD;ETHIOPIC SYLLABLE CCE;Lo;0;L;;;;;N;;;;; +2DAE;ETHIOPIC SYLLABLE CCO;Lo;0;L;;;;;N;;;;; +2DB0;ETHIOPIC SYLLABLE ZZA;Lo;0;L;;;;;N;;;;; +2DB1;ETHIOPIC SYLLABLE ZZU;Lo;0;L;;;;;N;;;;; +2DB2;ETHIOPIC SYLLABLE ZZI;Lo;0;L;;;;;N;;;;; +2DB3;ETHIOPIC SYLLABLE ZZAA;Lo;0;L;;;;;N;;;;; +2DB4;ETHIOPIC SYLLABLE ZZEE;Lo;0;L;;;;;N;;;;; +2DB5;ETHIOPIC SYLLABLE ZZE;Lo;0;L;;;;;N;;;;; +2DB6;ETHIOPIC SYLLABLE ZZO;Lo;0;L;;;;;N;;;;; +2DB8;ETHIOPIC SYLLABLE CCHA;Lo;0;L;;;;;N;;;;; +2DB9;ETHIOPIC SYLLABLE CCHU;Lo;0;L;;;;;N;;;;; +2DBA;ETHIOPIC SYLLABLE CCHI;Lo;0;L;;;;;N;;;;; +2DBB;ETHIOPIC SYLLABLE CCHAA;Lo;0;L;;;;;N;;;;; +2DBC;ETHIOPIC SYLLABLE CCHEE;Lo;0;L;;;;;N;;;;; +2DBD;ETHIOPIC SYLLABLE CCHE;Lo;0;L;;;;;N;;;;; +2DBE;ETHIOPIC SYLLABLE CCHO;Lo;0;L;;;;;N;;;;; +2DC0;ETHIOPIC SYLLABLE QYA;Lo;0;L;;;;;N;;;;; +2DC1;ETHIOPIC SYLLABLE QYU;Lo;0;L;;;;;N;;;;; +2DC2;ETHIOPIC SYLLABLE QYI;Lo;0;L;;;;;N;;;;; +2DC3;ETHIOPIC SYLLABLE QYAA;Lo;0;L;;;;;N;;;;; +2DC4;ETHIOPIC SYLLABLE QYEE;Lo;0;L;;;;;N;;;;; +2DC5;ETHIOPIC SYLLABLE QYE;Lo;0;L;;;;;N;;;;; +2DC6;ETHIOPIC SYLLABLE QYO;Lo;0;L;;;;;N;;;;; +2DC8;ETHIOPIC SYLLABLE KYA;Lo;0;L;;;;;N;;;;; +2DC9;ETHIOPIC SYLLABLE KYU;Lo;0;L;;;;;N;;;;; +2DCA;ETHIOPIC SYLLABLE KYI;Lo;0;L;;;;;N;;;;; +2DCB;ETHIOPIC SYLLABLE KYAA;Lo;0;L;;;;;N;;;;; +2DCC;ETHIOPIC SYLLABLE KYEE;Lo;0;L;;;;;N;;;;; +2DCD;ETHIOPIC SYLLABLE KYE;Lo;0;L;;;;;N;;;;; +2DCE;ETHIOPIC SYLLABLE KYO;Lo;0;L;;;;;N;;;;; +2DD0;ETHIOPIC SYLLABLE XYA;Lo;0;L;;;;;N;;;;; +2DD1;ETHIOPIC SYLLABLE XYU;Lo;0;L;;;;;N;;;;; +2DD2;ETHIOPIC SYLLABLE XYI;Lo;0;L;;;;;N;;;;; +2DD3;ETHIOPIC SYLLABLE XYAA;Lo;0;L;;;;;N;;;;; +2DD4;ETHIOPIC SYLLABLE XYEE;Lo;0;L;;;;;N;;;;; +2DD5;ETHIOPIC SYLLABLE XYE;Lo;0;L;;;;;N;;;;; +2DD6;ETHIOPIC SYLLABLE XYO;Lo;0;L;;;;;N;;;;; +2DD8;ETHIOPIC SYLLABLE GYA;Lo;0;L;;;;;N;;;;; +2DD9;ETHIOPIC SYLLABLE GYU;Lo;0;L;;;;;N;;;;; +2DDA;ETHIOPIC SYLLABLE GYI;Lo;0;L;;;;;N;;;;; +2DDB;ETHIOPIC SYLLABLE GYAA;Lo;0;L;;;;;N;;;;; +2DDC;ETHIOPIC SYLLABLE GYEE;Lo;0;L;;;;;N;;;;; +2DDD;ETHIOPIC SYLLABLE GYE;Lo;0;L;;;;;N;;;;; +2DDE;ETHIOPIC SYLLABLE GYO;Lo;0;L;;;;;N;;;;; +2DE0;COMBINING CYRILLIC LETTER BE;Mn;230;NSM;;;;;N;;;;; +2DE1;COMBINING CYRILLIC LETTER VE;Mn;230;NSM;;;;;N;;;;; +2DE2;COMBINING CYRILLIC LETTER GHE;Mn;230;NSM;;;;;N;;;;; +2DE3;COMBINING CYRILLIC LETTER DE;Mn;230;NSM;;;;;N;;;;; +2DE4;COMBINING CYRILLIC LETTER ZHE;Mn;230;NSM;;;;;N;;;;; +2DE5;COMBINING CYRILLIC LETTER ZE;Mn;230;NSM;;;;;N;;;;; +2DE6;COMBINING CYRILLIC LETTER KA;Mn;230;NSM;;;;;N;;;;; +2DE7;COMBINING CYRILLIC LETTER EL;Mn;230;NSM;;;;;N;;;;; +2DE8;COMBINING CYRILLIC LETTER EM;Mn;230;NSM;;;;;N;;;;; +2DE9;COMBINING CYRILLIC LETTER EN;Mn;230;NSM;;;;;N;;;;; +2DEA;COMBINING CYRILLIC LETTER O;Mn;230;NSM;;;;;N;;;;; +2DEB;COMBINING CYRILLIC LETTER PE;Mn;230;NSM;;;;;N;;;;; +2DEC;COMBINING CYRILLIC LETTER ER;Mn;230;NSM;;;;;N;;;;; +2DED;COMBINING CYRILLIC LETTER ES;Mn;230;NSM;;;;;N;;;;; +2DEE;COMBINING CYRILLIC LETTER TE;Mn;230;NSM;;;;;N;;;;; +2DEF;COMBINING CYRILLIC LETTER HA;Mn;230;NSM;;;;;N;;;;; +2DF0;COMBINING CYRILLIC LETTER TSE;Mn;230;NSM;;;;;N;;;;; +2DF1;COMBINING CYRILLIC LETTER CHE;Mn;230;NSM;;;;;N;;;;; +2DF2;COMBINING CYRILLIC LETTER SHA;Mn;230;NSM;;;;;N;;;;; +2DF3;COMBINING CYRILLIC LETTER SHCHA;Mn;230;NSM;;;;;N;;;;; +2DF4;COMBINING CYRILLIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +2DF5;COMBINING CYRILLIC LETTER ES-TE;Mn;230;NSM;;;;;N;;;;; +2DF6;COMBINING CYRILLIC LETTER A;Mn;230;NSM;;;;;N;;;;; +2DF7;COMBINING CYRILLIC LETTER IE;Mn;230;NSM;;;;;N;;;;; +2DF8;COMBINING CYRILLIC LETTER DJERV;Mn;230;NSM;;;;;N;;;;; +2DF9;COMBINING CYRILLIC LETTER MONOGRAPH UK;Mn;230;NSM;;;;;N;;;;; +2DFA;COMBINING CYRILLIC LETTER YAT;Mn;230;NSM;;;;;N;;;;; +2DFB;COMBINING CYRILLIC LETTER YU;Mn;230;NSM;;;;;N;;;;; +2DFC;COMBINING CYRILLIC LETTER IOTIFIED A;Mn;230;NSM;;;;;N;;;;; +2DFD;COMBINING CYRILLIC LETTER LITTLE YUS;Mn;230;NSM;;;;;N;;;;; +2DFE;COMBINING CYRILLIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; +2DFF;COMBINING CYRILLIC LETTER IOTIFIED BIG YUS;Mn;230;NSM;;;;;N;;;;; +2E00;RIGHT ANGLE SUBSTITUTION MARKER;Po;0;ON;;;;;N;;;;; +2E01;RIGHT ANGLE DOTTED SUBSTITUTION MARKER;Po;0;ON;;;;;N;;;;; +2E02;LEFT SUBSTITUTION BRACKET;Pi;0;ON;;;;;Y;;;;; +2E03;RIGHT SUBSTITUTION BRACKET;Pf;0;ON;;;;;Y;;;;; +2E04;LEFT DOTTED SUBSTITUTION BRACKET;Pi;0;ON;;;;;Y;;;;; +2E05;RIGHT DOTTED SUBSTITUTION BRACKET;Pf;0;ON;;;;;Y;;;;; +2E06;RAISED INTERPOLATION MARKER;Po;0;ON;;;;;N;;;;; +2E07;RAISED DOTTED INTERPOLATION MARKER;Po;0;ON;;;;;N;;;;; +2E08;DOTTED TRANSPOSITION MARKER;Po;0;ON;;;;;N;;;;; +2E09;LEFT TRANSPOSITION BRACKET;Pi;0;ON;;;;;Y;;;;; +2E0A;RIGHT TRANSPOSITION BRACKET;Pf;0;ON;;;;;Y;;;;; +2E0B;RAISED SQUARE;Po;0;ON;;;;;N;;;;; +2E0C;LEFT RAISED OMISSION BRACKET;Pi;0;ON;;;;;Y;;;;; +2E0D;RIGHT RAISED OMISSION BRACKET;Pf;0;ON;;;;;Y;;;;; +2E0E;EDITORIAL CORONIS;Po;0;ON;;;;;N;;;;; +2E0F;PARAGRAPHOS;Po;0;ON;;;;;N;;;;; +2E10;FORKED PARAGRAPHOS;Po;0;ON;;;;;N;;;;; +2E11;REVERSED FORKED PARAGRAPHOS;Po;0;ON;;;;;N;;;;; +2E12;HYPODIASTOLE;Po;0;ON;;;;;N;;;;; +2E13;DOTTED OBELOS;Po;0;ON;;;;;N;;;;; +2E14;DOWNWARDS ANCORA;Po;0;ON;;;;;N;;;;; +2E15;UPWARDS ANCORA;Po;0;ON;;;;;N;;;;; +2E16;DOTTED RIGHT-POINTING ANGLE;Po;0;ON;;;;;N;;;;; +2E17;DOUBLE OBLIQUE HYPHEN;Pd;0;ON;;;;;N;;;;; +2E18;INVERTED INTERROBANG;Po;0;ON;;;;;N;;;;; +2E19;PALM BRANCH;Po;0;ON;;;;;N;;;;; +2E1A;HYPHEN WITH DIAERESIS;Pd;0;ON;;;;;N;;;;; +2E1B;TILDE WITH RING ABOVE;Po;0;ON;;;;;N;;;;; +2E1C;LEFT LOW PARAPHRASE BRACKET;Pi;0;ON;;;;;Y;;;;; +2E1D;RIGHT LOW PARAPHRASE BRACKET;Pf;0;ON;;;;;Y;;;;; +2E1E;TILDE WITH DOT ABOVE;Po;0;ON;;;;;N;;;;; +2E1F;TILDE WITH DOT BELOW;Po;0;ON;;;;;N;;;;; +2E20;LEFT VERTICAL BAR WITH QUILL;Pi;0;ON;;;;;Y;;;;; +2E21;RIGHT VERTICAL BAR WITH QUILL;Pf;0;ON;;;;;Y;;;;; +2E22;TOP LEFT HALF BRACKET;Ps;0;ON;;;;;Y;;;;; +2E23;TOP RIGHT HALF BRACKET;Pe;0;ON;;;;;Y;;;;; +2E24;BOTTOM LEFT HALF BRACKET;Ps;0;ON;;;;;Y;;;;; +2E25;BOTTOM RIGHT HALF BRACKET;Pe;0;ON;;;;;Y;;;;; +2E26;LEFT SIDEWAYS U BRACKET;Ps;0;ON;;;;;Y;;;;; +2E27;RIGHT SIDEWAYS U BRACKET;Pe;0;ON;;;;;Y;;;;; +2E28;LEFT DOUBLE PARENTHESIS;Ps;0;ON;;;;;Y;;;;; +2E29;RIGHT DOUBLE PARENTHESIS;Pe;0;ON;;;;;Y;;;;; +2E2A;TWO DOTS OVER ONE DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +2E2B;ONE DOT OVER TWO DOTS PUNCTUATION;Po;0;ON;;;;;N;;;;; +2E2C;SQUARED FOUR DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +2E2D;FIVE DOT MARK;Po;0;ON;;;;;N;;;;; +2E2E;REVERSED QUESTION MARK;Po;0;ON;;;;;N;;;;; +2E2F;VERTICAL TILDE;Lm;0;ON;;;;;N;;;;; +2E30;RING POINT;Po;0;ON;;;;;N;;;;; +2E31;WORD SEPARATOR MIDDLE DOT;Po;0;ON;;;;;N;;;;; +2E32;TURNED COMMA;Po;0;ON;;;;;N;;;;; +2E33;RAISED DOT;Po;0;ON;;;;;N;;;;; +2E34;RAISED COMMA;Po;0;ON;;;;;N;;;;; +2E35;TURNED SEMICOLON;Po;0;ON;;;;;N;;;;; +2E36;DAGGER WITH LEFT GUARD;Po;0;ON;;;;;N;;;;; +2E37;DAGGER WITH RIGHT GUARD;Po;0;ON;;;;;N;;;;; +2E38;TURNED DAGGER;Po;0;ON;;;;;N;;;;; +2E39;TOP HALF SECTION SIGN;Po;0;ON;;;;;N;;;;; +2E3A;TWO-EM DASH;Pd;0;ON;;;;;N;;;;; +2E3B;THREE-EM DASH;Pd;0;ON;;;;;N;;;;; +2E80;CJK RADICAL REPEAT;So;0;ON;;;;;N;;;;; +2E81;CJK RADICAL CLIFF;So;0;ON;;;;;N;;;;; +2E82;CJK RADICAL SECOND ONE;So;0;ON;;;;;N;;;;; +2E83;CJK RADICAL SECOND TWO;So;0;ON;;;;;N;;;;; +2E84;CJK RADICAL SECOND THREE;So;0;ON;;;;;N;;;;; +2E85;CJK RADICAL PERSON;So;0;ON;;;;;N;;;;; +2E86;CJK RADICAL BOX;So;0;ON;;;;;N;;;;; +2E87;CJK RADICAL TABLE;So;0;ON;;;;;N;;;;; +2E88;CJK RADICAL KNIFE ONE;So;0;ON;;;;;N;;;;; +2E89;CJK RADICAL KNIFE TWO;So;0;ON;;;;;N;;;;; +2E8A;CJK RADICAL DIVINATION;So;0;ON;;;;;N;;;;; +2E8B;CJK RADICAL SEAL;So;0;ON;;;;;N;;;;; +2E8C;CJK RADICAL SMALL ONE;So;0;ON;;;;;N;;;;; +2E8D;CJK RADICAL SMALL TWO;So;0;ON;;;;;N;;;;; +2E8E;CJK RADICAL LAME ONE;So;0;ON;;;;;N;;;;; +2E8F;CJK RADICAL LAME TWO;So;0;ON;;;;;N;;;;; +2E90;CJK RADICAL LAME THREE;So;0;ON;;;;;N;;;;; +2E91;CJK RADICAL LAME FOUR;So;0;ON;;;;;N;;;;; +2E92;CJK RADICAL SNAKE;So;0;ON;;;;;N;;;;; +2E93;CJK RADICAL THREAD;So;0;ON;;;;;N;;;;; +2E94;CJK RADICAL SNOUT ONE;So;0;ON;;;;;N;;;;; +2E95;CJK RADICAL SNOUT TWO;So;0;ON;;;;;N;;;;; +2E96;CJK RADICAL HEART ONE;So;0;ON;;;;;N;;;;; +2E97;CJK RADICAL HEART TWO;So;0;ON;;;;;N;;;;; +2E98;CJK RADICAL HAND;So;0;ON;;;;;N;;;;; +2E99;CJK RADICAL RAP;So;0;ON;;;;;N;;;;; +2E9B;CJK RADICAL CHOKE;So;0;ON;;;;;N;;;;; +2E9C;CJK RADICAL SUN;So;0;ON;;;;;N;;;;; +2E9D;CJK RADICAL MOON;So;0;ON;;;;;N;;;;; +2E9E;CJK RADICAL DEATH;So;0;ON;;;;;N;;;;; +2E9F;CJK RADICAL MOTHER;So;0;ON; 6BCD;;;;N;;;;; +2EA0;CJK RADICAL CIVILIAN;So;0;ON;;;;;N;;;;; +2EA1;CJK RADICAL WATER ONE;So;0;ON;;;;;N;;;;; +2EA2;CJK RADICAL WATER TWO;So;0;ON;;;;;N;;;;; +2EA3;CJK RADICAL FIRE;So;0;ON;;;;;N;;;;; +2EA4;CJK RADICAL PAW ONE;So;0;ON;;;;;N;;;;; +2EA5;CJK RADICAL PAW TWO;So;0;ON;;;;;N;;;;; +2EA6;CJK RADICAL SIMPLIFIED HALF TREE TRUNK;So;0;ON;;;;;N;;;;; +2EA7;CJK RADICAL COW;So;0;ON;;;;;N;;;;; +2EA8;CJK RADICAL DOG;So;0;ON;;;;;N;;;;; +2EA9;CJK RADICAL JADE;So;0;ON;;;;;N;;;;; +2EAA;CJK RADICAL BOLT OF CLOTH;So;0;ON;;;;;N;;;;; +2EAB;CJK RADICAL EYE;So;0;ON;;;;;N;;;;; +2EAC;CJK RADICAL SPIRIT ONE;So;0;ON;;;;;N;;;;; +2EAD;CJK RADICAL SPIRIT TWO;So;0;ON;;;;;N;;;;; +2EAE;CJK RADICAL BAMBOO;So;0;ON;;;;;N;;;;; +2EAF;CJK RADICAL SILK;So;0;ON;;;;;N;;;;; +2EB0;CJK RADICAL C-SIMPLIFIED SILK;So;0;ON;;;;;N;;;;; +2EB1;CJK RADICAL NET ONE;So;0;ON;;;;;N;;;;; +2EB2;CJK RADICAL NET TWO;So;0;ON;;;;;N;;;;; +2EB3;CJK RADICAL NET THREE;So;0;ON;;;;;N;;;;; +2EB4;CJK RADICAL NET FOUR;So;0;ON;;;;;N;;;;; +2EB5;CJK RADICAL MESH;So;0;ON;;;;;N;;;;; +2EB6;CJK RADICAL SHEEP;So;0;ON;;;;;N;;;;; +2EB7;CJK RADICAL RAM;So;0;ON;;;;;N;;;;; +2EB8;CJK RADICAL EWE;So;0;ON;;;;;N;;;;; +2EB9;CJK RADICAL OLD;So;0;ON;;;;;N;;;;; +2EBA;CJK RADICAL BRUSH ONE;So;0;ON;;;;;N;;;;; +2EBB;CJK RADICAL BRUSH TWO;So;0;ON;;;;;N;;;;; +2EBC;CJK RADICAL MEAT;So;0;ON;;;;;N;;;;; +2EBD;CJK RADICAL MORTAR;So;0;ON;;;;;N;;;;; +2EBE;CJK RADICAL GRASS ONE;So;0;ON;;;;;N;;;;; +2EBF;CJK RADICAL GRASS TWO;So;0;ON;;;;;N;;;;; +2EC0;CJK RADICAL GRASS THREE;So;0;ON;;;;;N;;;;; +2EC1;CJK RADICAL TIGER;So;0;ON;;;;;N;;;;; +2EC2;CJK RADICAL CLOTHES;So;0;ON;;;;;N;;;;; +2EC3;CJK RADICAL WEST ONE;So;0;ON;;;;;N;;;;; +2EC4;CJK RADICAL WEST TWO;So;0;ON;;;;;N;;;;; +2EC5;CJK RADICAL C-SIMPLIFIED SEE;So;0;ON;;;;;N;;;;; +2EC6;CJK RADICAL SIMPLIFIED HORN;So;0;ON;;;;;N;;;;; +2EC7;CJK RADICAL HORN;So;0;ON;;;;;N;;;;; +2EC8;CJK RADICAL C-SIMPLIFIED SPEECH;So;0;ON;;;;;N;;;;; +2EC9;CJK RADICAL C-SIMPLIFIED SHELL;So;0;ON;;;;;N;;;;; +2ECA;CJK RADICAL FOOT;So;0;ON;;;;;N;;;;; +2ECB;CJK RADICAL C-SIMPLIFIED CART;So;0;ON;;;;;N;;;;; +2ECC;CJK RADICAL SIMPLIFIED WALK;So;0;ON;;;;;N;;;;; +2ECD;CJK RADICAL WALK ONE;So;0;ON;;;;;N;;;;; +2ECE;CJK RADICAL WALK TWO;So;0;ON;;;;;N;;;;; +2ECF;CJK RADICAL CITY;So;0;ON;;;;;N;;;;; +2ED0;CJK RADICAL C-SIMPLIFIED GOLD;So;0;ON;;;;;N;;;;; +2ED1;CJK RADICAL LONG ONE;So;0;ON;;;;;N;;;;; +2ED2;CJK RADICAL LONG TWO;So;0;ON;;;;;N;;;;; +2ED3;CJK RADICAL C-SIMPLIFIED LONG;So;0;ON;;;;;N;;;;; +2ED4;CJK RADICAL C-SIMPLIFIED GATE;So;0;ON;;;;;N;;;;; +2ED5;CJK RADICAL MOUND ONE;So;0;ON;;;;;N;;;;; +2ED6;CJK RADICAL MOUND TWO;So;0;ON;;;;;N;;;;; +2ED7;CJK RADICAL RAIN;So;0;ON;;;;;N;;;;; +2ED8;CJK RADICAL BLUE;So;0;ON;;;;;N;;;;; +2ED9;CJK RADICAL C-SIMPLIFIED TANNED LEATHER;So;0;ON;;;;;N;;;;; +2EDA;CJK RADICAL C-SIMPLIFIED LEAF;So;0;ON;;;;;N;;;;; +2EDB;CJK RADICAL C-SIMPLIFIED WIND;So;0;ON;;;;;N;;;;; +2EDC;CJK RADICAL C-SIMPLIFIED FLY;So;0;ON;;;;;N;;;;; +2EDD;CJK RADICAL EAT ONE;So;0;ON;;;;;N;;;;; +2EDE;CJK RADICAL EAT TWO;So;0;ON;;;;;N;;;;; +2EDF;CJK RADICAL EAT THREE;So;0;ON;;;;;N;;;;; +2EE0;CJK RADICAL C-SIMPLIFIED EAT;So;0;ON;;;;;N;;;;; +2EE1;CJK RADICAL HEAD;So;0;ON;;;;;N;;;;; +2EE2;CJK RADICAL C-SIMPLIFIED HORSE;So;0;ON;;;;;N;;;;; +2EE3;CJK RADICAL BONE;So;0;ON;;;;;N;;;;; +2EE4;CJK RADICAL GHOST;So;0;ON;;;;;N;;;;; +2EE5;CJK RADICAL C-SIMPLIFIED FISH;So;0;ON;;;;;N;;;;; +2EE6;CJK RADICAL C-SIMPLIFIED BIRD;So;0;ON;;;;;N;;;;; +2EE7;CJK RADICAL C-SIMPLIFIED SALT;So;0;ON;;;;;N;;;;; +2EE8;CJK RADICAL SIMPLIFIED WHEAT;So;0;ON;;;;;N;;;;; +2EE9;CJK RADICAL SIMPLIFIED YELLOW;So;0;ON;;;;;N;;;;; +2EEA;CJK RADICAL C-SIMPLIFIED FROG;So;0;ON;;;;;N;;;;; +2EEB;CJK RADICAL J-SIMPLIFIED EVEN;So;0;ON;;;;;N;;;;; +2EEC;CJK RADICAL C-SIMPLIFIED EVEN;So;0;ON;;;;;N;;;;; +2EED;CJK RADICAL J-SIMPLIFIED TOOTH;So;0;ON;;;;;N;;;;; +2EEE;CJK RADICAL C-SIMPLIFIED TOOTH;So;0;ON;;;;;N;;;;; +2EEF;CJK RADICAL J-SIMPLIFIED DRAGON;So;0;ON;;;;;N;;;;; +2EF0;CJK RADICAL C-SIMPLIFIED DRAGON;So;0;ON;;;;;N;;;;; +2EF1;CJK RADICAL TURTLE;So;0;ON;;;;;N;;;;; +2EF2;CJK RADICAL J-SIMPLIFIED TURTLE;So;0;ON;;;;;N;;;;; +2EF3;CJK RADICAL C-SIMPLIFIED TURTLE;So;0;ON; 9F9F;;;;N;;;;; +2F00;KANGXI RADICAL ONE;So;0;ON; 4E00;;;;N;;;;; +2F01;KANGXI RADICAL LINE;So;0;ON; 4E28;;;;N;;;;; +2F02;KANGXI RADICAL DOT;So;0;ON; 4E36;;;;N;;;;; +2F03;KANGXI RADICAL SLASH;So;0;ON; 4E3F;;;;N;;;;; +2F04;KANGXI RADICAL SECOND;So;0;ON; 4E59;;;;N;;;;; +2F05;KANGXI RADICAL HOOK;So;0;ON; 4E85;;;;N;;;;; +2F06;KANGXI RADICAL TWO;So;0;ON; 4E8C;;;;N;;;;; +2F07;KANGXI RADICAL LID;So;0;ON; 4EA0;;;;N;;;;; +2F08;KANGXI RADICAL MAN;So;0;ON; 4EBA;;;;N;;;;; +2F09;KANGXI RADICAL LEGS;So;0;ON; 513F;;;;N;;;;; +2F0A;KANGXI RADICAL ENTER;So;0;ON; 5165;;;;N;;;;; +2F0B;KANGXI RADICAL EIGHT;So;0;ON; 516B;;;;N;;;;; +2F0C;KANGXI RADICAL DOWN BOX;So;0;ON; 5182;;;;N;;;;; +2F0D;KANGXI RADICAL COVER;So;0;ON; 5196;;;;N;;;;; +2F0E;KANGXI RADICAL ICE;So;0;ON; 51AB;;;;N;;;;; +2F0F;KANGXI RADICAL TABLE;So;0;ON; 51E0;;;;N;;;;; +2F10;KANGXI RADICAL OPEN BOX;So;0;ON; 51F5;;;;N;;;;; +2F11;KANGXI RADICAL KNIFE;So;0;ON; 5200;;;;N;;;;; +2F12;KANGXI RADICAL POWER;So;0;ON; 529B;;;;N;;;;; +2F13;KANGXI RADICAL WRAP;So;0;ON; 52F9;;;;N;;;;; +2F14;KANGXI RADICAL SPOON;So;0;ON; 5315;;;;N;;;;; +2F15;KANGXI RADICAL RIGHT OPEN BOX;So;0;ON; 531A;;;;N;;;;; +2F16;KANGXI RADICAL HIDING ENCLOSURE;So;0;ON; 5338;;;;N;;;;; +2F17;KANGXI RADICAL TEN;So;0;ON; 5341;;;;N;;;;; +2F18;KANGXI RADICAL DIVINATION;So;0;ON; 535C;;;;N;;;;; +2F19;KANGXI RADICAL SEAL;So;0;ON; 5369;;;;N;;;;; +2F1A;KANGXI RADICAL CLIFF;So;0;ON; 5382;;;;N;;;;; +2F1B;KANGXI RADICAL PRIVATE;So;0;ON; 53B6;;;;N;;;;; +2F1C;KANGXI RADICAL AGAIN;So;0;ON; 53C8;;;;N;;;;; +2F1D;KANGXI RADICAL MOUTH;So;0;ON; 53E3;;;;N;;;;; +2F1E;KANGXI RADICAL ENCLOSURE;So;0;ON; 56D7;;;;N;;;;; +2F1F;KANGXI RADICAL EARTH;So;0;ON; 571F;;;;N;;;;; +2F20;KANGXI RADICAL SCHOLAR;So;0;ON; 58EB;;;;N;;;;; +2F21;KANGXI RADICAL GO;So;0;ON; 5902;;;;N;;;;; +2F22;KANGXI RADICAL GO SLOWLY;So;0;ON; 590A;;;;N;;;;; +2F23;KANGXI RADICAL EVENING;So;0;ON; 5915;;;;N;;;;; +2F24;KANGXI RADICAL BIG;So;0;ON; 5927;;;;N;;;;; +2F25;KANGXI RADICAL WOMAN;So;0;ON; 5973;;;;N;;;;; +2F26;KANGXI RADICAL CHILD;So;0;ON; 5B50;;;;N;;;;; +2F27;KANGXI RADICAL ROOF;So;0;ON; 5B80;;;;N;;;;; +2F28;KANGXI RADICAL INCH;So;0;ON; 5BF8;;;;N;;;;; +2F29;KANGXI RADICAL SMALL;So;0;ON; 5C0F;;;;N;;;;; +2F2A;KANGXI RADICAL LAME;So;0;ON; 5C22;;;;N;;;;; +2F2B;KANGXI RADICAL CORPSE;So;0;ON; 5C38;;;;N;;;;; +2F2C;KANGXI RADICAL SPROUT;So;0;ON; 5C6E;;;;N;;;;; +2F2D;KANGXI RADICAL MOUNTAIN;So;0;ON; 5C71;;;;N;;;;; +2F2E;KANGXI RADICAL RIVER;So;0;ON; 5DDB;;;;N;;;;; +2F2F;KANGXI RADICAL WORK;So;0;ON; 5DE5;;;;N;;;;; +2F30;KANGXI RADICAL ONESELF;So;0;ON; 5DF1;;;;N;;;;; +2F31;KANGXI RADICAL TURBAN;So;0;ON; 5DFE;;;;N;;;;; +2F32;KANGXI RADICAL DRY;So;0;ON; 5E72;;;;N;;;;; +2F33;KANGXI RADICAL SHORT THREAD;So;0;ON; 5E7A;;;;N;;;;; +2F34;KANGXI RADICAL DOTTED CLIFF;So;0;ON; 5E7F;;;;N;;;;; +2F35;KANGXI RADICAL LONG STRIDE;So;0;ON; 5EF4;;;;N;;;;; +2F36;KANGXI RADICAL TWO HANDS;So;0;ON; 5EFE;;;;N;;;;; +2F37;KANGXI RADICAL SHOOT;So;0;ON; 5F0B;;;;N;;;;; +2F38;KANGXI RADICAL BOW;So;0;ON; 5F13;;;;N;;;;; +2F39;KANGXI RADICAL SNOUT;So;0;ON; 5F50;;;;N;;;;; +2F3A;KANGXI RADICAL BRISTLE;So;0;ON; 5F61;;;;N;;;;; +2F3B;KANGXI RADICAL STEP;So;0;ON; 5F73;;;;N;;;;; +2F3C;KANGXI RADICAL HEART;So;0;ON; 5FC3;;;;N;;;;; +2F3D;KANGXI RADICAL HALBERD;So;0;ON; 6208;;;;N;;;;; +2F3E;KANGXI RADICAL DOOR;So;0;ON; 6236;;;;N;;;;; +2F3F;KANGXI RADICAL HAND;So;0;ON; 624B;;;;N;;;;; +2F40;KANGXI RADICAL BRANCH;So;0;ON; 652F;;;;N;;;;; +2F41;KANGXI RADICAL RAP;So;0;ON; 6534;;;;N;;;;; +2F42;KANGXI RADICAL SCRIPT;So;0;ON; 6587;;;;N;;;;; +2F43;KANGXI RADICAL DIPPER;So;0;ON; 6597;;;;N;;;;; +2F44;KANGXI RADICAL AXE;So;0;ON; 65A4;;;;N;;;;; +2F45;KANGXI RADICAL SQUARE;So;0;ON; 65B9;;;;N;;;;; +2F46;KANGXI RADICAL NOT;So;0;ON; 65E0;;;;N;;;;; +2F47;KANGXI RADICAL SUN;So;0;ON; 65E5;;;;N;;;;; +2F48;KANGXI RADICAL SAY;So;0;ON; 66F0;;;;N;;;;; +2F49;KANGXI RADICAL MOON;So;0;ON; 6708;;;;N;;;;; +2F4A;KANGXI RADICAL TREE;So;0;ON; 6728;;;;N;;;;; +2F4B;KANGXI RADICAL LACK;So;0;ON; 6B20;;;;N;;;;; +2F4C;KANGXI RADICAL STOP;So;0;ON; 6B62;;;;N;;;;; +2F4D;KANGXI RADICAL DEATH;So;0;ON; 6B79;;;;N;;;;; +2F4E;KANGXI RADICAL WEAPON;So;0;ON; 6BB3;;;;N;;;;; +2F4F;KANGXI RADICAL DO NOT;So;0;ON; 6BCB;;;;N;;;;; +2F50;KANGXI RADICAL COMPARE;So;0;ON; 6BD4;;;;N;;;;; +2F51;KANGXI RADICAL FUR;So;0;ON; 6BDB;;;;N;;;;; +2F52;KANGXI RADICAL CLAN;So;0;ON; 6C0F;;;;N;;;;; +2F53;KANGXI RADICAL STEAM;So;0;ON; 6C14;;;;N;;;;; +2F54;KANGXI RADICAL WATER;So;0;ON; 6C34;;;;N;;;;; +2F55;KANGXI RADICAL FIRE;So;0;ON; 706B;;;;N;;;;; +2F56;KANGXI RADICAL CLAW;So;0;ON; 722A;;;;N;;;;; +2F57;KANGXI RADICAL FATHER;So;0;ON; 7236;;;;N;;;;; +2F58;KANGXI RADICAL DOUBLE X;So;0;ON; 723B;;;;N;;;;; +2F59;KANGXI RADICAL HALF TREE TRUNK;So;0;ON; 723F;;;;N;;;;; +2F5A;KANGXI RADICAL SLICE;So;0;ON; 7247;;;;N;;;;; +2F5B;KANGXI RADICAL FANG;So;0;ON; 7259;;;;N;;;;; +2F5C;KANGXI RADICAL COW;So;0;ON; 725B;;;;N;;;;; +2F5D;KANGXI RADICAL DOG;So;0;ON; 72AC;;;;N;;;;; +2F5E;KANGXI RADICAL PROFOUND;So;0;ON; 7384;;;;N;;;;; +2F5F;KANGXI RADICAL JADE;So;0;ON; 7389;;;;N;;;;; +2F60;KANGXI RADICAL MELON;So;0;ON; 74DC;;;;N;;;;; +2F61;KANGXI RADICAL TILE;So;0;ON; 74E6;;;;N;;;;; +2F62;KANGXI RADICAL SWEET;So;0;ON; 7518;;;;N;;;;; +2F63;KANGXI RADICAL LIFE;So;0;ON; 751F;;;;N;;;;; +2F64;KANGXI RADICAL USE;So;0;ON; 7528;;;;N;;;;; +2F65;KANGXI RADICAL FIELD;So;0;ON; 7530;;;;N;;;;; +2F66;KANGXI RADICAL BOLT OF CLOTH;So;0;ON; 758B;;;;N;;;;; +2F67;KANGXI RADICAL SICKNESS;So;0;ON; 7592;;;;N;;;;; +2F68;KANGXI RADICAL DOTTED TENT;So;0;ON; 7676;;;;N;;;;; +2F69;KANGXI RADICAL WHITE;So;0;ON; 767D;;;;N;;;;; +2F6A;KANGXI RADICAL SKIN;So;0;ON; 76AE;;;;N;;;;; +2F6B;KANGXI RADICAL DISH;So;0;ON; 76BF;;;;N;;;;; +2F6C;KANGXI RADICAL EYE;So;0;ON; 76EE;;;;N;;;;; +2F6D;KANGXI RADICAL SPEAR;So;0;ON; 77DB;;;;N;;;;; +2F6E;KANGXI RADICAL ARROW;So;0;ON; 77E2;;;;N;;;;; +2F6F;KANGXI RADICAL STONE;So;0;ON; 77F3;;;;N;;;;; +2F70;KANGXI RADICAL SPIRIT;So;0;ON; 793A;;;;N;;;;; +2F71;KANGXI RADICAL TRACK;So;0;ON; 79B8;;;;N;;;;; +2F72;KANGXI RADICAL GRAIN;So;0;ON; 79BE;;;;N;;;;; +2F73;KANGXI RADICAL CAVE;So;0;ON; 7A74;;;;N;;;;; +2F74;KANGXI RADICAL STAND;So;0;ON; 7ACB;;;;N;;;;; +2F75;KANGXI RADICAL BAMBOO;So;0;ON; 7AF9;;;;N;;;;; +2F76;KANGXI RADICAL RICE;So;0;ON; 7C73;;;;N;;;;; +2F77;KANGXI RADICAL SILK;So;0;ON; 7CF8;;;;N;;;;; +2F78;KANGXI RADICAL JAR;So;0;ON; 7F36;;;;N;;;;; +2F79;KANGXI RADICAL NET;So;0;ON; 7F51;;;;N;;;;; +2F7A;KANGXI RADICAL SHEEP;So;0;ON; 7F8A;;;;N;;;;; +2F7B;KANGXI RADICAL FEATHER;So;0;ON; 7FBD;;;;N;;;;; +2F7C;KANGXI RADICAL OLD;So;0;ON; 8001;;;;N;;;;; +2F7D;KANGXI RADICAL AND;So;0;ON; 800C;;;;N;;;;; +2F7E;KANGXI RADICAL PLOW;So;0;ON; 8012;;;;N;;;;; +2F7F;KANGXI RADICAL EAR;So;0;ON; 8033;;;;N;;;;; +2F80;KANGXI RADICAL BRUSH;So;0;ON; 807F;;;;N;;;;; +2F81;KANGXI RADICAL MEAT;So;0;ON; 8089;;;;N;;;;; +2F82;KANGXI RADICAL MINISTER;So;0;ON; 81E3;;;;N;;;;; +2F83;KANGXI RADICAL SELF;So;0;ON; 81EA;;;;N;;;;; +2F84;KANGXI RADICAL ARRIVE;So;0;ON; 81F3;;;;N;;;;; +2F85;KANGXI RADICAL MORTAR;So;0;ON; 81FC;;;;N;;;;; +2F86;KANGXI RADICAL TONGUE;So;0;ON; 820C;;;;N;;;;; +2F87;KANGXI RADICAL OPPOSE;So;0;ON; 821B;;;;N;;;;; +2F88;KANGXI RADICAL BOAT;So;0;ON; 821F;;;;N;;;;; +2F89;KANGXI RADICAL STOPPING;So;0;ON; 826E;;;;N;;;;; +2F8A;KANGXI RADICAL COLOR;So;0;ON; 8272;;;;N;;;;; +2F8B;KANGXI RADICAL GRASS;So;0;ON; 8278;;;;N;;;;; +2F8C;KANGXI RADICAL TIGER;So;0;ON; 864D;;;;N;;;;; +2F8D;KANGXI RADICAL INSECT;So;0;ON; 866B;;;;N;;;;; +2F8E;KANGXI RADICAL BLOOD;So;0;ON; 8840;;;;N;;;;; +2F8F;KANGXI RADICAL WALK ENCLOSURE;So;0;ON; 884C;;;;N;;;;; +2F90;KANGXI RADICAL CLOTHES;So;0;ON; 8863;;;;N;;;;; +2F91;KANGXI RADICAL WEST;So;0;ON; 897E;;;;N;;;;; +2F92;KANGXI RADICAL SEE;So;0;ON; 898B;;;;N;;;;; +2F93;KANGXI RADICAL HORN;So;0;ON; 89D2;;;;N;;;;; +2F94;KANGXI RADICAL SPEECH;So;0;ON; 8A00;;;;N;;;;; +2F95;KANGXI RADICAL VALLEY;So;0;ON; 8C37;;;;N;;;;; +2F96;KANGXI RADICAL BEAN;So;0;ON; 8C46;;;;N;;;;; +2F97;KANGXI RADICAL PIG;So;0;ON; 8C55;;;;N;;;;; +2F98;KANGXI RADICAL BADGER;So;0;ON; 8C78;;;;N;;;;; +2F99;KANGXI RADICAL SHELL;So;0;ON; 8C9D;;;;N;;;;; +2F9A;KANGXI RADICAL RED;So;0;ON; 8D64;;;;N;;;;; +2F9B;KANGXI RADICAL RUN;So;0;ON; 8D70;;;;N;;;;; +2F9C;KANGXI RADICAL FOOT;So;0;ON; 8DB3;;;;N;;;;; +2F9D;KANGXI RADICAL BODY;So;0;ON; 8EAB;;;;N;;;;; +2F9E;KANGXI RADICAL CART;So;0;ON; 8ECA;;;;N;;;;; +2F9F;KANGXI RADICAL BITTER;So;0;ON; 8F9B;;;;N;;;;; +2FA0;KANGXI RADICAL MORNING;So;0;ON; 8FB0;;;;N;;;;; +2FA1;KANGXI RADICAL WALK;So;0;ON; 8FB5;;;;N;;;;; +2FA2;KANGXI RADICAL CITY;So;0;ON; 9091;;;;N;;;;; +2FA3;KANGXI RADICAL WINE;So;0;ON; 9149;;;;N;;;;; +2FA4;KANGXI RADICAL DISTINGUISH;So;0;ON; 91C6;;;;N;;;;; +2FA5;KANGXI RADICAL VILLAGE;So;0;ON; 91CC;;;;N;;;;; +2FA6;KANGXI RADICAL GOLD;So;0;ON; 91D1;;;;N;;;;; +2FA7;KANGXI RADICAL LONG;So;0;ON; 9577;;;;N;;;;; +2FA8;KANGXI RADICAL GATE;So;0;ON; 9580;;;;N;;;;; +2FA9;KANGXI RADICAL MOUND;So;0;ON; 961C;;;;N;;;;; +2FAA;KANGXI RADICAL SLAVE;So;0;ON; 96B6;;;;N;;;;; +2FAB;KANGXI RADICAL SHORT TAILED BIRD;So;0;ON; 96B9;;;;N;;;;; +2FAC;KANGXI RADICAL RAIN;So;0;ON; 96E8;;;;N;;;;; +2FAD;KANGXI RADICAL BLUE;So;0;ON; 9751;;;;N;;;;; +2FAE;KANGXI RADICAL WRONG;So;0;ON; 975E;;;;N;;;;; +2FAF;KANGXI RADICAL FACE;So;0;ON; 9762;;;;N;;;;; +2FB0;KANGXI RADICAL LEATHER;So;0;ON; 9769;;;;N;;;;; +2FB1;KANGXI RADICAL TANNED LEATHER;So;0;ON; 97CB;;;;N;;;;; +2FB2;KANGXI RADICAL LEEK;So;0;ON; 97ED;;;;N;;;;; +2FB3;KANGXI RADICAL SOUND;So;0;ON; 97F3;;;;N;;;;; +2FB4;KANGXI RADICAL LEAF;So;0;ON; 9801;;;;N;;;;; +2FB5;KANGXI RADICAL WIND;So;0;ON; 98A8;;;;N;;;;; +2FB6;KANGXI RADICAL FLY;So;0;ON; 98DB;;;;N;;;;; +2FB7;KANGXI RADICAL EAT;So;0;ON; 98DF;;;;N;;;;; +2FB8;KANGXI RADICAL HEAD;So;0;ON; 9996;;;;N;;;;; +2FB9;KANGXI RADICAL FRAGRANT;So;0;ON; 9999;;;;N;;;;; +2FBA;KANGXI RADICAL HORSE;So;0;ON; 99AC;;;;N;;;;; +2FBB;KANGXI RADICAL BONE;So;0;ON; 9AA8;;;;N;;;;; +2FBC;KANGXI RADICAL TALL;So;0;ON; 9AD8;;;;N;;;;; +2FBD;KANGXI RADICAL HAIR;So;0;ON; 9ADF;;;;N;;;;; +2FBE;KANGXI RADICAL FIGHT;So;0;ON; 9B25;;;;N;;;;; +2FBF;KANGXI RADICAL SACRIFICIAL WINE;So;0;ON; 9B2F;;;;N;;;;; +2FC0;KANGXI RADICAL CAULDRON;So;0;ON; 9B32;;;;N;;;;; +2FC1;KANGXI RADICAL GHOST;So;0;ON; 9B3C;;;;N;;;;; +2FC2;KANGXI RADICAL FISH;So;0;ON; 9B5A;;;;N;;;;; +2FC3;KANGXI RADICAL BIRD;So;0;ON; 9CE5;;;;N;;;;; +2FC4;KANGXI RADICAL SALT;So;0;ON; 9E75;;;;N;;;;; +2FC5;KANGXI RADICAL DEER;So;0;ON; 9E7F;;;;N;;;;; +2FC6;KANGXI RADICAL WHEAT;So;0;ON; 9EA5;;;;N;;;;; +2FC7;KANGXI RADICAL HEMP;So;0;ON; 9EBB;;;;N;;;;; +2FC8;KANGXI RADICAL YELLOW;So;0;ON; 9EC3;;;;N;;;;; +2FC9;KANGXI RADICAL MILLET;So;0;ON; 9ECD;;;;N;;;;; +2FCA;KANGXI RADICAL BLACK;So;0;ON; 9ED1;;;;N;;;;; +2FCB;KANGXI RADICAL EMBROIDERY;So;0;ON; 9EF9;;;;N;;;;; +2FCC;KANGXI RADICAL FROG;So;0;ON; 9EFD;;;;N;;;;; +2FCD;KANGXI RADICAL TRIPOD;So;0;ON; 9F0E;;;;N;;;;; +2FCE;KANGXI RADICAL DRUM;So;0;ON; 9F13;;;;N;;;;; +2FCF;KANGXI RADICAL RAT;So;0;ON; 9F20;;;;N;;;;; +2FD0;KANGXI RADICAL NOSE;So;0;ON; 9F3B;;;;N;;;;; +2FD1;KANGXI RADICAL EVEN;So;0;ON; 9F4A;;;;N;;;;; +2FD2;KANGXI RADICAL TOOTH;So;0;ON; 9F52;;;;N;;;;; +2FD3;KANGXI RADICAL DRAGON;So;0;ON; 9F8D;;;;N;;;;; +2FD4;KANGXI RADICAL TURTLE;So;0;ON; 9F9C;;;;N;;;;; +2FD5;KANGXI RADICAL FLUTE;So;0;ON; 9FA0;;;;N;;;;; +2FF0;IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT;So;0;ON;;;;;N;;;;; +2FF1;IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO BELOW;So;0;ON;;;;;N;;;;; +2FF2;IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO MIDDLE AND RIGHT;So;0;ON;;;;;N;;;;; +2FF3;IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO MIDDLE AND BELOW;So;0;ON;;;;;N;;;;; +2FF4;IDEOGRAPHIC DESCRIPTION CHARACTER FULL SURROUND;So;0;ON;;;;;N;;;;; +2FF5;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM ABOVE;So;0;ON;;;;;N;;;;; +2FF6;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM BELOW;So;0;ON;;;;;N;;;;; +2FF7;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LEFT;So;0;ON;;;;;N;;;;; +2FF8;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM UPPER LEFT;So;0;ON;;;;;N;;;;; +2FF9;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM UPPER RIGHT;So;0;ON;;;;;N;;;;; +2FFA;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER LEFT;So;0;ON;;;;;N;;;;; +2FFB;IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID;So;0;ON;;;;;N;;;;; +3000;IDEOGRAPHIC SPACE;Zs;0;WS; 0020;;;;N;;;;; +3001;IDEOGRAPHIC COMMA;Po;0;ON;;;;;N;;;;; +3002;IDEOGRAPHIC FULL STOP;Po;0;ON;;;;;N;IDEOGRAPHIC PERIOD;;;; +3003;DITTO MARK;Po;0;ON;;;;;N;;;;; +3004;JAPANESE INDUSTRIAL STANDARD SYMBOL;So;0;ON;;;;;N;;;;; +3005;IDEOGRAPHIC ITERATION MARK;Lm;0;L;;;;;N;;;;; +3006;IDEOGRAPHIC CLOSING MARK;Lo;0;L;;;;;N;;;;; +3007;IDEOGRAPHIC NUMBER ZERO;Nl;0;L;;;;0;N;;;;; +3008;LEFT ANGLE BRACKET;Ps;0;ON;;;;;Y;OPENING ANGLE BRACKET;;;; +3009;RIGHT ANGLE BRACKET;Pe;0;ON;;;;;Y;CLOSING ANGLE BRACKET;;;; +300A;LEFT DOUBLE ANGLE BRACKET;Ps;0;ON;;;;;Y;OPENING DOUBLE ANGLE BRACKET;;;; +300B;RIGHT DOUBLE ANGLE BRACKET;Pe;0;ON;;;;;Y;CLOSING DOUBLE ANGLE BRACKET;;;; +300C;LEFT CORNER BRACKET;Ps;0;ON;;;;;Y;OPENING CORNER BRACKET;;;; +300D;RIGHT CORNER BRACKET;Pe;0;ON;;;;;Y;CLOSING CORNER BRACKET;;;; +300E;LEFT WHITE CORNER BRACKET;Ps;0;ON;;;;;Y;OPENING WHITE CORNER BRACKET;;;; +300F;RIGHT WHITE CORNER BRACKET;Pe;0;ON;;;;;Y;CLOSING WHITE CORNER BRACKET;;;; +3010;LEFT BLACK LENTICULAR BRACKET;Ps;0;ON;;;;;Y;OPENING BLACK LENTICULAR BRACKET;;;; +3011;RIGHT BLACK LENTICULAR BRACKET;Pe;0;ON;;;;;Y;CLOSING BLACK LENTICULAR BRACKET;;;; +3012;POSTAL MARK;So;0;ON;;;;;N;;;;; +3013;GETA MARK;So;0;ON;;;;;N;;;;; +3014;LEFT TORTOISE SHELL BRACKET;Ps;0;ON;;;;;Y;OPENING TORTOISE SHELL BRACKET;;;; +3015;RIGHT TORTOISE SHELL BRACKET;Pe;0;ON;;;;;Y;CLOSING TORTOISE SHELL BRACKET;;;; +3016;LEFT WHITE LENTICULAR BRACKET;Ps;0;ON;;;;;Y;OPENING WHITE LENTICULAR BRACKET;;;; +3017;RIGHT WHITE LENTICULAR BRACKET;Pe;0;ON;;;;;Y;CLOSING WHITE LENTICULAR BRACKET;;;; +3018;LEFT WHITE TORTOISE SHELL BRACKET;Ps;0;ON;;;;;Y;OPENING WHITE TORTOISE SHELL BRACKET;;;; +3019;RIGHT WHITE TORTOISE SHELL BRACKET;Pe;0;ON;;;;;Y;CLOSING WHITE TORTOISE SHELL BRACKET;;;; +301A;LEFT WHITE SQUARE BRACKET;Ps;0;ON;;;;;Y;OPENING WHITE SQUARE BRACKET;;;; +301B;RIGHT WHITE SQUARE BRACKET;Pe;0;ON;;;;;Y;CLOSING WHITE SQUARE BRACKET;;;; +301C;WAVE DASH;Pd;0;ON;;;;;N;;;;; +301D;REVERSED DOUBLE PRIME QUOTATION MARK;Ps;0;ON;;;;;N;;;;; +301E;DOUBLE PRIME QUOTATION MARK;Pe;0;ON;;;;;N;;;;; +301F;LOW DOUBLE PRIME QUOTATION MARK;Pe;0;ON;;;;;N;;;;; +3020;POSTAL MARK FACE;So;0;ON;;;;;N;;;;; +3021;HANGZHOU NUMERAL ONE;Nl;0;L;;;;1;N;;;;; +3022;HANGZHOU NUMERAL TWO;Nl;0;L;;;;2;N;;;;; +3023;HANGZHOU NUMERAL THREE;Nl;0;L;;;;3;N;;;;; +3024;HANGZHOU NUMERAL FOUR;Nl;0;L;;;;4;N;;;;; +3025;HANGZHOU NUMERAL FIVE;Nl;0;L;;;;5;N;;;;; +3026;HANGZHOU NUMERAL SIX;Nl;0;L;;;;6;N;;;;; +3027;HANGZHOU NUMERAL SEVEN;Nl;0;L;;;;7;N;;;;; +3028;HANGZHOU NUMERAL EIGHT;Nl;0;L;;;;8;N;;;;; +3029;HANGZHOU NUMERAL NINE;Nl;0;L;;;;9;N;;;;; +302A;IDEOGRAPHIC LEVEL TONE MARK;Mn;218;NSM;;;;;N;;;;; +302B;IDEOGRAPHIC RISING TONE MARK;Mn;228;NSM;;;;;N;;;;; +302C;IDEOGRAPHIC DEPARTING TONE MARK;Mn;232;NSM;;;;;N;;;;; +302D;IDEOGRAPHIC ENTERING TONE MARK;Mn;222;NSM;;;;;N;;;;; +302E;HANGUL SINGLE DOT TONE MARK;Mc;224;L;;;;;N;;;;; +302F;HANGUL DOUBLE DOT TONE MARK;Mc;224;L;;;;;N;;;;; +3030;WAVY DASH;Pd;0;ON;;;;;N;;;;; +3031;VERTICAL KANA REPEAT MARK;Lm;0;L;;;;;N;;;;; +3032;VERTICAL KANA REPEAT WITH VOICED SOUND MARK;Lm;0;L;;;;;N;;;;; +3033;VERTICAL KANA REPEAT MARK UPPER HALF;Lm;0;L;;;;;N;;;;; +3034;VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HALF;Lm;0;L;;;;;N;;;;; +3035;VERTICAL KANA REPEAT MARK LOWER HALF;Lm;0;L;;;;;N;;;;; +3036;CIRCLED POSTAL MARK;So;0;ON; 3012;;;;N;;;;; +3037;IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL;So;0;ON;;;;;N;;;;; +3038;HANGZHOU NUMERAL TEN;Nl;0;L; 5341;;;10;N;;;;; +3039;HANGZHOU NUMERAL TWENTY;Nl;0;L; 5344;;;20;N;;;;; +303A;HANGZHOU NUMERAL THIRTY;Nl;0;L; 5345;;;30;N;;;;; +303B;VERTICAL IDEOGRAPHIC ITERATION MARK;Lm;0;L;;;;;N;;;;; +303C;MASU MARK;Lo;0;L;;;;;N;;;;; +303D;PART ALTERNATION MARK;Po;0;ON;;;;;N;;;;; +303E;IDEOGRAPHIC VARIATION INDICATOR;So;0;ON;;;;;N;;;;; +303F;IDEOGRAPHIC HALF FILL SPACE;So;0;ON;;;;;N;;;;; +3041;HIRAGANA LETTER SMALL A;Lo;0;L;;;;;N;;;;; +3042;HIRAGANA LETTER A;Lo;0;L;;;;;N;;;;; +3043;HIRAGANA LETTER SMALL I;Lo;0;L;;;;;N;;;;; +3044;HIRAGANA LETTER I;Lo;0;L;;;;;N;;;;; +3045;HIRAGANA LETTER SMALL U;Lo;0;L;;;;;N;;;;; +3046;HIRAGANA LETTER U;Lo;0;L;;;;;N;;;;; +3047;HIRAGANA LETTER SMALL E;Lo;0;L;;;;;N;;;;; +3048;HIRAGANA LETTER E;Lo;0;L;;;;;N;;;;; +3049;HIRAGANA LETTER SMALL O;Lo;0;L;;;;;N;;;;; +304A;HIRAGANA LETTER O;Lo;0;L;;;;;N;;;;; +304B;HIRAGANA LETTER KA;Lo;0;L;;;;;N;;;;; +304C;HIRAGANA LETTER GA;Lo;0;L;304B 3099;;;;N;;;;; +304D;HIRAGANA LETTER KI;Lo;0;L;;;;;N;;;;; +304E;HIRAGANA LETTER GI;Lo;0;L;304D 3099;;;;N;;;;; +304F;HIRAGANA LETTER KU;Lo;0;L;;;;;N;;;;; +3050;HIRAGANA LETTER GU;Lo;0;L;304F 3099;;;;N;;;;; +3051;HIRAGANA LETTER KE;Lo;0;L;;;;;N;;;;; +3052;HIRAGANA LETTER GE;Lo;0;L;3051 3099;;;;N;;;;; +3053;HIRAGANA LETTER KO;Lo;0;L;;;;;N;;;;; +3054;HIRAGANA LETTER GO;Lo;0;L;3053 3099;;;;N;;;;; +3055;HIRAGANA LETTER SA;Lo;0;L;;;;;N;;;;; +3056;HIRAGANA LETTER ZA;Lo;0;L;3055 3099;;;;N;;;;; +3057;HIRAGANA LETTER SI;Lo;0;L;;;;;N;;;;; +3058;HIRAGANA LETTER ZI;Lo;0;L;3057 3099;;;;N;;;;; +3059;HIRAGANA LETTER SU;Lo;0;L;;;;;N;;;;; +305A;HIRAGANA LETTER ZU;Lo;0;L;3059 3099;;;;N;;;;; +305B;HIRAGANA LETTER SE;Lo;0;L;;;;;N;;;;; +305C;HIRAGANA LETTER ZE;Lo;0;L;305B 3099;;;;N;;;;; +305D;HIRAGANA LETTER SO;Lo;0;L;;;;;N;;;;; +305E;HIRAGANA LETTER ZO;Lo;0;L;305D 3099;;;;N;;;;; +305F;HIRAGANA LETTER TA;Lo;0;L;;;;;N;;;;; +3060;HIRAGANA LETTER DA;Lo;0;L;305F 3099;;;;N;;;;; +3061;HIRAGANA LETTER TI;Lo;0;L;;;;;N;;;;; +3062;HIRAGANA LETTER DI;Lo;0;L;3061 3099;;;;N;;;;; +3063;HIRAGANA LETTER SMALL TU;Lo;0;L;;;;;N;;;;; +3064;HIRAGANA LETTER TU;Lo;0;L;;;;;N;;;;; +3065;HIRAGANA LETTER DU;Lo;0;L;3064 3099;;;;N;;;;; +3066;HIRAGANA LETTER TE;Lo;0;L;;;;;N;;;;; +3067;HIRAGANA LETTER DE;Lo;0;L;3066 3099;;;;N;;;;; +3068;HIRAGANA LETTER TO;Lo;0;L;;;;;N;;;;; +3069;HIRAGANA LETTER DO;Lo;0;L;3068 3099;;;;N;;;;; +306A;HIRAGANA LETTER NA;Lo;0;L;;;;;N;;;;; +306B;HIRAGANA LETTER NI;Lo;0;L;;;;;N;;;;; +306C;HIRAGANA LETTER NU;Lo;0;L;;;;;N;;;;; +306D;HIRAGANA LETTER NE;Lo;0;L;;;;;N;;;;; +306E;HIRAGANA LETTER NO;Lo;0;L;;;;;N;;;;; +306F;HIRAGANA LETTER HA;Lo;0;L;;;;;N;;;;; +3070;HIRAGANA LETTER BA;Lo;0;L;306F 3099;;;;N;;;;; +3071;HIRAGANA LETTER PA;Lo;0;L;306F 309A;;;;N;;;;; +3072;HIRAGANA LETTER HI;Lo;0;L;;;;;N;;;;; +3073;HIRAGANA LETTER BI;Lo;0;L;3072 3099;;;;N;;;;; +3074;HIRAGANA LETTER PI;Lo;0;L;3072 309A;;;;N;;;;; +3075;HIRAGANA LETTER HU;Lo;0;L;;;;;N;;;;; +3076;HIRAGANA LETTER BU;Lo;0;L;3075 3099;;;;N;;;;; +3077;HIRAGANA LETTER PU;Lo;0;L;3075 309A;;;;N;;;;; +3078;HIRAGANA LETTER HE;Lo;0;L;;;;;N;;;;; +3079;HIRAGANA LETTER BE;Lo;0;L;3078 3099;;;;N;;;;; +307A;HIRAGANA LETTER PE;Lo;0;L;3078 309A;;;;N;;;;; +307B;HIRAGANA LETTER HO;Lo;0;L;;;;;N;;;;; +307C;HIRAGANA LETTER BO;Lo;0;L;307B 3099;;;;N;;;;; +307D;HIRAGANA LETTER PO;Lo;0;L;307B 309A;;;;N;;;;; +307E;HIRAGANA LETTER MA;Lo;0;L;;;;;N;;;;; +307F;HIRAGANA LETTER MI;Lo;0;L;;;;;N;;;;; +3080;HIRAGANA LETTER MU;Lo;0;L;;;;;N;;;;; +3081;HIRAGANA LETTER ME;Lo;0;L;;;;;N;;;;; +3082;HIRAGANA LETTER MO;Lo;0;L;;;;;N;;;;; +3083;HIRAGANA LETTER SMALL YA;Lo;0;L;;;;;N;;;;; +3084;HIRAGANA LETTER YA;Lo;0;L;;;;;N;;;;; +3085;HIRAGANA LETTER SMALL YU;Lo;0;L;;;;;N;;;;; +3086;HIRAGANA LETTER YU;Lo;0;L;;;;;N;;;;; +3087;HIRAGANA LETTER SMALL YO;Lo;0;L;;;;;N;;;;; +3088;HIRAGANA LETTER YO;Lo;0;L;;;;;N;;;;; +3089;HIRAGANA LETTER RA;Lo;0;L;;;;;N;;;;; +308A;HIRAGANA LETTER RI;Lo;0;L;;;;;N;;;;; +308B;HIRAGANA LETTER RU;Lo;0;L;;;;;N;;;;; +308C;HIRAGANA LETTER RE;Lo;0;L;;;;;N;;;;; +308D;HIRAGANA LETTER RO;Lo;0;L;;;;;N;;;;; +308E;HIRAGANA LETTER SMALL WA;Lo;0;L;;;;;N;;;;; +308F;HIRAGANA LETTER WA;Lo;0;L;;;;;N;;;;; +3090;HIRAGANA LETTER WI;Lo;0;L;;;;;N;;;;; +3091;HIRAGANA LETTER WE;Lo;0;L;;;;;N;;;;; +3092;HIRAGANA LETTER WO;Lo;0;L;;;;;N;;;;; +3093;HIRAGANA LETTER N;Lo;0;L;;;;;N;;;;; +3094;HIRAGANA LETTER VU;Lo;0;L;3046 3099;;;;N;;;;; +3095;HIRAGANA LETTER SMALL KA;Lo;0;L;;;;;N;;;;; +3096;HIRAGANA LETTER SMALL KE;Lo;0;L;;;;;N;;;;; +3099;COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK;Mn;8;NSM;;;;;N;NON-SPACING KATAKANA-HIRAGANA VOICED SOUND MARK;;;; +309A;COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK;Mn;8;NSM;;;;;N;NON-SPACING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK;;;; +309B;KATAKANA-HIRAGANA VOICED SOUND MARK;Sk;0;ON; 0020 3099;;;;N;;;;; +309C;KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK;Sk;0;ON; 0020 309A;;;;N;;;;; +309D;HIRAGANA ITERATION MARK;Lm;0;L;;;;;N;;;;; +309E;HIRAGANA VOICED ITERATION MARK;Lm;0;L;309D 3099;;;;N;;;;; +309F;HIRAGANA DIGRAPH YORI;Lo;0;L; 3088 308A;;;;N;;;;; +30A0;KATAKANA-HIRAGANA DOUBLE HYPHEN;Pd;0;ON;;;;;N;;;;; +30A1;KATAKANA LETTER SMALL A;Lo;0;L;;;;;N;;;;; +30A2;KATAKANA LETTER A;Lo;0;L;;;;;N;;;;; +30A3;KATAKANA LETTER SMALL I;Lo;0;L;;;;;N;;;;; +30A4;KATAKANA LETTER I;Lo;0;L;;;;;N;;;;; +30A5;KATAKANA LETTER SMALL U;Lo;0;L;;;;;N;;;;; +30A6;KATAKANA LETTER U;Lo;0;L;;;;;N;;;;; +30A7;KATAKANA LETTER SMALL E;Lo;0;L;;;;;N;;;;; +30A8;KATAKANA LETTER E;Lo;0;L;;;;;N;;;;; +30A9;KATAKANA LETTER SMALL O;Lo;0;L;;;;;N;;;;; +30AA;KATAKANA LETTER O;Lo;0;L;;;;;N;;;;; +30AB;KATAKANA LETTER KA;Lo;0;L;;;;;N;;;;; +30AC;KATAKANA LETTER GA;Lo;0;L;30AB 3099;;;;N;;;;; +30AD;KATAKANA LETTER KI;Lo;0;L;;;;;N;;;;; +30AE;KATAKANA LETTER GI;Lo;0;L;30AD 3099;;;;N;;;;; +30AF;KATAKANA LETTER KU;Lo;0;L;;;;;N;;;;; +30B0;KATAKANA LETTER GU;Lo;0;L;30AF 3099;;;;N;;;;; +30B1;KATAKANA LETTER KE;Lo;0;L;;;;;N;;;;; +30B2;KATAKANA LETTER GE;Lo;0;L;30B1 3099;;;;N;;;;; +30B3;KATAKANA LETTER KO;Lo;0;L;;;;;N;;;;; +30B4;KATAKANA LETTER GO;Lo;0;L;30B3 3099;;;;N;;;;; +30B5;KATAKANA LETTER SA;Lo;0;L;;;;;N;;;;; +30B6;KATAKANA LETTER ZA;Lo;0;L;30B5 3099;;;;N;;;;; +30B7;KATAKANA LETTER SI;Lo;0;L;;;;;N;;;;; +30B8;KATAKANA LETTER ZI;Lo;0;L;30B7 3099;;;;N;;;;; +30B9;KATAKANA LETTER SU;Lo;0;L;;;;;N;;;;; +30BA;KATAKANA LETTER ZU;Lo;0;L;30B9 3099;;;;N;;;;; +30BB;KATAKANA LETTER SE;Lo;0;L;;;;;N;;;;; +30BC;KATAKANA LETTER ZE;Lo;0;L;30BB 3099;;;;N;;;;; +30BD;KATAKANA LETTER SO;Lo;0;L;;;;;N;;;;; +30BE;KATAKANA LETTER ZO;Lo;0;L;30BD 3099;;;;N;;;;; +30BF;KATAKANA LETTER TA;Lo;0;L;;;;;N;;;;; +30C0;KATAKANA LETTER DA;Lo;0;L;30BF 3099;;;;N;;;;; +30C1;KATAKANA LETTER TI;Lo;0;L;;;;;N;;;;; +30C2;KATAKANA LETTER DI;Lo;0;L;30C1 3099;;;;N;;;;; +30C3;KATAKANA LETTER SMALL TU;Lo;0;L;;;;;N;;;;; +30C4;KATAKANA LETTER TU;Lo;0;L;;;;;N;;;;; +30C5;KATAKANA LETTER DU;Lo;0;L;30C4 3099;;;;N;;;;; +30C6;KATAKANA LETTER TE;Lo;0;L;;;;;N;;;;; +30C7;KATAKANA LETTER DE;Lo;0;L;30C6 3099;;;;N;;;;; +30C8;KATAKANA LETTER TO;Lo;0;L;;;;;N;;;;; +30C9;KATAKANA LETTER DO;Lo;0;L;30C8 3099;;;;N;;;;; +30CA;KATAKANA LETTER NA;Lo;0;L;;;;;N;;;;; +30CB;KATAKANA LETTER NI;Lo;0;L;;;;;N;;;;; +30CC;KATAKANA LETTER NU;Lo;0;L;;;;;N;;;;; +30CD;KATAKANA LETTER NE;Lo;0;L;;;;;N;;;;; +30CE;KATAKANA LETTER NO;Lo;0;L;;;;;N;;;;; +30CF;KATAKANA LETTER HA;Lo;0;L;;;;;N;;;;; +30D0;KATAKANA LETTER BA;Lo;0;L;30CF 3099;;;;N;;;;; +30D1;KATAKANA LETTER PA;Lo;0;L;30CF 309A;;;;N;;;;; +30D2;KATAKANA LETTER HI;Lo;0;L;;;;;N;;;;; +30D3;KATAKANA LETTER BI;Lo;0;L;30D2 3099;;;;N;;;;; +30D4;KATAKANA LETTER PI;Lo;0;L;30D2 309A;;;;N;;;;; +30D5;KATAKANA LETTER HU;Lo;0;L;;;;;N;;;;; +30D6;KATAKANA LETTER BU;Lo;0;L;30D5 3099;;;;N;;;;; +30D7;KATAKANA LETTER PU;Lo;0;L;30D5 309A;;;;N;;;;; +30D8;KATAKANA LETTER HE;Lo;0;L;;;;;N;;;;; +30D9;KATAKANA LETTER BE;Lo;0;L;30D8 3099;;;;N;;;;; +30DA;KATAKANA LETTER PE;Lo;0;L;30D8 309A;;;;N;;;;; +30DB;KATAKANA LETTER HO;Lo;0;L;;;;;N;;;;; +30DC;KATAKANA LETTER BO;Lo;0;L;30DB 3099;;;;N;;;;; +30DD;KATAKANA LETTER PO;Lo;0;L;30DB 309A;;;;N;;;;; +30DE;KATAKANA LETTER MA;Lo;0;L;;;;;N;;;;; +30DF;KATAKANA LETTER MI;Lo;0;L;;;;;N;;;;; +30E0;KATAKANA LETTER MU;Lo;0;L;;;;;N;;;;; +30E1;KATAKANA LETTER ME;Lo;0;L;;;;;N;;;;; +30E2;KATAKANA LETTER MO;Lo;0;L;;;;;N;;;;; +30E3;KATAKANA LETTER SMALL YA;Lo;0;L;;;;;N;;;;; +30E4;KATAKANA LETTER YA;Lo;0;L;;;;;N;;;;; +30E5;KATAKANA LETTER SMALL YU;Lo;0;L;;;;;N;;;;; +30E6;KATAKANA LETTER YU;Lo;0;L;;;;;N;;;;; +30E7;KATAKANA LETTER SMALL YO;Lo;0;L;;;;;N;;;;; +30E8;KATAKANA LETTER YO;Lo;0;L;;;;;N;;;;; +30E9;KATAKANA LETTER RA;Lo;0;L;;;;;N;;;;; +30EA;KATAKANA LETTER RI;Lo;0;L;;;;;N;;;;; +30EB;KATAKANA LETTER RU;Lo;0;L;;;;;N;;;;; +30EC;KATAKANA LETTER RE;Lo;0;L;;;;;N;;;;; +30ED;KATAKANA LETTER RO;Lo;0;L;;;;;N;;;;; +30EE;KATAKANA LETTER SMALL WA;Lo;0;L;;;;;N;;;;; +30EF;KATAKANA LETTER WA;Lo;0;L;;;;;N;;;;; +30F0;KATAKANA LETTER WI;Lo;0;L;;;;;N;;;;; +30F1;KATAKANA LETTER WE;Lo;0;L;;;;;N;;;;; +30F2;KATAKANA LETTER WO;Lo;0;L;;;;;N;;;;; +30F3;KATAKANA LETTER N;Lo;0;L;;;;;N;;;;; +30F4;KATAKANA LETTER VU;Lo;0;L;30A6 3099;;;;N;;;;; +30F5;KATAKANA LETTER SMALL KA;Lo;0;L;;;;;N;;;;; +30F6;KATAKANA LETTER SMALL KE;Lo;0;L;;;;;N;;;;; +30F7;KATAKANA LETTER VA;Lo;0;L;30EF 3099;;;;N;;;;; +30F8;KATAKANA LETTER VI;Lo;0;L;30F0 3099;;;;N;;;;; +30F9;KATAKANA LETTER VE;Lo;0;L;30F1 3099;;;;N;;;;; +30FA;KATAKANA LETTER VO;Lo;0;L;30F2 3099;;;;N;;;;; +30FB;KATAKANA MIDDLE DOT;Po;0;ON;;;;;N;;;;; +30FC;KATAKANA-HIRAGANA PROLONGED SOUND MARK;Lm;0;L;;;;;N;;;;; +30FD;KATAKANA ITERATION MARK;Lm;0;L;;;;;N;;;;; +30FE;KATAKANA VOICED ITERATION MARK;Lm;0;L;30FD 3099;;;;N;;;;; +30FF;KATAKANA DIGRAPH KOTO;Lo;0;L; 30B3 30C8;;;;N;;;;; +3105;BOPOMOFO LETTER B;Lo;0;L;;;;;N;;;;; +3106;BOPOMOFO LETTER P;Lo;0;L;;;;;N;;;;; +3107;BOPOMOFO LETTER M;Lo;0;L;;;;;N;;;;; +3108;BOPOMOFO LETTER F;Lo;0;L;;;;;N;;;;; +3109;BOPOMOFO LETTER D;Lo;0;L;;;;;N;;;;; +310A;BOPOMOFO LETTER T;Lo;0;L;;;;;N;;;;; +310B;BOPOMOFO LETTER N;Lo;0;L;;;;;N;;;;; +310C;BOPOMOFO LETTER L;Lo;0;L;;;;;N;;;;; +310D;BOPOMOFO LETTER G;Lo;0;L;;;;;N;;;;; +310E;BOPOMOFO LETTER K;Lo;0;L;;;;;N;;;;; +310F;BOPOMOFO LETTER H;Lo;0;L;;;;;N;;;;; +3110;BOPOMOFO LETTER J;Lo;0;L;;;;;N;;;;; +3111;BOPOMOFO LETTER Q;Lo;0;L;;;;;N;;;;; +3112;BOPOMOFO LETTER X;Lo;0;L;;;;;N;;;;; +3113;BOPOMOFO LETTER ZH;Lo;0;L;;;;;N;;;;; +3114;BOPOMOFO LETTER CH;Lo;0;L;;;;;N;;;;; +3115;BOPOMOFO LETTER SH;Lo;0;L;;;;;N;;;;; +3116;BOPOMOFO LETTER R;Lo;0;L;;;;;N;;;;; +3117;BOPOMOFO LETTER Z;Lo;0;L;;;;;N;;;;; +3118;BOPOMOFO LETTER C;Lo;0;L;;;;;N;;;;; +3119;BOPOMOFO LETTER S;Lo;0;L;;;;;N;;;;; +311A;BOPOMOFO LETTER A;Lo;0;L;;;;;N;;;;; +311B;BOPOMOFO LETTER O;Lo;0;L;;;;;N;;;;; +311C;BOPOMOFO LETTER E;Lo;0;L;;;;;N;;;;; +311D;BOPOMOFO LETTER EH;Lo;0;L;;;;;N;;;;; +311E;BOPOMOFO LETTER AI;Lo;0;L;;;;;N;;;;; +311F;BOPOMOFO LETTER EI;Lo;0;L;;;;;N;;;;; +3120;BOPOMOFO LETTER AU;Lo;0;L;;;;;N;;;;; +3121;BOPOMOFO LETTER OU;Lo;0;L;;;;;N;;;;; +3122;BOPOMOFO LETTER AN;Lo;0;L;;;;;N;;;;; +3123;BOPOMOFO LETTER EN;Lo;0;L;;;;;N;;;;; +3124;BOPOMOFO LETTER ANG;Lo;0;L;;;;;N;;;;; +3125;BOPOMOFO LETTER ENG;Lo;0;L;;;;;N;;;;; +3126;BOPOMOFO LETTER ER;Lo;0;L;;;;;N;;;;; +3127;BOPOMOFO LETTER I;Lo;0;L;;;;;N;;;;; +3128;BOPOMOFO LETTER U;Lo;0;L;;;;;N;;;;; +3129;BOPOMOFO LETTER IU;Lo;0;L;;;;;N;;;;; +312A;BOPOMOFO LETTER V;Lo;0;L;;;;;N;;;;; +312B;BOPOMOFO LETTER NG;Lo;0;L;;;;;N;;;;; +312C;BOPOMOFO LETTER GN;Lo;0;L;;;;;N;;;;; +312D;BOPOMOFO LETTER IH;Lo;0;L;;;;;N;;;;; +3131;HANGUL LETTER KIYEOK;Lo;0;L; 1100;;;;N;HANGUL LETTER GIYEOG;;;; +3132;HANGUL LETTER SSANGKIYEOK;Lo;0;L; 1101;;;;N;HANGUL LETTER SSANG GIYEOG;;;; +3133;HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; +3134;HANGUL LETTER NIEUN;Lo;0;L; 1102;;;;N;;;;; +3135;HANGUL LETTER NIEUN-CIEUC;Lo;0;L; 11AC;;;;N;HANGUL LETTER NIEUN JIEUJ;;;; +3136;HANGUL LETTER NIEUN-HIEUH;Lo;0;L; 11AD;;;;N;HANGUL LETTER NIEUN HIEUH;;;; +3137;HANGUL LETTER TIKEUT;Lo;0;L; 1103;;;;N;HANGUL LETTER DIGEUD;;;; +3138;HANGUL LETTER SSANGTIKEUT;Lo;0;L; 1104;;;;N;HANGUL LETTER SSANG DIGEUD;;;; +3139;HANGUL LETTER RIEUL;Lo;0;L; 1105;;;;N;HANGUL LETTER LIEUL;;;; +313A;HANGUL LETTER RIEUL-KIYEOK;Lo;0;L; 11B0;;;;N;HANGUL LETTER LIEUL GIYEOG;;;; +313B;HANGUL LETTER RIEUL-MIEUM;Lo;0;L; 11B1;;;;N;HANGUL LETTER LIEUL MIEUM;;;; +313C;HANGUL LETTER RIEUL-PIEUP;Lo;0;L; 11B2;;;;N;HANGUL LETTER LIEUL BIEUB;;;; +313D;HANGUL LETTER RIEUL-SIOS;Lo;0;L; 11B3;;;;N;HANGUL LETTER LIEUL SIOS;;;; +313E;HANGUL LETTER RIEUL-THIEUTH;Lo;0;L; 11B4;;;;N;HANGUL LETTER LIEUL TIEUT;;;; +313F;HANGUL LETTER RIEUL-PHIEUPH;Lo;0;L; 11B5;;;;N;HANGUL LETTER LIEUL PIEUP;;;; +3140;HANGUL LETTER RIEUL-HIEUH;Lo;0;L; 111A;;;;N;HANGUL LETTER LIEUL HIEUH;;;; +3141;HANGUL LETTER MIEUM;Lo;0;L; 1106;;;;N;;;;; +3142;HANGUL LETTER PIEUP;Lo;0;L; 1107;;;;N;HANGUL LETTER BIEUB;;;; +3143;HANGUL LETTER SSANGPIEUP;Lo;0;L; 1108;;;;N;HANGUL LETTER SSANG BIEUB;;;; +3144;HANGUL LETTER PIEUP-SIOS;Lo;0;L; 1121;;;;N;HANGUL LETTER BIEUB SIOS;;;; +3145;HANGUL LETTER SIOS;Lo;0;L; 1109;;;;N;;;;; +3146;HANGUL LETTER SSANGSIOS;Lo;0;L; 110A;;;;N;HANGUL LETTER SSANG SIOS;;;; +3147;HANGUL LETTER IEUNG;Lo;0;L; 110B;;;;N;;;;; +3148;HANGUL LETTER CIEUC;Lo;0;L; 110C;;;;N;HANGUL LETTER JIEUJ;;;; +3149;HANGUL LETTER SSANGCIEUC;Lo;0;L; 110D;;;;N;HANGUL LETTER SSANG JIEUJ;;;; +314A;HANGUL LETTER CHIEUCH;Lo;0;L; 110E;;;;N;HANGUL LETTER CIEUC;;;; +314B;HANGUL LETTER KHIEUKH;Lo;0;L; 110F;;;;N;HANGUL LETTER KIYEOK;;;; +314C;HANGUL LETTER THIEUTH;Lo;0;L; 1110;;;;N;HANGUL LETTER TIEUT;;;; +314D;HANGUL LETTER PHIEUPH;Lo;0;L; 1111;;;;N;HANGUL LETTER PIEUP;;;; +314E;HANGUL LETTER HIEUH;Lo;0;L; 1112;;;;N;;;;; +314F;HANGUL LETTER A;Lo;0;L; 1161;;;;N;;;;; +3150;HANGUL LETTER AE;Lo;0;L; 1162;;;;N;;;;; +3151;HANGUL LETTER YA;Lo;0;L; 1163;;;;N;;;;; +3152;HANGUL LETTER YAE;Lo;0;L; 1164;;;;N;;;;; +3153;HANGUL LETTER EO;Lo;0;L; 1165;;;;N;;;;; +3154;HANGUL LETTER E;Lo;0;L; 1166;;;;N;;;;; +3155;HANGUL LETTER YEO;Lo;0;L; 1167;;;;N;;;;; +3156;HANGUL LETTER YE;Lo;0;L; 1168;;;;N;;;;; +3157;HANGUL LETTER O;Lo;0;L; 1169;;;;N;;;;; +3158;HANGUL LETTER WA;Lo;0;L; 116A;;;;N;;;;; +3159;HANGUL LETTER WAE;Lo;0;L; 116B;;;;N;;;;; +315A;HANGUL LETTER OE;Lo;0;L; 116C;;;;N;;;;; +315B;HANGUL LETTER YO;Lo;0;L; 116D;;;;N;;;;; +315C;HANGUL LETTER U;Lo;0;L; 116E;;;;N;;;;; +315D;HANGUL LETTER WEO;Lo;0;L; 116F;;;;N;;;;; +315E;HANGUL LETTER WE;Lo;0;L; 1170;;;;N;;;;; +315F;HANGUL LETTER WI;Lo;0;L; 1171;;;;N;;;;; +3160;HANGUL LETTER YU;Lo;0;L; 1172;;;;N;;;;; +3161;HANGUL LETTER EU;Lo;0;L; 1173;;;;N;;;;; +3162;HANGUL LETTER YI;Lo;0;L; 1174;;;;N;;;;; +3163;HANGUL LETTER I;Lo;0;L; 1175;;;;N;;;;; +3164;HANGUL FILLER;Lo;0;L; 1160;;;;N;HANGUL CAE OM;;;; +3165;HANGUL LETTER SSANGNIEUN;Lo;0;L; 1114;;;;N;HANGUL LETTER SSANG NIEUN;;;; +3166;HANGUL LETTER NIEUN-TIKEUT;Lo;0;L; 1115;;;;N;HANGUL LETTER NIEUN DIGEUD;;;; +3167;HANGUL LETTER NIEUN-SIOS;Lo;0;L; 11C7;;;;N;HANGUL LETTER NIEUN SIOS;;;; +3168;HANGUL LETTER NIEUN-PANSIOS;Lo;0;L; 11C8;;;;N;HANGUL LETTER NIEUN BAN CHI EUM;;;; +3169;HANGUL LETTER RIEUL-KIYEOK-SIOS;Lo;0;L; 11CC;;;;N;HANGUL LETTER LIEUL GIYEOG SIOS;;;; +316A;HANGUL LETTER RIEUL-TIKEUT;Lo;0;L; 11CE;;;;N;HANGUL LETTER LIEUL DIGEUD;;;; +316B;HANGUL LETTER RIEUL-PIEUP-SIOS;Lo;0;L; 11D3;;;;N;HANGUL LETTER LIEUL BIEUB SIOS;;;; +316C;HANGUL LETTER RIEUL-PANSIOS;Lo;0;L; 11D7;;;;N;HANGUL LETTER LIEUL BAN CHI EUM;;;; +316D;HANGUL LETTER RIEUL-YEORINHIEUH;Lo;0;L; 11D9;;;;N;HANGUL LETTER LIEUL YEOLIN HIEUH;;;; +316E;HANGUL LETTER MIEUM-PIEUP;Lo;0;L; 111C;;;;N;HANGUL LETTER MIEUM BIEUB;;;; +316F;HANGUL LETTER MIEUM-SIOS;Lo;0;L; 11DD;;;;N;HANGUL LETTER MIEUM SIOS;;;; +3170;HANGUL LETTER MIEUM-PANSIOS;Lo;0;L; 11DF;;;;N;HANGUL LETTER BIEUB BAN CHI EUM;;;; +3171;HANGUL LETTER KAPYEOUNMIEUM;Lo;0;L; 111D;;;;N;HANGUL LETTER MIEUM SUN GYEONG EUM;;;; +3172;HANGUL LETTER PIEUP-KIYEOK;Lo;0;L; 111E;;;;N;HANGUL LETTER BIEUB GIYEOG;;;; +3173;HANGUL LETTER PIEUP-TIKEUT;Lo;0;L; 1120;;;;N;HANGUL LETTER BIEUB DIGEUD;;;; +3174;HANGUL LETTER PIEUP-SIOS-KIYEOK;Lo;0;L; 1122;;;;N;HANGUL LETTER BIEUB SIOS GIYEOG;;;; +3175;HANGUL LETTER PIEUP-SIOS-TIKEUT;Lo;0;L; 1123;;;;N;HANGUL LETTER BIEUB SIOS DIGEUD;;;; +3176;HANGUL LETTER PIEUP-CIEUC;Lo;0;L; 1127;;;;N;HANGUL LETTER BIEUB JIEUJ;;;; +3177;HANGUL LETTER PIEUP-THIEUTH;Lo;0;L; 1129;;;;N;HANGUL LETTER BIEUB TIEUT;;;; +3178;HANGUL LETTER KAPYEOUNPIEUP;Lo;0;L; 112B;;;;N;HANGUL LETTER BIEUB SUN GYEONG EUM;;;; +3179;HANGUL LETTER KAPYEOUNSSANGPIEUP;Lo;0;L; 112C;;;;N;HANGUL LETTER SSANG BIEUB SUN GYEONG EUM;;;; +317A;HANGUL LETTER SIOS-KIYEOK;Lo;0;L; 112D;;;;N;HANGUL LETTER SIOS GIYEOG;;;; +317B;HANGUL LETTER SIOS-NIEUN;Lo;0;L; 112E;;;;N;HANGUL LETTER SIOS NIEUN;;;; +317C;HANGUL LETTER SIOS-TIKEUT;Lo;0;L; 112F;;;;N;HANGUL LETTER SIOS DIGEUD;;;; +317D;HANGUL LETTER SIOS-PIEUP;Lo;0;L; 1132;;;;N;HANGUL LETTER SIOS BIEUB;;;; +317E;HANGUL LETTER SIOS-CIEUC;Lo;0;L; 1136;;;;N;HANGUL LETTER SIOS JIEUJ;;;; +317F;HANGUL LETTER PANSIOS;Lo;0;L; 1140;;;;N;HANGUL LETTER BAN CHI EUM;;;; +3180;HANGUL LETTER SSANGIEUNG;Lo;0;L; 1147;;;;N;HANGUL LETTER SSANG IEUNG;;;; +3181;HANGUL LETTER YESIEUNG;Lo;0;L; 114C;;;;N;HANGUL LETTER NGIEUNG;;;; +3182;HANGUL LETTER YESIEUNG-SIOS;Lo;0;L; 11F1;;;;N;HANGUL LETTER NGIEUNG SIOS;;;; +3183;HANGUL LETTER YESIEUNG-PANSIOS;Lo;0;L; 11F2;;;;N;HANGUL LETTER NGIEUNG BAN CHI EUM;;;; +3184;HANGUL LETTER KAPYEOUNPHIEUPH;Lo;0;L; 1157;;;;N;HANGUL LETTER PIEUP SUN GYEONG EUM;;;; +3185;HANGUL LETTER SSANGHIEUH;Lo;0;L; 1158;;;;N;HANGUL LETTER SSANG HIEUH;;;; +3186;HANGUL LETTER YEORINHIEUH;Lo;0;L; 1159;;;;N;HANGUL LETTER YEOLIN HIEUH;;;; +3187;HANGUL LETTER YO-YA;Lo;0;L; 1184;;;;N;HANGUL LETTER YOYA;;;; +3188;HANGUL LETTER YO-YAE;Lo;0;L; 1185;;;;N;HANGUL LETTER YOYAE;;;; +3189;HANGUL LETTER YO-I;Lo;0;L; 1188;;;;N;HANGUL LETTER YOI;;;; +318A;HANGUL LETTER YU-YEO;Lo;0;L; 1191;;;;N;HANGUL LETTER YUYEO;;;; +318B;HANGUL LETTER YU-YE;Lo;0;L; 1192;;;;N;HANGUL LETTER YUYE;;;; +318C;HANGUL LETTER YU-I;Lo;0;L; 1194;;;;N;HANGUL LETTER YUI;;;; +318D;HANGUL LETTER ARAEA;Lo;0;L; 119E;;;;N;HANGUL LETTER ALAE A;;;; +318E;HANGUL LETTER ARAEAE;Lo;0;L; 11A1;;;;N;HANGUL LETTER ALAE AE;;;; +3190;IDEOGRAPHIC ANNOTATION LINKING MARK;So;0;L;;;;;N;KANBUN TATETEN;;;; +3191;IDEOGRAPHIC ANNOTATION REVERSE MARK;So;0;L;;;;;N;KAERITEN RE;;;; +3192;IDEOGRAPHIC ANNOTATION ONE MARK;No;0;L; 4E00;;;1;N;KAERITEN ITI;;;; +3193;IDEOGRAPHIC ANNOTATION TWO MARK;No;0;L; 4E8C;;;2;N;KAERITEN NI;;;; +3194;IDEOGRAPHIC ANNOTATION THREE MARK;No;0;L; 4E09;;;3;N;KAERITEN SAN;;;; +3195;IDEOGRAPHIC ANNOTATION FOUR MARK;No;0;L; 56DB;;;4;N;KAERITEN SI;;;; +3196;IDEOGRAPHIC ANNOTATION TOP MARK;So;0;L; 4E0A;;;;N;KAERITEN ZYOU;;;; +3197;IDEOGRAPHIC ANNOTATION MIDDLE MARK;So;0;L; 4E2D;;;;N;KAERITEN TYUU;;;; +3198;IDEOGRAPHIC ANNOTATION BOTTOM MARK;So;0;L; 4E0B;;;;N;KAERITEN GE;;;; +3199;IDEOGRAPHIC ANNOTATION FIRST MARK;So;0;L; 7532;;;;N;KAERITEN KOU;;;; +319A;IDEOGRAPHIC ANNOTATION SECOND MARK;So;0;L; 4E59;;;;N;KAERITEN OTU;;;; +319B;IDEOGRAPHIC ANNOTATION THIRD MARK;So;0;L; 4E19;;;;N;KAERITEN HEI;;;; +319C;IDEOGRAPHIC ANNOTATION FOURTH MARK;So;0;L; 4E01;;;;N;KAERITEN TEI;;;; +319D;IDEOGRAPHIC ANNOTATION HEAVEN MARK;So;0;L; 5929;;;;N;KAERITEN TEN;;;; +319E;IDEOGRAPHIC ANNOTATION EARTH MARK;So;0;L; 5730;;;;N;KAERITEN TI;;;; +319F;IDEOGRAPHIC ANNOTATION MAN MARK;So;0;L; 4EBA;;;;N;KAERITEN ZIN;;;; +31A0;BOPOMOFO LETTER BU;Lo;0;L;;;;;N;;;;; +31A1;BOPOMOFO LETTER ZI;Lo;0;L;;;;;N;;;;; +31A2;BOPOMOFO LETTER JI;Lo;0;L;;;;;N;;;;; +31A3;BOPOMOFO LETTER GU;Lo;0;L;;;;;N;;;;; +31A4;BOPOMOFO LETTER EE;Lo;0;L;;;;;N;;;;; +31A5;BOPOMOFO LETTER ENN;Lo;0;L;;;;;N;;;;; +31A6;BOPOMOFO LETTER OO;Lo;0;L;;;;;N;;;;; +31A7;BOPOMOFO LETTER ONN;Lo;0;L;;;;;N;;;;; +31A8;BOPOMOFO LETTER IR;Lo;0;L;;;;;N;;;;; +31A9;BOPOMOFO LETTER ANN;Lo;0;L;;;;;N;;;;; +31AA;BOPOMOFO LETTER INN;Lo;0;L;;;;;N;;;;; +31AB;BOPOMOFO LETTER UNN;Lo;0;L;;;;;N;;;;; +31AC;BOPOMOFO LETTER IM;Lo;0;L;;;;;N;;;;; +31AD;BOPOMOFO LETTER NGG;Lo;0;L;;;;;N;;;;; +31AE;BOPOMOFO LETTER AINN;Lo;0;L;;;;;N;;;;; +31AF;BOPOMOFO LETTER AUNN;Lo;0;L;;;;;N;;;;; +31B0;BOPOMOFO LETTER AM;Lo;0;L;;;;;N;;;;; +31B1;BOPOMOFO LETTER OM;Lo;0;L;;;;;N;;;;; +31B2;BOPOMOFO LETTER ONG;Lo;0;L;;;;;N;;;;; +31B3;BOPOMOFO LETTER INNN;Lo;0;L;;;;;N;;;;; +31B4;BOPOMOFO FINAL LETTER P;Lo;0;L;;;;;N;;;;; +31B5;BOPOMOFO FINAL LETTER T;Lo;0;L;;;;;N;;;;; +31B6;BOPOMOFO FINAL LETTER K;Lo;0;L;;;;;N;;;;; +31B7;BOPOMOFO FINAL LETTER H;Lo;0;L;;;;;N;;;;; +31B8;BOPOMOFO LETTER GH;Lo;0;L;;;;;N;;;;; +31B9;BOPOMOFO LETTER LH;Lo;0;L;;;;;N;;;;; +31BA;BOPOMOFO LETTER ZY;Lo;0;L;;;;;N;;;;; +31C0;CJK STROKE T;So;0;ON;;;;;N;;;;; +31C1;CJK STROKE WG;So;0;ON;;;;;N;;;;; +31C2;CJK STROKE XG;So;0;ON;;;;;N;;;;; +31C3;CJK STROKE BXG;So;0;ON;;;;;N;;;;; +31C4;CJK STROKE SW;So;0;ON;;;;;N;;;;; +31C5;CJK STROKE HZZ;So;0;ON;;;;;N;;;;; +31C6;CJK STROKE HZG;So;0;ON;;;;;N;;;;; +31C7;CJK STROKE HP;So;0;ON;;;;;N;;;;; +31C8;CJK STROKE HZWG;So;0;ON;;;;;N;;;;; +31C9;CJK STROKE SZWG;So;0;ON;;;;;N;;;;; +31CA;CJK STROKE HZT;So;0;ON;;;;;N;;;;; +31CB;CJK STROKE HZZP;So;0;ON;;;;;N;;;;; +31CC;CJK STROKE HPWG;So;0;ON;;;;;N;;;;; +31CD;CJK STROKE HZW;So;0;ON;;;;;N;;;;; +31CE;CJK STROKE HZZZ;So;0;ON;;;;;N;;;;; +31CF;CJK STROKE N;So;0;ON;;;;;N;;;;; +31D0;CJK STROKE H;So;0;ON;;;;;N;;;;; +31D1;CJK STROKE S;So;0;ON;;;;;N;;;;; +31D2;CJK STROKE P;So;0;ON;;;;;N;;;;; +31D3;CJK STROKE SP;So;0;ON;;;;;N;;;;; +31D4;CJK STROKE D;So;0;ON;;;;;N;;;;; +31D5;CJK STROKE HZ;So;0;ON;;;;;N;;;;; +31D6;CJK STROKE HG;So;0;ON;;;;;N;;;;; +31D7;CJK STROKE SZ;So;0;ON;;;;;N;;;;; +31D8;CJK STROKE SWZ;So;0;ON;;;;;N;;;;; +31D9;CJK STROKE ST;So;0;ON;;;;;N;;;;; +31DA;CJK STROKE SG;So;0;ON;;;;;N;;;;; +31DB;CJK STROKE PD;So;0;ON;;;;;N;;;;; +31DC;CJK STROKE PZ;So;0;ON;;;;;N;;;;; +31DD;CJK STROKE TN;So;0;ON;;;;;N;;;;; +31DE;CJK STROKE SZZ;So;0;ON;;;;;N;;;;; +31DF;CJK STROKE SWG;So;0;ON;;;;;N;;;;; +31E0;CJK STROKE HXWG;So;0;ON;;;;;N;;;;; +31E1;CJK STROKE HZZZG;So;0;ON;;;;;N;;;;; +31E2;CJK STROKE PG;So;0;ON;;;;;N;;;;; +31E3;CJK STROKE Q;So;0;ON;;;;;N;;;;; +31F0;KATAKANA LETTER SMALL KU;Lo;0;L;;;;;N;;;;; +31F1;KATAKANA LETTER SMALL SI;Lo;0;L;;;;;N;;;;; +31F2;KATAKANA LETTER SMALL SU;Lo;0;L;;;;;N;;;;; +31F3;KATAKANA LETTER SMALL TO;Lo;0;L;;;;;N;;;;; +31F4;KATAKANA LETTER SMALL NU;Lo;0;L;;;;;N;;;;; +31F5;KATAKANA LETTER SMALL HA;Lo;0;L;;;;;N;;;;; +31F6;KATAKANA LETTER SMALL HI;Lo;0;L;;;;;N;;;;; +31F7;KATAKANA LETTER SMALL HU;Lo;0;L;;;;;N;;;;; +31F8;KATAKANA LETTER SMALL HE;Lo;0;L;;;;;N;;;;; +31F9;KATAKANA LETTER SMALL HO;Lo;0;L;;;;;N;;;;; +31FA;KATAKANA LETTER SMALL MU;Lo;0;L;;;;;N;;;;; +31FB;KATAKANA LETTER SMALL RA;Lo;0;L;;;;;N;;;;; +31FC;KATAKANA LETTER SMALL RI;Lo;0;L;;;;;N;;;;; +31FD;KATAKANA LETTER SMALL RU;Lo;0;L;;;;;N;;;;; +31FE;KATAKANA LETTER SMALL RE;Lo;0;L;;;;;N;;;;; +31FF;KATAKANA LETTER SMALL RO;Lo;0;L;;;;;N;;;;; +3200;PARENTHESIZED HANGUL KIYEOK;So;0;L; 0028 1100 0029;;;;N;PARENTHESIZED HANGUL GIYEOG;;;; +3201;PARENTHESIZED HANGUL NIEUN;So;0;L; 0028 1102 0029;;;;N;;;;; +3202;PARENTHESIZED HANGUL TIKEUT;So;0;L; 0028 1103 0029;;;;N;PARENTHESIZED HANGUL DIGEUD;;;; +3203;PARENTHESIZED HANGUL RIEUL;So;0;L; 0028 1105 0029;;;;N;PARENTHESIZED HANGUL LIEUL;;;; +3204;PARENTHESIZED HANGUL MIEUM;So;0;L; 0028 1106 0029;;;;N;;;;; +3205;PARENTHESIZED HANGUL PIEUP;So;0;L; 0028 1107 0029;;;;N;PARENTHESIZED HANGUL BIEUB;;;; +3206;PARENTHESIZED HANGUL SIOS;So;0;L; 0028 1109 0029;;;;N;;;;; +3207;PARENTHESIZED HANGUL IEUNG;So;0;L; 0028 110B 0029;;;;N;;;;; +3208;PARENTHESIZED HANGUL CIEUC;So;0;L; 0028 110C 0029;;;;N;PARENTHESIZED HANGUL JIEUJ;;;; +3209;PARENTHESIZED HANGUL CHIEUCH;So;0;L; 0028 110E 0029;;;;N;PARENTHESIZED HANGUL CIEUC;;;; +320A;PARENTHESIZED HANGUL KHIEUKH;So;0;L; 0028 110F 0029;;;;N;PARENTHESIZED HANGUL KIYEOK;;;; +320B;PARENTHESIZED HANGUL THIEUTH;So;0;L; 0028 1110 0029;;;;N;PARENTHESIZED HANGUL TIEUT;;;; +320C;PARENTHESIZED HANGUL PHIEUPH;So;0;L; 0028 1111 0029;;;;N;PARENTHESIZED HANGUL PIEUP;;;; +320D;PARENTHESIZED HANGUL HIEUH;So;0;L; 0028 1112 0029;;;;N;;;;; +320E;PARENTHESIZED HANGUL KIYEOK A;So;0;L; 0028 1100 1161 0029;;;;N;PARENTHESIZED HANGUL GA;;;; +320F;PARENTHESIZED HANGUL NIEUN A;So;0;L; 0028 1102 1161 0029;;;;N;PARENTHESIZED HANGUL NA;;;; +3210;PARENTHESIZED HANGUL TIKEUT A;So;0;L; 0028 1103 1161 0029;;;;N;PARENTHESIZED HANGUL DA;;;; +3211;PARENTHESIZED HANGUL RIEUL A;So;0;L; 0028 1105 1161 0029;;;;N;PARENTHESIZED HANGUL LA;;;; +3212;PARENTHESIZED HANGUL MIEUM A;So;0;L; 0028 1106 1161 0029;;;;N;PARENTHESIZED HANGUL MA;;;; +3213;PARENTHESIZED HANGUL PIEUP A;So;0;L; 0028 1107 1161 0029;;;;N;PARENTHESIZED HANGUL BA;;;; +3214;PARENTHESIZED HANGUL SIOS A;So;0;L; 0028 1109 1161 0029;;;;N;PARENTHESIZED HANGUL SA;;;; +3215;PARENTHESIZED HANGUL IEUNG A;So;0;L; 0028 110B 1161 0029;;;;N;PARENTHESIZED HANGUL A;;;; +3216;PARENTHESIZED HANGUL CIEUC A;So;0;L; 0028 110C 1161 0029;;;;N;PARENTHESIZED HANGUL JA;;;; +3217;PARENTHESIZED HANGUL CHIEUCH A;So;0;L; 0028 110E 1161 0029;;;;N;PARENTHESIZED HANGUL CA;;;; +3218;PARENTHESIZED HANGUL KHIEUKH A;So;0;L; 0028 110F 1161 0029;;;;N;PARENTHESIZED HANGUL KA;;;; +3219;PARENTHESIZED HANGUL THIEUTH A;So;0;L; 0028 1110 1161 0029;;;;N;PARENTHESIZED HANGUL TA;;;; +321A;PARENTHESIZED HANGUL PHIEUPH A;So;0;L; 0028 1111 1161 0029;;;;N;PARENTHESIZED HANGUL PA;;;; +321B;PARENTHESIZED HANGUL HIEUH A;So;0;L; 0028 1112 1161 0029;;;;N;PARENTHESIZED HANGUL HA;;;; +321C;PARENTHESIZED HANGUL CIEUC U;So;0;L; 0028 110C 116E 0029;;;;N;PARENTHESIZED HANGUL JU;;;; +321D;PARENTHESIZED KOREAN CHARACTER OJEON;So;0;ON; 0028 110B 1169 110C 1165 11AB 0029;;;;N;;;;; +321E;PARENTHESIZED KOREAN CHARACTER O HU;So;0;ON; 0028 110B 1169 1112 116E 0029;;;;N;;;;; +3220;PARENTHESIZED IDEOGRAPH ONE;No;0;L; 0028 4E00 0029;;;1;N;;;;; +3221;PARENTHESIZED IDEOGRAPH TWO;No;0;L; 0028 4E8C 0029;;;2;N;;;;; +3222;PARENTHESIZED IDEOGRAPH THREE;No;0;L; 0028 4E09 0029;;;3;N;;;;; +3223;PARENTHESIZED IDEOGRAPH FOUR;No;0;L; 0028 56DB 0029;;;4;N;;;;; +3224;PARENTHESIZED IDEOGRAPH FIVE;No;0;L; 0028 4E94 0029;;;5;N;;;;; +3225;PARENTHESIZED IDEOGRAPH SIX;No;0;L; 0028 516D 0029;;;6;N;;;;; +3226;PARENTHESIZED IDEOGRAPH SEVEN;No;0;L; 0028 4E03 0029;;;7;N;;;;; +3227;PARENTHESIZED IDEOGRAPH EIGHT;No;0;L; 0028 516B 0029;;;8;N;;;;; +3228;PARENTHESIZED IDEOGRAPH NINE;No;0;L; 0028 4E5D 0029;;;9;N;;;;; +3229;PARENTHESIZED IDEOGRAPH TEN;No;0;L; 0028 5341 0029;;;10;N;;;;; +322A;PARENTHESIZED IDEOGRAPH MOON;So;0;L; 0028 6708 0029;;;;N;;;;; +322B;PARENTHESIZED IDEOGRAPH FIRE;So;0;L; 0028 706B 0029;;;;N;;;;; +322C;PARENTHESIZED IDEOGRAPH WATER;So;0;L; 0028 6C34 0029;;;;N;;;;; +322D;PARENTHESIZED IDEOGRAPH WOOD;So;0;L; 0028 6728 0029;;;;N;;;;; +322E;PARENTHESIZED IDEOGRAPH METAL;So;0;L; 0028 91D1 0029;;;;N;;;;; +322F;PARENTHESIZED IDEOGRAPH EARTH;So;0;L; 0028 571F 0029;;;;N;;;;; +3230;PARENTHESIZED IDEOGRAPH SUN;So;0;L; 0028 65E5 0029;;;;N;;;;; +3231;PARENTHESIZED IDEOGRAPH STOCK;So;0;L; 0028 682A 0029;;;;N;;;;; +3232;PARENTHESIZED IDEOGRAPH HAVE;So;0;L; 0028 6709 0029;;;;N;;;;; +3233;PARENTHESIZED IDEOGRAPH SOCIETY;So;0;L; 0028 793E 0029;;;;N;;;;; +3234;PARENTHESIZED IDEOGRAPH NAME;So;0;L; 0028 540D 0029;;;;N;;;;; +3235;PARENTHESIZED IDEOGRAPH SPECIAL;So;0;L; 0028 7279 0029;;;;N;;;;; +3236;PARENTHESIZED IDEOGRAPH FINANCIAL;So;0;L; 0028 8CA1 0029;;;;N;;;;; +3237;PARENTHESIZED IDEOGRAPH CONGRATULATION;So;0;L; 0028 795D 0029;;;;N;;;;; +3238;PARENTHESIZED IDEOGRAPH LABOR;So;0;L; 0028 52B4 0029;;;;N;;;;; +3239;PARENTHESIZED IDEOGRAPH REPRESENT;So;0;L; 0028 4EE3 0029;;;;N;;;;; +323A;PARENTHESIZED IDEOGRAPH CALL;So;0;L; 0028 547C 0029;;;;N;;;;; +323B;PARENTHESIZED IDEOGRAPH STUDY;So;0;L; 0028 5B66 0029;;;;N;;;;; +323C;PARENTHESIZED IDEOGRAPH SUPERVISE;So;0;L; 0028 76E3 0029;;;;N;;;;; +323D;PARENTHESIZED IDEOGRAPH ENTERPRISE;So;0;L; 0028 4F01 0029;;;;N;;;;; +323E;PARENTHESIZED IDEOGRAPH RESOURCE;So;0;L; 0028 8CC7 0029;;;;N;;;;; +323F;PARENTHESIZED IDEOGRAPH ALLIANCE;So;0;L; 0028 5354 0029;;;;N;;;;; +3240;PARENTHESIZED IDEOGRAPH FESTIVAL;So;0;L; 0028 796D 0029;;;;N;;;;; +3241;PARENTHESIZED IDEOGRAPH REST;So;0;L; 0028 4F11 0029;;;;N;;;;; +3242;PARENTHESIZED IDEOGRAPH SELF;So;0;L; 0028 81EA 0029;;;;N;;;;; +3243;PARENTHESIZED IDEOGRAPH REACH;So;0;L; 0028 81F3 0029;;;;N;;;;; +3244;CIRCLED IDEOGRAPH QUESTION;So;0;L; 554F;;;;N;;;;; +3245;CIRCLED IDEOGRAPH KINDERGARTEN;So;0;L; 5E7C;;;;N;;;;; +3246;CIRCLED IDEOGRAPH SCHOOL;So;0;L; 6587;;;;N;;;;; +3247;CIRCLED IDEOGRAPH KOTO;So;0;L; 7B8F;;;;N;;;;; +3248;CIRCLED NUMBER TEN ON BLACK SQUARE;No;0;L;;;;10;N;;;;; +3249;CIRCLED NUMBER TWENTY ON BLACK SQUARE;No;0;L;;;;20;N;;;;; +324A;CIRCLED NUMBER THIRTY ON BLACK SQUARE;No;0;L;;;;30;N;;;;; +324B;CIRCLED NUMBER FORTY ON BLACK SQUARE;No;0;L;;;;40;N;;;;; +324C;CIRCLED NUMBER FIFTY ON BLACK SQUARE;No;0;L;;;;50;N;;;;; +324D;CIRCLED NUMBER SIXTY ON BLACK SQUARE;No;0;L;;;;60;N;;;;; +324E;CIRCLED NUMBER SEVENTY ON BLACK SQUARE;No;0;L;;;;70;N;;;;; +324F;CIRCLED NUMBER EIGHTY ON BLACK SQUARE;No;0;L;;;;80;N;;;;; +3250;PARTNERSHIP SIGN;So;0;ON; 0050 0054 0045;;;;N;;;;; +3251;CIRCLED NUMBER TWENTY ONE;No;0;ON; 0032 0031;;;21;N;;;;; +3252;CIRCLED NUMBER TWENTY TWO;No;0;ON; 0032 0032;;;22;N;;;;; +3253;CIRCLED NUMBER TWENTY THREE;No;0;ON; 0032 0033;;;23;N;;;;; +3254;CIRCLED NUMBER TWENTY FOUR;No;0;ON; 0032 0034;;;24;N;;;;; +3255;CIRCLED NUMBER TWENTY FIVE;No;0;ON; 0032 0035;;;25;N;;;;; +3256;CIRCLED NUMBER TWENTY SIX;No;0;ON; 0032 0036;;;26;N;;;;; +3257;CIRCLED NUMBER TWENTY SEVEN;No;0;ON; 0032 0037;;;27;N;;;;; +3258;CIRCLED NUMBER TWENTY EIGHT;No;0;ON; 0032 0038;;;28;N;;;;; +3259;CIRCLED NUMBER TWENTY NINE;No;0;ON; 0032 0039;;;29;N;;;;; +325A;CIRCLED NUMBER THIRTY;No;0;ON; 0033 0030;;;30;N;;;;; +325B;CIRCLED NUMBER THIRTY ONE;No;0;ON; 0033 0031;;;31;N;;;;; +325C;CIRCLED NUMBER THIRTY TWO;No;0;ON; 0033 0032;;;32;N;;;;; +325D;CIRCLED NUMBER THIRTY THREE;No;0;ON; 0033 0033;;;33;N;;;;; +325E;CIRCLED NUMBER THIRTY FOUR;No;0;ON; 0033 0034;;;34;N;;;;; +325F;CIRCLED NUMBER THIRTY FIVE;No;0;ON; 0033 0035;;;35;N;;;;; +3260;CIRCLED HANGUL KIYEOK;So;0;L; 1100;;;;N;CIRCLED HANGUL GIYEOG;;;; +3261;CIRCLED HANGUL NIEUN;So;0;L; 1102;;;;N;;;;; +3262;CIRCLED HANGUL TIKEUT;So;0;L; 1103;;;;N;CIRCLED HANGUL DIGEUD;;;; +3263;CIRCLED HANGUL RIEUL;So;0;L; 1105;;;;N;CIRCLED HANGUL LIEUL;;;; +3264;CIRCLED HANGUL MIEUM;So;0;L; 1106;;;;N;;;;; +3265;CIRCLED HANGUL PIEUP;So;0;L; 1107;;;;N;CIRCLED HANGUL BIEUB;;;; +3266;CIRCLED HANGUL SIOS;So;0;L; 1109;;;;N;;;;; +3267;CIRCLED HANGUL IEUNG;So;0;L; 110B;;;;N;;;;; +3268;CIRCLED HANGUL CIEUC;So;0;L; 110C;;;;N;CIRCLED HANGUL JIEUJ;;;; +3269;CIRCLED HANGUL CHIEUCH;So;0;L; 110E;;;;N;CIRCLED HANGUL CIEUC;;;; +326A;CIRCLED HANGUL KHIEUKH;So;0;L; 110F;;;;N;CIRCLED HANGUL KIYEOK;;;; +326B;CIRCLED HANGUL THIEUTH;So;0;L; 1110;;;;N;CIRCLED HANGUL TIEUT;;;; +326C;CIRCLED HANGUL PHIEUPH;So;0;L; 1111;;;;N;CIRCLED HANGUL PIEUP;;;; +326D;CIRCLED HANGUL HIEUH;So;0;L; 1112;;;;N;;;;; +326E;CIRCLED HANGUL KIYEOK A;So;0;L; 1100 1161;;;;N;CIRCLED HANGUL GA;;;; +326F;CIRCLED HANGUL NIEUN A;So;0;L; 1102 1161;;;;N;CIRCLED HANGUL NA;;;; +3270;CIRCLED HANGUL TIKEUT A;So;0;L; 1103 1161;;;;N;CIRCLED HANGUL DA;;;; +3271;CIRCLED HANGUL RIEUL A;So;0;L; 1105 1161;;;;N;CIRCLED HANGUL LA;;;; +3272;CIRCLED HANGUL MIEUM A;So;0;L; 1106 1161;;;;N;CIRCLED HANGUL MA;;;; +3273;CIRCLED HANGUL PIEUP A;So;0;L; 1107 1161;;;;N;CIRCLED HANGUL BA;;;; +3274;CIRCLED HANGUL SIOS A;So;0;L; 1109 1161;;;;N;CIRCLED HANGUL SA;;;; +3275;CIRCLED HANGUL IEUNG A;So;0;L; 110B 1161;;;;N;CIRCLED HANGUL A;;;; +3276;CIRCLED HANGUL CIEUC A;So;0;L; 110C 1161;;;;N;CIRCLED HANGUL JA;;;; +3277;CIRCLED HANGUL CHIEUCH A;So;0;L; 110E 1161;;;;N;CIRCLED HANGUL CA;;;; +3278;CIRCLED HANGUL KHIEUKH A;So;0;L; 110F 1161;;;;N;CIRCLED HANGUL KA;;;; +3279;CIRCLED HANGUL THIEUTH A;So;0;L; 1110 1161;;;;N;CIRCLED HANGUL TA;;;; +327A;CIRCLED HANGUL PHIEUPH A;So;0;L; 1111 1161;;;;N;CIRCLED HANGUL PA;;;; +327B;CIRCLED HANGUL HIEUH A;So;0;L; 1112 1161;;;;N;CIRCLED HANGUL HA;;;; +327C;CIRCLED KOREAN CHARACTER CHAMKO;So;0;ON; 110E 1161 11B7 1100 1169;;;;N;;;;; +327D;CIRCLED KOREAN CHARACTER JUEUI;So;0;ON; 110C 116E 110B 1174;;;;N;;;;; +327E;CIRCLED HANGUL IEUNG U;So;0;ON; 110B 116E;;;;N;;;;; +327F;KOREAN STANDARD SYMBOL;So;0;L;;;;;N;;;;; +3280;CIRCLED IDEOGRAPH ONE;No;0;L; 4E00;;;1;N;;;;; +3281;CIRCLED IDEOGRAPH TWO;No;0;L; 4E8C;;;2;N;;;;; +3282;CIRCLED IDEOGRAPH THREE;No;0;L; 4E09;;;3;N;;;;; +3283;CIRCLED IDEOGRAPH FOUR;No;0;L; 56DB;;;4;N;;;;; +3284;CIRCLED IDEOGRAPH FIVE;No;0;L; 4E94;;;5;N;;;;; +3285;CIRCLED IDEOGRAPH SIX;No;0;L; 516D;;;6;N;;;;; +3286;CIRCLED IDEOGRAPH SEVEN;No;0;L; 4E03;;;7;N;;;;; +3287;CIRCLED IDEOGRAPH EIGHT;No;0;L; 516B;;;8;N;;;;; +3288;CIRCLED IDEOGRAPH NINE;No;0;L; 4E5D;;;9;N;;;;; +3289;CIRCLED IDEOGRAPH TEN;No;0;L; 5341;;;10;N;;;;; +328A;CIRCLED IDEOGRAPH MOON;So;0;L; 6708;;;;N;;;;; +328B;CIRCLED IDEOGRAPH FIRE;So;0;L; 706B;;;;N;;;;; +328C;CIRCLED IDEOGRAPH WATER;So;0;L; 6C34;;;;N;;;;; +328D;CIRCLED IDEOGRAPH WOOD;So;0;L; 6728;;;;N;;;;; +328E;CIRCLED IDEOGRAPH METAL;So;0;L; 91D1;;;;N;;;;; +328F;CIRCLED IDEOGRAPH EARTH;So;0;L; 571F;;;;N;;;;; +3290;CIRCLED IDEOGRAPH SUN;So;0;L; 65E5;;;;N;;;;; +3291;CIRCLED IDEOGRAPH STOCK;So;0;L; 682A;;;;N;;;;; +3292;CIRCLED IDEOGRAPH HAVE;So;0;L; 6709;;;;N;;;;; +3293;CIRCLED IDEOGRAPH SOCIETY;So;0;L; 793E;;;;N;;;;; +3294;CIRCLED IDEOGRAPH NAME;So;0;L; 540D;;;;N;;;;; +3295;CIRCLED IDEOGRAPH SPECIAL;So;0;L; 7279;;;;N;;;;; +3296;CIRCLED IDEOGRAPH FINANCIAL;So;0;L; 8CA1;;;;N;;;;; +3297;CIRCLED IDEOGRAPH CONGRATULATION;So;0;L; 795D;;;;N;;;;; +3298;CIRCLED IDEOGRAPH LABOR;So;0;L; 52B4;;;;N;;;;; +3299;CIRCLED IDEOGRAPH SECRET;So;0;L; 79D8;;;;N;;;;; +329A;CIRCLED IDEOGRAPH MALE;So;0;L; 7537;;;;N;;;;; +329B;CIRCLED IDEOGRAPH FEMALE;So;0;L; 5973;;;;N;;;;; +329C;CIRCLED IDEOGRAPH SUITABLE;So;0;L; 9069;;;;N;;;;; +329D;CIRCLED IDEOGRAPH EXCELLENT;So;0;L; 512A;;;;N;;;;; +329E;CIRCLED IDEOGRAPH PRINT;So;0;L; 5370;;;;N;;;;; +329F;CIRCLED IDEOGRAPH ATTENTION;So;0;L; 6CE8;;;;N;;;;; +32A0;CIRCLED IDEOGRAPH ITEM;So;0;L; 9805;;;;N;;;;; +32A1;CIRCLED IDEOGRAPH REST;So;0;L; 4F11;;;;N;;;;; +32A2;CIRCLED IDEOGRAPH COPY;So;0;L; 5199;;;;N;;;;; +32A3;CIRCLED IDEOGRAPH CORRECT;So;0;L; 6B63;;;;N;;;;; +32A4;CIRCLED IDEOGRAPH HIGH;So;0;L; 4E0A;;;;N;;;;; +32A5;CIRCLED IDEOGRAPH CENTRE;So;0;L; 4E2D;;;;N;CIRCLED IDEOGRAPH CENTER;;;; +32A6;CIRCLED IDEOGRAPH LOW;So;0;L; 4E0B;;;;N;;;;; +32A7;CIRCLED IDEOGRAPH LEFT;So;0;L; 5DE6;;;;N;;;;; +32A8;CIRCLED IDEOGRAPH RIGHT;So;0;L; 53F3;;;;N;;;;; +32A9;CIRCLED IDEOGRAPH MEDICINE;So;0;L; 533B;;;;N;;;;; +32AA;CIRCLED IDEOGRAPH RELIGION;So;0;L; 5B97;;;;N;;;;; +32AB;CIRCLED IDEOGRAPH STUDY;So;0;L; 5B66;;;;N;;;;; +32AC;CIRCLED IDEOGRAPH SUPERVISE;So;0;L; 76E3;;;;N;;;;; +32AD;CIRCLED IDEOGRAPH ENTERPRISE;So;0;L; 4F01;;;;N;;;;; +32AE;CIRCLED IDEOGRAPH RESOURCE;So;0;L; 8CC7;;;;N;;;;; +32AF;CIRCLED IDEOGRAPH ALLIANCE;So;0;L; 5354;;;;N;;;;; +32B0;CIRCLED IDEOGRAPH NIGHT;So;0;L; 591C;;;;N;;;;; +32B1;CIRCLED NUMBER THIRTY SIX;No;0;ON; 0033 0036;;;36;N;;;;; +32B2;CIRCLED NUMBER THIRTY SEVEN;No;0;ON; 0033 0037;;;37;N;;;;; +32B3;CIRCLED NUMBER THIRTY EIGHT;No;0;ON; 0033 0038;;;38;N;;;;; +32B4;CIRCLED NUMBER THIRTY NINE;No;0;ON; 0033 0039;;;39;N;;;;; +32B5;CIRCLED NUMBER FORTY;No;0;ON; 0034 0030;;;40;N;;;;; +32B6;CIRCLED NUMBER FORTY ONE;No;0;ON; 0034 0031;;;41;N;;;;; +32B7;CIRCLED NUMBER FORTY TWO;No;0;ON; 0034 0032;;;42;N;;;;; +32B8;CIRCLED NUMBER FORTY THREE;No;0;ON; 0034 0033;;;43;N;;;;; +32B9;CIRCLED NUMBER FORTY FOUR;No;0;ON; 0034 0034;;;44;N;;;;; +32BA;CIRCLED NUMBER FORTY FIVE;No;0;ON; 0034 0035;;;45;N;;;;; +32BB;CIRCLED NUMBER FORTY SIX;No;0;ON; 0034 0036;;;46;N;;;;; +32BC;CIRCLED NUMBER FORTY SEVEN;No;0;ON; 0034 0037;;;47;N;;;;; +32BD;CIRCLED NUMBER FORTY EIGHT;No;0;ON; 0034 0038;;;48;N;;;;; +32BE;CIRCLED NUMBER FORTY NINE;No;0;ON; 0034 0039;;;49;N;;;;; +32BF;CIRCLED NUMBER FIFTY;No;0;ON; 0035 0030;;;50;N;;;;; +32C0;IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY;So;0;L; 0031 6708;;;;N;;;;; +32C1;IDEOGRAPHIC TELEGRAPH SYMBOL FOR FEBRUARY;So;0;L; 0032 6708;;;;N;;;;; +32C2;IDEOGRAPHIC TELEGRAPH SYMBOL FOR MARCH;So;0;L; 0033 6708;;;;N;;;;; +32C3;IDEOGRAPHIC TELEGRAPH SYMBOL FOR APRIL;So;0;L; 0034 6708;;;;N;;;;; +32C4;IDEOGRAPHIC TELEGRAPH SYMBOL FOR MAY;So;0;L; 0035 6708;;;;N;;;;; +32C5;IDEOGRAPHIC TELEGRAPH SYMBOL FOR JUNE;So;0;L; 0036 6708;;;;N;;;;; +32C6;IDEOGRAPHIC TELEGRAPH SYMBOL FOR JULY;So;0;L; 0037 6708;;;;N;;;;; +32C7;IDEOGRAPHIC TELEGRAPH SYMBOL FOR AUGUST;So;0;L; 0038 6708;;;;N;;;;; +32C8;IDEOGRAPHIC TELEGRAPH SYMBOL FOR SEPTEMBER;So;0;L; 0039 6708;;;;N;;;;; +32C9;IDEOGRAPHIC TELEGRAPH SYMBOL FOR OCTOBER;So;0;L; 0031 0030 6708;;;;N;;;;; +32CA;IDEOGRAPHIC TELEGRAPH SYMBOL FOR NOVEMBER;So;0;L; 0031 0031 6708;;;;N;;;;; +32CB;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DECEMBER;So;0;L; 0031 0032 6708;;;;N;;;;; +32CC;SQUARE HG;So;0;ON; 0048 0067;;;;N;;;;; +32CD;SQUARE ERG;So;0;ON; 0065 0072 0067;;;;N;;;;; +32CE;SQUARE EV;So;0;ON; 0065 0056;;;;N;;;;; +32CF;LIMITED LIABILITY SIGN;So;0;ON; 004C 0054 0044;;;;N;;;;; +32D0;CIRCLED KATAKANA A;So;0;L; 30A2;;;;N;;;;; +32D1;CIRCLED KATAKANA I;So;0;L; 30A4;;;;N;;;;; +32D2;CIRCLED KATAKANA U;So;0;L; 30A6;;;;N;;;;; +32D3;CIRCLED KATAKANA E;So;0;L; 30A8;;;;N;;;;; +32D4;CIRCLED KATAKANA O;So;0;L; 30AA;;;;N;;;;; +32D5;CIRCLED KATAKANA KA;So;0;L; 30AB;;;;N;;;;; +32D6;CIRCLED KATAKANA KI;So;0;L; 30AD;;;;N;;;;; +32D7;CIRCLED KATAKANA KU;So;0;L; 30AF;;;;N;;;;; +32D8;CIRCLED KATAKANA KE;So;0;L; 30B1;;;;N;;;;; +32D9;CIRCLED KATAKANA KO;So;0;L; 30B3;;;;N;;;;; +32DA;CIRCLED KATAKANA SA;So;0;L; 30B5;;;;N;;;;; +32DB;CIRCLED KATAKANA SI;So;0;L; 30B7;;;;N;;;;; +32DC;CIRCLED KATAKANA SU;So;0;L; 30B9;;;;N;;;;; +32DD;CIRCLED KATAKANA SE;So;0;L; 30BB;;;;N;;;;; +32DE;CIRCLED KATAKANA SO;So;0;L; 30BD;;;;N;;;;; +32DF;CIRCLED KATAKANA TA;So;0;L; 30BF;;;;N;;;;; +32E0;CIRCLED KATAKANA TI;So;0;L; 30C1;;;;N;;;;; +32E1;CIRCLED KATAKANA TU;So;0;L; 30C4;;;;N;;;;; +32E2;CIRCLED KATAKANA TE;So;0;L; 30C6;;;;N;;;;; +32E3;CIRCLED KATAKANA TO;So;0;L; 30C8;;;;N;;;;; +32E4;CIRCLED KATAKANA NA;So;0;L; 30CA;;;;N;;;;; +32E5;CIRCLED KATAKANA NI;So;0;L; 30CB;;;;N;;;;; +32E6;CIRCLED KATAKANA NU;So;0;L; 30CC;;;;N;;;;; +32E7;CIRCLED KATAKANA NE;So;0;L; 30CD;;;;N;;;;; +32E8;CIRCLED KATAKANA NO;So;0;L; 30CE;;;;N;;;;; +32E9;CIRCLED KATAKANA HA;So;0;L; 30CF;;;;N;;;;; +32EA;CIRCLED KATAKANA HI;So;0;L; 30D2;;;;N;;;;; +32EB;CIRCLED KATAKANA HU;So;0;L; 30D5;;;;N;;;;; +32EC;CIRCLED KATAKANA HE;So;0;L; 30D8;;;;N;;;;; +32ED;CIRCLED KATAKANA HO;So;0;L; 30DB;;;;N;;;;; +32EE;CIRCLED KATAKANA MA;So;0;L; 30DE;;;;N;;;;; +32EF;CIRCLED KATAKANA MI;So;0;L; 30DF;;;;N;;;;; +32F0;CIRCLED KATAKANA MU;So;0;L; 30E0;;;;N;;;;; +32F1;CIRCLED KATAKANA ME;So;0;L; 30E1;;;;N;;;;; +32F2;CIRCLED KATAKANA MO;So;0;L; 30E2;;;;N;;;;; +32F3;CIRCLED KATAKANA YA;So;0;L; 30E4;;;;N;;;;; +32F4;CIRCLED KATAKANA YU;So;0;L; 30E6;;;;N;;;;; +32F5;CIRCLED KATAKANA YO;So;0;L; 30E8;;;;N;;;;; +32F6;CIRCLED KATAKANA RA;So;0;L; 30E9;;;;N;;;;; +32F7;CIRCLED KATAKANA RI;So;0;L; 30EA;;;;N;;;;; +32F8;CIRCLED KATAKANA RU;So;0;L; 30EB;;;;N;;;;; +32F9;CIRCLED KATAKANA RE;So;0;L; 30EC;;;;N;;;;; +32FA;CIRCLED KATAKANA RO;So;0;L; 30ED;;;;N;;;;; +32FB;CIRCLED KATAKANA WA;So;0;L; 30EF;;;;N;;;;; +32FC;CIRCLED KATAKANA WI;So;0;L; 30F0;;;;N;;;;; +32FD;CIRCLED KATAKANA WE;So;0;L; 30F1;;;;N;;;;; +32FE;CIRCLED KATAKANA WO;So;0;L; 30F2;;;;N;;;;; +3300;SQUARE APAATO;So;0;L; 30A2 30D1 30FC 30C8;;;;N;SQUARED APAATO;;;; +3301;SQUARE ARUHUA;So;0;L; 30A2 30EB 30D5 30A1;;;;N;SQUARED ARUHUA;;;; +3302;SQUARE ANPEA;So;0;L; 30A2 30F3 30DA 30A2;;;;N;SQUARED ANPEA;;;; +3303;SQUARE AARU;So;0;L; 30A2 30FC 30EB;;;;N;SQUARED AARU;;;; +3304;SQUARE ININGU;So;0;L; 30A4 30CB 30F3 30B0;;;;N;SQUARED ININGU;;;; +3305;SQUARE INTI;So;0;L; 30A4 30F3 30C1;;;;N;SQUARED INTI;;;; +3306;SQUARE UON;So;0;L; 30A6 30A9 30F3;;;;N;SQUARED UON;;;; +3307;SQUARE ESUKUUDO;So;0;L; 30A8 30B9 30AF 30FC 30C9;;;;N;SQUARED ESUKUUDO;;;; +3308;SQUARE EEKAA;So;0;L; 30A8 30FC 30AB 30FC;;;;N;SQUARED EEKAA;;;; +3309;SQUARE ONSU;So;0;L; 30AA 30F3 30B9;;;;N;SQUARED ONSU;;;; +330A;SQUARE OOMU;So;0;L; 30AA 30FC 30E0;;;;N;SQUARED OOMU;;;; +330B;SQUARE KAIRI;So;0;L; 30AB 30A4 30EA;;;;N;SQUARED KAIRI;;;; +330C;SQUARE KARATTO;So;0;L; 30AB 30E9 30C3 30C8;;;;N;SQUARED KARATTO;;;; +330D;SQUARE KARORII;So;0;L; 30AB 30ED 30EA 30FC;;;;N;SQUARED KARORII;;;; +330E;SQUARE GARON;So;0;L; 30AC 30ED 30F3;;;;N;SQUARED GARON;;;; +330F;SQUARE GANMA;So;0;L; 30AC 30F3 30DE;;;;N;SQUARED GANMA;;;; +3310;SQUARE GIGA;So;0;L; 30AE 30AC;;;;N;SQUARED GIGA;;;; +3311;SQUARE GINII;So;0;L; 30AE 30CB 30FC;;;;N;SQUARED GINII;;;; +3312;SQUARE KYURII;So;0;L; 30AD 30E5 30EA 30FC;;;;N;SQUARED KYURII;;;; +3313;SQUARE GIRUDAA;So;0;L; 30AE 30EB 30C0 30FC;;;;N;SQUARED GIRUDAA;;;; +3314;SQUARE KIRO;So;0;L; 30AD 30ED;;;;N;SQUARED KIRO;;;; +3315;SQUARE KIROGURAMU;So;0;L; 30AD 30ED 30B0 30E9 30E0;;;;N;SQUARED KIROGURAMU;;;; +3316;SQUARE KIROMEETORU;So;0;L; 30AD 30ED 30E1 30FC 30C8 30EB;;;;N;SQUARED KIROMEETORU;;;; +3317;SQUARE KIROWATTO;So;0;L; 30AD 30ED 30EF 30C3 30C8;;;;N;SQUARED KIROWATTO;;;; +3318;SQUARE GURAMU;So;0;L; 30B0 30E9 30E0;;;;N;SQUARED GURAMU;;;; +3319;SQUARE GURAMUTON;So;0;L; 30B0 30E9 30E0 30C8 30F3;;;;N;SQUARED GURAMUTON;;;; +331A;SQUARE KURUZEIRO;So;0;L; 30AF 30EB 30BC 30A4 30ED;;;;N;SQUARED KURUZEIRO;;;; +331B;SQUARE KUROONE;So;0;L; 30AF 30ED 30FC 30CD;;;;N;SQUARED KUROONE;;;; +331C;SQUARE KEESU;So;0;L; 30B1 30FC 30B9;;;;N;SQUARED KEESU;;;; +331D;SQUARE KORUNA;So;0;L; 30B3 30EB 30CA;;;;N;SQUARED KORUNA;;;; +331E;SQUARE KOOPO;So;0;L; 30B3 30FC 30DD;;;;N;SQUARED KOOPO;;;; +331F;SQUARE SAIKURU;So;0;L; 30B5 30A4 30AF 30EB;;;;N;SQUARED SAIKURU;;;; +3320;SQUARE SANTIIMU;So;0;L; 30B5 30F3 30C1 30FC 30E0;;;;N;SQUARED SANTIIMU;;;; +3321;SQUARE SIRINGU;So;0;L; 30B7 30EA 30F3 30B0;;;;N;SQUARED SIRINGU;;;; +3322;SQUARE SENTI;So;0;L; 30BB 30F3 30C1;;;;N;SQUARED SENTI;;;; +3323;SQUARE SENTO;So;0;L; 30BB 30F3 30C8;;;;N;SQUARED SENTO;;;; +3324;SQUARE DAASU;So;0;L; 30C0 30FC 30B9;;;;N;SQUARED DAASU;;;; +3325;SQUARE DESI;So;0;L; 30C7 30B7;;;;N;SQUARED DESI;;;; +3326;SQUARE DORU;So;0;L; 30C9 30EB;;;;N;SQUARED DORU;;;; +3327;SQUARE TON;So;0;L; 30C8 30F3;;;;N;SQUARED TON;;;; +3328;SQUARE NANO;So;0;L; 30CA 30CE;;;;N;SQUARED NANO;;;; +3329;SQUARE NOTTO;So;0;L; 30CE 30C3 30C8;;;;N;SQUARED NOTTO;;;; +332A;SQUARE HAITU;So;0;L; 30CF 30A4 30C4;;;;N;SQUARED HAITU;;;; +332B;SQUARE PAASENTO;So;0;L; 30D1 30FC 30BB 30F3 30C8;;;;N;SQUARED PAASENTO;;;; +332C;SQUARE PAATU;So;0;L; 30D1 30FC 30C4;;;;N;SQUARED PAATU;;;; +332D;SQUARE BAARERU;So;0;L; 30D0 30FC 30EC 30EB;;;;N;SQUARED BAARERU;;;; +332E;SQUARE PIASUTORU;So;0;L; 30D4 30A2 30B9 30C8 30EB;;;;N;SQUARED PIASUTORU;;;; +332F;SQUARE PIKURU;So;0;L; 30D4 30AF 30EB;;;;N;SQUARED PIKURU;;;; +3330;SQUARE PIKO;So;0;L; 30D4 30B3;;;;N;SQUARED PIKO;;;; +3331;SQUARE BIRU;So;0;L; 30D3 30EB;;;;N;SQUARED BIRU;;;; +3332;SQUARE HUARADDO;So;0;L; 30D5 30A1 30E9 30C3 30C9;;;;N;SQUARED HUARADDO;;;; +3333;SQUARE HUIITO;So;0;L; 30D5 30A3 30FC 30C8;;;;N;SQUARED HUIITO;;;; +3334;SQUARE BUSSYERU;So;0;L; 30D6 30C3 30B7 30A7 30EB;;;;N;SQUARED BUSSYERU;;;; +3335;SQUARE HURAN;So;0;L; 30D5 30E9 30F3;;;;N;SQUARED HURAN;;;; +3336;SQUARE HEKUTAARU;So;0;L; 30D8 30AF 30BF 30FC 30EB;;;;N;SQUARED HEKUTAARU;;;; +3337;SQUARE PESO;So;0;L; 30DA 30BD;;;;N;SQUARED PESO;;;; +3338;SQUARE PENIHI;So;0;L; 30DA 30CB 30D2;;;;N;SQUARED PENIHI;;;; +3339;SQUARE HERUTU;So;0;L; 30D8 30EB 30C4;;;;N;SQUARED HERUTU;;;; +333A;SQUARE PENSU;So;0;L; 30DA 30F3 30B9;;;;N;SQUARED PENSU;;;; +333B;SQUARE PEEZI;So;0;L; 30DA 30FC 30B8;;;;N;SQUARED PEEZI;;;; +333C;SQUARE BEETA;So;0;L; 30D9 30FC 30BF;;;;N;SQUARED BEETA;;;; +333D;SQUARE POINTO;So;0;L; 30DD 30A4 30F3 30C8;;;;N;SQUARED POINTO;;;; +333E;SQUARE BORUTO;So;0;L; 30DC 30EB 30C8;;;;N;SQUARED BORUTO;;;; +333F;SQUARE HON;So;0;L; 30DB 30F3;;;;N;SQUARED HON;;;; +3340;SQUARE PONDO;So;0;L; 30DD 30F3 30C9;;;;N;SQUARED PONDO;;;; +3341;SQUARE HOORU;So;0;L; 30DB 30FC 30EB;;;;N;SQUARED HOORU;;;; +3342;SQUARE HOON;So;0;L; 30DB 30FC 30F3;;;;N;SQUARED HOON;;;; +3343;SQUARE MAIKURO;So;0;L; 30DE 30A4 30AF 30ED;;;;N;SQUARED MAIKURO;;;; +3344;SQUARE MAIRU;So;0;L; 30DE 30A4 30EB;;;;N;SQUARED MAIRU;;;; +3345;SQUARE MAHHA;So;0;L; 30DE 30C3 30CF;;;;N;SQUARED MAHHA;;;; +3346;SQUARE MARUKU;So;0;L; 30DE 30EB 30AF;;;;N;SQUARED MARUKU;;;; +3347;SQUARE MANSYON;So;0;L; 30DE 30F3 30B7 30E7 30F3;;;;N;SQUARED MANSYON;;;; +3348;SQUARE MIKURON;So;0;L; 30DF 30AF 30ED 30F3;;;;N;SQUARED MIKURON;;;; +3349;SQUARE MIRI;So;0;L; 30DF 30EA;;;;N;SQUARED MIRI;;;; +334A;SQUARE MIRIBAARU;So;0;L; 30DF 30EA 30D0 30FC 30EB;;;;N;SQUARED MIRIBAARU;;;; +334B;SQUARE MEGA;So;0;L; 30E1 30AC;;;;N;SQUARED MEGA;;;; +334C;SQUARE MEGATON;So;0;L; 30E1 30AC 30C8 30F3;;;;N;SQUARED MEGATON;;;; +334D;SQUARE MEETORU;So;0;L; 30E1 30FC 30C8 30EB;;;;N;SQUARED MEETORU;;;; +334E;SQUARE YAADO;So;0;L; 30E4 30FC 30C9;;;;N;SQUARED YAADO;;;; +334F;SQUARE YAARU;So;0;L; 30E4 30FC 30EB;;;;N;SQUARED YAARU;;;; +3350;SQUARE YUAN;So;0;L; 30E6 30A2 30F3;;;;N;SQUARED YUAN;;;; +3351;SQUARE RITTORU;So;0;L; 30EA 30C3 30C8 30EB;;;;N;SQUARED RITTORU;;;; +3352;SQUARE RIRA;So;0;L; 30EA 30E9;;;;N;SQUARED RIRA;;;; +3353;SQUARE RUPII;So;0;L; 30EB 30D4 30FC;;;;N;SQUARED RUPII;;;; +3354;SQUARE RUUBURU;So;0;L; 30EB 30FC 30D6 30EB;;;;N;SQUARED RUUBURU;;;; +3355;SQUARE REMU;So;0;L; 30EC 30E0;;;;N;SQUARED REMU;;;; +3356;SQUARE RENTOGEN;So;0;L; 30EC 30F3 30C8 30B2 30F3;;;;N;SQUARED RENTOGEN;;;; +3357;SQUARE WATTO;So;0;L; 30EF 30C3 30C8;;;;N;SQUARED WATTO;;;; +3358;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR ZERO;So;0;L; 0030 70B9;;;;N;;;;; +3359;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR ONE;So;0;L; 0031 70B9;;;;N;;;;; +335A;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWO;So;0;L; 0032 70B9;;;;N;;;;; +335B;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR THREE;So;0;L; 0033 70B9;;;;N;;;;; +335C;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FOUR;So;0;L; 0034 70B9;;;;N;;;;; +335D;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FIVE;So;0;L; 0035 70B9;;;;N;;;;; +335E;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SIX;So;0;L; 0036 70B9;;;;N;;;;; +335F;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SEVEN;So;0;L; 0037 70B9;;;;N;;;;; +3360;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR EIGHT;So;0;L; 0038 70B9;;;;N;;;;; +3361;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR NINE;So;0;L; 0039 70B9;;;;N;;;;; +3362;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TEN;So;0;L; 0031 0030 70B9;;;;N;;;;; +3363;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR ELEVEN;So;0;L; 0031 0031 70B9;;;;N;;;;; +3364;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWELVE;So;0;L; 0031 0032 70B9;;;;N;;;;; +3365;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR THIRTEEN;So;0;L; 0031 0033 70B9;;;;N;;;;; +3366;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FOURTEEN;So;0;L; 0031 0034 70B9;;;;N;;;;; +3367;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FIFTEEN;So;0;L; 0031 0035 70B9;;;;N;;;;; +3368;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SIXTEEN;So;0;L; 0031 0036 70B9;;;;N;;;;; +3369;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SEVENTEEN;So;0;L; 0031 0037 70B9;;;;N;;;;; +336A;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR EIGHTEEN;So;0;L; 0031 0038 70B9;;;;N;;;;; +336B;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR NINETEEN;So;0;L; 0031 0039 70B9;;;;N;;;;; +336C;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY;So;0;L; 0032 0030 70B9;;;;N;;;;; +336D;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-ONE;So;0;L; 0032 0031 70B9;;;;N;;;;; +336E;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-TWO;So;0;L; 0032 0032 70B9;;;;N;;;;; +336F;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-THREE;So;0;L; 0032 0033 70B9;;;;N;;;;; +3370;IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-FOUR;So;0;L; 0032 0034 70B9;;;;N;;;;; +3371;SQUARE HPA;So;0;L; 0068 0050 0061;;;;N;;;;; +3372;SQUARE DA;So;0;L; 0064 0061;;;;N;;;;; +3373;SQUARE AU;So;0;L; 0041 0055;;;;N;;;;; +3374;SQUARE BAR;So;0;L; 0062 0061 0072;;;;N;;;;; +3375;SQUARE OV;So;0;L; 006F 0056;;;;N;;;;; +3376;SQUARE PC;So;0;L; 0070 0063;;;;N;;;;; +3377;SQUARE DM;So;0;ON; 0064 006D;;;;N;;;;; +3378;SQUARE DM SQUARED;So;0;ON; 0064 006D 00B2;;;;N;;;;; +3379;SQUARE DM CUBED;So;0;ON; 0064 006D 00B3;;;;N;;;;; +337A;SQUARE IU;So;0;ON; 0049 0055;;;;N;;;;; +337B;SQUARE ERA NAME HEISEI;So;0;L; 5E73 6210;;;;N;SQUARED TWO IDEOGRAPHS ERA NAME HEISEI;;;; +337C;SQUARE ERA NAME SYOUWA;So;0;L; 662D 548C;;;;N;SQUARED TWO IDEOGRAPHS ERA NAME SYOUWA;;;; +337D;SQUARE ERA NAME TAISYOU;So;0;L; 5927 6B63;;;;N;SQUARED TWO IDEOGRAPHS ERA NAME TAISYOU;;;; +337E;SQUARE ERA NAME MEIZI;So;0;L; 660E 6CBB;;;;N;SQUARED TWO IDEOGRAPHS ERA NAME MEIZI;;;; +337F;SQUARE CORPORATION;So;0;L; 682A 5F0F 4F1A 793E;;;;N;SQUARED FOUR IDEOGRAPHS CORPORATION;;;; +3380;SQUARE PA AMPS;So;0;L; 0070 0041;;;;N;SQUARED PA AMPS;;;; +3381;SQUARE NA;So;0;L; 006E 0041;;;;N;SQUARED NA;;;; +3382;SQUARE MU A;So;0;L; 03BC 0041;;;;N;SQUARED MU A;;;; +3383;SQUARE MA;So;0;L; 006D 0041;;;;N;SQUARED MA;;;; +3384;SQUARE KA;So;0;L; 006B 0041;;;;N;SQUARED KA;;;; +3385;SQUARE KB;So;0;L; 004B 0042;;;;N;SQUARED KB;;;; +3386;SQUARE MB;So;0;L; 004D 0042;;;;N;SQUARED MB;;;; +3387;SQUARE GB;So;0;L; 0047 0042;;;;N;SQUARED GB;;;; +3388;SQUARE CAL;So;0;L; 0063 0061 006C;;;;N;SQUARED CAL;;;; +3389;SQUARE KCAL;So;0;L; 006B 0063 0061 006C;;;;N;SQUARED KCAL;;;; +338A;SQUARE PF;So;0;L; 0070 0046;;;;N;SQUARED PF;;;; +338B;SQUARE NF;So;0;L; 006E 0046;;;;N;SQUARED NF;;;; +338C;SQUARE MU F;So;0;L; 03BC 0046;;;;N;SQUARED MU F;;;; +338D;SQUARE MU G;So;0;L; 03BC 0067;;;;N;SQUARED MU G;;;; +338E;SQUARE MG;So;0;L; 006D 0067;;;;N;SQUARED MG;;;; +338F;SQUARE KG;So;0;L; 006B 0067;;;;N;SQUARED KG;;;; +3390;SQUARE HZ;So;0;L; 0048 007A;;;;N;SQUARED HZ;;;; +3391;SQUARE KHZ;So;0;L; 006B 0048 007A;;;;N;SQUARED KHZ;;;; +3392;SQUARE MHZ;So;0;L; 004D 0048 007A;;;;N;SQUARED MHZ;;;; +3393;SQUARE GHZ;So;0;L; 0047 0048 007A;;;;N;SQUARED GHZ;;;; +3394;SQUARE THZ;So;0;L; 0054 0048 007A;;;;N;SQUARED THZ;;;; +3395;SQUARE MU L;So;0;L; 03BC 2113;;;;N;SQUARED MU L;;;; +3396;SQUARE ML;So;0;L; 006D 2113;;;;N;SQUARED ML;;;; +3397;SQUARE DL;So;0;L; 0064 2113;;;;N;SQUARED DL;;;; +3398;SQUARE KL;So;0;L; 006B 2113;;;;N;SQUARED KL;;;; +3399;SQUARE FM;So;0;L; 0066 006D;;;;N;SQUARED FM;;;; +339A;SQUARE NM;So;0;L; 006E 006D;;;;N;SQUARED NM;;;; +339B;SQUARE MU M;So;0;L; 03BC 006D;;;;N;SQUARED MU M;;;; +339C;SQUARE MM;So;0;L; 006D 006D;;;;N;SQUARED MM;;;; +339D;SQUARE CM;So;0;L; 0063 006D;;;;N;SQUARED CM;;;; +339E;SQUARE KM;So;0;L; 006B 006D;;;;N;SQUARED KM;;;; +339F;SQUARE MM SQUARED;So;0;L; 006D 006D 00B2;;;;N;SQUARED MM SQUARED;;;; +33A0;SQUARE CM SQUARED;So;0;L; 0063 006D 00B2;;;;N;SQUARED CM SQUARED;;;; +33A1;SQUARE M SQUARED;So;0;L; 006D 00B2;;;;N;SQUARED M SQUARED;;;; +33A2;SQUARE KM SQUARED;So;0;L; 006B 006D 00B2;;;;N;SQUARED KM SQUARED;;;; +33A3;SQUARE MM CUBED;So;0;L; 006D 006D 00B3;;;;N;SQUARED MM CUBED;;;; +33A4;SQUARE CM CUBED;So;0;L; 0063 006D 00B3;;;;N;SQUARED CM CUBED;;;; +33A5;SQUARE M CUBED;So;0;L; 006D 00B3;;;;N;SQUARED M CUBED;;;; +33A6;SQUARE KM CUBED;So;0;L; 006B 006D 00B3;;;;N;SQUARED KM CUBED;;;; +33A7;SQUARE M OVER S;So;0;L; 006D 2215 0073;;;;N;SQUARED M OVER S;;;; +33A8;SQUARE M OVER S SQUARED;So;0;L; 006D 2215 0073 00B2;;;;N;SQUARED M OVER S SQUARED;;;; +33A9;SQUARE PA;So;0;L; 0050 0061;;;;N;SQUARED PA;;;; +33AA;SQUARE KPA;So;0;L; 006B 0050 0061;;;;N;SQUARED KPA;;;; +33AB;SQUARE MPA;So;0;L; 004D 0050 0061;;;;N;SQUARED MPA;;;; +33AC;SQUARE GPA;So;0;L; 0047 0050 0061;;;;N;SQUARED GPA;;;; +33AD;SQUARE RAD;So;0;L; 0072 0061 0064;;;;N;SQUARED RAD;;;; +33AE;SQUARE RAD OVER S;So;0;L; 0072 0061 0064 2215 0073;;;;N;SQUARED RAD OVER S;;;; +33AF;SQUARE RAD OVER S SQUARED;So;0;L; 0072 0061 0064 2215 0073 00B2;;;;N;SQUARED RAD OVER S SQUARED;;;; +33B0;SQUARE PS;So;0;L; 0070 0073;;;;N;SQUARED PS;;;; +33B1;SQUARE NS;So;0;L; 006E 0073;;;;N;SQUARED NS;;;; +33B2;SQUARE MU S;So;0;L; 03BC 0073;;;;N;SQUARED MU S;;;; +33B3;SQUARE MS;So;0;L; 006D 0073;;;;N;SQUARED MS;;;; +33B4;SQUARE PV;So;0;L; 0070 0056;;;;N;SQUARED PV;;;; +33B5;SQUARE NV;So;0;L; 006E 0056;;;;N;SQUARED NV;;;; +33B6;SQUARE MU V;So;0;L; 03BC 0056;;;;N;SQUARED MU V;;;; +33B7;SQUARE MV;So;0;L; 006D 0056;;;;N;SQUARED MV;;;; +33B8;SQUARE KV;So;0;L; 006B 0056;;;;N;SQUARED KV;;;; +33B9;SQUARE MV MEGA;So;0;L; 004D 0056;;;;N;SQUARED MV MEGA;;;; +33BA;SQUARE PW;So;0;L; 0070 0057;;;;N;SQUARED PW;;;; +33BB;SQUARE NW;So;0;L; 006E 0057;;;;N;SQUARED NW;;;; +33BC;SQUARE MU W;So;0;L; 03BC 0057;;;;N;SQUARED MU W;;;; +33BD;SQUARE MW;So;0;L; 006D 0057;;;;N;SQUARED MW;;;; +33BE;SQUARE KW;So;0;L; 006B 0057;;;;N;SQUARED KW;;;; +33BF;SQUARE MW MEGA;So;0;L; 004D 0057;;;;N;SQUARED MW MEGA;;;; +33C0;SQUARE K OHM;So;0;L; 006B 03A9;;;;N;SQUARED K OHM;;;; +33C1;SQUARE M OHM;So;0;L; 004D 03A9;;;;N;SQUARED M OHM;;;; +33C2;SQUARE AM;So;0;L; 0061 002E 006D 002E;;;;N;SQUARED AM;;;; +33C3;SQUARE BQ;So;0;L; 0042 0071;;;;N;SQUARED BQ;;;; +33C4;SQUARE CC;So;0;L; 0063 0063;;;;N;SQUARED CC;;;; +33C5;SQUARE CD;So;0;L; 0063 0064;;;;N;SQUARED CD;;;; +33C6;SQUARE C OVER KG;So;0;L; 0043 2215 006B 0067;;;;N;SQUARED C OVER KG;;;; +33C7;SQUARE CO;So;0;L; 0043 006F 002E;;;;N;SQUARED CO;;;; +33C8;SQUARE DB;So;0;L; 0064 0042;;;;N;SQUARED DB;;;; +33C9;SQUARE GY;So;0;L; 0047 0079;;;;N;SQUARED GY;;;; +33CA;SQUARE HA;So;0;L; 0068 0061;;;;N;SQUARED HA;;;; +33CB;SQUARE HP;So;0;L; 0048 0050;;;;N;SQUARED HP;;;; +33CC;SQUARE IN;So;0;L; 0069 006E;;;;N;SQUARED IN;;;; +33CD;SQUARE KK;So;0;L; 004B 004B;;;;N;SQUARED KK;;;; +33CE;SQUARE KM CAPITAL;So;0;L; 004B 004D;;;;N;SQUARED KM CAPITAL;;;; +33CF;SQUARE KT;So;0;L; 006B 0074;;;;N;SQUARED KT;;;; +33D0;SQUARE LM;So;0;L; 006C 006D;;;;N;SQUARED LM;;;; +33D1;SQUARE LN;So;0;L; 006C 006E;;;;N;SQUARED LN;;;; +33D2;SQUARE LOG;So;0;L; 006C 006F 0067;;;;N;SQUARED LOG;;;; +33D3;SQUARE LX;So;0;L; 006C 0078;;;;N;SQUARED LX;;;; +33D4;SQUARE MB SMALL;So;0;L; 006D 0062;;;;N;SQUARED MB SMALL;;;; +33D5;SQUARE MIL;So;0;L; 006D 0069 006C;;;;N;SQUARED MIL;;;; +33D6;SQUARE MOL;So;0;L; 006D 006F 006C;;;;N;SQUARED MOL;;;; +33D7;SQUARE PH;So;0;L; 0050 0048;;;;N;SQUARED PH;;;; +33D8;SQUARE PM;So;0;L; 0070 002E 006D 002E;;;;N;SQUARED PM;;;; +33D9;SQUARE PPM;So;0;L; 0050 0050 004D;;;;N;SQUARED PPM;;;; +33DA;SQUARE PR;So;0;L; 0050 0052;;;;N;SQUARED PR;;;; +33DB;SQUARE SR;So;0;L; 0073 0072;;;;N;SQUARED SR;;;; +33DC;SQUARE SV;So;0;L; 0053 0076;;;;N;SQUARED SV;;;; +33DD;SQUARE WB;So;0;L; 0057 0062;;;;N;SQUARED WB;;;; +33DE;SQUARE V OVER M;So;0;ON; 0056 2215 006D;;;;N;;;;; +33DF;SQUARE A OVER M;So;0;ON; 0041 2215 006D;;;;N;;;;; +33E0;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY ONE;So;0;L; 0031 65E5;;;;N;;;;; +33E1;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWO;So;0;L; 0032 65E5;;;;N;;;;; +33E2;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THREE;So;0;L; 0033 65E5;;;;N;;;;; +33E3;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FOUR;So;0;L; 0034 65E5;;;;N;;;;; +33E4;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FIVE;So;0;L; 0035 65E5;;;;N;;;;; +33E5;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SIX;So;0;L; 0036 65E5;;;;N;;;;; +33E6;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SEVEN;So;0;L; 0037 65E5;;;;N;;;;; +33E7;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY EIGHT;So;0;L; 0038 65E5;;;;N;;;;; +33E8;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY NINE;So;0;L; 0039 65E5;;;;N;;;;; +33E9;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TEN;So;0;L; 0031 0030 65E5;;;;N;;;;; +33EA;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY ELEVEN;So;0;L; 0031 0031 65E5;;;;N;;;;; +33EB;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWELVE;So;0;L; 0031 0032 65E5;;;;N;;;;; +33EC;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTEEN;So;0;L; 0031 0033 65E5;;;;N;;;;; +33ED;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FOURTEEN;So;0;L; 0031 0034 65E5;;;;N;;;;; +33EE;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FIFTEEN;So;0;L; 0031 0035 65E5;;;;N;;;;; +33EF;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SIXTEEN;So;0;L; 0031 0036 65E5;;;;N;;;;; +33F0;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SEVENTEEN;So;0;L; 0031 0037 65E5;;;;N;;;;; +33F1;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY EIGHTEEN;So;0;L; 0031 0038 65E5;;;;N;;;;; +33F2;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY NINETEEN;So;0;L; 0031 0039 65E5;;;;N;;;;; +33F3;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY;So;0;L; 0032 0030 65E5;;;;N;;;;; +33F4;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-ONE;So;0;L; 0032 0031 65E5;;;;N;;;;; +33F5;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-TWO;So;0;L; 0032 0032 65E5;;;;N;;;;; +33F6;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-THREE;So;0;L; 0032 0033 65E5;;;;N;;;;; +33F7;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-FOUR;So;0;L; 0032 0034 65E5;;;;N;;;;; +33F8;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-FIVE;So;0;L; 0032 0035 65E5;;;;N;;;;; +33F9;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-SIX;So;0;L; 0032 0036 65E5;;;;N;;;;; +33FA;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-SEVEN;So;0;L; 0032 0037 65E5;;;;N;;;;; +33FB;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-EIGHT;So;0;L; 0032 0038 65E5;;;;N;;;;; +33FC;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-NINE;So;0;L; 0032 0039 65E5;;;;N;;;;; +33FD;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY;So;0;L; 0033 0030 65E5;;;;N;;;;; +33FE;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE;So;0;L; 0033 0031 65E5;;;;N;;;;; +33FF;SQUARE GAL;So;0;ON; 0067 0061 006C;;;;N;;;;; +3400;;Lo;0;L;;;;;N;;;;; +4DB5;;Lo;0;L;;;;;N;;;;; +4DC0;HEXAGRAM FOR THE CREATIVE HEAVEN;So;0;ON;;;;;N;;;;; +4DC1;HEXAGRAM FOR THE RECEPTIVE EARTH;So;0;ON;;;;;N;;;;; +4DC2;HEXAGRAM FOR DIFFICULTY AT THE BEGINNING;So;0;ON;;;;;N;;;;; +4DC3;HEXAGRAM FOR YOUTHFUL FOLLY;So;0;ON;;;;;N;;;;; +4DC4;HEXAGRAM FOR WAITING;So;0;ON;;;;;N;;;;; +4DC5;HEXAGRAM FOR CONFLICT;So;0;ON;;;;;N;;;;; +4DC6;HEXAGRAM FOR THE ARMY;So;0;ON;;;;;N;;;;; +4DC7;HEXAGRAM FOR HOLDING TOGETHER;So;0;ON;;;;;N;;;;; +4DC8;HEXAGRAM FOR SMALL TAMING;So;0;ON;;;;;N;;;;; +4DC9;HEXAGRAM FOR TREADING;So;0;ON;;;;;N;;;;; +4DCA;HEXAGRAM FOR PEACE;So;0;ON;;;;;N;;;;; +4DCB;HEXAGRAM FOR STANDSTILL;So;0;ON;;;;;N;;;;; +4DCC;HEXAGRAM FOR FELLOWSHIP;So;0;ON;;;;;N;;;;; +4DCD;HEXAGRAM FOR GREAT POSSESSION;So;0;ON;;;;;N;;;;; +4DCE;HEXAGRAM FOR MODESTY;So;0;ON;;;;;N;;;;; +4DCF;HEXAGRAM FOR ENTHUSIASM;So;0;ON;;;;;N;;;;; +4DD0;HEXAGRAM FOR FOLLOWING;So;0;ON;;;;;N;;;;; +4DD1;HEXAGRAM FOR WORK ON THE DECAYED;So;0;ON;;;;;N;;;;; +4DD2;HEXAGRAM FOR APPROACH;So;0;ON;;;;;N;;;;; +4DD3;HEXAGRAM FOR CONTEMPLATION;So;0;ON;;;;;N;;;;; +4DD4;HEXAGRAM FOR BITING THROUGH;So;0;ON;;;;;N;;;;; +4DD5;HEXAGRAM FOR GRACE;So;0;ON;;;;;N;;;;; +4DD6;HEXAGRAM FOR SPLITTING APART;So;0;ON;;;;;N;;;;; +4DD7;HEXAGRAM FOR RETURN;So;0;ON;;;;;N;;;;; +4DD8;HEXAGRAM FOR INNOCENCE;So;0;ON;;;;;N;;;;; +4DD9;HEXAGRAM FOR GREAT TAMING;So;0;ON;;;;;N;;;;; +4DDA;HEXAGRAM FOR MOUTH CORNERS;So;0;ON;;;;;N;;;;; +4DDB;HEXAGRAM FOR GREAT PREPONDERANCE;So;0;ON;;;;;N;;;;; +4DDC;HEXAGRAM FOR THE ABYSMAL WATER;So;0;ON;;;;;N;;;;; +4DDD;HEXAGRAM FOR THE CLINGING FIRE;So;0;ON;;;;;N;;;;; +4DDE;HEXAGRAM FOR INFLUENCE;So;0;ON;;;;;N;;;;; +4DDF;HEXAGRAM FOR DURATION;So;0;ON;;;;;N;;;;; +4DE0;HEXAGRAM FOR RETREAT;So;0;ON;;;;;N;;;;; +4DE1;HEXAGRAM FOR GREAT POWER;So;0;ON;;;;;N;;;;; +4DE2;HEXAGRAM FOR PROGRESS;So;0;ON;;;;;N;;;;; +4DE3;HEXAGRAM FOR DARKENING OF THE LIGHT;So;0;ON;;;;;N;;;;; +4DE4;HEXAGRAM FOR THE FAMILY;So;0;ON;;;;;N;;;;; +4DE5;HEXAGRAM FOR OPPOSITION;So;0;ON;;;;;N;;;;; +4DE6;HEXAGRAM FOR OBSTRUCTION;So;0;ON;;;;;N;;;;; +4DE7;HEXAGRAM FOR DELIVERANCE;So;0;ON;;;;;N;;;;; +4DE8;HEXAGRAM FOR DECREASE;So;0;ON;;;;;N;;;;; +4DE9;HEXAGRAM FOR INCREASE;So;0;ON;;;;;N;;;;; +4DEA;HEXAGRAM FOR BREAKTHROUGH;So;0;ON;;;;;N;;;;; +4DEB;HEXAGRAM FOR COMING TO MEET;So;0;ON;;;;;N;;;;; +4DEC;HEXAGRAM FOR GATHERING TOGETHER;So;0;ON;;;;;N;;;;; +4DED;HEXAGRAM FOR PUSHING UPWARD;So;0;ON;;;;;N;;;;; +4DEE;HEXAGRAM FOR OPPRESSION;So;0;ON;;;;;N;;;;; +4DEF;HEXAGRAM FOR THE WELL;So;0;ON;;;;;N;;;;; +4DF0;HEXAGRAM FOR REVOLUTION;So;0;ON;;;;;N;;;;; +4DF1;HEXAGRAM FOR THE CAULDRON;So;0;ON;;;;;N;;;;; +4DF2;HEXAGRAM FOR THE AROUSING THUNDER;So;0;ON;;;;;N;;;;; +4DF3;HEXAGRAM FOR THE KEEPING STILL MOUNTAIN;So;0;ON;;;;;N;;;;; +4DF4;HEXAGRAM FOR DEVELOPMENT;So;0;ON;;;;;N;;;;; +4DF5;HEXAGRAM FOR THE MARRYING MAIDEN;So;0;ON;;;;;N;;;;; +4DF6;HEXAGRAM FOR ABUNDANCE;So;0;ON;;;;;N;;;;; +4DF7;HEXAGRAM FOR THE WANDERER;So;0;ON;;;;;N;;;;; +4DF8;HEXAGRAM FOR THE GENTLE WIND;So;0;ON;;;;;N;;;;; +4DF9;HEXAGRAM FOR THE JOYOUS LAKE;So;0;ON;;;;;N;;;;; +4DFA;HEXAGRAM FOR DISPERSION;So;0;ON;;;;;N;;;;; +4DFB;HEXAGRAM FOR LIMITATION;So;0;ON;;;;;N;;;;; +4DFC;HEXAGRAM FOR INNER TRUTH;So;0;ON;;;;;N;;;;; +4DFD;HEXAGRAM FOR SMALL PREPONDERANCE;So;0;ON;;;;;N;;;;; +4DFE;HEXAGRAM FOR AFTER COMPLETION;So;0;ON;;;;;N;;;;; +4DFF;HEXAGRAM FOR BEFORE COMPLETION;So;0;ON;;;;;N;;;;; +4E00;;Lo;0;L;;;;;N;;;;; +9FCC;;Lo;0;L;;;;;N;;;;; +A000;YI SYLLABLE IT;Lo;0;L;;;;;N;;;;; +A001;YI SYLLABLE IX;Lo;0;L;;;;;N;;;;; +A002;YI SYLLABLE I;Lo;0;L;;;;;N;;;;; +A003;YI SYLLABLE IP;Lo;0;L;;;;;N;;;;; +A004;YI SYLLABLE IET;Lo;0;L;;;;;N;;;;; +A005;YI SYLLABLE IEX;Lo;0;L;;;;;N;;;;; +A006;YI SYLLABLE IE;Lo;0;L;;;;;N;;;;; +A007;YI SYLLABLE IEP;Lo;0;L;;;;;N;;;;; +A008;YI SYLLABLE AT;Lo;0;L;;;;;N;;;;; +A009;YI SYLLABLE AX;Lo;0;L;;;;;N;;;;; +A00A;YI SYLLABLE A;Lo;0;L;;;;;N;;;;; +A00B;YI SYLLABLE AP;Lo;0;L;;;;;N;;;;; +A00C;YI SYLLABLE UOX;Lo;0;L;;;;;N;;;;; +A00D;YI SYLLABLE UO;Lo;0;L;;;;;N;;;;; +A00E;YI SYLLABLE UOP;Lo;0;L;;;;;N;;;;; +A00F;YI SYLLABLE OT;Lo;0;L;;;;;N;;;;; +A010;YI SYLLABLE OX;Lo;0;L;;;;;N;;;;; +A011;YI SYLLABLE O;Lo;0;L;;;;;N;;;;; +A012;YI SYLLABLE OP;Lo;0;L;;;;;N;;;;; +A013;YI SYLLABLE EX;Lo;0;L;;;;;N;;;;; +A014;YI SYLLABLE E;Lo;0;L;;;;;N;;;;; +A015;YI SYLLABLE WU;Lm;0;L;;;;;N;;;;; +A016;YI SYLLABLE BIT;Lo;0;L;;;;;N;;;;; +A017;YI SYLLABLE BIX;Lo;0;L;;;;;N;;;;; +A018;YI SYLLABLE BI;Lo;0;L;;;;;N;;;;; +A019;YI SYLLABLE BIP;Lo;0;L;;;;;N;;;;; +A01A;YI SYLLABLE BIET;Lo;0;L;;;;;N;;;;; +A01B;YI SYLLABLE BIEX;Lo;0;L;;;;;N;;;;; +A01C;YI SYLLABLE BIE;Lo;0;L;;;;;N;;;;; +A01D;YI SYLLABLE BIEP;Lo;0;L;;;;;N;;;;; +A01E;YI SYLLABLE BAT;Lo;0;L;;;;;N;;;;; +A01F;YI SYLLABLE BAX;Lo;0;L;;;;;N;;;;; +A020;YI SYLLABLE BA;Lo;0;L;;;;;N;;;;; +A021;YI SYLLABLE BAP;Lo;0;L;;;;;N;;;;; +A022;YI SYLLABLE BUOX;Lo;0;L;;;;;N;;;;; +A023;YI SYLLABLE BUO;Lo;0;L;;;;;N;;;;; +A024;YI SYLLABLE BUOP;Lo;0;L;;;;;N;;;;; +A025;YI SYLLABLE BOT;Lo;0;L;;;;;N;;;;; +A026;YI SYLLABLE BOX;Lo;0;L;;;;;N;;;;; +A027;YI SYLLABLE BO;Lo;0;L;;;;;N;;;;; +A028;YI SYLLABLE BOP;Lo;0;L;;;;;N;;;;; +A029;YI SYLLABLE BEX;Lo;0;L;;;;;N;;;;; +A02A;YI SYLLABLE BE;Lo;0;L;;;;;N;;;;; +A02B;YI SYLLABLE BEP;Lo;0;L;;;;;N;;;;; +A02C;YI SYLLABLE BUT;Lo;0;L;;;;;N;;;;; +A02D;YI SYLLABLE BUX;Lo;0;L;;;;;N;;;;; +A02E;YI SYLLABLE BU;Lo;0;L;;;;;N;;;;; +A02F;YI SYLLABLE BUP;Lo;0;L;;;;;N;;;;; +A030;YI SYLLABLE BURX;Lo;0;L;;;;;N;;;;; +A031;YI SYLLABLE BUR;Lo;0;L;;;;;N;;;;; +A032;YI SYLLABLE BYT;Lo;0;L;;;;;N;;;;; +A033;YI SYLLABLE BYX;Lo;0;L;;;;;N;;;;; +A034;YI SYLLABLE BY;Lo;0;L;;;;;N;;;;; +A035;YI SYLLABLE BYP;Lo;0;L;;;;;N;;;;; +A036;YI SYLLABLE BYRX;Lo;0;L;;;;;N;;;;; +A037;YI SYLLABLE BYR;Lo;0;L;;;;;N;;;;; +A038;YI SYLLABLE PIT;Lo;0;L;;;;;N;;;;; +A039;YI SYLLABLE PIX;Lo;0;L;;;;;N;;;;; +A03A;YI SYLLABLE PI;Lo;0;L;;;;;N;;;;; +A03B;YI SYLLABLE PIP;Lo;0;L;;;;;N;;;;; +A03C;YI SYLLABLE PIEX;Lo;0;L;;;;;N;;;;; +A03D;YI SYLLABLE PIE;Lo;0;L;;;;;N;;;;; +A03E;YI SYLLABLE PIEP;Lo;0;L;;;;;N;;;;; +A03F;YI SYLLABLE PAT;Lo;0;L;;;;;N;;;;; +A040;YI SYLLABLE PAX;Lo;0;L;;;;;N;;;;; +A041;YI SYLLABLE PA;Lo;0;L;;;;;N;;;;; +A042;YI SYLLABLE PAP;Lo;0;L;;;;;N;;;;; +A043;YI SYLLABLE PUOX;Lo;0;L;;;;;N;;;;; +A044;YI SYLLABLE PUO;Lo;0;L;;;;;N;;;;; +A045;YI SYLLABLE PUOP;Lo;0;L;;;;;N;;;;; +A046;YI SYLLABLE POT;Lo;0;L;;;;;N;;;;; +A047;YI SYLLABLE POX;Lo;0;L;;;;;N;;;;; +A048;YI SYLLABLE PO;Lo;0;L;;;;;N;;;;; +A049;YI SYLLABLE POP;Lo;0;L;;;;;N;;;;; +A04A;YI SYLLABLE PUT;Lo;0;L;;;;;N;;;;; +A04B;YI SYLLABLE PUX;Lo;0;L;;;;;N;;;;; +A04C;YI SYLLABLE PU;Lo;0;L;;;;;N;;;;; +A04D;YI SYLLABLE PUP;Lo;0;L;;;;;N;;;;; +A04E;YI SYLLABLE PURX;Lo;0;L;;;;;N;;;;; +A04F;YI SYLLABLE PUR;Lo;0;L;;;;;N;;;;; +A050;YI SYLLABLE PYT;Lo;0;L;;;;;N;;;;; +A051;YI SYLLABLE PYX;Lo;0;L;;;;;N;;;;; +A052;YI SYLLABLE PY;Lo;0;L;;;;;N;;;;; +A053;YI SYLLABLE PYP;Lo;0;L;;;;;N;;;;; +A054;YI SYLLABLE PYRX;Lo;0;L;;;;;N;;;;; +A055;YI SYLLABLE PYR;Lo;0;L;;;;;N;;;;; +A056;YI SYLLABLE BBIT;Lo;0;L;;;;;N;;;;; +A057;YI SYLLABLE BBIX;Lo;0;L;;;;;N;;;;; +A058;YI SYLLABLE BBI;Lo;0;L;;;;;N;;;;; +A059;YI SYLLABLE BBIP;Lo;0;L;;;;;N;;;;; +A05A;YI SYLLABLE BBIET;Lo;0;L;;;;;N;;;;; +A05B;YI SYLLABLE BBIEX;Lo;0;L;;;;;N;;;;; +A05C;YI SYLLABLE BBIE;Lo;0;L;;;;;N;;;;; +A05D;YI SYLLABLE BBIEP;Lo;0;L;;;;;N;;;;; +A05E;YI SYLLABLE BBAT;Lo;0;L;;;;;N;;;;; +A05F;YI SYLLABLE BBAX;Lo;0;L;;;;;N;;;;; +A060;YI SYLLABLE BBA;Lo;0;L;;;;;N;;;;; +A061;YI SYLLABLE BBAP;Lo;0;L;;;;;N;;;;; +A062;YI SYLLABLE BBUOX;Lo;0;L;;;;;N;;;;; +A063;YI SYLLABLE BBUO;Lo;0;L;;;;;N;;;;; +A064;YI SYLLABLE BBUOP;Lo;0;L;;;;;N;;;;; +A065;YI SYLLABLE BBOT;Lo;0;L;;;;;N;;;;; +A066;YI SYLLABLE BBOX;Lo;0;L;;;;;N;;;;; +A067;YI SYLLABLE BBO;Lo;0;L;;;;;N;;;;; +A068;YI SYLLABLE BBOP;Lo;0;L;;;;;N;;;;; +A069;YI SYLLABLE BBEX;Lo;0;L;;;;;N;;;;; +A06A;YI SYLLABLE BBE;Lo;0;L;;;;;N;;;;; +A06B;YI SYLLABLE BBEP;Lo;0;L;;;;;N;;;;; +A06C;YI SYLLABLE BBUT;Lo;0;L;;;;;N;;;;; +A06D;YI SYLLABLE BBUX;Lo;0;L;;;;;N;;;;; +A06E;YI SYLLABLE BBU;Lo;0;L;;;;;N;;;;; +A06F;YI SYLLABLE BBUP;Lo;0;L;;;;;N;;;;; +A070;YI SYLLABLE BBURX;Lo;0;L;;;;;N;;;;; +A071;YI SYLLABLE BBUR;Lo;0;L;;;;;N;;;;; +A072;YI SYLLABLE BBYT;Lo;0;L;;;;;N;;;;; +A073;YI SYLLABLE BBYX;Lo;0;L;;;;;N;;;;; +A074;YI SYLLABLE BBY;Lo;0;L;;;;;N;;;;; +A075;YI SYLLABLE BBYP;Lo;0;L;;;;;N;;;;; +A076;YI SYLLABLE NBIT;Lo;0;L;;;;;N;;;;; +A077;YI SYLLABLE NBIX;Lo;0;L;;;;;N;;;;; +A078;YI SYLLABLE NBI;Lo;0;L;;;;;N;;;;; +A079;YI SYLLABLE NBIP;Lo;0;L;;;;;N;;;;; +A07A;YI SYLLABLE NBIEX;Lo;0;L;;;;;N;;;;; +A07B;YI SYLLABLE NBIE;Lo;0;L;;;;;N;;;;; +A07C;YI SYLLABLE NBIEP;Lo;0;L;;;;;N;;;;; +A07D;YI SYLLABLE NBAT;Lo;0;L;;;;;N;;;;; +A07E;YI SYLLABLE NBAX;Lo;0;L;;;;;N;;;;; +A07F;YI SYLLABLE NBA;Lo;0;L;;;;;N;;;;; +A080;YI SYLLABLE NBAP;Lo;0;L;;;;;N;;;;; +A081;YI SYLLABLE NBOT;Lo;0;L;;;;;N;;;;; +A082;YI SYLLABLE NBOX;Lo;0;L;;;;;N;;;;; +A083;YI SYLLABLE NBO;Lo;0;L;;;;;N;;;;; +A084;YI SYLLABLE NBOP;Lo;0;L;;;;;N;;;;; +A085;YI SYLLABLE NBUT;Lo;0;L;;;;;N;;;;; +A086;YI SYLLABLE NBUX;Lo;0;L;;;;;N;;;;; +A087;YI SYLLABLE NBU;Lo;0;L;;;;;N;;;;; +A088;YI SYLLABLE NBUP;Lo;0;L;;;;;N;;;;; +A089;YI SYLLABLE NBURX;Lo;0;L;;;;;N;;;;; +A08A;YI SYLLABLE NBUR;Lo;0;L;;;;;N;;;;; +A08B;YI SYLLABLE NBYT;Lo;0;L;;;;;N;;;;; +A08C;YI SYLLABLE NBYX;Lo;0;L;;;;;N;;;;; +A08D;YI SYLLABLE NBY;Lo;0;L;;;;;N;;;;; +A08E;YI SYLLABLE NBYP;Lo;0;L;;;;;N;;;;; +A08F;YI SYLLABLE NBYRX;Lo;0;L;;;;;N;;;;; +A090;YI SYLLABLE NBYR;Lo;0;L;;;;;N;;;;; +A091;YI SYLLABLE HMIT;Lo;0;L;;;;;N;;;;; +A092;YI SYLLABLE HMIX;Lo;0;L;;;;;N;;;;; +A093;YI SYLLABLE HMI;Lo;0;L;;;;;N;;;;; +A094;YI SYLLABLE HMIP;Lo;0;L;;;;;N;;;;; +A095;YI SYLLABLE HMIEX;Lo;0;L;;;;;N;;;;; +A096;YI SYLLABLE HMIE;Lo;0;L;;;;;N;;;;; +A097;YI SYLLABLE HMIEP;Lo;0;L;;;;;N;;;;; +A098;YI SYLLABLE HMAT;Lo;0;L;;;;;N;;;;; +A099;YI SYLLABLE HMAX;Lo;0;L;;;;;N;;;;; +A09A;YI SYLLABLE HMA;Lo;0;L;;;;;N;;;;; +A09B;YI SYLLABLE HMAP;Lo;0;L;;;;;N;;;;; +A09C;YI SYLLABLE HMUOX;Lo;0;L;;;;;N;;;;; +A09D;YI SYLLABLE HMUO;Lo;0;L;;;;;N;;;;; +A09E;YI SYLLABLE HMUOP;Lo;0;L;;;;;N;;;;; +A09F;YI SYLLABLE HMOT;Lo;0;L;;;;;N;;;;; +A0A0;YI SYLLABLE HMOX;Lo;0;L;;;;;N;;;;; +A0A1;YI SYLLABLE HMO;Lo;0;L;;;;;N;;;;; +A0A2;YI SYLLABLE HMOP;Lo;0;L;;;;;N;;;;; +A0A3;YI SYLLABLE HMUT;Lo;0;L;;;;;N;;;;; +A0A4;YI SYLLABLE HMUX;Lo;0;L;;;;;N;;;;; +A0A5;YI SYLLABLE HMU;Lo;0;L;;;;;N;;;;; +A0A6;YI SYLLABLE HMUP;Lo;0;L;;;;;N;;;;; +A0A7;YI SYLLABLE HMURX;Lo;0;L;;;;;N;;;;; +A0A8;YI SYLLABLE HMUR;Lo;0;L;;;;;N;;;;; +A0A9;YI SYLLABLE HMYX;Lo;0;L;;;;;N;;;;; +A0AA;YI SYLLABLE HMY;Lo;0;L;;;;;N;;;;; +A0AB;YI SYLLABLE HMYP;Lo;0;L;;;;;N;;;;; +A0AC;YI SYLLABLE HMYRX;Lo;0;L;;;;;N;;;;; +A0AD;YI SYLLABLE HMYR;Lo;0;L;;;;;N;;;;; +A0AE;YI SYLLABLE MIT;Lo;0;L;;;;;N;;;;; +A0AF;YI SYLLABLE MIX;Lo;0;L;;;;;N;;;;; +A0B0;YI SYLLABLE MI;Lo;0;L;;;;;N;;;;; +A0B1;YI SYLLABLE MIP;Lo;0;L;;;;;N;;;;; +A0B2;YI SYLLABLE MIEX;Lo;0;L;;;;;N;;;;; +A0B3;YI SYLLABLE MIE;Lo;0;L;;;;;N;;;;; +A0B4;YI SYLLABLE MIEP;Lo;0;L;;;;;N;;;;; +A0B5;YI SYLLABLE MAT;Lo;0;L;;;;;N;;;;; +A0B6;YI SYLLABLE MAX;Lo;0;L;;;;;N;;;;; +A0B7;YI SYLLABLE MA;Lo;0;L;;;;;N;;;;; +A0B8;YI SYLLABLE MAP;Lo;0;L;;;;;N;;;;; +A0B9;YI SYLLABLE MUOT;Lo;0;L;;;;;N;;;;; +A0BA;YI SYLLABLE MUOX;Lo;0;L;;;;;N;;;;; +A0BB;YI SYLLABLE MUO;Lo;0;L;;;;;N;;;;; +A0BC;YI SYLLABLE MUOP;Lo;0;L;;;;;N;;;;; +A0BD;YI SYLLABLE MOT;Lo;0;L;;;;;N;;;;; +A0BE;YI SYLLABLE MOX;Lo;0;L;;;;;N;;;;; +A0BF;YI SYLLABLE MO;Lo;0;L;;;;;N;;;;; +A0C0;YI SYLLABLE MOP;Lo;0;L;;;;;N;;;;; +A0C1;YI SYLLABLE MEX;Lo;0;L;;;;;N;;;;; +A0C2;YI SYLLABLE ME;Lo;0;L;;;;;N;;;;; +A0C3;YI SYLLABLE MUT;Lo;0;L;;;;;N;;;;; +A0C4;YI SYLLABLE MUX;Lo;0;L;;;;;N;;;;; +A0C5;YI SYLLABLE MU;Lo;0;L;;;;;N;;;;; +A0C6;YI SYLLABLE MUP;Lo;0;L;;;;;N;;;;; +A0C7;YI SYLLABLE MURX;Lo;0;L;;;;;N;;;;; +A0C8;YI SYLLABLE MUR;Lo;0;L;;;;;N;;;;; +A0C9;YI SYLLABLE MYT;Lo;0;L;;;;;N;;;;; +A0CA;YI SYLLABLE MYX;Lo;0;L;;;;;N;;;;; +A0CB;YI SYLLABLE MY;Lo;0;L;;;;;N;;;;; +A0CC;YI SYLLABLE MYP;Lo;0;L;;;;;N;;;;; +A0CD;YI SYLLABLE FIT;Lo;0;L;;;;;N;;;;; +A0CE;YI SYLLABLE FIX;Lo;0;L;;;;;N;;;;; +A0CF;YI SYLLABLE FI;Lo;0;L;;;;;N;;;;; +A0D0;YI SYLLABLE FIP;Lo;0;L;;;;;N;;;;; +A0D1;YI SYLLABLE FAT;Lo;0;L;;;;;N;;;;; +A0D2;YI SYLLABLE FAX;Lo;0;L;;;;;N;;;;; +A0D3;YI SYLLABLE FA;Lo;0;L;;;;;N;;;;; +A0D4;YI SYLLABLE FAP;Lo;0;L;;;;;N;;;;; +A0D5;YI SYLLABLE FOX;Lo;0;L;;;;;N;;;;; +A0D6;YI SYLLABLE FO;Lo;0;L;;;;;N;;;;; +A0D7;YI SYLLABLE FOP;Lo;0;L;;;;;N;;;;; +A0D8;YI SYLLABLE FUT;Lo;0;L;;;;;N;;;;; +A0D9;YI SYLLABLE FUX;Lo;0;L;;;;;N;;;;; +A0DA;YI SYLLABLE FU;Lo;0;L;;;;;N;;;;; +A0DB;YI SYLLABLE FUP;Lo;0;L;;;;;N;;;;; +A0DC;YI SYLLABLE FURX;Lo;0;L;;;;;N;;;;; +A0DD;YI SYLLABLE FUR;Lo;0;L;;;;;N;;;;; +A0DE;YI SYLLABLE FYT;Lo;0;L;;;;;N;;;;; +A0DF;YI SYLLABLE FYX;Lo;0;L;;;;;N;;;;; +A0E0;YI SYLLABLE FY;Lo;0;L;;;;;N;;;;; +A0E1;YI SYLLABLE FYP;Lo;0;L;;;;;N;;;;; +A0E2;YI SYLLABLE VIT;Lo;0;L;;;;;N;;;;; +A0E3;YI SYLLABLE VIX;Lo;0;L;;;;;N;;;;; +A0E4;YI SYLLABLE VI;Lo;0;L;;;;;N;;;;; +A0E5;YI SYLLABLE VIP;Lo;0;L;;;;;N;;;;; +A0E6;YI SYLLABLE VIET;Lo;0;L;;;;;N;;;;; +A0E7;YI SYLLABLE VIEX;Lo;0;L;;;;;N;;;;; +A0E8;YI SYLLABLE VIE;Lo;0;L;;;;;N;;;;; +A0E9;YI SYLLABLE VIEP;Lo;0;L;;;;;N;;;;; +A0EA;YI SYLLABLE VAT;Lo;0;L;;;;;N;;;;; +A0EB;YI SYLLABLE VAX;Lo;0;L;;;;;N;;;;; +A0EC;YI SYLLABLE VA;Lo;0;L;;;;;N;;;;; +A0ED;YI SYLLABLE VAP;Lo;0;L;;;;;N;;;;; +A0EE;YI SYLLABLE VOT;Lo;0;L;;;;;N;;;;; +A0EF;YI SYLLABLE VOX;Lo;0;L;;;;;N;;;;; +A0F0;YI SYLLABLE VO;Lo;0;L;;;;;N;;;;; +A0F1;YI SYLLABLE VOP;Lo;0;L;;;;;N;;;;; +A0F2;YI SYLLABLE VEX;Lo;0;L;;;;;N;;;;; +A0F3;YI SYLLABLE VEP;Lo;0;L;;;;;N;;;;; +A0F4;YI SYLLABLE VUT;Lo;0;L;;;;;N;;;;; +A0F5;YI SYLLABLE VUX;Lo;0;L;;;;;N;;;;; +A0F6;YI SYLLABLE VU;Lo;0;L;;;;;N;;;;; +A0F7;YI SYLLABLE VUP;Lo;0;L;;;;;N;;;;; +A0F8;YI SYLLABLE VURX;Lo;0;L;;;;;N;;;;; +A0F9;YI SYLLABLE VUR;Lo;0;L;;;;;N;;;;; +A0FA;YI SYLLABLE VYT;Lo;0;L;;;;;N;;;;; +A0FB;YI SYLLABLE VYX;Lo;0;L;;;;;N;;;;; +A0FC;YI SYLLABLE VY;Lo;0;L;;;;;N;;;;; +A0FD;YI SYLLABLE VYP;Lo;0;L;;;;;N;;;;; +A0FE;YI SYLLABLE VYRX;Lo;0;L;;;;;N;;;;; +A0FF;YI SYLLABLE VYR;Lo;0;L;;;;;N;;;;; +A100;YI SYLLABLE DIT;Lo;0;L;;;;;N;;;;; +A101;YI SYLLABLE DIX;Lo;0;L;;;;;N;;;;; +A102;YI SYLLABLE DI;Lo;0;L;;;;;N;;;;; +A103;YI SYLLABLE DIP;Lo;0;L;;;;;N;;;;; +A104;YI SYLLABLE DIEX;Lo;0;L;;;;;N;;;;; +A105;YI SYLLABLE DIE;Lo;0;L;;;;;N;;;;; +A106;YI SYLLABLE DIEP;Lo;0;L;;;;;N;;;;; +A107;YI SYLLABLE DAT;Lo;0;L;;;;;N;;;;; +A108;YI SYLLABLE DAX;Lo;0;L;;;;;N;;;;; +A109;YI SYLLABLE DA;Lo;0;L;;;;;N;;;;; +A10A;YI SYLLABLE DAP;Lo;0;L;;;;;N;;;;; +A10B;YI SYLLABLE DUOX;Lo;0;L;;;;;N;;;;; +A10C;YI SYLLABLE DUO;Lo;0;L;;;;;N;;;;; +A10D;YI SYLLABLE DOT;Lo;0;L;;;;;N;;;;; +A10E;YI SYLLABLE DOX;Lo;0;L;;;;;N;;;;; +A10F;YI SYLLABLE DO;Lo;0;L;;;;;N;;;;; +A110;YI SYLLABLE DOP;Lo;0;L;;;;;N;;;;; +A111;YI SYLLABLE DEX;Lo;0;L;;;;;N;;;;; +A112;YI SYLLABLE DE;Lo;0;L;;;;;N;;;;; +A113;YI SYLLABLE DEP;Lo;0;L;;;;;N;;;;; +A114;YI SYLLABLE DUT;Lo;0;L;;;;;N;;;;; +A115;YI SYLLABLE DUX;Lo;0;L;;;;;N;;;;; +A116;YI SYLLABLE DU;Lo;0;L;;;;;N;;;;; +A117;YI SYLLABLE DUP;Lo;0;L;;;;;N;;;;; +A118;YI SYLLABLE DURX;Lo;0;L;;;;;N;;;;; +A119;YI SYLLABLE DUR;Lo;0;L;;;;;N;;;;; +A11A;YI SYLLABLE TIT;Lo;0;L;;;;;N;;;;; +A11B;YI SYLLABLE TIX;Lo;0;L;;;;;N;;;;; +A11C;YI SYLLABLE TI;Lo;0;L;;;;;N;;;;; +A11D;YI SYLLABLE TIP;Lo;0;L;;;;;N;;;;; +A11E;YI SYLLABLE TIEX;Lo;0;L;;;;;N;;;;; +A11F;YI SYLLABLE TIE;Lo;0;L;;;;;N;;;;; +A120;YI SYLLABLE TIEP;Lo;0;L;;;;;N;;;;; +A121;YI SYLLABLE TAT;Lo;0;L;;;;;N;;;;; +A122;YI SYLLABLE TAX;Lo;0;L;;;;;N;;;;; +A123;YI SYLLABLE TA;Lo;0;L;;;;;N;;;;; +A124;YI SYLLABLE TAP;Lo;0;L;;;;;N;;;;; +A125;YI SYLLABLE TUOT;Lo;0;L;;;;;N;;;;; +A126;YI SYLLABLE TUOX;Lo;0;L;;;;;N;;;;; +A127;YI SYLLABLE TUO;Lo;0;L;;;;;N;;;;; +A128;YI SYLLABLE TUOP;Lo;0;L;;;;;N;;;;; +A129;YI SYLLABLE TOT;Lo;0;L;;;;;N;;;;; +A12A;YI SYLLABLE TOX;Lo;0;L;;;;;N;;;;; +A12B;YI SYLLABLE TO;Lo;0;L;;;;;N;;;;; +A12C;YI SYLLABLE TOP;Lo;0;L;;;;;N;;;;; +A12D;YI SYLLABLE TEX;Lo;0;L;;;;;N;;;;; +A12E;YI SYLLABLE TE;Lo;0;L;;;;;N;;;;; +A12F;YI SYLLABLE TEP;Lo;0;L;;;;;N;;;;; +A130;YI SYLLABLE TUT;Lo;0;L;;;;;N;;;;; +A131;YI SYLLABLE TUX;Lo;0;L;;;;;N;;;;; +A132;YI SYLLABLE TU;Lo;0;L;;;;;N;;;;; +A133;YI SYLLABLE TUP;Lo;0;L;;;;;N;;;;; +A134;YI SYLLABLE TURX;Lo;0;L;;;;;N;;;;; +A135;YI SYLLABLE TUR;Lo;0;L;;;;;N;;;;; +A136;YI SYLLABLE DDIT;Lo;0;L;;;;;N;;;;; +A137;YI SYLLABLE DDIX;Lo;0;L;;;;;N;;;;; +A138;YI SYLLABLE DDI;Lo;0;L;;;;;N;;;;; +A139;YI SYLLABLE DDIP;Lo;0;L;;;;;N;;;;; +A13A;YI SYLLABLE DDIEX;Lo;0;L;;;;;N;;;;; +A13B;YI SYLLABLE DDIE;Lo;0;L;;;;;N;;;;; +A13C;YI SYLLABLE DDIEP;Lo;0;L;;;;;N;;;;; +A13D;YI SYLLABLE DDAT;Lo;0;L;;;;;N;;;;; +A13E;YI SYLLABLE DDAX;Lo;0;L;;;;;N;;;;; +A13F;YI SYLLABLE DDA;Lo;0;L;;;;;N;;;;; +A140;YI SYLLABLE DDAP;Lo;0;L;;;;;N;;;;; +A141;YI SYLLABLE DDUOX;Lo;0;L;;;;;N;;;;; +A142;YI SYLLABLE DDUO;Lo;0;L;;;;;N;;;;; +A143;YI SYLLABLE DDUOP;Lo;0;L;;;;;N;;;;; +A144;YI SYLLABLE DDOT;Lo;0;L;;;;;N;;;;; +A145;YI SYLLABLE DDOX;Lo;0;L;;;;;N;;;;; +A146;YI SYLLABLE DDO;Lo;0;L;;;;;N;;;;; +A147;YI SYLLABLE DDOP;Lo;0;L;;;;;N;;;;; +A148;YI SYLLABLE DDEX;Lo;0;L;;;;;N;;;;; +A149;YI SYLLABLE DDE;Lo;0;L;;;;;N;;;;; +A14A;YI SYLLABLE DDEP;Lo;0;L;;;;;N;;;;; +A14B;YI SYLLABLE DDUT;Lo;0;L;;;;;N;;;;; +A14C;YI SYLLABLE DDUX;Lo;0;L;;;;;N;;;;; +A14D;YI SYLLABLE DDU;Lo;0;L;;;;;N;;;;; +A14E;YI SYLLABLE DDUP;Lo;0;L;;;;;N;;;;; +A14F;YI SYLLABLE DDURX;Lo;0;L;;;;;N;;;;; +A150;YI SYLLABLE DDUR;Lo;0;L;;;;;N;;;;; +A151;YI SYLLABLE NDIT;Lo;0;L;;;;;N;;;;; +A152;YI SYLLABLE NDIX;Lo;0;L;;;;;N;;;;; +A153;YI SYLLABLE NDI;Lo;0;L;;;;;N;;;;; +A154;YI SYLLABLE NDIP;Lo;0;L;;;;;N;;;;; +A155;YI SYLLABLE NDIEX;Lo;0;L;;;;;N;;;;; +A156;YI SYLLABLE NDIE;Lo;0;L;;;;;N;;;;; +A157;YI SYLLABLE NDAT;Lo;0;L;;;;;N;;;;; +A158;YI SYLLABLE NDAX;Lo;0;L;;;;;N;;;;; +A159;YI SYLLABLE NDA;Lo;0;L;;;;;N;;;;; +A15A;YI SYLLABLE NDAP;Lo;0;L;;;;;N;;;;; +A15B;YI SYLLABLE NDOT;Lo;0;L;;;;;N;;;;; +A15C;YI SYLLABLE NDOX;Lo;0;L;;;;;N;;;;; +A15D;YI SYLLABLE NDO;Lo;0;L;;;;;N;;;;; +A15E;YI SYLLABLE NDOP;Lo;0;L;;;;;N;;;;; +A15F;YI SYLLABLE NDEX;Lo;0;L;;;;;N;;;;; +A160;YI SYLLABLE NDE;Lo;0;L;;;;;N;;;;; +A161;YI SYLLABLE NDEP;Lo;0;L;;;;;N;;;;; +A162;YI SYLLABLE NDUT;Lo;0;L;;;;;N;;;;; +A163;YI SYLLABLE NDUX;Lo;0;L;;;;;N;;;;; +A164;YI SYLLABLE NDU;Lo;0;L;;;;;N;;;;; +A165;YI SYLLABLE NDUP;Lo;0;L;;;;;N;;;;; +A166;YI SYLLABLE NDURX;Lo;0;L;;;;;N;;;;; +A167;YI SYLLABLE NDUR;Lo;0;L;;;;;N;;;;; +A168;YI SYLLABLE HNIT;Lo;0;L;;;;;N;;;;; +A169;YI SYLLABLE HNIX;Lo;0;L;;;;;N;;;;; +A16A;YI SYLLABLE HNI;Lo;0;L;;;;;N;;;;; +A16B;YI SYLLABLE HNIP;Lo;0;L;;;;;N;;;;; +A16C;YI SYLLABLE HNIET;Lo;0;L;;;;;N;;;;; +A16D;YI SYLLABLE HNIEX;Lo;0;L;;;;;N;;;;; +A16E;YI SYLLABLE HNIE;Lo;0;L;;;;;N;;;;; +A16F;YI SYLLABLE HNIEP;Lo;0;L;;;;;N;;;;; +A170;YI SYLLABLE HNAT;Lo;0;L;;;;;N;;;;; +A171;YI SYLLABLE HNAX;Lo;0;L;;;;;N;;;;; +A172;YI SYLLABLE HNA;Lo;0;L;;;;;N;;;;; +A173;YI SYLLABLE HNAP;Lo;0;L;;;;;N;;;;; +A174;YI SYLLABLE HNUOX;Lo;0;L;;;;;N;;;;; +A175;YI SYLLABLE HNUO;Lo;0;L;;;;;N;;;;; +A176;YI SYLLABLE HNOT;Lo;0;L;;;;;N;;;;; +A177;YI SYLLABLE HNOX;Lo;0;L;;;;;N;;;;; +A178;YI SYLLABLE HNOP;Lo;0;L;;;;;N;;;;; +A179;YI SYLLABLE HNEX;Lo;0;L;;;;;N;;;;; +A17A;YI SYLLABLE HNE;Lo;0;L;;;;;N;;;;; +A17B;YI SYLLABLE HNEP;Lo;0;L;;;;;N;;;;; +A17C;YI SYLLABLE HNUT;Lo;0;L;;;;;N;;;;; +A17D;YI SYLLABLE NIT;Lo;0;L;;;;;N;;;;; +A17E;YI SYLLABLE NIX;Lo;0;L;;;;;N;;;;; +A17F;YI SYLLABLE NI;Lo;0;L;;;;;N;;;;; +A180;YI SYLLABLE NIP;Lo;0;L;;;;;N;;;;; +A181;YI SYLLABLE NIEX;Lo;0;L;;;;;N;;;;; +A182;YI SYLLABLE NIE;Lo;0;L;;;;;N;;;;; +A183;YI SYLLABLE NIEP;Lo;0;L;;;;;N;;;;; +A184;YI SYLLABLE NAX;Lo;0;L;;;;;N;;;;; +A185;YI SYLLABLE NA;Lo;0;L;;;;;N;;;;; +A186;YI SYLLABLE NAP;Lo;0;L;;;;;N;;;;; +A187;YI SYLLABLE NUOX;Lo;0;L;;;;;N;;;;; +A188;YI SYLLABLE NUO;Lo;0;L;;;;;N;;;;; +A189;YI SYLLABLE NUOP;Lo;0;L;;;;;N;;;;; +A18A;YI SYLLABLE NOT;Lo;0;L;;;;;N;;;;; +A18B;YI SYLLABLE NOX;Lo;0;L;;;;;N;;;;; +A18C;YI SYLLABLE NO;Lo;0;L;;;;;N;;;;; +A18D;YI SYLLABLE NOP;Lo;0;L;;;;;N;;;;; +A18E;YI SYLLABLE NEX;Lo;0;L;;;;;N;;;;; +A18F;YI SYLLABLE NE;Lo;0;L;;;;;N;;;;; +A190;YI SYLLABLE NEP;Lo;0;L;;;;;N;;;;; +A191;YI SYLLABLE NUT;Lo;0;L;;;;;N;;;;; +A192;YI SYLLABLE NUX;Lo;0;L;;;;;N;;;;; +A193;YI SYLLABLE NU;Lo;0;L;;;;;N;;;;; +A194;YI SYLLABLE NUP;Lo;0;L;;;;;N;;;;; +A195;YI SYLLABLE NURX;Lo;0;L;;;;;N;;;;; +A196;YI SYLLABLE NUR;Lo;0;L;;;;;N;;;;; +A197;YI SYLLABLE HLIT;Lo;0;L;;;;;N;;;;; +A198;YI SYLLABLE HLIX;Lo;0;L;;;;;N;;;;; +A199;YI SYLLABLE HLI;Lo;0;L;;;;;N;;;;; +A19A;YI SYLLABLE HLIP;Lo;0;L;;;;;N;;;;; +A19B;YI SYLLABLE HLIEX;Lo;0;L;;;;;N;;;;; +A19C;YI SYLLABLE HLIE;Lo;0;L;;;;;N;;;;; +A19D;YI SYLLABLE HLIEP;Lo;0;L;;;;;N;;;;; +A19E;YI SYLLABLE HLAT;Lo;0;L;;;;;N;;;;; +A19F;YI SYLLABLE HLAX;Lo;0;L;;;;;N;;;;; +A1A0;YI SYLLABLE HLA;Lo;0;L;;;;;N;;;;; +A1A1;YI SYLLABLE HLAP;Lo;0;L;;;;;N;;;;; +A1A2;YI SYLLABLE HLUOX;Lo;0;L;;;;;N;;;;; +A1A3;YI SYLLABLE HLUO;Lo;0;L;;;;;N;;;;; +A1A4;YI SYLLABLE HLUOP;Lo;0;L;;;;;N;;;;; +A1A5;YI SYLLABLE HLOX;Lo;0;L;;;;;N;;;;; +A1A6;YI SYLLABLE HLO;Lo;0;L;;;;;N;;;;; +A1A7;YI SYLLABLE HLOP;Lo;0;L;;;;;N;;;;; +A1A8;YI SYLLABLE HLEX;Lo;0;L;;;;;N;;;;; +A1A9;YI SYLLABLE HLE;Lo;0;L;;;;;N;;;;; +A1AA;YI SYLLABLE HLEP;Lo;0;L;;;;;N;;;;; +A1AB;YI SYLLABLE HLUT;Lo;0;L;;;;;N;;;;; +A1AC;YI SYLLABLE HLUX;Lo;0;L;;;;;N;;;;; +A1AD;YI SYLLABLE HLU;Lo;0;L;;;;;N;;;;; +A1AE;YI SYLLABLE HLUP;Lo;0;L;;;;;N;;;;; +A1AF;YI SYLLABLE HLURX;Lo;0;L;;;;;N;;;;; +A1B0;YI SYLLABLE HLUR;Lo;0;L;;;;;N;;;;; +A1B1;YI SYLLABLE HLYT;Lo;0;L;;;;;N;;;;; +A1B2;YI SYLLABLE HLYX;Lo;0;L;;;;;N;;;;; +A1B3;YI SYLLABLE HLY;Lo;0;L;;;;;N;;;;; +A1B4;YI SYLLABLE HLYP;Lo;0;L;;;;;N;;;;; +A1B5;YI SYLLABLE HLYRX;Lo;0;L;;;;;N;;;;; +A1B6;YI SYLLABLE HLYR;Lo;0;L;;;;;N;;;;; +A1B7;YI SYLLABLE LIT;Lo;0;L;;;;;N;;;;; +A1B8;YI SYLLABLE LIX;Lo;0;L;;;;;N;;;;; +A1B9;YI SYLLABLE LI;Lo;0;L;;;;;N;;;;; +A1BA;YI SYLLABLE LIP;Lo;0;L;;;;;N;;;;; +A1BB;YI SYLLABLE LIET;Lo;0;L;;;;;N;;;;; +A1BC;YI SYLLABLE LIEX;Lo;0;L;;;;;N;;;;; +A1BD;YI SYLLABLE LIE;Lo;0;L;;;;;N;;;;; +A1BE;YI SYLLABLE LIEP;Lo;0;L;;;;;N;;;;; +A1BF;YI SYLLABLE LAT;Lo;0;L;;;;;N;;;;; +A1C0;YI SYLLABLE LAX;Lo;0;L;;;;;N;;;;; +A1C1;YI SYLLABLE LA;Lo;0;L;;;;;N;;;;; +A1C2;YI SYLLABLE LAP;Lo;0;L;;;;;N;;;;; +A1C3;YI SYLLABLE LUOT;Lo;0;L;;;;;N;;;;; +A1C4;YI SYLLABLE LUOX;Lo;0;L;;;;;N;;;;; +A1C5;YI SYLLABLE LUO;Lo;0;L;;;;;N;;;;; +A1C6;YI SYLLABLE LUOP;Lo;0;L;;;;;N;;;;; +A1C7;YI SYLLABLE LOT;Lo;0;L;;;;;N;;;;; +A1C8;YI SYLLABLE LOX;Lo;0;L;;;;;N;;;;; +A1C9;YI SYLLABLE LO;Lo;0;L;;;;;N;;;;; +A1CA;YI SYLLABLE LOP;Lo;0;L;;;;;N;;;;; +A1CB;YI SYLLABLE LEX;Lo;0;L;;;;;N;;;;; +A1CC;YI SYLLABLE LE;Lo;0;L;;;;;N;;;;; +A1CD;YI SYLLABLE LEP;Lo;0;L;;;;;N;;;;; +A1CE;YI SYLLABLE LUT;Lo;0;L;;;;;N;;;;; +A1CF;YI SYLLABLE LUX;Lo;0;L;;;;;N;;;;; +A1D0;YI SYLLABLE LU;Lo;0;L;;;;;N;;;;; +A1D1;YI SYLLABLE LUP;Lo;0;L;;;;;N;;;;; +A1D2;YI SYLLABLE LURX;Lo;0;L;;;;;N;;;;; +A1D3;YI SYLLABLE LUR;Lo;0;L;;;;;N;;;;; +A1D4;YI SYLLABLE LYT;Lo;0;L;;;;;N;;;;; +A1D5;YI SYLLABLE LYX;Lo;0;L;;;;;N;;;;; +A1D6;YI SYLLABLE LY;Lo;0;L;;;;;N;;;;; +A1D7;YI SYLLABLE LYP;Lo;0;L;;;;;N;;;;; +A1D8;YI SYLLABLE LYRX;Lo;0;L;;;;;N;;;;; +A1D9;YI SYLLABLE LYR;Lo;0;L;;;;;N;;;;; +A1DA;YI SYLLABLE GIT;Lo;0;L;;;;;N;;;;; +A1DB;YI SYLLABLE GIX;Lo;0;L;;;;;N;;;;; +A1DC;YI SYLLABLE GI;Lo;0;L;;;;;N;;;;; +A1DD;YI SYLLABLE GIP;Lo;0;L;;;;;N;;;;; +A1DE;YI SYLLABLE GIET;Lo;0;L;;;;;N;;;;; +A1DF;YI SYLLABLE GIEX;Lo;0;L;;;;;N;;;;; +A1E0;YI SYLLABLE GIE;Lo;0;L;;;;;N;;;;; +A1E1;YI SYLLABLE GIEP;Lo;0;L;;;;;N;;;;; +A1E2;YI SYLLABLE GAT;Lo;0;L;;;;;N;;;;; +A1E3;YI SYLLABLE GAX;Lo;0;L;;;;;N;;;;; +A1E4;YI SYLLABLE GA;Lo;0;L;;;;;N;;;;; +A1E5;YI SYLLABLE GAP;Lo;0;L;;;;;N;;;;; +A1E6;YI SYLLABLE GUOT;Lo;0;L;;;;;N;;;;; +A1E7;YI SYLLABLE GUOX;Lo;0;L;;;;;N;;;;; +A1E8;YI SYLLABLE GUO;Lo;0;L;;;;;N;;;;; +A1E9;YI SYLLABLE GUOP;Lo;0;L;;;;;N;;;;; +A1EA;YI SYLLABLE GOT;Lo;0;L;;;;;N;;;;; +A1EB;YI SYLLABLE GOX;Lo;0;L;;;;;N;;;;; +A1EC;YI SYLLABLE GO;Lo;0;L;;;;;N;;;;; +A1ED;YI SYLLABLE GOP;Lo;0;L;;;;;N;;;;; +A1EE;YI SYLLABLE GET;Lo;0;L;;;;;N;;;;; +A1EF;YI SYLLABLE GEX;Lo;0;L;;;;;N;;;;; +A1F0;YI SYLLABLE GE;Lo;0;L;;;;;N;;;;; +A1F1;YI SYLLABLE GEP;Lo;0;L;;;;;N;;;;; +A1F2;YI SYLLABLE GUT;Lo;0;L;;;;;N;;;;; +A1F3;YI SYLLABLE GUX;Lo;0;L;;;;;N;;;;; +A1F4;YI SYLLABLE GU;Lo;0;L;;;;;N;;;;; +A1F5;YI SYLLABLE GUP;Lo;0;L;;;;;N;;;;; +A1F6;YI SYLLABLE GURX;Lo;0;L;;;;;N;;;;; +A1F7;YI SYLLABLE GUR;Lo;0;L;;;;;N;;;;; +A1F8;YI SYLLABLE KIT;Lo;0;L;;;;;N;;;;; +A1F9;YI SYLLABLE KIX;Lo;0;L;;;;;N;;;;; +A1FA;YI SYLLABLE KI;Lo;0;L;;;;;N;;;;; +A1FB;YI SYLLABLE KIP;Lo;0;L;;;;;N;;;;; +A1FC;YI SYLLABLE KIEX;Lo;0;L;;;;;N;;;;; +A1FD;YI SYLLABLE KIE;Lo;0;L;;;;;N;;;;; +A1FE;YI SYLLABLE KIEP;Lo;0;L;;;;;N;;;;; +A1FF;YI SYLLABLE KAT;Lo;0;L;;;;;N;;;;; +A200;YI SYLLABLE KAX;Lo;0;L;;;;;N;;;;; +A201;YI SYLLABLE KA;Lo;0;L;;;;;N;;;;; +A202;YI SYLLABLE KAP;Lo;0;L;;;;;N;;;;; +A203;YI SYLLABLE KUOX;Lo;0;L;;;;;N;;;;; +A204;YI SYLLABLE KUO;Lo;0;L;;;;;N;;;;; +A205;YI SYLLABLE KUOP;Lo;0;L;;;;;N;;;;; +A206;YI SYLLABLE KOT;Lo;0;L;;;;;N;;;;; +A207;YI SYLLABLE KOX;Lo;0;L;;;;;N;;;;; +A208;YI SYLLABLE KO;Lo;0;L;;;;;N;;;;; +A209;YI SYLLABLE KOP;Lo;0;L;;;;;N;;;;; +A20A;YI SYLLABLE KET;Lo;0;L;;;;;N;;;;; +A20B;YI SYLLABLE KEX;Lo;0;L;;;;;N;;;;; +A20C;YI SYLLABLE KE;Lo;0;L;;;;;N;;;;; +A20D;YI SYLLABLE KEP;Lo;0;L;;;;;N;;;;; +A20E;YI SYLLABLE KUT;Lo;0;L;;;;;N;;;;; +A20F;YI SYLLABLE KUX;Lo;0;L;;;;;N;;;;; +A210;YI SYLLABLE KU;Lo;0;L;;;;;N;;;;; +A211;YI SYLLABLE KUP;Lo;0;L;;;;;N;;;;; +A212;YI SYLLABLE KURX;Lo;0;L;;;;;N;;;;; +A213;YI SYLLABLE KUR;Lo;0;L;;;;;N;;;;; +A214;YI SYLLABLE GGIT;Lo;0;L;;;;;N;;;;; +A215;YI SYLLABLE GGIX;Lo;0;L;;;;;N;;;;; +A216;YI SYLLABLE GGI;Lo;0;L;;;;;N;;;;; +A217;YI SYLLABLE GGIEX;Lo;0;L;;;;;N;;;;; +A218;YI SYLLABLE GGIE;Lo;0;L;;;;;N;;;;; +A219;YI SYLLABLE GGIEP;Lo;0;L;;;;;N;;;;; +A21A;YI SYLLABLE GGAT;Lo;0;L;;;;;N;;;;; +A21B;YI SYLLABLE GGAX;Lo;0;L;;;;;N;;;;; +A21C;YI SYLLABLE GGA;Lo;0;L;;;;;N;;;;; +A21D;YI SYLLABLE GGAP;Lo;0;L;;;;;N;;;;; +A21E;YI SYLLABLE GGUOT;Lo;0;L;;;;;N;;;;; +A21F;YI SYLLABLE GGUOX;Lo;0;L;;;;;N;;;;; +A220;YI SYLLABLE GGUO;Lo;0;L;;;;;N;;;;; +A221;YI SYLLABLE GGUOP;Lo;0;L;;;;;N;;;;; +A222;YI SYLLABLE GGOT;Lo;0;L;;;;;N;;;;; +A223;YI SYLLABLE GGOX;Lo;0;L;;;;;N;;;;; +A224;YI SYLLABLE GGO;Lo;0;L;;;;;N;;;;; +A225;YI SYLLABLE GGOP;Lo;0;L;;;;;N;;;;; +A226;YI SYLLABLE GGET;Lo;0;L;;;;;N;;;;; +A227;YI SYLLABLE GGEX;Lo;0;L;;;;;N;;;;; +A228;YI SYLLABLE GGE;Lo;0;L;;;;;N;;;;; +A229;YI SYLLABLE GGEP;Lo;0;L;;;;;N;;;;; +A22A;YI SYLLABLE GGUT;Lo;0;L;;;;;N;;;;; +A22B;YI SYLLABLE GGUX;Lo;0;L;;;;;N;;;;; +A22C;YI SYLLABLE GGU;Lo;0;L;;;;;N;;;;; +A22D;YI SYLLABLE GGUP;Lo;0;L;;;;;N;;;;; +A22E;YI SYLLABLE GGURX;Lo;0;L;;;;;N;;;;; +A22F;YI SYLLABLE GGUR;Lo;0;L;;;;;N;;;;; +A230;YI SYLLABLE MGIEX;Lo;0;L;;;;;N;;;;; +A231;YI SYLLABLE MGIE;Lo;0;L;;;;;N;;;;; +A232;YI SYLLABLE MGAT;Lo;0;L;;;;;N;;;;; +A233;YI SYLLABLE MGAX;Lo;0;L;;;;;N;;;;; +A234;YI SYLLABLE MGA;Lo;0;L;;;;;N;;;;; +A235;YI SYLLABLE MGAP;Lo;0;L;;;;;N;;;;; +A236;YI SYLLABLE MGUOX;Lo;0;L;;;;;N;;;;; +A237;YI SYLLABLE MGUO;Lo;0;L;;;;;N;;;;; +A238;YI SYLLABLE MGUOP;Lo;0;L;;;;;N;;;;; +A239;YI SYLLABLE MGOT;Lo;0;L;;;;;N;;;;; +A23A;YI SYLLABLE MGOX;Lo;0;L;;;;;N;;;;; +A23B;YI SYLLABLE MGO;Lo;0;L;;;;;N;;;;; +A23C;YI SYLLABLE MGOP;Lo;0;L;;;;;N;;;;; +A23D;YI SYLLABLE MGEX;Lo;0;L;;;;;N;;;;; +A23E;YI SYLLABLE MGE;Lo;0;L;;;;;N;;;;; +A23F;YI SYLLABLE MGEP;Lo;0;L;;;;;N;;;;; +A240;YI SYLLABLE MGUT;Lo;0;L;;;;;N;;;;; +A241;YI SYLLABLE MGUX;Lo;0;L;;;;;N;;;;; +A242;YI SYLLABLE MGU;Lo;0;L;;;;;N;;;;; +A243;YI SYLLABLE MGUP;Lo;0;L;;;;;N;;;;; +A244;YI SYLLABLE MGURX;Lo;0;L;;;;;N;;;;; +A245;YI SYLLABLE MGUR;Lo;0;L;;;;;N;;;;; +A246;YI SYLLABLE HXIT;Lo;0;L;;;;;N;;;;; +A247;YI SYLLABLE HXIX;Lo;0;L;;;;;N;;;;; +A248;YI SYLLABLE HXI;Lo;0;L;;;;;N;;;;; +A249;YI SYLLABLE HXIP;Lo;0;L;;;;;N;;;;; +A24A;YI SYLLABLE HXIET;Lo;0;L;;;;;N;;;;; +A24B;YI SYLLABLE HXIEX;Lo;0;L;;;;;N;;;;; +A24C;YI SYLLABLE HXIE;Lo;0;L;;;;;N;;;;; +A24D;YI SYLLABLE HXIEP;Lo;0;L;;;;;N;;;;; +A24E;YI SYLLABLE HXAT;Lo;0;L;;;;;N;;;;; +A24F;YI SYLLABLE HXAX;Lo;0;L;;;;;N;;;;; +A250;YI SYLLABLE HXA;Lo;0;L;;;;;N;;;;; +A251;YI SYLLABLE HXAP;Lo;0;L;;;;;N;;;;; +A252;YI SYLLABLE HXUOT;Lo;0;L;;;;;N;;;;; +A253;YI SYLLABLE HXUOX;Lo;0;L;;;;;N;;;;; +A254;YI SYLLABLE HXUO;Lo;0;L;;;;;N;;;;; +A255;YI SYLLABLE HXUOP;Lo;0;L;;;;;N;;;;; +A256;YI SYLLABLE HXOT;Lo;0;L;;;;;N;;;;; +A257;YI SYLLABLE HXOX;Lo;0;L;;;;;N;;;;; +A258;YI SYLLABLE HXO;Lo;0;L;;;;;N;;;;; +A259;YI SYLLABLE HXOP;Lo;0;L;;;;;N;;;;; +A25A;YI SYLLABLE HXEX;Lo;0;L;;;;;N;;;;; +A25B;YI SYLLABLE HXE;Lo;0;L;;;;;N;;;;; +A25C;YI SYLLABLE HXEP;Lo;0;L;;;;;N;;;;; +A25D;YI SYLLABLE NGIEX;Lo;0;L;;;;;N;;;;; +A25E;YI SYLLABLE NGIE;Lo;0;L;;;;;N;;;;; +A25F;YI SYLLABLE NGIEP;Lo;0;L;;;;;N;;;;; +A260;YI SYLLABLE NGAT;Lo;0;L;;;;;N;;;;; +A261;YI SYLLABLE NGAX;Lo;0;L;;;;;N;;;;; +A262;YI SYLLABLE NGA;Lo;0;L;;;;;N;;;;; +A263;YI SYLLABLE NGAP;Lo;0;L;;;;;N;;;;; +A264;YI SYLLABLE NGUOT;Lo;0;L;;;;;N;;;;; +A265;YI SYLLABLE NGUOX;Lo;0;L;;;;;N;;;;; +A266;YI SYLLABLE NGUO;Lo;0;L;;;;;N;;;;; +A267;YI SYLLABLE NGOT;Lo;0;L;;;;;N;;;;; +A268;YI SYLLABLE NGOX;Lo;0;L;;;;;N;;;;; +A269;YI SYLLABLE NGO;Lo;0;L;;;;;N;;;;; +A26A;YI SYLLABLE NGOP;Lo;0;L;;;;;N;;;;; +A26B;YI SYLLABLE NGEX;Lo;0;L;;;;;N;;;;; +A26C;YI SYLLABLE NGE;Lo;0;L;;;;;N;;;;; +A26D;YI SYLLABLE NGEP;Lo;0;L;;;;;N;;;;; +A26E;YI SYLLABLE HIT;Lo;0;L;;;;;N;;;;; +A26F;YI SYLLABLE HIEX;Lo;0;L;;;;;N;;;;; +A270;YI SYLLABLE HIE;Lo;0;L;;;;;N;;;;; +A271;YI SYLLABLE HAT;Lo;0;L;;;;;N;;;;; +A272;YI SYLLABLE HAX;Lo;0;L;;;;;N;;;;; +A273;YI SYLLABLE HA;Lo;0;L;;;;;N;;;;; +A274;YI SYLLABLE HAP;Lo;0;L;;;;;N;;;;; +A275;YI SYLLABLE HUOT;Lo;0;L;;;;;N;;;;; +A276;YI SYLLABLE HUOX;Lo;0;L;;;;;N;;;;; +A277;YI SYLLABLE HUO;Lo;0;L;;;;;N;;;;; +A278;YI SYLLABLE HUOP;Lo;0;L;;;;;N;;;;; +A279;YI SYLLABLE HOT;Lo;0;L;;;;;N;;;;; +A27A;YI SYLLABLE HOX;Lo;0;L;;;;;N;;;;; +A27B;YI SYLLABLE HO;Lo;0;L;;;;;N;;;;; +A27C;YI SYLLABLE HOP;Lo;0;L;;;;;N;;;;; +A27D;YI SYLLABLE HEX;Lo;0;L;;;;;N;;;;; +A27E;YI SYLLABLE HE;Lo;0;L;;;;;N;;;;; +A27F;YI SYLLABLE HEP;Lo;0;L;;;;;N;;;;; +A280;YI SYLLABLE WAT;Lo;0;L;;;;;N;;;;; +A281;YI SYLLABLE WAX;Lo;0;L;;;;;N;;;;; +A282;YI SYLLABLE WA;Lo;0;L;;;;;N;;;;; +A283;YI SYLLABLE WAP;Lo;0;L;;;;;N;;;;; +A284;YI SYLLABLE WUOX;Lo;0;L;;;;;N;;;;; +A285;YI SYLLABLE WUO;Lo;0;L;;;;;N;;;;; +A286;YI SYLLABLE WUOP;Lo;0;L;;;;;N;;;;; +A287;YI SYLLABLE WOX;Lo;0;L;;;;;N;;;;; +A288;YI SYLLABLE WO;Lo;0;L;;;;;N;;;;; +A289;YI SYLLABLE WOP;Lo;0;L;;;;;N;;;;; +A28A;YI SYLLABLE WEX;Lo;0;L;;;;;N;;;;; +A28B;YI SYLLABLE WE;Lo;0;L;;;;;N;;;;; +A28C;YI SYLLABLE WEP;Lo;0;L;;;;;N;;;;; +A28D;YI SYLLABLE ZIT;Lo;0;L;;;;;N;;;;; +A28E;YI SYLLABLE ZIX;Lo;0;L;;;;;N;;;;; +A28F;YI SYLLABLE ZI;Lo;0;L;;;;;N;;;;; +A290;YI SYLLABLE ZIP;Lo;0;L;;;;;N;;;;; +A291;YI SYLLABLE ZIEX;Lo;0;L;;;;;N;;;;; +A292;YI SYLLABLE ZIE;Lo;0;L;;;;;N;;;;; +A293;YI SYLLABLE ZIEP;Lo;0;L;;;;;N;;;;; +A294;YI SYLLABLE ZAT;Lo;0;L;;;;;N;;;;; +A295;YI SYLLABLE ZAX;Lo;0;L;;;;;N;;;;; +A296;YI SYLLABLE ZA;Lo;0;L;;;;;N;;;;; +A297;YI SYLLABLE ZAP;Lo;0;L;;;;;N;;;;; +A298;YI SYLLABLE ZUOX;Lo;0;L;;;;;N;;;;; +A299;YI SYLLABLE ZUO;Lo;0;L;;;;;N;;;;; +A29A;YI SYLLABLE ZUOP;Lo;0;L;;;;;N;;;;; +A29B;YI SYLLABLE ZOT;Lo;0;L;;;;;N;;;;; +A29C;YI SYLLABLE ZOX;Lo;0;L;;;;;N;;;;; +A29D;YI SYLLABLE ZO;Lo;0;L;;;;;N;;;;; +A29E;YI SYLLABLE ZOP;Lo;0;L;;;;;N;;;;; +A29F;YI SYLLABLE ZEX;Lo;0;L;;;;;N;;;;; +A2A0;YI SYLLABLE ZE;Lo;0;L;;;;;N;;;;; +A2A1;YI SYLLABLE ZEP;Lo;0;L;;;;;N;;;;; +A2A2;YI SYLLABLE ZUT;Lo;0;L;;;;;N;;;;; +A2A3;YI SYLLABLE ZUX;Lo;0;L;;;;;N;;;;; +A2A4;YI SYLLABLE ZU;Lo;0;L;;;;;N;;;;; +A2A5;YI SYLLABLE ZUP;Lo;0;L;;;;;N;;;;; +A2A6;YI SYLLABLE ZURX;Lo;0;L;;;;;N;;;;; +A2A7;YI SYLLABLE ZUR;Lo;0;L;;;;;N;;;;; +A2A8;YI SYLLABLE ZYT;Lo;0;L;;;;;N;;;;; +A2A9;YI SYLLABLE ZYX;Lo;0;L;;;;;N;;;;; +A2AA;YI SYLLABLE ZY;Lo;0;L;;;;;N;;;;; +A2AB;YI SYLLABLE ZYP;Lo;0;L;;;;;N;;;;; +A2AC;YI SYLLABLE ZYRX;Lo;0;L;;;;;N;;;;; +A2AD;YI SYLLABLE ZYR;Lo;0;L;;;;;N;;;;; +A2AE;YI SYLLABLE CIT;Lo;0;L;;;;;N;;;;; +A2AF;YI SYLLABLE CIX;Lo;0;L;;;;;N;;;;; +A2B0;YI SYLLABLE CI;Lo;0;L;;;;;N;;;;; +A2B1;YI SYLLABLE CIP;Lo;0;L;;;;;N;;;;; +A2B2;YI SYLLABLE CIET;Lo;0;L;;;;;N;;;;; +A2B3;YI SYLLABLE CIEX;Lo;0;L;;;;;N;;;;; +A2B4;YI SYLLABLE CIE;Lo;0;L;;;;;N;;;;; +A2B5;YI SYLLABLE CIEP;Lo;0;L;;;;;N;;;;; +A2B6;YI SYLLABLE CAT;Lo;0;L;;;;;N;;;;; +A2B7;YI SYLLABLE CAX;Lo;0;L;;;;;N;;;;; +A2B8;YI SYLLABLE CA;Lo;0;L;;;;;N;;;;; +A2B9;YI SYLLABLE CAP;Lo;0;L;;;;;N;;;;; +A2BA;YI SYLLABLE CUOX;Lo;0;L;;;;;N;;;;; +A2BB;YI SYLLABLE CUO;Lo;0;L;;;;;N;;;;; +A2BC;YI SYLLABLE CUOP;Lo;0;L;;;;;N;;;;; +A2BD;YI SYLLABLE COT;Lo;0;L;;;;;N;;;;; +A2BE;YI SYLLABLE COX;Lo;0;L;;;;;N;;;;; +A2BF;YI SYLLABLE CO;Lo;0;L;;;;;N;;;;; +A2C0;YI SYLLABLE COP;Lo;0;L;;;;;N;;;;; +A2C1;YI SYLLABLE CEX;Lo;0;L;;;;;N;;;;; +A2C2;YI SYLLABLE CE;Lo;0;L;;;;;N;;;;; +A2C3;YI SYLLABLE CEP;Lo;0;L;;;;;N;;;;; +A2C4;YI SYLLABLE CUT;Lo;0;L;;;;;N;;;;; +A2C5;YI SYLLABLE CUX;Lo;0;L;;;;;N;;;;; +A2C6;YI SYLLABLE CU;Lo;0;L;;;;;N;;;;; +A2C7;YI SYLLABLE CUP;Lo;0;L;;;;;N;;;;; +A2C8;YI SYLLABLE CURX;Lo;0;L;;;;;N;;;;; +A2C9;YI SYLLABLE CUR;Lo;0;L;;;;;N;;;;; +A2CA;YI SYLLABLE CYT;Lo;0;L;;;;;N;;;;; +A2CB;YI SYLLABLE CYX;Lo;0;L;;;;;N;;;;; +A2CC;YI SYLLABLE CY;Lo;0;L;;;;;N;;;;; +A2CD;YI SYLLABLE CYP;Lo;0;L;;;;;N;;;;; +A2CE;YI SYLLABLE CYRX;Lo;0;L;;;;;N;;;;; +A2CF;YI SYLLABLE CYR;Lo;0;L;;;;;N;;;;; +A2D0;YI SYLLABLE ZZIT;Lo;0;L;;;;;N;;;;; +A2D1;YI SYLLABLE ZZIX;Lo;0;L;;;;;N;;;;; +A2D2;YI SYLLABLE ZZI;Lo;0;L;;;;;N;;;;; +A2D3;YI SYLLABLE ZZIP;Lo;0;L;;;;;N;;;;; +A2D4;YI SYLLABLE ZZIET;Lo;0;L;;;;;N;;;;; +A2D5;YI SYLLABLE ZZIEX;Lo;0;L;;;;;N;;;;; +A2D6;YI SYLLABLE ZZIE;Lo;0;L;;;;;N;;;;; +A2D7;YI SYLLABLE ZZIEP;Lo;0;L;;;;;N;;;;; +A2D8;YI SYLLABLE ZZAT;Lo;0;L;;;;;N;;;;; +A2D9;YI SYLLABLE ZZAX;Lo;0;L;;;;;N;;;;; +A2DA;YI SYLLABLE ZZA;Lo;0;L;;;;;N;;;;; +A2DB;YI SYLLABLE ZZAP;Lo;0;L;;;;;N;;;;; +A2DC;YI SYLLABLE ZZOX;Lo;0;L;;;;;N;;;;; +A2DD;YI SYLLABLE ZZO;Lo;0;L;;;;;N;;;;; +A2DE;YI SYLLABLE ZZOP;Lo;0;L;;;;;N;;;;; +A2DF;YI SYLLABLE ZZEX;Lo;0;L;;;;;N;;;;; +A2E0;YI SYLLABLE ZZE;Lo;0;L;;;;;N;;;;; +A2E1;YI SYLLABLE ZZEP;Lo;0;L;;;;;N;;;;; +A2E2;YI SYLLABLE ZZUX;Lo;0;L;;;;;N;;;;; +A2E3;YI SYLLABLE ZZU;Lo;0;L;;;;;N;;;;; +A2E4;YI SYLLABLE ZZUP;Lo;0;L;;;;;N;;;;; +A2E5;YI SYLLABLE ZZURX;Lo;0;L;;;;;N;;;;; +A2E6;YI SYLLABLE ZZUR;Lo;0;L;;;;;N;;;;; +A2E7;YI SYLLABLE ZZYT;Lo;0;L;;;;;N;;;;; +A2E8;YI SYLLABLE ZZYX;Lo;0;L;;;;;N;;;;; +A2E9;YI SYLLABLE ZZY;Lo;0;L;;;;;N;;;;; +A2EA;YI SYLLABLE ZZYP;Lo;0;L;;;;;N;;;;; +A2EB;YI SYLLABLE ZZYRX;Lo;0;L;;;;;N;;;;; +A2EC;YI SYLLABLE ZZYR;Lo;0;L;;;;;N;;;;; +A2ED;YI SYLLABLE NZIT;Lo;0;L;;;;;N;;;;; +A2EE;YI SYLLABLE NZIX;Lo;0;L;;;;;N;;;;; +A2EF;YI SYLLABLE NZI;Lo;0;L;;;;;N;;;;; +A2F0;YI SYLLABLE NZIP;Lo;0;L;;;;;N;;;;; +A2F1;YI SYLLABLE NZIEX;Lo;0;L;;;;;N;;;;; +A2F2;YI SYLLABLE NZIE;Lo;0;L;;;;;N;;;;; +A2F3;YI SYLLABLE NZIEP;Lo;0;L;;;;;N;;;;; +A2F4;YI SYLLABLE NZAT;Lo;0;L;;;;;N;;;;; +A2F5;YI SYLLABLE NZAX;Lo;0;L;;;;;N;;;;; +A2F6;YI SYLLABLE NZA;Lo;0;L;;;;;N;;;;; +A2F7;YI SYLLABLE NZAP;Lo;0;L;;;;;N;;;;; +A2F8;YI SYLLABLE NZUOX;Lo;0;L;;;;;N;;;;; +A2F9;YI SYLLABLE NZUO;Lo;0;L;;;;;N;;;;; +A2FA;YI SYLLABLE NZOX;Lo;0;L;;;;;N;;;;; +A2FB;YI SYLLABLE NZOP;Lo;0;L;;;;;N;;;;; +A2FC;YI SYLLABLE NZEX;Lo;0;L;;;;;N;;;;; +A2FD;YI SYLLABLE NZE;Lo;0;L;;;;;N;;;;; +A2FE;YI SYLLABLE NZUX;Lo;0;L;;;;;N;;;;; +A2FF;YI SYLLABLE NZU;Lo;0;L;;;;;N;;;;; +A300;YI SYLLABLE NZUP;Lo;0;L;;;;;N;;;;; +A301;YI SYLLABLE NZURX;Lo;0;L;;;;;N;;;;; +A302;YI SYLLABLE NZUR;Lo;0;L;;;;;N;;;;; +A303;YI SYLLABLE NZYT;Lo;0;L;;;;;N;;;;; +A304;YI SYLLABLE NZYX;Lo;0;L;;;;;N;;;;; +A305;YI SYLLABLE NZY;Lo;0;L;;;;;N;;;;; +A306;YI SYLLABLE NZYP;Lo;0;L;;;;;N;;;;; +A307;YI SYLLABLE NZYRX;Lo;0;L;;;;;N;;;;; +A308;YI SYLLABLE NZYR;Lo;0;L;;;;;N;;;;; +A309;YI SYLLABLE SIT;Lo;0;L;;;;;N;;;;; +A30A;YI SYLLABLE SIX;Lo;0;L;;;;;N;;;;; +A30B;YI SYLLABLE SI;Lo;0;L;;;;;N;;;;; +A30C;YI SYLLABLE SIP;Lo;0;L;;;;;N;;;;; +A30D;YI SYLLABLE SIEX;Lo;0;L;;;;;N;;;;; +A30E;YI SYLLABLE SIE;Lo;0;L;;;;;N;;;;; +A30F;YI SYLLABLE SIEP;Lo;0;L;;;;;N;;;;; +A310;YI SYLLABLE SAT;Lo;0;L;;;;;N;;;;; +A311;YI SYLLABLE SAX;Lo;0;L;;;;;N;;;;; +A312;YI SYLLABLE SA;Lo;0;L;;;;;N;;;;; +A313;YI SYLLABLE SAP;Lo;0;L;;;;;N;;;;; +A314;YI SYLLABLE SUOX;Lo;0;L;;;;;N;;;;; +A315;YI SYLLABLE SUO;Lo;0;L;;;;;N;;;;; +A316;YI SYLLABLE SUOP;Lo;0;L;;;;;N;;;;; +A317;YI SYLLABLE SOT;Lo;0;L;;;;;N;;;;; +A318;YI SYLLABLE SOX;Lo;0;L;;;;;N;;;;; +A319;YI SYLLABLE SO;Lo;0;L;;;;;N;;;;; +A31A;YI SYLLABLE SOP;Lo;0;L;;;;;N;;;;; +A31B;YI SYLLABLE SEX;Lo;0;L;;;;;N;;;;; +A31C;YI SYLLABLE SE;Lo;0;L;;;;;N;;;;; +A31D;YI SYLLABLE SEP;Lo;0;L;;;;;N;;;;; +A31E;YI SYLLABLE SUT;Lo;0;L;;;;;N;;;;; +A31F;YI SYLLABLE SUX;Lo;0;L;;;;;N;;;;; +A320;YI SYLLABLE SU;Lo;0;L;;;;;N;;;;; +A321;YI SYLLABLE SUP;Lo;0;L;;;;;N;;;;; +A322;YI SYLLABLE SURX;Lo;0;L;;;;;N;;;;; +A323;YI SYLLABLE SUR;Lo;0;L;;;;;N;;;;; +A324;YI SYLLABLE SYT;Lo;0;L;;;;;N;;;;; +A325;YI SYLLABLE SYX;Lo;0;L;;;;;N;;;;; +A326;YI SYLLABLE SY;Lo;0;L;;;;;N;;;;; +A327;YI SYLLABLE SYP;Lo;0;L;;;;;N;;;;; +A328;YI SYLLABLE SYRX;Lo;0;L;;;;;N;;;;; +A329;YI SYLLABLE SYR;Lo;0;L;;;;;N;;;;; +A32A;YI SYLLABLE SSIT;Lo;0;L;;;;;N;;;;; +A32B;YI SYLLABLE SSIX;Lo;0;L;;;;;N;;;;; +A32C;YI SYLLABLE SSI;Lo;0;L;;;;;N;;;;; +A32D;YI SYLLABLE SSIP;Lo;0;L;;;;;N;;;;; +A32E;YI SYLLABLE SSIEX;Lo;0;L;;;;;N;;;;; +A32F;YI SYLLABLE SSIE;Lo;0;L;;;;;N;;;;; +A330;YI SYLLABLE SSIEP;Lo;0;L;;;;;N;;;;; +A331;YI SYLLABLE SSAT;Lo;0;L;;;;;N;;;;; +A332;YI SYLLABLE SSAX;Lo;0;L;;;;;N;;;;; +A333;YI SYLLABLE SSA;Lo;0;L;;;;;N;;;;; +A334;YI SYLLABLE SSAP;Lo;0;L;;;;;N;;;;; +A335;YI SYLLABLE SSOT;Lo;0;L;;;;;N;;;;; +A336;YI SYLLABLE SSOX;Lo;0;L;;;;;N;;;;; +A337;YI SYLLABLE SSO;Lo;0;L;;;;;N;;;;; +A338;YI SYLLABLE SSOP;Lo;0;L;;;;;N;;;;; +A339;YI SYLLABLE SSEX;Lo;0;L;;;;;N;;;;; +A33A;YI SYLLABLE SSE;Lo;0;L;;;;;N;;;;; +A33B;YI SYLLABLE SSEP;Lo;0;L;;;;;N;;;;; +A33C;YI SYLLABLE SSUT;Lo;0;L;;;;;N;;;;; +A33D;YI SYLLABLE SSUX;Lo;0;L;;;;;N;;;;; +A33E;YI SYLLABLE SSU;Lo;0;L;;;;;N;;;;; +A33F;YI SYLLABLE SSUP;Lo;0;L;;;;;N;;;;; +A340;YI SYLLABLE SSYT;Lo;0;L;;;;;N;;;;; +A341;YI SYLLABLE SSYX;Lo;0;L;;;;;N;;;;; +A342;YI SYLLABLE SSY;Lo;0;L;;;;;N;;;;; +A343;YI SYLLABLE SSYP;Lo;0;L;;;;;N;;;;; +A344;YI SYLLABLE SSYRX;Lo;0;L;;;;;N;;;;; +A345;YI SYLLABLE SSYR;Lo;0;L;;;;;N;;;;; +A346;YI SYLLABLE ZHAT;Lo;0;L;;;;;N;;;;; +A347;YI SYLLABLE ZHAX;Lo;0;L;;;;;N;;;;; +A348;YI SYLLABLE ZHA;Lo;0;L;;;;;N;;;;; +A349;YI SYLLABLE ZHAP;Lo;0;L;;;;;N;;;;; +A34A;YI SYLLABLE ZHUOX;Lo;0;L;;;;;N;;;;; +A34B;YI SYLLABLE ZHUO;Lo;0;L;;;;;N;;;;; +A34C;YI SYLLABLE ZHUOP;Lo;0;L;;;;;N;;;;; +A34D;YI SYLLABLE ZHOT;Lo;0;L;;;;;N;;;;; +A34E;YI SYLLABLE ZHOX;Lo;0;L;;;;;N;;;;; +A34F;YI SYLLABLE ZHO;Lo;0;L;;;;;N;;;;; +A350;YI SYLLABLE ZHOP;Lo;0;L;;;;;N;;;;; +A351;YI SYLLABLE ZHET;Lo;0;L;;;;;N;;;;; +A352;YI SYLLABLE ZHEX;Lo;0;L;;;;;N;;;;; +A353;YI SYLLABLE ZHE;Lo;0;L;;;;;N;;;;; +A354;YI SYLLABLE ZHEP;Lo;0;L;;;;;N;;;;; +A355;YI SYLLABLE ZHUT;Lo;0;L;;;;;N;;;;; +A356;YI SYLLABLE ZHUX;Lo;0;L;;;;;N;;;;; +A357;YI SYLLABLE ZHU;Lo;0;L;;;;;N;;;;; +A358;YI SYLLABLE ZHUP;Lo;0;L;;;;;N;;;;; +A359;YI SYLLABLE ZHURX;Lo;0;L;;;;;N;;;;; +A35A;YI SYLLABLE ZHUR;Lo;0;L;;;;;N;;;;; +A35B;YI SYLLABLE ZHYT;Lo;0;L;;;;;N;;;;; +A35C;YI SYLLABLE ZHYX;Lo;0;L;;;;;N;;;;; +A35D;YI SYLLABLE ZHY;Lo;0;L;;;;;N;;;;; +A35E;YI SYLLABLE ZHYP;Lo;0;L;;;;;N;;;;; +A35F;YI SYLLABLE ZHYRX;Lo;0;L;;;;;N;;;;; +A360;YI SYLLABLE ZHYR;Lo;0;L;;;;;N;;;;; +A361;YI SYLLABLE CHAT;Lo;0;L;;;;;N;;;;; +A362;YI SYLLABLE CHAX;Lo;0;L;;;;;N;;;;; +A363;YI SYLLABLE CHA;Lo;0;L;;;;;N;;;;; +A364;YI SYLLABLE CHAP;Lo;0;L;;;;;N;;;;; +A365;YI SYLLABLE CHUOT;Lo;0;L;;;;;N;;;;; +A366;YI SYLLABLE CHUOX;Lo;0;L;;;;;N;;;;; +A367;YI SYLLABLE CHUO;Lo;0;L;;;;;N;;;;; +A368;YI SYLLABLE CHUOP;Lo;0;L;;;;;N;;;;; +A369;YI SYLLABLE CHOT;Lo;0;L;;;;;N;;;;; +A36A;YI SYLLABLE CHOX;Lo;0;L;;;;;N;;;;; +A36B;YI SYLLABLE CHO;Lo;0;L;;;;;N;;;;; +A36C;YI SYLLABLE CHOP;Lo;0;L;;;;;N;;;;; +A36D;YI SYLLABLE CHET;Lo;0;L;;;;;N;;;;; +A36E;YI SYLLABLE CHEX;Lo;0;L;;;;;N;;;;; +A36F;YI SYLLABLE CHE;Lo;0;L;;;;;N;;;;; +A370;YI SYLLABLE CHEP;Lo;0;L;;;;;N;;;;; +A371;YI SYLLABLE CHUX;Lo;0;L;;;;;N;;;;; +A372;YI SYLLABLE CHU;Lo;0;L;;;;;N;;;;; +A373;YI SYLLABLE CHUP;Lo;0;L;;;;;N;;;;; +A374;YI SYLLABLE CHURX;Lo;0;L;;;;;N;;;;; +A375;YI SYLLABLE CHUR;Lo;0;L;;;;;N;;;;; +A376;YI SYLLABLE CHYT;Lo;0;L;;;;;N;;;;; +A377;YI SYLLABLE CHYX;Lo;0;L;;;;;N;;;;; +A378;YI SYLLABLE CHY;Lo;0;L;;;;;N;;;;; +A379;YI SYLLABLE CHYP;Lo;0;L;;;;;N;;;;; +A37A;YI SYLLABLE CHYRX;Lo;0;L;;;;;N;;;;; +A37B;YI SYLLABLE CHYR;Lo;0;L;;;;;N;;;;; +A37C;YI SYLLABLE RRAX;Lo;0;L;;;;;N;;;;; +A37D;YI SYLLABLE RRA;Lo;0;L;;;;;N;;;;; +A37E;YI SYLLABLE RRUOX;Lo;0;L;;;;;N;;;;; +A37F;YI SYLLABLE RRUO;Lo;0;L;;;;;N;;;;; +A380;YI SYLLABLE RROT;Lo;0;L;;;;;N;;;;; +A381;YI SYLLABLE RROX;Lo;0;L;;;;;N;;;;; +A382;YI SYLLABLE RRO;Lo;0;L;;;;;N;;;;; +A383;YI SYLLABLE RROP;Lo;0;L;;;;;N;;;;; +A384;YI SYLLABLE RRET;Lo;0;L;;;;;N;;;;; +A385;YI SYLLABLE RREX;Lo;0;L;;;;;N;;;;; +A386;YI SYLLABLE RRE;Lo;0;L;;;;;N;;;;; +A387;YI SYLLABLE RREP;Lo;0;L;;;;;N;;;;; +A388;YI SYLLABLE RRUT;Lo;0;L;;;;;N;;;;; +A389;YI SYLLABLE RRUX;Lo;0;L;;;;;N;;;;; +A38A;YI SYLLABLE RRU;Lo;0;L;;;;;N;;;;; +A38B;YI SYLLABLE RRUP;Lo;0;L;;;;;N;;;;; +A38C;YI SYLLABLE RRURX;Lo;0;L;;;;;N;;;;; +A38D;YI SYLLABLE RRUR;Lo;0;L;;;;;N;;;;; +A38E;YI SYLLABLE RRYT;Lo;0;L;;;;;N;;;;; +A38F;YI SYLLABLE RRYX;Lo;0;L;;;;;N;;;;; +A390;YI SYLLABLE RRY;Lo;0;L;;;;;N;;;;; +A391;YI SYLLABLE RRYP;Lo;0;L;;;;;N;;;;; +A392;YI SYLLABLE RRYRX;Lo;0;L;;;;;N;;;;; +A393;YI SYLLABLE RRYR;Lo;0;L;;;;;N;;;;; +A394;YI SYLLABLE NRAT;Lo;0;L;;;;;N;;;;; +A395;YI SYLLABLE NRAX;Lo;0;L;;;;;N;;;;; +A396;YI SYLLABLE NRA;Lo;0;L;;;;;N;;;;; +A397;YI SYLLABLE NRAP;Lo;0;L;;;;;N;;;;; +A398;YI SYLLABLE NROX;Lo;0;L;;;;;N;;;;; +A399;YI SYLLABLE NRO;Lo;0;L;;;;;N;;;;; +A39A;YI SYLLABLE NROP;Lo;0;L;;;;;N;;;;; +A39B;YI SYLLABLE NRET;Lo;0;L;;;;;N;;;;; +A39C;YI SYLLABLE NREX;Lo;0;L;;;;;N;;;;; +A39D;YI SYLLABLE NRE;Lo;0;L;;;;;N;;;;; +A39E;YI SYLLABLE NREP;Lo;0;L;;;;;N;;;;; +A39F;YI SYLLABLE NRUT;Lo;0;L;;;;;N;;;;; +A3A0;YI SYLLABLE NRUX;Lo;0;L;;;;;N;;;;; +A3A1;YI SYLLABLE NRU;Lo;0;L;;;;;N;;;;; +A3A2;YI SYLLABLE NRUP;Lo;0;L;;;;;N;;;;; +A3A3;YI SYLLABLE NRURX;Lo;0;L;;;;;N;;;;; +A3A4;YI SYLLABLE NRUR;Lo;0;L;;;;;N;;;;; +A3A5;YI SYLLABLE NRYT;Lo;0;L;;;;;N;;;;; +A3A6;YI SYLLABLE NRYX;Lo;0;L;;;;;N;;;;; +A3A7;YI SYLLABLE NRY;Lo;0;L;;;;;N;;;;; +A3A8;YI SYLLABLE NRYP;Lo;0;L;;;;;N;;;;; +A3A9;YI SYLLABLE NRYRX;Lo;0;L;;;;;N;;;;; +A3AA;YI SYLLABLE NRYR;Lo;0;L;;;;;N;;;;; +A3AB;YI SYLLABLE SHAT;Lo;0;L;;;;;N;;;;; +A3AC;YI SYLLABLE SHAX;Lo;0;L;;;;;N;;;;; +A3AD;YI SYLLABLE SHA;Lo;0;L;;;;;N;;;;; +A3AE;YI SYLLABLE SHAP;Lo;0;L;;;;;N;;;;; +A3AF;YI SYLLABLE SHUOX;Lo;0;L;;;;;N;;;;; +A3B0;YI SYLLABLE SHUO;Lo;0;L;;;;;N;;;;; +A3B1;YI SYLLABLE SHUOP;Lo;0;L;;;;;N;;;;; +A3B2;YI SYLLABLE SHOT;Lo;0;L;;;;;N;;;;; +A3B3;YI SYLLABLE SHOX;Lo;0;L;;;;;N;;;;; +A3B4;YI SYLLABLE SHO;Lo;0;L;;;;;N;;;;; +A3B5;YI SYLLABLE SHOP;Lo;0;L;;;;;N;;;;; +A3B6;YI SYLLABLE SHET;Lo;0;L;;;;;N;;;;; +A3B7;YI SYLLABLE SHEX;Lo;0;L;;;;;N;;;;; +A3B8;YI SYLLABLE SHE;Lo;0;L;;;;;N;;;;; +A3B9;YI SYLLABLE SHEP;Lo;0;L;;;;;N;;;;; +A3BA;YI SYLLABLE SHUT;Lo;0;L;;;;;N;;;;; +A3BB;YI SYLLABLE SHUX;Lo;0;L;;;;;N;;;;; +A3BC;YI SYLLABLE SHU;Lo;0;L;;;;;N;;;;; +A3BD;YI SYLLABLE SHUP;Lo;0;L;;;;;N;;;;; +A3BE;YI SYLLABLE SHURX;Lo;0;L;;;;;N;;;;; +A3BF;YI SYLLABLE SHUR;Lo;0;L;;;;;N;;;;; +A3C0;YI SYLLABLE SHYT;Lo;0;L;;;;;N;;;;; +A3C1;YI SYLLABLE SHYX;Lo;0;L;;;;;N;;;;; +A3C2;YI SYLLABLE SHY;Lo;0;L;;;;;N;;;;; +A3C3;YI SYLLABLE SHYP;Lo;0;L;;;;;N;;;;; +A3C4;YI SYLLABLE SHYRX;Lo;0;L;;;;;N;;;;; +A3C5;YI SYLLABLE SHYR;Lo;0;L;;;;;N;;;;; +A3C6;YI SYLLABLE RAT;Lo;0;L;;;;;N;;;;; +A3C7;YI SYLLABLE RAX;Lo;0;L;;;;;N;;;;; +A3C8;YI SYLLABLE RA;Lo;0;L;;;;;N;;;;; +A3C9;YI SYLLABLE RAP;Lo;0;L;;;;;N;;;;; +A3CA;YI SYLLABLE RUOX;Lo;0;L;;;;;N;;;;; +A3CB;YI SYLLABLE RUO;Lo;0;L;;;;;N;;;;; +A3CC;YI SYLLABLE RUOP;Lo;0;L;;;;;N;;;;; +A3CD;YI SYLLABLE ROT;Lo;0;L;;;;;N;;;;; +A3CE;YI SYLLABLE ROX;Lo;0;L;;;;;N;;;;; +A3CF;YI SYLLABLE RO;Lo;0;L;;;;;N;;;;; +A3D0;YI SYLLABLE ROP;Lo;0;L;;;;;N;;;;; +A3D1;YI SYLLABLE REX;Lo;0;L;;;;;N;;;;; +A3D2;YI SYLLABLE RE;Lo;0;L;;;;;N;;;;; +A3D3;YI SYLLABLE REP;Lo;0;L;;;;;N;;;;; +A3D4;YI SYLLABLE RUT;Lo;0;L;;;;;N;;;;; +A3D5;YI SYLLABLE RUX;Lo;0;L;;;;;N;;;;; +A3D6;YI SYLLABLE RU;Lo;0;L;;;;;N;;;;; +A3D7;YI SYLLABLE RUP;Lo;0;L;;;;;N;;;;; +A3D8;YI SYLLABLE RURX;Lo;0;L;;;;;N;;;;; +A3D9;YI SYLLABLE RUR;Lo;0;L;;;;;N;;;;; +A3DA;YI SYLLABLE RYT;Lo;0;L;;;;;N;;;;; +A3DB;YI SYLLABLE RYX;Lo;0;L;;;;;N;;;;; +A3DC;YI SYLLABLE RY;Lo;0;L;;;;;N;;;;; +A3DD;YI SYLLABLE RYP;Lo;0;L;;;;;N;;;;; +A3DE;YI SYLLABLE RYRX;Lo;0;L;;;;;N;;;;; +A3DF;YI SYLLABLE RYR;Lo;0;L;;;;;N;;;;; +A3E0;YI SYLLABLE JIT;Lo;0;L;;;;;N;;;;; +A3E1;YI SYLLABLE JIX;Lo;0;L;;;;;N;;;;; +A3E2;YI SYLLABLE JI;Lo;0;L;;;;;N;;;;; +A3E3;YI SYLLABLE JIP;Lo;0;L;;;;;N;;;;; +A3E4;YI SYLLABLE JIET;Lo;0;L;;;;;N;;;;; +A3E5;YI SYLLABLE JIEX;Lo;0;L;;;;;N;;;;; +A3E6;YI SYLLABLE JIE;Lo;0;L;;;;;N;;;;; +A3E7;YI SYLLABLE JIEP;Lo;0;L;;;;;N;;;;; +A3E8;YI SYLLABLE JUOT;Lo;0;L;;;;;N;;;;; +A3E9;YI SYLLABLE JUOX;Lo;0;L;;;;;N;;;;; +A3EA;YI SYLLABLE JUO;Lo;0;L;;;;;N;;;;; +A3EB;YI SYLLABLE JUOP;Lo;0;L;;;;;N;;;;; +A3EC;YI SYLLABLE JOT;Lo;0;L;;;;;N;;;;; +A3ED;YI SYLLABLE JOX;Lo;0;L;;;;;N;;;;; +A3EE;YI SYLLABLE JO;Lo;0;L;;;;;N;;;;; +A3EF;YI SYLLABLE JOP;Lo;0;L;;;;;N;;;;; +A3F0;YI SYLLABLE JUT;Lo;0;L;;;;;N;;;;; +A3F1;YI SYLLABLE JUX;Lo;0;L;;;;;N;;;;; +A3F2;YI SYLLABLE JU;Lo;0;L;;;;;N;;;;; +A3F3;YI SYLLABLE JUP;Lo;0;L;;;;;N;;;;; +A3F4;YI SYLLABLE JURX;Lo;0;L;;;;;N;;;;; +A3F5;YI SYLLABLE JUR;Lo;0;L;;;;;N;;;;; +A3F6;YI SYLLABLE JYT;Lo;0;L;;;;;N;;;;; +A3F7;YI SYLLABLE JYX;Lo;0;L;;;;;N;;;;; +A3F8;YI SYLLABLE JY;Lo;0;L;;;;;N;;;;; +A3F9;YI SYLLABLE JYP;Lo;0;L;;;;;N;;;;; +A3FA;YI SYLLABLE JYRX;Lo;0;L;;;;;N;;;;; +A3FB;YI SYLLABLE JYR;Lo;0;L;;;;;N;;;;; +A3FC;YI SYLLABLE QIT;Lo;0;L;;;;;N;;;;; +A3FD;YI SYLLABLE QIX;Lo;0;L;;;;;N;;;;; +A3FE;YI SYLLABLE QI;Lo;0;L;;;;;N;;;;; +A3FF;YI SYLLABLE QIP;Lo;0;L;;;;;N;;;;; +A400;YI SYLLABLE QIET;Lo;0;L;;;;;N;;;;; +A401;YI SYLLABLE QIEX;Lo;0;L;;;;;N;;;;; +A402;YI SYLLABLE QIE;Lo;0;L;;;;;N;;;;; +A403;YI SYLLABLE QIEP;Lo;0;L;;;;;N;;;;; +A404;YI SYLLABLE QUOT;Lo;0;L;;;;;N;;;;; +A405;YI SYLLABLE QUOX;Lo;0;L;;;;;N;;;;; +A406;YI SYLLABLE QUO;Lo;0;L;;;;;N;;;;; +A407;YI SYLLABLE QUOP;Lo;0;L;;;;;N;;;;; +A408;YI SYLLABLE QOT;Lo;0;L;;;;;N;;;;; +A409;YI SYLLABLE QOX;Lo;0;L;;;;;N;;;;; +A40A;YI SYLLABLE QO;Lo;0;L;;;;;N;;;;; +A40B;YI SYLLABLE QOP;Lo;0;L;;;;;N;;;;; +A40C;YI SYLLABLE QUT;Lo;0;L;;;;;N;;;;; +A40D;YI SYLLABLE QUX;Lo;0;L;;;;;N;;;;; +A40E;YI SYLLABLE QU;Lo;0;L;;;;;N;;;;; +A40F;YI SYLLABLE QUP;Lo;0;L;;;;;N;;;;; +A410;YI SYLLABLE QURX;Lo;0;L;;;;;N;;;;; +A411;YI SYLLABLE QUR;Lo;0;L;;;;;N;;;;; +A412;YI SYLLABLE QYT;Lo;0;L;;;;;N;;;;; +A413;YI SYLLABLE QYX;Lo;0;L;;;;;N;;;;; +A414;YI SYLLABLE QY;Lo;0;L;;;;;N;;;;; +A415;YI SYLLABLE QYP;Lo;0;L;;;;;N;;;;; +A416;YI SYLLABLE QYRX;Lo;0;L;;;;;N;;;;; +A417;YI SYLLABLE QYR;Lo;0;L;;;;;N;;;;; +A418;YI SYLLABLE JJIT;Lo;0;L;;;;;N;;;;; +A419;YI SYLLABLE JJIX;Lo;0;L;;;;;N;;;;; +A41A;YI SYLLABLE JJI;Lo;0;L;;;;;N;;;;; +A41B;YI SYLLABLE JJIP;Lo;0;L;;;;;N;;;;; +A41C;YI SYLLABLE JJIET;Lo;0;L;;;;;N;;;;; +A41D;YI SYLLABLE JJIEX;Lo;0;L;;;;;N;;;;; +A41E;YI SYLLABLE JJIE;Lo;0;L;;;;;N;;;;; +A41F;YI SYLLABLE JJIEP;Lo;0;L;;;;;N;;;;; +A420;YI SYLLABLE JJUOX;Lo;0;L;;;;;N;;;;; +A421;YI SYLLABLE JJUO;Lo;0;L;;;;;N;;;;; +A422;YI SYLLABLE JJUOP;Lo;0;L;;;;;N;;;;; +A423;YI SYLLABLE JJOT;Lo;0;L;;;;;N;;;;; +A424;YI SYLLABLE JJOX;Lo;0;L;;;;;N;;;;; +A425;YI SYLLABLE JJO;Lo;0;L;;;;;N;;;;; +A426;YI SYLLABLE JJOP;Lo;0;L;;;;;N;;;;; +A427;YI SYLLABLE JJUT;Lo;0;L;;;;;N;;;;; +A428;YI SYLLABLE JJUX;Lo;0;L;;;;;N;;;;; +A429;YI SYLLABLE JJU;Lo;0;L;;;;;N;;;;; +A42A;YI SYLLABLE JJUP;Lo;0;L;;;;;N;;;;; +A42B;YI SYLLABLE JJURX;Lo;0;L;;;;;N;;;;; +A42C;YI SYLLABLE JJUR;Lo;0;L;;;;;N;;;;; +A42D;YI SYLLABLE JJYT;Lo;0;L;;;;;N;;;;; +A42E;YI SYLLABLE JJYX;Lo;0;L;;;;;N;;;;; +A42F;YI SYLLABLE JJY;Lo;0;L;;;;;N;;;;; +A430;YI SYLLABLE JJYP;Lo;0;L;;;;;N;;;;; +A431;YI SYLLABLE NJIT;Lo;0;L;;;;;N;;;;; +A432;YI SYLLABLE NJIX;Lo;0;L;;;;;N;;;;; +A433;YI SYLLABLE NJI;Lo;0;L;;;;;N;;;;; +A434;YI SYLLABLE NJIP;Lo;0;L;;;;;N;;;;; +A435;YI SYLLABLE NJIET;Lo;0;L;;;;;N;;;;; +A436;YI SYLLABLE NJIEX;Lo;0;L;;;;;N;;;;; +A437;YI SYLLABLE NJIE;Lo;0;L;;;;;N;;;;; +A438;YI SYLLABLE NJIEP;Lo;0;L;;;;;N;;;;; +A439;YI SYLLABLE NJUOX;Lo;0;L;;;;;N;;;;; +A43A;YI SYLLABLE NJUO;Lo;0;L;;;;;N;;;;; +A43B;YI SYLLABLE NJOT;Lo;0;L;;;;;N;;;;; +A43C;YI SYLLABLE NJOX;Lo;0;L;;;;;N;;;;; +A43D;YI SYLLABLE NJO;Lo;0;L;;;;;N;;;;; +A43E;YI SYLLABLE NJOP;Lo;0;L;;;;;N;;;;; +A43F;YI SYLLABLE NJUX;Lo;0;L;;;;;N;;;;; +A440;YI SYLLABLE NJU;Lo;0;L;;;;;N;;;;; +A441;YI SYLLABLE NJUP;Lo;0;L;;;;;N;;;;; +A442;YI SYLLABLE NJURX;Lo;0;L;;;;;N;;;;; +A443;YI SYLLABLE NJUR;Lo;0;L;;;;;N;;;;; +A444;YI SYLLABLE NJYT;Lo;0;L;;;;;N;;;;; +A445;YI SYLLABLE NJYX;Lo;0;L;;;;;N;;;;; +A446;YI SYLLABLE NJY;Lo;0;L;;;;;N;;;;; +A447;YI SYLLABLE NJYP;Lo;0;L;;;;;N;;;;; +A448;YI SYLLABLE NJYRX;Lo;0;L;;;;;N;;;;; +A449;YI SYLLABLE NJYR;Lo;0;L;;;;;N;;;;; +A44A;YI SYLLABLE NYIT;Lo;0;L;;;;;N;;;;; +A44B;YI SYLLABLE NYIX;Lo;0;L;;;;;N;;;;; +A44C;YI SYLLABLE NYI;Lo;0;L;;;;;N;;;;; +A44D;YI SYLLABLE NYIP;Lo;0;L;;;;;N;;;;; +A44E;YI SYLLABLE NYIET;Lo;0;L;;;;;N;;;;; +A44F;YI SYLLABLE NYIEX;Lo;0;L;;;;;N;;;;; +A450;YI SYLLABLE NYIE;Lo;0;L;;;;;N;;;;; +A451;YI SYLLABLE NYIEP;Lo;0;L;;;;;N;;;;; +A452;YI SYLLABLE NYUOX;Lo;0;L;;;;;N;;;;; +A453;YI SYLLABLE NYUO;Lo;0;L;;;;;N;;;;; +A454;YI SYLLABLE NYUOP;Lo;0;L;;;;;N;;;;; +A455;YI SYLLABLE NYOT;Lo;0;L;;;;;N;;;;; +A456;YI SYLLABLE NYOX;Lo;0;L;;;;;N;;;;; +A457;YI SYLLABLE NYO;Lo;0;L;;;;;N;;;;; +A458;YI SYLLABLE NYOP;Lo;0;L;;;;;N;;;;; +A459;YI SYLLABLE NYUT;Lo;0;L;;;;;N;;;;; +A45A;YI SYLLABLE NYUX;Lo;0;L;;;;;N;;;;; +A45B;YI SYLLABLE NYU;Lo;0;L;;;;;N;;;;; +A45C;YI SYLLABLE NYUP;Lo;0;L;;;;;N;;;;; +A45D;YI SYLLABLE XIT;Lo;0;L;;;;;N;;;;; +A45E;YI SYLLABLE XIX;Lo;0;L;;;;;N;;;;; +A45F;YI SYLLABLE XI;Lo;0;L;;;;;N;;;;; +A460;YI SYLLABLE XIP;Lo;0;L;;;;;N;;;;; +A461;YI SYLLABLE XIET;Lo;0;L;;;;;N;;;;; +A462;YI SYLLABLE XIEX;Lo;0;L;;;;;N;;;;; +A463;YI SYLLABLE XIE;Lo;0;L;;;;;N;;;;; +A464;YI SYLLABLE XIEP;Lo;0;L;;;;;N;;;;; +A465;YI SYLLABLE XUOX;Lo;0;L;;;;;N;;;;; +A466;YI SYLLABLE XUO;Lo;0;L;;;;;N;;;;; +A467;YI SYLLABLE XOT;Lo;0;L;;;;;N;;;;; +A468;YI SYLLABLE XOX;Lo;0;L;;;;;N;;;;; +A469;YI SYLLABLE XO;Lo;0;L;;;;;N;;;;; +A46A;YI SYLLABLE XOP;Lo;0;L;;;;;N;;;;; +A46B;YI SYLLABLE XYT;Lo;0;L;;;;;N;;;;; +A46C;YI SYLLABLE XYX;Lo;0;L;;;;;N;;;;; +A46D;YI SYLLABLE XY;Lo;0;L;;;;;N;;;;; +A46E;YI SYLLABLE XYP;Lo;0;L;;;;;N;;;;; +A46F;YI SYLLABLE XYRX;Lo;0;L;;;;;N;;;;; +A470;YI SYLLABLE XYR;Lo;0;L;;;;;N;;;;; +A471;YI SYLLABLE YIT;Lo;0;L;;;;;N;;;;; +A472;YI SYLLABLE YIX;Lo;0;L;;;;;N;;;;; +A473;YI SYLLABLE YI;Lo;0;L;;;;;N;;;;; +A474;YI SYLLABLE YIP;Lo;0;L;;;;;N;;;;; +A475;YI SYLLABLE YIET;Lo;0;L;;;;;N;;;;; +A476;YI SYLLABLE YIEX;Lo;0;L;;;;;N;;;;; +A477;YI SYLLABLE YIE;Lo;0;L;;;;;N;;;;; +A478;YI SYLLABLE YIEP;Lo;0;L;;;;;N;;;;; +A479;YI SYLLABLE YUOT;Lo;0;L;;;;;N;;;;; +A47A;YI SYLLABLE YUOX;Lo;0;L;;;;;N;;;;; +A47B;YI SYLLABLE YUO;Lo;0;L;;;;;N;;;;; +A47C;YI SYLLABLE YUOP;Lo;0;L;;;;;N;;;;; +A47D;YI SYLLABLE YOT;Lo;0;L;;;;;N;;;;; +A47E;YI SYLLABLE YOX;Lo;0;L;;;;;N;;;;; +A47F;YI SYLLABLE YO;Lo;0;L;;;;;N;;;;; +A480;YI SYLLABLE YOP;Lo;0;L;;;;;N;;;;; +A481;YI SYLLABLE YUT;Lo;0;L;;;;;N;;;;; +A482;YI SYLLABLE YUX;Lo;0;L;;;;;N;;;;; +A483;YI SYLLABLE YU;Lo;0;L;;;;;N;;;;; +A484;YI SYLLABLE YUP;Lo;0;L;;;;;N;;;;; +A485;YI SYLLABLE YURX;Lo;0;L;;;;;N;;;;; +A486;YI SYLLABLE YUR;Lo;0;L;;;;;N;;;;; +A487;YI SYLLABLE YYT;Lo;0;L;;;;;N;;;;; +A488;YI SYLLABLE YYX;Lo;0;L;;;;;N;;;;; +A489;YI SYLLABLE YY;Lo;0;L;;;;;N;;;;; +A48A;YI SYLLABLE YYP;Lo;0;L;;;;;N;;;;; +A48B;YI SYLLABLE YYRX;Lo;0;L;;;;;N;;;;; +A48C;YI SYLLABLE YYR;Lo;0;L;;;;;N;;;;; +A490;YI RADICAL QOT;So;0;ON;;;;;N;;;;; +A491;YI RADICAL LI;So;0;ON;;;;;N;;;;; +A492;YI RADICAL KIT;So;0;ON;;;;;N;;;;; +A493;YI RADICAL NYIP;So;0;ON;;;;;N;;;;; +A494;YI RADICAL CYP;So;0;ON;;;;;N;;;;; +A495;YI RADICAL SSI;So;0;ON;;;;;N;;;;; +A496;YI RADICAL GGOP;So;0;ON;;;;;N;;;;; +A497;YI RADICAL GEP;So;0;ON;;;;;N;;;;; +A498;YI RADICAL MI;So;0;ON;;;;;N;;;;; +A499;YI RADICAL HXIT;So;0;ON;;;;;N;;;;; +A49A;YI RADICAL LYR;So;0;ON;;;;;N;;;;; +A49B;YI RADICAL BBUT;So;0;ON;;;;;N;;;;; +A49C;YI RADICAL MOP;So;0;ON;;;;;N;;;;; +A49D;YI RADICAL YO;So;0;ON;;;;;N;;;;; +A49E;YI RADICAL PUT;So;0;ON;;;;;N;;;;; +A49F;YI RADICAL HXUO;So;0;ON;;;;;N;;;;; +A4A0;YI RADICAL TAT;So;0;ON;;;;;N;;;;; +A4A1;YI RADICAL GA;So;0;ON;;;;;N;;;;; +A4A2;YI RADICAL ZUP;So;0;ON;;;;;N;;;;; +A4A3;YI RADICAL CYT;So;0;ON;;;;;N;;;;; +A4A4;YI RADICAL DDUR;So;0;ON;;;;;N;;;;; +A4A5;YI RADICAL BUR;So;0;ON;;;;;N;;;;; +A4A6;YI RADICAL GGUO;So;0;ON;;;;;N;;;;; +A4A7;YI RADICAL NYOP;So;0;ON;;;;;N;;;;; +A4A8;YI RADICAL TU;So;0;ON;;;;;N;;;;; +A4A9;YI RADICAL OP;So;0;ON;;;;;N;;;;; +A4AA;YI RADICAL JJUT;So;0;ON;;;;;N;;;;; +A4AB;YI RADICAL ZOT;So;0;ON;;;;;N;;;;; +A4AC;YI RADICAL PYT;So;0;ON;;;;;N;;;;; +A4AD;YI RADICAL HMO;So;0;ON;;;;;N;;;;; +A4AE;YI RADICAL YIT;So;0;ON;;;;;N;;;;; +A4AF;YI RADICAL VUR;So;0;ON;;;;;N;;;;; +A4B0;YI RADICAL SHY;So;0;ON;;;;;N;;;;; +A4B1;YI RADICAL VEP;So;0;ON;;;;;N;;;;; +A4B2;YI RADICAL ZA;So;0;ON;;;;;N;;;;; +A4B3;YI RADICAL JO;So;0;ON;;;;;N;;;;; +A4B4;YI RADICAL NZUP;So;0;ON;;;;;N;;;;; +A4B5;YI RADICAL JJY;So;0;ON;;;;;N;;;;; +A4B6;YI RADICAL GOT;So;0;ON;;;;;N;;;;; +A4B7;YI RADICAL JJIE;So;0;ON;;;;;N;;;;; +A4B8;YI RADICAL WO;So;0;ON;;;;;N;;;;; +A4B9;YI RADICAL DU;So;0;ON;;;;;N;;;;; +A4BA;YI RADICAL SHUR;So;0;ON;;;;;N;;;;; +A4BB;YI RADICAL LIE;So;0;ON;;;;;N;;;;; +A4BC;YI RADICAL CY;So;0;ON;;;;;N;;;;; +A4BD;YI RADICAL CUOP;So;0;ON;;;;;N;;;;; +A4BE;YI RADICAL CIP;So;0;ON;;;;;N;;;;; +A4BF;YI RADICAL HXOP;So;0;ON;;;;;N;;;;; +A4C0;YI RADICAL SHAT;So;0;ON;;;;;N;;;;; +A4C1;YI RADICAL ZUR;So;0;ON;;;;;N;;;;; +A4C2;YI RADICAL SHOP;So;0;ON;;;;;N;;;;; +A4C3;YI RADICAL CHE;So;0;ON;;;;;N;;;;; +A4C4;YI RADICAL ZZIET;So;0;ON;;;;;N;;;;; +A4C5;YI RADICAL NBIE;So;0;ON;;;;;N;;;;; +A4C6;YI RADICAL KE;So;0;ON;;;;;N;;;;; +A4D0;LISU LETTER BA;Lo;0;L;;;;;N;;;;; +A4D1;LISU LETTER PA;Lo;0;L;;;;;N;;;;; +A4D2;LISU LETTER PHA;Lo;0;L;;;;;N;;;;; +A4D3;LISU LETTER DA;Lo;0;L;;;;;N;;;;; +A4D4;LISU LETTER TA;Lo;0;L;;;;;N;;;;; +A4D5;LISU LETTER THA;Lo;0;L;;;;;N;;;;; +A4D6;LISU LETTER GA;Lo;0;L;;;;;N;;;;; +A4D7;LISU LETTER KA;Lo;0;L;;;;;N;;;;; +A4D8;LISU LETTER KHA;Lo;0;L;;;;;N;;;;; +A4D9;LISU LETTER JA;Lo;0;L;;;;;N;;;;; +A4DA;LISU LETTER CA;Lo;0;L;;;;;N;;;;; +A4DB;LISU LETTER CHA;Lo;0;L;;;;;N;;;;; +A4DC;LISU LETTER DZA;Lo;0;L;;;;;N;;;;; +A4DD;LISU LETTER TSA;Lo;0;L;;;;;N;;;;; +A4DE;LISU LETTER TSHA;Lo;0;L;;;;;N;;;;; +A4DF;LISU LETTER MA;Lo;0;L;;;;;N;;;;; +A4E0;LISU LETTER NA;Lo;0;L;;;;;N;;;;; +A4E1;LISU LETTER LA;Lo;0;L;;;;;N;;;;; +A4E2;LISU LETTER SA;Lo;0;L;;;;;N;;;;; +A4E3;LISU LETTER ZHA;Lo;0;L;;;;;N;;;;; +A4E4;LISU LETTER ZA;Lo;0;L;;;;;N;;;;; +A4E5;LISU LETTER NGA;Lo;0;L;;;;;N;;;;; +A4E6;LISU LETTER HA;Lo;0;L;;;;;N;;;;; +A4E7;LISU LETTER XA;Lo;0;L;;;;;N;;;;; +A4E8;LISU LETTER HHA;Lo;0;L;;;;;N;;;;; +A4E9;LISU LETTER FA;Lo;0;L;;;;;N;;;;; +A4EA;LISU LETTER WA;Lo;0;L;;;;;N;;;;; +A4EB;LISU LETTER SHA;Lo;0;L;;;;;N;;;;; +A4EC;LISU LETTER YA;Lo;0;L;;;;;N;;;;; +A4ED;LISU LETTER GHA;Lo;0;L;;;;;N;;;;; +A4EE;LISU LETTER A;Lo;0;L;;;;;N;;;;; +A4EF;LISU LETTER AE;Lo;0;L;;;;;N;;;;; +A4F0;LISU LETTER E;Lo;0;L;;;;;N;;;;; +A4F1;LISU LETTER EU;Lo;0;L;;;;;N;;;;; +A4F2;LISU LETTER I;Lo;0;L;;;;;N;;;;; +A4F3;LISU LETTER O;Lo;0;L;;;;;N;;;;; +A4F4;LISU LETTER U;Lo;0;L;;;;;N;;;;; +A4F5;LISU LETTER UE;Lo;0;L;;;;;N;;;;; +A4F6;LISU LETTER UH;Lo;0;L;;;;;N;;;;; +A4F7;LISU LETTER OE;Lo;0;L;;;;;N;;;;; +A4F8;LISU LETTER TONE MYA TI;Lm;0;L;;;;;N;;;;; +A4F9;LISU LETTER TONE NA PO;Lm;0;L;;;;;N;;;;; +A4FA;LISU LETTER TONE MYA CYA;Lm;0;L;;;;;N;;;;; +A4FB;LISU LETTER TONE MYA BO;Lm;0;L;;;;;N;;;;; +A4FC;LISU LETTER TONE MYA NA;Lm;0;L;;;;;N;;;;; +A4FD;LISU LETTER TONE MYA JEU;Lm;0;L;;;;;N;;;;; +A4FE;LISU PUNCTUATION COMMA;Po;0;L;;;;;N;;;;; +A4FF;LISU PUNCTUATION FULL STOP;Po;0;L;;;;;N;;;;; +A500;VAI SYLLABLE EE;Lo;0;L;;;;;N;;;;; +A501;VAI SYLLABLE EEN;Lo;0;L;;;;;N;;;;; +A502;VAI SYLLABLE HEE;Lo;0;L;;;;;N;;;;; +A503;VAI SYLLABLE WEE;Lo;0;L;;;;;N;;;;; +A504;VAI SYLLABLE WEEN;Lo;0;L;;;;;N;;;;; +A505;VAI SYLLABLE PEE;Lo;0;L;;;;;N;;;;; +A506;VAI SYLLABLE BHEE;Lo;0;L;;;;;N;;;;; +A507;VAI SYLLABLE BEE;Lo;0;L;;;;;N;;;;; +A508;VAI SYLLABLE MBEE;Lo;0;L;;;;;N;;;;; +A509;VAI SYLLABLE KPEE;Lo;0;L;;;;;N;;;;; +A50A;VAI SYLLABLE MGBEE;Lo;0;L;;;;;N;;;;; +A50B;VAI SYLLABLE GBEE;Lo;0;L;;;;;N;;;;; +A50C;VAI SYLLABLE FEE;Lo;0;L;;;;;N;;;;; +A50D;VAI SYLLABLE VEE;Lo;0;L;;;;;N;;;;; +A50E;VAI SYLLABLE TEE;Lo;0;L;;;;;N;;;;; +A50F;VAI SYLLABLE THEE;Lo;0;L;;;;;N;;;;; +A510;VAI SYLLABLE DHEE;Lo;0;L;;;;;N;;;;; +A511;VAI SYLLABLE DHHEE;Lo;0;L;;;;;N;;;;; +A512;VAI SYLLABLE LEE;Lo;0;L;;;;;N;;;;; +A513;VAI SYLLABLE REE;Lo;0;L;;;;;N;;;;; +A514;VAI SYLLABLE DEE;Lo;0;L;;;;;N;;;;; +A515;VAI SYLLABLE NDEE;Lo;0;L;;;;;N;;;;; +A516;VAI SYLLABLE SEE;Lo;0;L;;;;;N;;;;; +A517;VAI SYLLABLE SHEE;Lo;0;L;;;;;N;;;;; +A518;VAI SYLLABLE ZEE;Lo;0;L;;;;;N;;;;; +A519;VAI SYLLABLE ZHEE;Lo;0;L;;;;;N;;;;; +A51A;VAI SYLLABLE CEE;Lo;0;L;;;;;N;;;;; +A51B;VAI SYLLABLE JEE;Lo;0;L;;;;;N;;;;; +A51C;VAI SYLLABLE NJEE;Lo;0;L;;;;;N;;;;; +A51D;VAI SYLLABLE YEE;Lo;0;L;;;;;N;;;;; +A51E;VAI SYLLABLE KEE;Lo;0;L;;;;;N;;;;; +A51F;VAI SYLLABLE NGGEE;Lo;0;L;;;;;N;;;;; +A520;VAI SYLLABLE GEE;Lo;0;L;;;;;N;;;;; +A521;VAI SYLLABLE MEE;Lo;0;L;;;;;N;;;;; +A522;VAI SYLLABLE NEE;Lo;0;L;;;;;N;;;;; +A523;VAI SYLLABLE NYEE;Lo;0;L;;;;;N;;;;; +A524;VAI SYLLABLE I;Lo;0;L;;;;;N;;;;; +A525;VAI SYLLABLE IN;Lo;0;L;;;;;N;;;;; +A526;VAI SYLLABLE HI;Lo;0;L;;;;;N;;;;; +A527;VAI SYLLABLE HIN;Lo;0;L;;;;;N;;;;; +A528;VAI SYLLABLE WI;Lo;0;L;;;;;N;;;;; +A529;VAI SYLLABLE WIN;Lo;0;L;;;;;N;;;;; +A52A;VAI SYLLABLE PI;Lo;0;L;;;;;N;;;;; +A52B;VAI SYLLABLE BHI;Lo;0;L;;;;;N;;;;; +A52C;VAI SYLLABLE BI;Lo;0;L;;;;;N;;;;; +A52D;VAI SYLLABLE MBI;Lo;0;L;;;;;N;;;;; +A52E;VAI SYLLABLE KPI;Lo;0;L;;;;;N;;;;; +A52F;VAI SYLLABLE MGBI;Lo;0;L;;;;;N;;;;; +A530;VAI SYLLABLE GBI;Lo;0;L;;;;;N;;;;; +A531;VAI SYLLABLE FI;Lo;0;L;;;;;N;;;;; +A532;VAI SYLLABLE VI;Lo;0;L;;;;;N;;;;; +A533;VAI SYLLABLE TI;Lo;0;L;;;;;N;;;;; +A534;VAI SYLLABLE THI;Lo;0;L;;;;;N;;;;; +A535;VAI SYLLABLE DHI;Lo;0;L;;;;;N;;;;; +A536;VAI SYLLABLE DHHI;Lo;0;L;;;;;N;;;;; +A537;VAI SYLLABLE LI;Lo;0;L;;;;;N;;;;; +A538;VAI SYLLABLE RI;Lo;0;L;;;;;N;;;;; +A539;VAI SYLLABLE DI;Lo;0;L;;;;;N;;;;; +A53A;VAI SYLLABLE NDI;Lo;0;L;;;;;N;;;;; +A53B;VAI SYLLABLE SI;Lo;0;L;;;;;N;;;;; +A53C;VAI SYLLABLE SHI;Lo;0;L;;;;;N;;;;; +A53D;VAI SYLLABLE ZI;Lo;0;L;;;;;N;;;;; +A53E;VAI SYLLABLE ZHI;Lo;0;L;;;;;N;;;;; +A53F;VAI SYLLABLE CI;Lo;0;L;;;;;N;;;;; +A540;VAI SYLLABLE JI;Lo;0;L;;;;;N;;;;; +A541;VAI SYLLABLE NJI;Lo;0;L;;;;;N;;;;; +A542;VAI SYLLABLE YI;Lo;0;L;;;;;N;;;;; +A543;VAI SYLLABLE KI;Lo;0;L;;;;;N;;;;; +A544;VAI SYLLABLE NGGI;Lo;0;L;;;;;N;;;;; +A545;VAI SYLLABLE GI;Lo;0;L;;;;;N;;;;; +A546;VAI SYLLABLE MI;Lo;0;L;;;;;N;;;;; +A547;VAI SYLLABLE NI;Lo;0;L;;;;;N;;;;; +A548;VAI SYLLABLE NYI;Lo;0;L;;;;;N;;;;; +A549;VAI SYLLABLE A;Lo;0;L;;;;;N;;;;; +A54A;VAI SYLLABLE AN;Lo;0;L;;;;;N;;;;; +A54B;VAI SYLLABLE NGAN;Lo;0;L;;;;;N;;;;; +A54C;VAI SYLLABLE HA;Lo;0;L;;;;;N;;;;; +A54D;VAI SYLLABLE HAN;Lo;0;L;;;;;N;;;;; +A54E;VAI SYLLABLE WA;Lo;0;L;;;;;N;;;;; +A54F;VAI SYLLABLE WAN;Lo;0;L;;;;;N;;;;; +A550;VAI SYLLABLE PA;Lo;0;L;;;;;N;;;;; +A551;VAI SYLLABLE BHA;Lo;0;L;;;;;N;;;;; +A552;VAI SYLLABLE BA;Lo;0;L;;;;;N;;;;; +A553;VAI SYLLABLE MBA;Lo;0;L;;;;;N;;;;; +A554;VAI SYLLABLE KPA;Lo;0;L;;;;;N;;;;; +A555;VAI SYLLABLE KPAN;Lo;0;L;;;;;N;;;;; +A556;VAI SYLLABLE MGBA;Lo;0;L;;;;;N;;;;; +A557;VAI SYLLABLE GBA;Lo;0;L;;;;;N;;;;; +A558;VAI SYLLABLE FA;Lo;0;L;;;;;N;;;;; +A559;VAI SYLLABLE VA;Lo;0;L;;;;;N;;;;; +A55A;VAI SYLLABLE TA;Lo;0;L;;;;;N;;;;; +A55B;VAI SYLLABLE THA;Lo;0;L;;;;;N;;;;; +A55C;VAI SYLLABLE DHA;Lo;0;L;;;;;N;;;;; +A55D;VAI SYLLABLE DHHA;Lo;0;L;;;;;N;;;;; +A55E;VAI SYLLABLE LA;Lo;0;L;;;;;N;;;;; +A55F;VAI SYLLABLE RA;Lo;0;L;;;;;N;;;;; +A560;VAI SYLLABLE DA;Lo;0;L;;;;;N;;;;; +A561;VAI SYLLABLE NDA;Lo;0;L;;;;;N;;;;; +A562;VAI SYLLABLE SA;Lo;0;L;;;;;N;;;;; +A563;VAI SYLLABLE SHA;Lo;0;L;;;;;N;;;;; +A564;VAI SYLLABLE ZA;Lo;0;L;;;;;N;;;;; +A565;VAI SYLLABLE ZHA;Lo;0;L;;;;;N;;;;; +A566;VAI SYLLABLE CA;Lo;0;L;;;;;N;;;;; +A567;VAI SYLLABLE JA;Lo;0;L;;;;;N;;;;; +A568;VAI SYLLABLE NJA;Lo;0;L;;;;;N;;;;; +A569;VAI SYLLABLE YA;Lo;0;L;;;;;N;;;;; +A56A;VAI SYLLABLE KA;Lo;0;L;;;;;N;;;;; +A56B;VAI SYLLABLE KAN;Lo;0;L;;;;;N;;;;; +A56C;VAI SYLLABLE NGGA;Lo;0;L;;;;;N;;;;; +A56D;VAI SYLLABLE GA;Lo;0;L;;;;;N;;;;; +A56E;VAI SYLLABLE MA;Lo;0;L;;;;;N;;;;; +A56F;VAI SYLLABLE NA;Lo;0;L;;;;;N;;;;; +A570;VAI SYLLABLE NYA;Lo;0;L;;;;;N;;;;; +A571;VAI SYLLABLE OO;Lo;0;L;;;;;N;;;;; +A572;VAI SYLLABLE OON;Lo;0;L;;;;;N;;;;; +A573;VAI SYLLABLE HOO;Lo;0;L;;;;;N;;;;; +A574;VAI SYLLABLE WOO;Lo;0;L;;;;;N;;;;; +A575;VAI SYLLABLE WOON;Lo;0;L;;;;;N;;;;; +A576;VAI SYLLABLE POO;Lo;0;L;;;;;N;;;;; +A577;VAI SYLLABLE BHOO;Lo;0;L;;;;;N;;;;; +A578;VAI SYLLABLE BOO;Lo;0;L;;;;;N;;;;; +A579;VAI SYLLABLE MBOO;Lo;0;L;;;;;N;;;;; +A57A;VAI SYLLABLE KPOO;Lo;0;L;;;;;N;;;;; +A57B;VAI SYLLABLE MGBOO;Lo;0;L;;;;;N;;;;; +A57C;VAI SYLLABLE GBOO;Lo;0;L;;;;;N;;;;; +A57D;VAI SYLLABLE FOO;Lo;0;L;;;;;N;;;;; +A57E;VAI SYLLABLE VOO;Lo;0;L;;;;;N;;;;; +A57F;VAI SYLLABLE TOO;Lo;0;L;;;;;N;;;;; +A580;VAI SYLLABLE THOO;Lo;0;L;;;;;N;;;;; +A581;VAI SYLLABLE DHOO;Lo;0;L;;;;;N;;;;; +A582;VAI SYLLABLE DHHOO;Lo;0;L;;;;;N;;;;; +A583;VAI SYLLABLE LOO;Lo;0;L;;;;;N;;;;; +A584;VAI SYLLABLE ROO;Lo;0;L;;;;;N;;;;; +A585;VAI SYLLABLE DOO;Lo;0;L;;;;;N;;;;; +A586;VAI SYLLABLE NDOO;Lo;0;L;;;;;N;;;;; +A587;VAI SYLLABLE SOO;Lo;0;L;;;;;N;;;;; +A588;VAI SYLLABLE SHOO;Lo;0;L;;;;;N;;;;; +A589;VAI SYLLABLE ZOO;Lo;0;L;;;;;N;;;;; +A58A;VAI SYLLABLE ZHOO;Lo;0;L;;;;;N;;;;; +A58B;VAI SYLLABLE COO;Lo;0;L;;;;;N;;;;; +A58C;VAI SYLLABLE JOO;Lo;0;L;;;;;N;;;;; +A58D;VAI SYLLABLE NJOO;Lo;0;L;;;;;N;;;;; +A58E;VAI SYLLABLE YOO;Lo;0;L;;;;;N;;;;; +A58F;VAI SYLLABLE KOO;Lo;0;L;;;;;N;;;;; +A590;VAI SYLLABLE NGGOO;Lo;0;L;;;;;N;;;;; +A591;VAI SYLLABLE GOO;Lo;0;L;;;;;N;;;;; +A592;VAI SYLLABLE MOO;Lo;0;L;;;;;N;;;;; +A593;VAI SYLLABLE NOO;Lo;0;L;;;;;N;;;;; +A594;VAI SYLLABLE NYOO;Lo;0;L;;;;;N;;;;; +A595;VAI SYLLABLE U;Lo;0;L;;;;;N;;;;; +A596;VAI SYLLABLE UN;Lo;0;L;;;;;N;;;;; +A597;VAI SYLLABLE HU;Lo;0;L;;;;;N;;;;; +A598;VAI SYLLABLE HUN;Lo;0;L;;;;;N;;;;; +A599;VAI SYLLABLE WU;Lo;0;L;;;;;N;;;;; +A59A;VAI SYLLABLE WUN;Lo;0;L;;;;;N;;;;; +A59B;VAI SYLLABLE PU;Lo;0;L;;;;;N;;;;; +A59C;VAI SYLLABLE BHU;Lo;0;L;;;;;N;;;;; +A59D;VAI SYLLABLE BU;Lo;0;L;;;;;N;;;;; +A59E;VAI SYLLABLE MBU;Lo;0;L;;;;;N;;;;; +A59F;VAI SYLLABLE KPU;Lo;0;L;;;;;N;;;;; +A5A0;VAI SYLLABLE MGBU;Lo;0;L;;;;;N;;;;; +A5A1;VAI SYLLABLE GBU;Lo;0;L;;;;;N;;;;; +A5A2;VAI SYLLABLE FU;Lo;0;L;;;;;N;;;;; +A5A3;VAI SYLLABLE VU;Lo;0;L;;;;;N;;;;; +A5A4;VAI SYLLABLE TU;Lo;0;L;;;;;N;;;;; +A5A5;VAI SYLLABLE THU;Lo;0;L;;;;;N;;;;; +A5A6;VAI SYLLABLE DHU;Lo;0;L;;;;;N;;;;; +A5A7;VAI SYLLABLE DHHU;Lo;0;L;;;;;N;;;;; +A5A8;VAI SYLLABLE LU;Lo;0;L;;;;;N;;;;; +A5A9;VAI SYLLABLE RU;Lo;0;L;;;;;N;;;;; +A5AA;VAI SYLLABLE DU;Lo;0;L;;;;;N;;;;; +A5AB;VAI SYLLABLE NDU;Lo;0;L;;;;;N;;;;; +A5AC;VAI SYLLABLE SU;Lo;0;L;;;;;N;;;;; +A5AD;VAI SYLLABLE SHU;Lo;0;L;;;;;N;;;;; +A5AE;VAI SYLLABLE ZU;Lo;0;L;;;;;N;;;;; +A5AF;VAI SYLLABLE ZHU;Lo;0;L;;;;;N;;;;; +A5B0;VAI SYLLABLE CU;Lo;0;L;;;;;N;;;;; +A5B1;VAI SYLLABLE JU;Lo;0;L;;;;;N;;;;; +A5B2;VAI SYLLABLE NJU;Lo;0;L;;;;;N;;;;; +A5B3;VAI SYLLABLE YU;Lo;0;L;;;;;N;;;;; +A5B4;VAI SYLLABLE KU;Lo;0;L;;;;;N;;;;; +A5B5;VAI SYLLABLE NGGU;Lo;0;L;;;;;N;;;;; +A5B6;VAI SYLLABLE GU;Lo;0;L;;;;;N;;;;; +A5B7;VAI SYLLABLE MU;Lo;0;L;;;;;N;;;;; +A5B8;VAI SYLLABLE NU;Lo;0;L;;;;;N;;;;; +A5B9;VAI SYLLABLE NYU;Lo;0;L;;;;;N;;;;; +A5BA;VAI SYLLABLE O;Lo;0;L;;;;;N;;;;; +A5BB;VAI SYLLABLE ON;Lo;0;L;;;;;N;;;;; +A5BC;VAI SYLLABLE NGON;Lo;0;L;;;;;N;;;;; +A5BD;VAI SYLLABLE HO;Lo;0;L;;;;;N;;;;; +A5BE;VAI SYLLABLE HON;Lo;0;L;;;;;N;;;;; +A5BF;VAI SYLLABLE WO;Lo;0;L;;;;;N;;;;; +A5C0;VAI SYLLABLE WON;Lo;0;L;;;;;N;;;;; +A5C1;VAI SYLLABLE PO;Lo;0;L;;;;;N;;;;; +A5C2;VAI SYLLABLE BHO;Lo;0;L;;;;;N;;;;; +A5C3;VAI SYLLABLE BO;Lo;0;L;;;;;N;;;;; +A5C4;VAI SYLLABLE MBO;Lo;0;L;;;;;N;;;;; +A5C5;VAI SYLLABLE KPO;Lo;0;L;;;;;N;;;;; +A5C6;VAI SYLLABLE MGBO;Lo;0;L;;;;;N;;;;; +A5C7;VAI SYLLABLE GBO;Lo;0;L;;;;;N;;;;; +A5C8;VAI SYLLABLE GBON;Lo;0;L;;;;;N;;;;; +A5C9;VAI SYLLABLE FO;Lo;0;L;;;;;N;;;;; +A5CA;VAI SYLLABLE VO;Lo;0;L;;;;;N;;;;; +A5CB;VAI SYLLABLE TO;Lo;0;L;;;;;N;;;;; +A5CC;VAI SYLLABLE THO;Lo;0;L;;;;;N;;;;; +A5CD;VAI SYLLABLE DHO;Lo;0;L;;;;;N;;;;; +A5CE;VAI SYLLABLE DHHO;Lo;0;L;;;;;N;;;;; +A5CF;VAI SYLLABLE LO;Lo;0;L;;;;;N;;;;; +A5D0;VAI SYLLABLE RO;Lo;0;L;;;;;N;;;;; +A5D1;VAI SYLLABLE DO;Lo;0;L;;;;;N;;;;; +A5D2;VAI SYLLABLE NDO;Lo;0;L;;;;;N;;;;; +A5D3;VAI SYLLABLE SO;Lo;0;L;;;;;N;;;;; +A5D4;VAI SYLLABLE SHO;Lo;0;L;;;;;N;;;;; +A5D5;VAI SYLLABLE ZO;Lo;0;L;;;;;N;;;;; +A5D6;VAI SYLLABLE ZHO;Lo;0;L;;;;;N;;;;; +A5D7;VAI SYLLABLE CO;Lo;0;L;;;;;N;;;;; +A5D8;VAI SYLLABLE JO;Lo;0;L;;;;;N;;;;; +A5D9;VAI SYLLABLE NJO;Lo;0;L;;;;;N;;;;; +A5DA;VAI SYLLABLE YO;Lo;0;L;;;;;N;;;;; +A5DB;VAI SYLLABLE KO;Lo;0;L;;;;;N;;;;; +A5DC;VAI SYLLABLE NGGO;Lo;0;L;;;;;N;;;;; +A5DD;VAI SYLLABLE GO;Lo;0;L;;;;;N;;;;; +A5DE;VAI SYLLABLE MO;Lo;0;L;;;;;N;;;;; +A5DF;VAI SYLLABLE NO;Lo;0;L;;;;;N;;;;; +A5E0;VAI SYLLABLE NYO;Lo;0;L;;;;;N;;;;; +A5E1;VAI SYLLABLE E;Lo;0;L;;;;;N;;;;; +A5E2;VAI SYLLABLE EN;Lo;0;L;;;;;N;;;;; +A5E3;VAI SYLLABLE NGEN;Lo;0;L;;;;;N;;;;; +A5E4;VAI SYLLABLE HE;Lo;0;L;;;;;N;;;;; +A5E5;VAI SYLLABLE HEN;Lo;0;L;;;;;N;;;;; +A5E6;VAI SYLLABLE WE;Lo;0;L;;;;;N;;;;; +A5E7;VAI SYLLABLE WEN;Lo;0;L;;;;;N;;;;; +A5E8;VAI SYLLABLE PE;Lo;0;L;;;;;N;;;;; +A5E9;VAI SYLLABLE BHE;Lo;0;L;;;;;N;;;;; +A5EA;VAI SYLLABLE BE;Lo;0;L;;;;;N;;;;; +A5EB;VAI SYLLABLE MBE;Lo;0;L;;;;;N;;;;; +A5EC;VAI SYLLABLE KPE;Lo;0;L;;;;;N;;;;; +A5ED;VAI SYLLABLE KPEN;Lo;0;L;;;;;N;;;;; +A5EE;VAI SYLLABLE MGBE;Lo;0;L;;;;;N;;;;; +A5EF;VAI SYLLABLE GBE;Lo;0;L;;;;;N;;;;; +A5F0;VAI SYLLABLE GBEN;Lo;0;L;;;;;N;;;;; +A5F1;VAI SYLLABLE FE;Lo;0;L;;;;;N;;;;; +A5F2;VAI SYLLABLE VE;Lo;0;L;;;;;N;;;;; +A5F3;VAI SYLLABLE TE;Lo;0;L;;;;;N;;;;; +A5F4;VAI SYLLABLE THE;Lo;0;L;;;;;N;;;;; +A5F5;VAI SYLLABLE DHE;Lo;0;L;;;;;N;;;;; +A5F6;VAI SYLLABLE DHHE;Lo;0;L;;;;;N;;;;; +A5F7;VAI SYLLABLE LE;Lo;0;L;;;;;N;;;;; +A5F8;VAI SYLLABLE RE;Lo;0;L;;;;;N;;;;; +A5F9;VAI SYLLABLE DE;Lo;0;L;;;;;N;;;;; +A5FA;VAI SYLLABLE NDE;Lo;0;L;;;;;N;;;;; +A5FB;VAI SYLLABLE SE;Lo;0;L;;;;;N;;;;; +A5FC;VAI SYLLABLE SHE;Lo;0;L;;;;;N;;;;; +A5FD;VAI SYLLABLE ZE;Lo;0;L;;;;;N;;;;; +A5FE;VAI SYLLABLE ZHE;Lo;0;L;;;;;N;;;;; +A5FF;VAI SYLLABLE CE;Lo;0;L;;;;;N;;;;; +A600;VAI SYLLABLE JE;Lo;0;L;;;;;N;;;;; +A601;VAI SYLLABLE NJE;Lo;0;L;;;;;N;;;;; +A602;VAI SYLLABLE YE;Lo;0;L;;;;;N;;;;; +A603;VAI SYLLABLE KE;Lo;0;L;;;;;N;;;;; +A604;VAI SYLLABLE NGGE;Lo;0;L;;;;;N;;;;; +A605;VAI SYLLABLE NGGEN;Lo;0;L;;;;;N;;;;; +A606;VAI SYLLABLE GE;Lo;0;L;;;;;N;;;;; +A607;VAI SYLLABLE GEN;Lo;0;L;;;;;N;;;;; +A608;VAI SYLLABLE ME;Lo;0;L;;;;;N;;;;; +A609;VAI SYLLABLE NE;Lo;0;L;;;;;N;;;;; +A60A;VAI SYLLABLE NYE;Lo;0;L;;;;;N;;;;; +A60B;VAI SYLLABLE NG;Lo;0;L;;;;;N;;;;; +A60C;VAI SYLLABLE LENGTHENER;Lm;0;L;;;;;N;;;;; +A60D;VAI COMMA;Po;0;ON;;;;;N;;;;; +A60E;VAI FULL STOP;Po;0;ON;;;;;N;;;;; +A60F;VAI QUESTION MARK;Po;0;ON;;;;;N;;;;; +A610;VAI SYLLABLE NDOLE FA;Lo;0;L;;;;;N;;;;; +A611;VAI SYLLABLE NDOLE KA;Lo;0;L;;;;;N;;;;; +A612;VAI SYLLABLE NDOLE SOO;Lo;0;L;;;;;N;;;;; +A613;VAI SYMBOL FEENG;Lo;0;L;;;;;N;;;;; +A614;VAI SYMBOL KEENG;Lo;0;L;;;;;N;;;;; +A615;VAI SYMBOL TING;Lo;0;L;;;;;N;;;;; +A616;VAI SYMBOL NII;Lo;0;L;;;;;N;;;;; +A617;VAI SYMBOL BANG;Lo;0;L;;;;;N;;;;; +A618;VAI SYMBOL FAA;Lo;0;L;;;;;N;;;;; +A619;VAI SYMBOL TAA;Lo;0;L;;;;;N;;;;; +A61A;VAI SYMBOL DANG;Lo;0;L;;;;;N;;;;; +A61B;VAI SYMBOL DOONG;Lo;0;L;;;;;N;;;;; +A61C;VAI SYMBOL KUNG;Lo;0;L;;;;;N;;;;; +A61D;VAI SYMBOL TONG;Lo;0;L;;;;;N;;;;; +A61E;VAI SYMBOL DO-O;Lo;0;L;;;;;N;;;;; +A61F;VAI SYMBOL JONG;Lo;0;L;;;;;N;;;;; +A620;VAI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +A621;VAI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +A622;VAI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +A623;VAI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +A624;VAI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +A625;VAI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +A626;VAI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +A627;VAI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +A628;VAI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +A629;VAI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +A62A;VAI SYLLABLE NDOLE MA;Lo;0;L;;;;;N;;;;; +A62B;VAI SYLLABLE NDOLE DO;Lo;0;L;;;;;N;;;;; +A640;CYRILLIC CAPITAL LETTER ZEMLYA;Lu;0;L;;;;;N;;;;A641; +A641;CYRILLIC SMALL LETTER ZEMLYA;Ll;0;L;;;;;N;;;A640;;A640 +A642;CYRILLIC CAPITAL LETTER DZELO;Lu;0;L;;;;;N;;;;A643; +A643;CYRILLIC SMALL LETTER DZELO;Ll;0;L;;;;;N;;;A642;;A642 +A644;CYRILLIC CAPITAL LETTER REVERSED DZE;Lu;0;L;;;;;N;;;;A645; +A645;CYRILLIC SMALL LETTER REVERSED DZE;Ll;0;L;;;;;N;;;A644;;A644 +A646;CYRILLIC CAPITAL LETTER IOTA;Lu;0;L;;;;;N;;;;A647; +A647;CYRILLIC SMALL LETTER IOTA;Ll;0;L;;;;;N;;;A646;;A646 +A648;CYRILLIC CAPITAL LETTER DJERV;Lu;0;L;;;;;N;;;;A649; +A649;CYRILLIC SMALL LETTER DJERV;Ll;0;L;;;;;N;;;A648;;A648 +A64A;CYRILLIC CAPITAL LETTER MONOGRAPH UK;Lu;0;L;;;;;N;;;;A64B; +A64B;CYRILLIC SMALL LETTER MONOGRAPH UK;Ll;0;L;;;;;N;;;A64A;;A64A +A64C;CYRILLIC CAPITAL LETTER BROAD OMEGA;Lu;0;L;;;;;N;;;;A64D; +A64D;CYRILLIC SMALL LETTER BROAD OMEGA;Ll;0;L;;;;;N;;;A64C;;A64C +A64E;CYRILLIC CAPITAL LETTER NEUTRAL YER;Lu;0;L;;;;;N;;;;A64F; +A64F;CYRILLIC SMALL LETTER NEUTRAL YER;Ll;0;L;;;;;N;;;A64E;;A64E +A650;CYRILLIC CAPITAL LETTER YERU WITH BACK YER;Lu;0;L;;;;;N;;;;A651; +A651;CYRILLIC SMALL LETTER YERU WITH BACK YER;Ll;0;L;;;;;N;;;A650;;A650 +A652;CYRILLIC CAPITAL LETTER IOTIFIED YAT;Lu;0;L;;;;;N;;;;A653; +A653;CYRILLIC SMALL LETTER IOTIFIED YAT;Ll;0;L;;;;;N;;;A652;;A652 +A654;CYRILLIC CAPITAL LETTER REVERSED YU;Lu;0;L;;;;;N;;;;A655; +A655;CYRILLIC SMALL LETTER REVERSED YU;Ll;0;L;;;;;N;;;A654;;A654 +A656;CYRILLIC CAPITAL LETTER IOTIFIED A;Lu;0;L;;;;;N;;;;A657; +A657;CYRILLIC SMALL LETTER IOTIFIED A;Ll;0;L;;;;;N;;;A656;;A656 +A658;CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS;Lu;0;L;;;;;N;;;;A659; +A659;CYRILLIC SMALL LETTER CLOSED LITTLE YUS;Ll;0;L;;;;;N;;;A658;;A658 +A65A;CYRILLIC CAPITAL LETTER BLENDED YUS;Lu;0;L;;;;;N;;;;A65B; +A65B;CYRILLIC SMALL LETTER BLENDED YUS;Ll;0;L;;;;;N;;;A65A;;A65A +A65C;CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS;Lu;0;L;;;;;N;;;;A65D; +A65D;CYRILLIC SMALL LETTER IOTIFIED CLOSED LITTLE YUS;Ll;0;L;;;;;N;;;A65C;;A65C +A65E;CYRILLIC CAPITAL LETTER YN;Lu;0;L;;;;;N;;;;A65F; +A65F;CYRILLIC SMALL LETTER YN;Ll;0;L;;;;;N;;;A65E;;A65E +A660;CYRILLIC CAPITAL LETTER REVERSED TSE;Lu;0;L;;;;;N;;;;A661; +A661;CYRILLIC SMALL LETTER REVERSED TSE;Ll;0;L;;;;;N;;;A660;;A660 +A662;CYRILLIC CAPITAL LETTER SOFT DE;Lu;0;L;;;;;N;;;;A663; +A663;CYRILLIC SMALL LETTER SOFT DE;Ll;0;L;;;;;N;;;A662;;A662 +A664;CYRILLIC CAPITAL LETTER SOFT EL;Lu;0;L;;;;;N;;;;A665; +A665;CYRILLIC SMALL LETTER SOFT EL;Ll;0;L;;;;;N;;;A664;;A664 +A666;CYRILLIC CAPITAL LETTER SOFT EM;Lu;0;L;;;;;N;;;;A667; +A667;CYRILLIC SMALL LETTER SOFT EM;Ll;0;L;;;;;N;;;A666;;A666 +A668;CYRILLIC CAPITAL LETTER MONOCULAR O;Lu;0;L;;;;;N;;;;A669; +A669;CYRILLIC SMALL LETTER MONOCULAR O;Ll;0;L;;;;;N;;;A668;;A668 +A66A;CYRILLIC CAPITAL LETTER BINOCULAR O;Lu;0;L;;;;;N;;;;A66B; +A66B;CYRILLIC SMALL LETTER BINOCULAR O;Ll;0;L;;;;;N;;;A66A;;A66A +A66C;CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O;Lu;0;L;;;;;N;;;;A66D; +A66D;CYRILLIC SMALL LETTER DOUBLE MONOCULAR O;Ll;0;L;;;;;N;;;A66C;;A66C +A66E;CYRILLIC LETTER MULTIOCULAR O;Lo;0;L;;;;;N;;;;; +A66F;COMBINING CYRILLIC VZMET;Mn;230;NSM;;;;;N;;;;; +A670;COMBINING CYRILLIC TEN MILLIONS SIGN;Me;0;NSM;;;;;N;;;;; +A671;COMBINING CYRILLIC HUNDRED MILLIONS SIGN;Me;0;NSM;;;;;N;;;;; +A672;COMBINING CYRILLIC THOUSAND MILLIONS SIGN;Me;0;NSM;;;;;N;;;;; +A673;SLAVONIC ASTERISK;Po;0;ON;;;;;N;;;;; +A674;COMBINING CYRILLIC LETTER UKRAINIAN IE;Mn;230;NSM;;;;;N;;;;; +A675;COMBINING CYRILLIC LETTER I;Mn;230;NSM;;;;;N;;;;; +A676;COMBINING CYRILLIC LETTER YI;Mn;230;NSM;;;;;N;;;;; +A677;COMBINING CYRILLIC LETTER U;Mn;230;NSM;;;;;N;;;;; +A678;COMBINING CYRILLIC LETTER HARD SIGN;Mn;230;NSM;;;;;N;;;;; +A679;COMBINING CYRILLIC LETTER YERU;Mn;230;NSM;;;;;N;;;;; +A67A;COMBINING CYRILLIC LETTER SOFT SIGN;Mn;230;NSM;;;;;N;;;;; +A67B;COMBINING CYRILLIC LETTER OMEGA;Mn;230;NSM;;;;;N;;;;; +A67C;COMBINING CYRILLIC KAVYKA;Mn;230;NSM;;;;;N;;;;; +A67D;COMBINING CYRILLIC PAYEROK;Mn;230;NSM;;;;;N;;;;; +A67E;CYRILLIC KAVYKA;Po;0;ON;;;;;N;;;;; +A67F;CYRILLIC PAYEROK;Lm;0;ON;;;;;N;;;;; +A680;CYRILLIC CAPITAL LETTER DWE;Lu;0;L;;;;;N;;;;A681; +A681;CYRILLIC SMALL LETTER DWE;Ll;0;L;;;;;N;;;A680;;A680 +A682;CYRILLIC CAPITAL LETTER DZWE;Lu;0;L;;;;;N;;;;A683; +A683;CYRILLIC SMALL LETTER DZWE;Ll;0;L;;;;;N;;;A682;;A682 +A684;CYRILLIC CAPITAL LETTER ZHWE;Lu;0;L;;;;;N;;;;A685; +A685;CYRILLIC SMALL LETTER ZHWE;Ll;0;L;;;;;N;;;A684;;A684 +A686;CYRILLIC CAPITAL LETTER CCHE;Lu;0;L;;;;;N;;;;A687; +A687;CYRILLIC SMALL LETTER CCHE;Ll;0;L;;;;;N;;;A686;;A686 +A688;CYRILLIC CAPITAL LETTER DZZE;Lu;0;L;;;;;N;;;;A689; +A689;CYRILLIC SMALL LETTER DZZE;Ll;0;L;;;;;N;;;A688;;A688 +A68A;CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK;Lu;0;L;;;;;N;;;;A68B; +A68B;CYRILLIC SMALL LETTER TE WITH MIDDLE HOOK;Ll;0;L;;;;;N;;;A68A;;A68A +A68C;CYRILLIC CAPITAL LETTER TWE;Lu;0;L;;;;;N;;;;A68D; +A68D;CYRILLIC SMALL LETTER TWE;Ll;0;L;;;;;N;;;A68C;;A68C +A68E;CYRILLIC CAPITAL LETTER TSWE;Lu;0;L;;;;;N;;;;A68F; +A68F;CYRILLIC SMALL LETTER TSWE;Ll;0;L;;;;;N;;;A68E;;A68E +A690;CYRILLIC CAPITAL LETTER TSSE;Lu;0;L;;;;;N;;;;A691; +A691;CYRILLIC SMALL LETTER TSSE;Ll;0;L;;;;;N;;;A690;;A690 +A692;CYRILLIC CAPITAL LETTER TCHE;Lu;0;L;;;;;N;;;;A693; +A693;CYRILLIC SMALL LETTER TCHE;Ll;0;L;;;;;N;;;A692;;A692 +A694;CYRILLIC CAPITAL LETTER HWE;Lu;0;L;;;;;N;;;;A695; +A695;CYRILLIC SMALL LETTER HWE;Ll;0;L;;;;;N;;;A694;;A694 +A696;CYRILLIC CAPITAL LETTER SHWE;Lu;0;L;;;;;N;;;;A697; +A697;CYRILLIC SMALL LETTER SHWE;Ll;0;L;;;;;N;;;A696;;A696 +A69F;COMBINING CYRILLIC LETTER IOTIFIED E;Mn;230;NSM;;;;;N;;;;; +A6A0;BAMUM LETTER A;Lo;0;L;;;;;N;;;;; +A6A1;BAMUM LETTER KA;Lo;0;L;;;;;N;;;;; +A6A2;BAMUM LETTER U;Lo;0;L;;;;;N;;;;; +A6A3;BAMUM LETTER KU;Lo;0;L;;;;;N;;;;; +A6A4;BAMUM LETTER EE;Lo;0;L;;;;;N;;;;; +A6A5;BAMUM LETTER REE;Lo;0;L;;;;;N;;;;; +A6A6;BAMUM LETTER TAE;Lo;0;L;;;;;N;;;;; +A6A7;BAMUM LETTER O;Lo;0;L;;;;;N;;;;; +A6A8;BAMUM LETTER NYI;Lo;0;L;;;;;N;;;;; +A6A9;BAMUM LETTER I;Lo;0;L;;;;;N;;;;; +A6AA;BAMUM LETTER LA;Lo;0;L;;;;;N;;;;; +A6AB;BAMUM LETTER PA;Lo;0;L;;;;;N;;;;; +A6AC;BAMUM LETTER RII;Lo;0;L;;;;;N;;;;; +A6AD;BAMUM LETTER RIEE;Lo;0;L;;;;;N;;;;; +A6AE;BAMUM LETTER LEEEE;Lo;0;L;;;;;N;;;;; +A6AF;BAMUM LETTER MEEEE;Lo;0;L;;;;;N;;;;; +A6B0;BAMUM LETTER TAA;Lo;0;L;;;;;N;;;;; +A6B1;BAMUM LETTER NDAA;Lo;0;L;;;;;N;;;;; +A6B2;BAMUM LETTER NJAEM;Lo;0;L;;;;;N;;;;; +A6B3;BAMUM LETTER M;Lo;0;L;;;;;N;;;;; +A6B4;BAMUM LETTER SUU;Lo;0;L;;;;;N;;;;; +A6B5;BAMUM LETTER MU;Lo;0;L;;;;;N;;;;; +A6B6;BAMUM LETTER SHII;Lo;0;L;;;;;N;;;;; +A6B7;BAMUM LETTER SI;Lo;0;L;;;;;N;;;;; +A6B8;BAMUM LETTER SHEUX;Lo;0;L;;;;;N;;;;; +A6B9;BAMUM LETTER SEUX;Lo;0;L;;;;;N;;;;; +A6BA;BAMUM LETTER KYEE;Lo;0;L;;;;;N;;;;; +A6BB;BAMUM LETTER KET;Lo;0;L;;;;;N;;;;; +A6BC;BAMUM LETTER NUAE;Lo;0;L;;;;;N;;;;; +A6BD;BAMUM LETTER NU;Lo;0;L;;;;;N;;;;; +A6BE;BAMUM LETTER NJUAE;Lo;0;L;;;;;N;;;;; +A6BF;BAMUM LETTER YOQ;Lo;0;L;;;;;N;;;;; +A6C0;BAMUM LETTER SHU;Lo;0;L;;;;;N;;;;; +A6C1;BAMUM LETTER YUQ;Lo;0;L;;;;;N;;;;; +A6C2;BAMUM LETTER YA;Lo;0;L;;;;;N;;;;; +A6C3;BAMUM LETTER NSHA;Lo;0;L;;;;;N;;;;; +A6C4;BAMUM LETTER KEUX;Lo;0;L;;;;;N;;;;; +A6C5;BAMUM LETTER PEUX;Lo;0;L;;;;;N;;;;; +A6C6;BAMUM LETTER NJEE;Lo;0;L;;;;;N;;;;; +A6C7;BAMUM LETTER NTEE;Lo;0;L;;;;;N;;;;; +A6C8;BAMUM LETTER PUE;Lo;0;L;;;;;N;;;;; +A6C9;BAMUM LETTER WUE;Lo;0;L;;;;;N;;;;; +A6CA;BAMUM LETTER PEE;Lo;0;L;;;;;N;;;;; +A6CB;BAMUM LETTER FEE;Lo;0;L;;;;;N;;;;; +A6CC;BAMUM LETTER RU;Lo;0;L;;;;;N;;;;; +A6CD;BAMUM LETTER LU;Lo;0;L;;;;;N;;;;; +A6CE;BAMUM LETTER MI;Lo;0;L;;;;;N;;;;; +A6CF;BAMUM LETTER NI;Lo;0;L;;;;;N;;;;; +A6D0;BAMUM LETTER REUX;Lo;0;L;;;;;N;;;;; +A6D1;BAMUM LETTER RAE;Lo;0;L;;;;;N;;;;; +A6D2;BAMUM LETTER KEN;Lo;0;L;;;;;N;;;;; +A6D3;BAMUM LETTER NGKWAEN;Lo;0;L;;;;;N;;;;; +A6D4;BAMUM LETTER NGGA;Lo;0;L;;;;;N;;;;; +A6D5;BAMUM LETTER NGA;Lo;0;L;;;;;N;;;;; +A6D6;BAMUM LETTER SHO;Lo;0;L;;;;;N;;;;; +A6D7;BAMUM LETTER PUAE;Lo;0;L;;;;;N;;;;; +A6D8;BAMUM LETTER FU;Lo;0;L;;;;;N;;;;; +A6D9;BAMUM LETTER FOM;Lo;0;L;;;;;N;;;;; +A6DA;BAMUM LETTER WA;Lo;0;L;;;;;N;;;;; +A6DB;BAMUM LETTER NA;Lo;0;L;;;;;N;;;;; +A6DC;BAMUM LETTER LI;Lo;0;L;;;;;N;;;;; +A6DD;BAMUM LETTER PI;Lo;0;L;;;;;N;;;;; +A6DE;BAMUM LETTER LOQ;Lo;0;L;;;;;N;;;;; +A6DF;BAMUM LETTER KO;Lo;0;L;;;;;N;;;;; +A6E0;BAMUM LETTER MBEN;Lo;0;L;;;;;N;;;;; +A6E1;BAMUM LETTER REN;Lo;0;L;;;;;N;;;;; +A6E2;BAMUM LETTER MEN;Lo;0;L;;;;;N;;;;; +A6E3;BAMUM LETTER MA;Lo;0;L;;;;;N;;;;; +A6E4;BAMUM LETTER TI;Lo;0;L;;;;;N;;;;; +A6E5;BAMUM LETTER KI;Lo;0;L;;;;;N;;;;; +A6E6;BAMUM LETTER MO;Nl;0;L;;;;1;N;;;;; +A6E7;BAMUM LETTER MBAA;Nl;0;L;;;;2;N;;;;; +A6E8;BAMUM LETTER TET;Nl;0;L;;;;3;N;;;;; +A6E9;BAMUM LETTER KPA;Nl;0;L;;;;4;N;;;;; +A6EA;BAMUM LETTER TEN;Nl;0;L;;;;5;N;;;;; +A6EB;BAMUM LETTER NTUU;Nl;0;L;;;;6;N;;;;; +A6EC;BAMUM LETTER SAMBA;Nl;0;L;;;;7;N;;;;; +A6ED;BAMUM LETTER FAAMAE;Nl;0;L;;;;8;N;;;;; +A6EE;BAMUM LETTER KOVUU;Nl;0;L;;;;9;N;;;;; +A6EF;BAMUM LETTER KOGHOM;Nl;0;L;;;;0;N;;;;; +A6F0;BAMUM COMBINING MARK KOQNDON;Mn;230;NSM;;;;;N;;;;; +A6F1;BAMUM COMBINING MARK TUKWENTIS;Mn;230;NSM;;;;;N;;;;; +A6F2;BAMUM NJAEMLI;Po;0;L;;;;;N;;;;; +A6F3;BAMUM FULL STOP;Po;0;L;;;;;N;;;;; +A6F4;BAMUM COLON;Po;0;L;;;;;N;;;;; +A6F5;BAMUM COMMA;Po;0;L;;;;;N;;;;; +A6F6;BAMUM SEMICOLON;Po;0;L;;;;;N;;;;; +A6F7;BAMUM QUESTION MARK;Po;0;L;;;;;N;;;;; +A700;MODIFIER LETTER CHINESE TONE YIN PING;Sk;0;ON;;;;;N;;;;; +A701;MODIFIER LETTER CHINESE TONE YANG PING;Sk;0;ON;;;;;N;;;;; +A702;MODIFIER LETTER CHINESE TONE YIN SHANG;Sk;0;ON;;;;;N;;;;; +A703;MODIFIER LETTER CHINESE TONE YANG SHANG;Sk;0;ON;;;;;N;;;;; +A704;MODIFIER LETTER CHINESE TONE YIN QU;Sk;0;ON;;;;;N;;;;; +A705;MODIFIER LETTER CHINESE TONE YANG QU;Sk;0;ON;;;;;N;;;;; +A706;MODIFIER LETTER CHINESE TONE YIN RU;Sk;0;ON;;;;;N;;;;; +A707;MODIFIER LETTER CHINESE TONE YANG RU;Sk;0;ON;;;;;N;;;;; +A708;MODIFIER LETTER EXTRA-HIGH DOTTED TONE BAR;Sk;0;ON;;;;;N;;;;; +A709;MODIFIER LETTER HIGH DOTTED TONE BAR;Sk;0;ON;;;;;N;;;;; +A70A;MODIFIER LETTER MID DOTTED TONE BAR;Sk;0;ON;;;;;N;;;;; +A70B;MODIFIER LETTER LOW DOTTED TONE BAR;Sk;0;ON;;;;;N;;;;; +A70C;MODIFIER LETTER EXTRA-LOW DOTTED TONE BAR;Sk;0;ON;;;;;N;;;;; +A70D;MODIFIER LETTER EXTRA-HIGH DOTTED LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A70E;MODIFIER LETTER HIGH DOTTED LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A70F;MODIFIER LETTER MID DOTTED LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A710;MODIFIER LETTER LOW DOTTED LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A711;MODIFIER LETTER EXTRA-LOW DOTTED LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A712;MODIFIER LETTER EXTRA-HIGH LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A713;MODIFIER LETTER HIGH LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A714;MODIFIER LETTER MID LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A715;MODIFIER LETTER LOW LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A716;MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR;Sk;0;ON;;;;;N;;;;; +A717;MODIFIER LETTER DOT VERTICAL BAR;Lm;0;ON;;;;;N;;;;; +A718;MODIFIER LETTER DOT SLASH;Lm;0;ON;;;;;N;;;;; +A719;MODIFIER LETTER DOT HORIZONTAL BAR;Lm;0;ON;;;;;N;;;;; +A71A;MODIFIER LETTER LOWER RIGHT CORNER ANGLE;Lm;0;ON;;;;;N;;;;; +A71B;MODIFIER LETTER RAISED UP ARROW;Lm;0;ON;;;;;N;;;;; +A71C;MODIFIER LETTER RAISED DOWN ARROW;Lm;0;ON;;;;;N;;;;; +A71D;MODIFIER LETTER RAISED EXCLAMATION MARK;Lm;0;ON;;;;;N;;;;; +A71E;MODIFIER LETTER RAISED INVERTED EXCLAMATION MARK;Lm;0;ON;;;;;N;;;;; +A71F;MODIFIER LETTER LOW INVERTED EXCLAMATION MARK;Lm;0;ON;;;;;N;;;;; +A720;MODIFIER LETTER STRESS AND HIGH TONE;Sk;0;ON;;;;;N;;;;; +A721;MODIFIER LETTER STRESS AND LOW TONE;Sk;0;ON;;;;;N;;;;; +A722;LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF;Lu;0;L;;;;;N;;;;A723; +A723;LATIN SMALL LETTER EGYPTOLOGICAL ALEF;Ll;0;L;;;;;N;;;A722;;A722 +A724;LATIN CAPITAL LETTER EGYPTOLOGICAL AIN;Lu;0;L;;;;;N;;;;A725; +A725;LATIN SMALL LETTER EGYPTOLOGICAL AIN;Ll;0;L;;;;;N;;;A724;;A724 +A726;LATIN CAPITAL LETTER HENG;Lu;0;L;;;;;N;;;;A727; +A727;LATIN SMALL LETTER HENG;Ll;0;L;;;;;N;;;A726;;A726 +A728;LATIN CAPITAL LETTER TZ;Lu;0;L;;;;;N;;;;A729; +A729;LATIN SMALL LETTER TZ;Ll;0;L;;;;;N;;;A728;;A728 +A72A;LATIN CAPITAL LETTER TRESILLO;Lu;0;L;;;;;N;;;;A72B; +A72B;LATIN SMALL LETTER TRESILLO;Ll;0;L;;;;;N;;;A72A;;A72A +A72C;LATIN CAPITAL LETTER CUATRILLO;Lu;0;L;;;;;N;;;;A72D; +A72D;LATIN SMALL LETTER CUATRILLO;Ll;0;L;;;;;N;;;A72C;;A72C +A72E;LATIN CAPITAL LETTER CUATRILLO WITH COMMA;Lu;0;L;;;;;N;;;;A72F; +A72F;LATIN SMALL LETTER CUATRILLO WITH COMMA;Ll;0;L;;;;;N;;;A72E;;A72E +A730;LATIN LETTER SMALL CAPITAL F;Ll;0;L;;;;;N;;;;; +A731;LATIN LETTER SMALL CAPITAL S;Ll;0;L;;;;;N;;;;; +A732;LATIN CAPITAL LETTER AA;Lu;0;L;;;;;N;;;;A733; +A733;LATIN SMALL LETTER AA;Ll;0;L;;;;;N;;;A732;;A732 +A734;LATIN CAPITAL LETTER AO;Lu;0;L;;;;;N;;;;A735; +A735;LATIN SMALL LETTER AO;Ll;0;L;;;;;N;;;A734;;A734 +A736;LATIN CAPITAL LETTER AU;Lu;0;L;;;;;N;;;;A737; +A737;LATIN SMALL LETTER AU;Ll;0;L;;;;;N;;;A736;;A736 +A738;LATIN CAPITAL LETTER AV;Lu;0;L;;;;;N;;;;A739; +A739;LATIN SMALL LETTER AV;Ll;0;L;;;;;N;;;A738;;A738 +A73A;LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR;Lu;0;L;;;;;N;;;;A73B; +A73B;LATIN SMALL LETTER AV WITH HORIZONTAL BAR;Ll;0;L;;;;;N;;;A73A;;A73A +A73C;LATIN CAPITAL LETTER AY;Lu;0;L;;;;;N;;;;A73D; +A73D;LATIN SMALL LETTER AY;Ll;0;L;;;;;N;;;A73C;;A73C +A73E;LATIN CAPITAL LETTER REVERSED C WITH DOT;Lu;0;L;;;;;N;;;;A73F; +A73F;LATIN SMALL LETTER REVERSED C WITH DOT;Ll;0;L;;;;;N;;;A73E;;A73E +A740;LATIN CAPITAL LETTER K WITH STROKE;Lu;0;L;;;;;N;;;;A741; +A741;LATIN SMALL LETTER K WITH STROKE;Ll;0;L;;;;;N;;;A740;;A740 +A742;LATIN CAPITAL LETTER K WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A743; +A743;LATIN SMALL LETTER K WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;A742;;A742 +A744;LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A745; +A745;LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE;Ll;0;L;;;;;N;;;A744;;A744 +A746;LATIN CAPITAL LETTER BROKEN L;Lu;0;L;;;;;N;;;;A747; +A747;LATIN SMALL LETTER BROKEN L;Ll;0;L;;;;;N;;;A746;;A746 +A748;LATIN CAPITAL LETTER L WITH HIGH STROKE;Lu;0;L;;;;;N;;;;A749; +A749;LATIN SMALL LETTER L WITH HIGH STROKE;Ll;0;L;;;;;N;;;A748;;A748 +A74A;LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY;Lu;0;L;;;;;N;;;;A74B; +A74B;LATIN SMALL LETTER O WITH LONG STROKE OVERLAY;Ll;0;L;;;;;N;;;A74A;;A74A +A74C;LATIN CAPITAL LETTER O WITH LOOP;Lu;0;L;;;;;N;;;;A74D; +A74D;LATIN SMALL LETTER O WITH LOOP;Ll;0;L;;;;;N;;;A74C;;A74C +A74E;LATIN CAPITAL LETTER OO;Lu;0;L;;;;;N;;;;A74F; +A74F;LATIN SMALL LETTER OO;Ll;0;L;;;;;N;;;A74E;;A74E +A750;LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER;Lu;0;L;;;;;N;;;;A751; +A751;LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER;Ll;0;L;;;;;N;;;A750;;A750 +A752;LATIN CAPITAL LETTER P WITH FLOURISH;Lu;0;L;;;;;N;;;;A753; +A753;LATIN SMALL LETTER P WITH FLOURISH;Ll;0;L;;;;;N;;;A752;;A752 +A754;LATIN CAPITAL LETTER P WITH SQUIRREL TAIL;Lu;0;L;;;;;N;;;;A755; +A755;LATIN SMALL LETTER P WITH SQUIRREL TAIL;Ll;0;L;;;;;N;;;A754;;A754 +A756;LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER;Lu;0;L;;;;;N;;;;A757; +A757;LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER;Ll;0;L;;;;;N;;;A756;;A756 +A758;LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A759; +A759;LATIN SMALL LETTER Q WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;A758;;A758 +A75A;LATIN CAPITAL LETTER R ROTUNDA;Lu;0;L;;;;;N;;;;A75B; +A75B;LATIN SMALL LETTER R ROTUNDA;Ll;0;L;;;;;N;;;A75A;;A75A +A75C;LATIN CAPITAL LETTER RUM ROTUNDA;Lu;0;L;;;;;N;;;;A75D; +A75D;LATIN SMALL LETTER RUM ROTUNDA;Ll;0;L;;;;;N;;;A75C;;A75C +A75E;LATIN CAPITAL LETTER V WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A75F; +A75F;LATIN SMALL LETTER V WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;A75E;;A75E +A760;LATIN CAPITAL LETTER VY;Lu;0;L;;;;;N;;;;A761; +A761;LATIN SMALL LETTER VY;Ll;0;L;;;;;N;;;A760;;A760 +A762;LATIN CAPITAL LETTER VISIGOTHIC Z;Lu;0;L;;;;;N;;;;A763; +A763;LATIN SMALL LETTER VISIGOTHIC Z;Ll;0;L;;;;;N;;;A762;;A762 +A764;LATIN CAPITAL LETTER THORN WITH STROKE;Lu;0;L;;;;;N;;;;A765; +A765;LATIN SMALL LETTER THORN WITH STROKE;Ll;0;L;;;;;N;;;A764;;A764 +A766;LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER;Lu;0;L;;;;;N;;;;A767; +A767;LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER;Ll;0;L;;;;;N;;;A766;;A766 +A768;LATIN CAPITAL LETTER VEND;Lu;0;L;;;;;N;;;;A769; +A769;LATIN SMALL LETTER VEND;Ll;0;L;;;;;N;;;A768;;A768 +A76A;LATIN CAPITAL LETTER ET;Lu;0;L;;;;;N;;;;A76B; +A76B;LATIN SMALL LETTER ET;Ll;0;L;;;;;N;;;A76A;;A76A +A76C;LATIN CAPITAL LETTER IS;Lu;0;L;;;;;N;;;;A76D; +A76D;LATIN SMALL LETTER IS;Ll;0;L;;;;;N;;;A76C;;A76C +A76E;LATIN CAPITAL LETTER CON;Lu;0;L;;;;;N;;;;A76F; +A76F;LATIN SMALL LETTER CON;Ll;0;L;;;;;N;;;A76E;;A76E +A770;MODIFIER LETTER US;Lm;0;L; A76F;;;;N;;;;; +A771;LATIN SMALL LETTER DUM;Ll;0;L;;;;;N;;;;; +A772;LATIN SMALL LETTER LUM;Ll;0;L;;;;;N;;;;; +A773;LATIN SMALL LETTER MUM;Ll;0;L;;;;;N;;;;; +A774;LATIN SMALL LETTER NUM;Ll;0;L;;;;;N;;;;; +A775;LATIN SMALL LETTER RUM;Ll;0;L;;;;;N;;;;; +A776;LATIN LETTER SMALL CAPITAL RUM;Ll;0;L;;;;;N;;;;; +A777;LATIN SMALL LETTER TUM;Ll;0;L;;;;;N;;;;; +A778;LATIN SMALL LETTER UM;Ll;0;L;;;;;N;;;;; +A779;LATIN CAPITAL LETTER INSULAR D;Lu;0;L;;;;;N;;;;A77A; +A77A;LATIN SMALL LETTER INSULAR D;Ll;0;L;;;;;N;;;A779;;A779 +A77B;LATIN CAPITAL LETTER INSULAR F;Lu;0;L;;;;;N;;;;A77C; +A77C;LATIN SMALL LETTER INSULAR F;Ll;0;L;;;;;N;;;A77B;;A77B +A77D;LATIN CAPITAL LETTER INSULAR G;Lu;0;L;;;;;N;;;;1D79; +A77E;LATIN CAPITAL LETTER TURNED INSULAR G;Lu;0;L;;;;;N;;;;A77F; +A77F;LATIN SMALL LETTER TURNED INSULAR G;Ll;0;L;;;;;N;;;A77E;;A77E +A780;LATIN CAPITAL LETTER TURNED L;Lu;0;L;;;;;N;;;;A781; +A781;LATIN SMALL LETTER TURNED L;Ll;0;L;;;;;N;;;A780;;A780 +A782;LATIN CAPITAL LETTER INSULAR R;Lu;0;L;;;;;N;;;;A783; +A783;LATIN SMALL LETTER INSULAR R;Ll;0;L;;;;;N;;;A782;;A782 +A784;LATIN CAPITAL LETTER INSULAR S;Lu;0;L;;;;;N;;;;A785; +A785;LATIN SMALL LETTER INSULAR S;Ll;0;L;;;;;N;;;A784;;A784 +A786;LATIN CAPITAL LETTER INSULAR T;Lu;0;L;;;;;N;;;;A787; +A787;LATIN SMALL LETTER INSULAR T;Ll;0;L;;;;;N;;;A786;;A786 +A788;MODIFIER LETTER LOW CIRCUMFLEX ACCENT;Lm;0;ON;;;;;N;;;;; +A789;MODIFIER LETTER COLON;Sk;0;L;;;;;N;;;;; +A78A;MODIFIER LETTER SHORT EQUALS SIGN;Sk;0;L;;;;;N;;;;; +A78B;LATIN CAPITAL LETTER SALTILLO;Lu;0;L;;;;;N;;;;A78C; +A78C;LATIN SMALL LETTER SALTILLO;Ll;0;L;;;;;N;;;A78B;;A78B +A78D;LATIN CAPITAL LETTER TURNED H;Lu;0;L;;;;;N;;;;0265; +A78E;LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT;Ll;0;L;;;;;N;;;;; +A790;LATIN CAPITAL LETTER N WITH DESCENDER;Lu;0;L;;;;;N;;;;A791; +A791;LATIN SMALL LETTER N WITH DESCENDER;Ll;0;L;;;;;N;;;A790;;A790 +A792;LATIN CAPITAL LETTER C WITH BAR;Lu;0;L;;;;;N;;;;A793; +A793;LATIN SMALL LETTER C WITH BAR;Ll;0;L;;;;;N;;;A792;;A792 +A7A0;LATIN CAPITAL LETTER G WITH OBLIQUE STROKE;Lu;0;L;;;;;N;;;;A7A1; +A7A1;LATIN SMALL LETTER G WITH OBLIQUE STROKE;Ll;0;L;;;;;N;;;A7A0;;A7A0 +A7A2;LATIN CAPITAL LETTER K WITH OBLIQUE STROKE;Lu;0;L;;;;;N;;;;A7A3; +A7A3;LATIN SMALL LETTER K WITH OBLIQUE STROKE;Ll;0;L;;;;;N;;;A7A2;;A7A2 +A7A4;LATIN CAPITAL LETTER N WITH OBLIQUE STROKE;Lu;0;L;;;;;N;;;;A7A5; +A7A5;LATIN SMALL LETTER N WITH OBLIQUE STROKE;Ll;0;L;;;;;N;;;A7A4;;A7A4 +A7A6;LATIN CAPITAL LETTER R WITH OBLIQUE STROKE;Lu;0;L;;;;;N;;;;A7A7; +A7A7;LATIN SMALL LETTER R WITH OBLIQUE STROKE;Ll;0;L;;;;;N;;;A7A6;;A7A6 +A7A8;LATIN CAPITAL LETTER S WITH OBLIQUE STROKE;Lu;0;L;;;;;N;;;;A7A9; +A7A9;LATIN SMALL LETTER S WITH OBLIQUE STROKE;Ll;0;L;;;;;N;;;A7A8;;A7A8 +A7AA;LATIN CAPITAL LETTER H WITH HOOK;Lu;0;L;;;;;N;;;;0266; +A7F8;MODIFIER LETTER CAPITAL H WITH STROKE;Lm;0;L; 0126;;;;N;;;;; +A7F9;MODIFIER LETTER SMALL LIGATURE OE;Lm;0;L; 0153;;;;N;;;;; +A7FA;LATIN LETTER SMALL CAPITAL TURNED M;Ll;0;L;;;;;N;;;;; +A7FB;LATIN EPIGRAPHIC LETTER REVERSED F;Lo;0;L;;;;;N;;;;; +A7FC;LATIN EPIGRAPHIC LETTER REVERSED P;Lo;0;L;;;;;N;;;;; +A7FD;LATIN EPIGRAPHIC LETTER INVERTED M;Lo;0;L;;;;;N;;;;; +A7FE;LATIN EPIGRAPHIC LETTER I LONGA;Lo;0;L;;;;;N;;;;; +A7FF;LATIN EPIGRAPHIC LETTER ARCHAIC M;Lo;0;L;;;;;N;;;;; +A800;SYLOTI NAGRI LETTER A;Lo;0;L;;;;;N;;;;; +A801;SYLOTI NAGRI LETTER I;Lo;0;L;;;;;N;;;;; +A802;SYLOTI NAGRI SIGN DVISVARA;Mn;0;NSM;;;;;N;;;;; +A803;SYLOTI NAGRI LETTER U;Lo;0;L;;;;;N;;;;; +A804;SYLOTI NAGRI LETTER E;Lo;0;L;;;;;N;;;;; +A805;SYLOTI NAGRI LETTER O;Lo;0;L;;;;;N;;;;; +A806;SYLOTI NAGRI SIGN HASANTA;Mn;9;NSM;;;;;N;;;;; +A807;SYLOTI NAGRI LETTER KO;Lo;0;L;;;;;N;;;;; +A808;SYLOTI NAGRI LETTER KHO;Lo;0;L;;;;;N;;;;; +A809;SYLOTI NAGRI LETTER GO;Lo;0;L;;;;;N;;;;; +A80A;SYLOTI NAGRI LETTER GHO;Lo;0;L;;;;;N;;;;; +A80B;SYLOTI NAGRI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +A80C;SYLOTI NAGRI LETTER CO;Lo;0;L;;;;;N;;;;; +A80D;SYLOTI NAGRI LETTER CHO;Lo;0;L;;;;;N;;;;; +A80E;SYLOTI NAGRI LETTER JO;Lo;0;L;;;;;N;;;;; +A80F;SYLOTI NAGRI LETTER JHO;Lo;0;L;;;;;N;;;;; +A810;SYLOTI NAGRI LETTER TTO;Lo;0;L;;;;;N;;;;; +A811;SYLOTI NAGRI LETTER TTHO;Lo;0;L;;;;;N;;;;; +A812;SYLOTI NAGRI LETTER DDO;Lo;0;L;;;;;N;;;;; +A813;SYLOTI NAGRI LETTER DDHO;Lo;0;L;;;;;N;;;;; +A814;SYLOTI NAGRI LETTER TO;Lo;0;L;;;;;N;;;;; +A815;SYLOTI NAGRI LETTER THO;Lo;0;L;;;;;N;;;;; +A816;SYLOTI NAGRI LETTER DO;Lo;0;L;;;;;N;;;;; +A817;SYLOTI NAGRI LETTER DHO;Lo;0;L;;;;;N;;;;; +A818;SYLOTI NAGRI LETTER NO;Lo;0;L;;;;;N;;;;; +A819;SYLOTI NAGRI LETTER PO;Lo;0;L;;;;;N;;;;; +A81A;SYLOTI NAGRI LETTER PHO;Lo;0;L;;;;;N;;;;; +A81B;SYLOTI NAGRI LETTER BO;Lo;0;L;;;;;N;;;;; +A81C;SYLOTI NAGRI LETTER BHO;Lo;0;L;;;;;N;;;;; +A81D;SYLOTI NAGRI LETTER MO;Lo;0;L;;;;;N;;;;; +A81E;SYLOTI NAGRI LETTER RO;Lo;0;L;;;;;N;;;;; +A81F;SYLOTI NAGRI LETTER LO;Lo;0;L;;;;;N;;;;; +A820;SYLOTI NAGRI LETTER RRO;Lo;0;L;;;;;N;;;;; +A821;SYLOTI NAGRI LETTER SO;Lo;0;L;;;;;N;;;;; +A822;SYLOTI NAGRI LETTER HO;Lo;0;L;;;;;N;;;;; +A823;SYLOTI NAGRI VOWEL SIGN A;Mc;0;L;;;;;N;;;;; +A824;SYLOTI NAGRI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +A825;SYLOTI NAGRI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +A826;SYLOTI NAGRI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +A827;SYLOTI NAGRI VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +A828;SYLOTI NAGRI POETRY MARK-1;So;0;ON;;;;;N;;;;; +A829;SYLOTI NAGRI POETRY MARK-2;So;0;ON;;;;;N;;;;; +A82A;SYLOTI NAGRI POETRY MARK-3;So;0;ON;;;;;N;;;;; +A82B;SYLOTI NAGRI POETRY MARK-4;So;0;ON;;;;;N;;;;; +A830;NORTH INDIC FRACTION ONE QUARTER;No;0;L;;;;1/4;N;;;;; +A831;NORTH INDIC FRACTION ONE HALF;No;0;L;;;;1/2;N;;;;; +A832;NORTH INDIC FRACTION THREE QUARTERS;No;0;L;;;;3/4;N;;;;; +A833;NORTH INDIC FRACTION ONE SIXTEENTH;No;0;L;;;;1/16;N;;;;; +A834;NORTH INDIC FRACTION ONE EIGHTH;No;0;L;;;;1/8;N;;;;; +A835;NORTH INDIC FRACTION THREE SIXTEENTHS;No;0;L;;;;3/16;N;;;;; +A836;NORTH INDIC QUARTER MARK;So;0;L;;;;;N;;;;; +A837;NORTH INDIC PLACEHOLDER MARK;So;0;L;;;;;N;;;;; +A838;NORTH INDIC RUPEE MARK;Sc;0;ET;;;;;N;;;;; +A839;NORTH INDIC QUANTITY MARK;So;0;ET;;;;;N;;;;; +A840;PHAGS-PA LETTER KA;Lo;0;L;;;;;N;;;;; +A841;PHAGS-PA LETTER KHA;Lo;0;L;;;;;N;;;;; +A842;PHAGS-PA LETTER GA;Lo;0;L;;;;;N;;;;; +A843;PHAGS-PA LETTER NGA;Lo;0;L;;;;;N;;;;; +A844;PHAGS-PA LETTER CA;Lo;0;L;;;;;N;;;;; +A845;PHAGS-PA LETTER CHA;Lo;0;L;;;;;N;;;;; +A846;PHAGS-PA LETTER JA;Lo;0;L;;;;;N;;;;; +A847;PHAGS-PA LETTER NYA;Lo;0;L;;;;;N;;;;; +A848;PHAGS-PA LETTER TA;Lo;0;L;;;;;N;;;;; +A849;PHAGS-PA LETTER THA;Lo;0;L;;;;;N;;;;; +A84A;PHAGS-PA LETTER DA;Lo;0;L;;;;;N;;;;; +A84B;PHAGS-PA LETTER NA;Lo;0;L;;;;;N;;;;; +A84C;PHAGS-PA LETTER PA;Lo;0;L;;;;;N;;;;; +A84D;PHAGS-PA LETTER PHA;Lo;0;L;;;;;N;;;;; +A84E;PHAGS-PA LETTER BA;Lo;0;L;;;;;N;;;;; +A84F;PHAGS-PA LETTER MA;Lo;0;L;;;;;N;;;;; +A850;PHAGS-PA LETTER TSA;Lo;0;L;;;;;N;;;;; +A851;PHAGS-PA LETTER TSHA;Lo;0;L;;;;;N;;;;; +A852;PHAGS-PA LETTER DZA;Lo;0;L;;;;;N;;;;; +A853;PHAGS-PA LETTER WA;Lo;0;L;;;;;N;;;;; +A854;PHAGS-PA LETTER ZHA;Lo;0;L;;;;;N;;;;; +A855;PHAGS-PA LETTER ZA;Lo;0;L;;;;;N;;;;; +A856;PHAGS-PA LETTER SMALL A;Lo;0;L;;;;;N;;;;; +A857;PHAGS-PA LETTER YA;Lo;0;L;;;;;N;;;;; +A858;PHAGS-PA LETTER RA;Lo;0;L;;;;;N;;;;; +A859;PHAGS-PA LETTER LA;Lo;0;L;;;;;N;;;;; +A85A;PHAGS-PA LETTER SHA;Lo;0;L;;;;;N;;;;; +A85B;PHAGS-PA LETTER SA;Lo;0;L;;;;;N;;;;; +A85C;PHAGS-PA LETTER HA;Lo;0;L;;;;;N;;;;; +A85D;PHAGS-PA LETTER A;Lo;0;L;;;;;N;;;;; +A85E;PHAGS-PA LETTER I;Lo;0;L;;;;;N;;;;; +A85F;PHAGS-PA LETTER U;Lo;0;L;;;;;N;;;;; +A860;PHAGS-PA LETTER E;Lo;0;L;;;;;N;;;;; +A861;PHAGS-PA LETTER O;Lo;0;L;;;;;N;;;;; +A862;PHAGS-PA LETTER QA;Lo;0;L;;;;;N;;;;; +A863;PHAGS-PA LETTER XA;Lo;0;L;;;;;N;;;;; +A864;PHAGS-PA LETTER FA;Lo;0;L;;;;;N;;;;; +A865;PHAGS-PA LETTER GGA;Lo;0;L;;;;;N;;;;; +A866;PHAGS-PA LETTER EE;Lo;0;L;;;;;N;;;;; +A867;PHAGS-PA SUBJOINED LETTER WA;Lo;0;L;;;;;N;;;;; +A868;PHAGS-PA SUBJOINED LETTER YA;Lo;0;L;;;;;N;;;;; +A869;PHAGS-PA LETTER TTA;Lo;0;L;;;;;N;;;;; +A86A;PHAGS-PA LETTER TTHA;Lo;0;L;;;;;N;;;;; +A86B;PHAGS-PA LETTER DDA;Lo;0;L;;;;;N;;;;; +A86C;PHAGS-PA LETTER NNA;Lo;0;L;;;;;N;;;;; +A86D;PHAGS-PA LETTER ALTERNATE YA;Lo;0;L;;;;;N;;;;; +A86E;PHAGS-PA LETTER VOICELESS SHA;Lo;0;L;;;;;N;;;;; +A86F;PHAGS-PA LETTER VOICED HA;Lo;0;L;;;;;N;;;;; +A870;PHAGS-PA LETTER ASPIRATED FA;Lo;0;L;;;;;N;;;;; +A871;PHAGS-PA SUBJOINED LETTER RA;Lo;0;L;;;;;N;;;;; +A872;PHAGS-PA SUPERFIXED LETTER RA;Lo;0;L;;;;;N;;;;; +A873;PHAGS-PA LETTER CANDRABINDU;Lo;0;L;;;;;N;;;;; +A874;PHAGS-PA SINGLE HEAD MARK;Po;0;ON;;;;;N;;;;; +A875;PHAGS-PA DOUBLE HEAD MARK;Po;0;ON;;;;;N;;;;; +A876;PHAGS-PA MARK SHAD;Po;0;ON;;;;;N;;;;; +A877;PHAGS-PA MARK DOUBLE SHAD;Po;0;ON;;;;;N;;;;; +A880;SAURASHTRA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +A881;SAURASHTRA SIGN VISARGA;Mc;0;L;;;;;N;;;;; +A882;SAURASHTRA LETTER A;Lo;0;L;;;;;N;;;;; +A883;SAURASHTRA LETTER AA;Lo;0;L;;;;;N;;;;; +A884;SAURASHTRA LETTER I;Lo;0;L;;;;;N;;;;; +A885;SAURASHTRA LETTER II;Lo;0;L;;;;;N;;;;; +A886;SAURASHTRA LETTER U;Lo;0;L;;;;;N;;;;; +A887;SAURASHTRA LETTER UU;Lo;0;L;;;;;N;;;;; +A888;SAURASHTRA LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +A889;SAURASHTRA LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +A88A;SAURASHTRA LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +A88B;SAURASHTRA LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +A88C;SAURASHTRA LETTER E;Lo;0;L;;;;;N;;;;; +A88D;SAURASHTRA LETTER EE;Lo;0;L;;;;;N;;;;; +A88E;SAURASHTRA LETTER AI;Lo;0;L;;;;;N;;;;; +A88F;SAURASHTRA LETTER O;Lo;0;L;;;;;N;;;;; +A890;SAURASHTRA LETTER OO;Lo;0;L;;;;;N;;;;; +A891;SAURASHTRA LETTER AU;Lo;0;L;;;;;N;;;;; +A892;SAURASHTRA LETTER KA;Lo;0;L;;;;;N;;;;; +A893;SAURASHTRA LETTER KHA;Lo;0;L;;;;;N;;;;; +A894;SAURASHTRA LETTER GA;Lo;0;L;;;;;N;;;;; +A895;SAURASHTRA LETTER GHA;Lo;0;L;;;;;N;;;;; +A896;SAURASHTRA LETTER NGA;Lo;0;L;;;;;N;;;;; +A897;SAURASHTRA LETTER CA;Lo;0;L;;;;;N;;;;; +A898;SAURASHTRA LETTER CHA;Lo;0;L;;;;;N;;;;; +A899;SAURASHTRA LETTER JA;Lo;0;L;;;;;N;;;;; +A89A;SAURASHTRA LETTER JHA;Lo;0;L;;;;;N;;;;; +A89B;SAURASHTRA LETTER NYA;Lo;0;L;;;;;N;;;;; +A89C;SAURASHTRA LETTER TTA;Lo;0;L;;;;;N;;;;; +A89D;SAURASHTRA LETTER TTHA;Lo;0;L;;;;;N;;;;; +A89E;SAURASHTRA LETTER DDA;Lo;0;L;;;;;N;;;;; +A89F;SAURASHTRA LETTER DDHA;Lo;0;L;;;;;N;;;;; +A8A0;SAURASHTRA LETTER NNA;Lo;0;L;;;;;N;;;;; +A8A1;SAURASHTRA LETTER TA;Lo;0;L;;;;;N;;;;; +A8A2;SAURASHTRA LETTER THA;Lo;0;L;;;;;N;;;;; +A8A3;SAURASHTRA LETTER DA;Lo;0;L;;;;;N;;;;; +A8A4;SAURASHTRA LETTER DHA;Lo;0;L;;;;;N;;;;; +A8A5;SAURASHTRA LETTER NA;Lo;0;L;;;;;N;;;;; +A8A6;SAURASHTRA LETTER PA;Lo;0;L;;;;;N;;;;; +A8A7;SAURASHTRA LETTER PHA;Lo;0;L;;;;;N;;;;; +A8A8;SAURASHTRA LETTER BA;Lo;0;L;;;;;N;;;;; +A8A9;SAURASHTRA LETTER BHA;Lo;0;L;;;;;N;;;;; +A8AA;SAURASHTRA LETTER MA;Lo;0;L;;;;;N;;;;; +A8AB;SAURASHTRA LETTER YA;Lo;0;L;;;;;N;;;;; +A8AC;SAURASHTRA LETTER RA;Lo;0;L;;;;;N;;;;; +A8AD;SAURASHTRA LETTER LA;Lo;0;L;;;;;N;;;;; +A8AE;SAURASHTRA LETTER VA;Lo;0;L;;;;;N;;;;; +A8AF;SAURASHTRA LETTER SHA;Lo;0;L;;;;;N;;;;; +A8B0;SAURASHTRA LETTER SSA;Lo;0;L;;;;;N;;;;; +A8B1;SAURASHTRA LETTER SA;Lo;0;L;;;;;N;;;;; +A8B2;SAURASHTRA LETTER HA;Lo;0;L;;;;;N;;;;; +A8B3;SAURASHTRA LETTER LLA;Lo;0;L;;;;;N;;;;; +A8B4;SAURASHTRA CONSONANT SIGN HAARU;Mc;0;L;;;;;N;;;;; +A8B5;SAURASHTRA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +A8B6;SAURASHTRA VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +A8B7;SAURASHTRA VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +A8B8;SAURASHTRA VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +A8B9;SAURASHTRA VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +A8BA;SAURASHTRA VOWEL SIGN VOCALIC R;Mc;0;L;;;;;N;;;;; +A8BB;SAURASHTRA VOWEL SIGN VOCALIC RR;Mc;0;L;;;;;N;;;;; +A8BC;SAURASHTRA VOWEL SIGN VOCALIC L;Mc;0;L;;;;;N;;;;; +A8BD;SAURASHTRA VOWEL SIGN VOCALIC LL;Mc;0;L;;;;;N;;;;; +A8BE;SAURASHTRA VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +A8BF;SAURASHTRA VOWEL SIGN EE;Mc;0;L;;;;;N;;;;; +A8C0;SAURASHTRA VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +A8C1;SAURASHTRA VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +A8C2;SAURASHTRA VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +A8C3;SAURASHTRA VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +A8C4;SAURASHTRA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +A8CE;SAURASHTRA DANDA;Po;0;L;;;;;N;;;;; +A8CF;SAURASHTRA DOUBLE DANDA;Po;0;L;;;;;N;;;;; +A8D0;SAURASHTRA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +A8D1;SAURASHTRA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +A8D2;SAURASHTRA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +A8D3;SAURASHTRA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +A8D4;SAURASHTRA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +A8D5;SAURASHTRA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +A8D6;SAURASHTRA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +A8D7;SAURASHTRA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +A8D8;SAURASHTRA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +A8D9;SAURASHTRA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +A8E0;COMBINING DEVANAGARI DIGIT ZERO;Mn;230;NSM;;;;;N;;;;; +A8E1;COMBINING DEVANAGARI DIGIT ONE;Mn;230;NSM;;;;;N;;;;; +A8E2;COMBINING DEVANAGARI DIGIT TWO;Mn;230;NSM;;;;;N;;;;; +A8E3;COMBINING DEVANAGARI DIGIT THREE;Mn;230;NSM;;;;;N;;;;; +A8E4;COMBINING DEVANAGARI DIGIT FOUR;Mn;230;NSM;;;;;N;;;;; +A8E5;COMBINING DEVANAGARI DIGIT FIVE;Mn;230;NSM;;;;;N;;;;; +A8E6;COMBINING DEVANAGARI DIGIT SIX;Mn;230;NSM;;;;;N;;;;; +A8E7;COMBINING DEVANAGARI DIGIT SEVEN;Mn;230;NSM;;;;;N;;;;; +A8E8;COMBINING DEVANAGARI DIGIT EIGHT;Mn;230;NSM;;;;;N;;;;; +A8E9;COMBINING DEVANAGARI DIGIT NINE;Mn;230;NSM;;;;;N;;;;; +A8EA;COMBINING DEVANAGARI LETTER A;Mn;230;NSM;;;;;N;;;;; +A8EB;COMBINING DEVANAGARI LETTER U;Mn;230;NSM;;;;;N;;;;; +A8EC;COMBINING DEVANAGARI LETTER KA;Mn;230;NSM;;;;;N;;;;; +A8ED;COMBINING DEVANAGARI LETTER NA;Mn;230;NSM;;;;;N;;;;; +A8EE;COMBINING DEVANAGARI LETTER PA;Mn;230;NSM;;;;;N;;;;; +A8EF;COMBINING DEVANAGARI LETTER RA;Mn;230;NSM;;;;;N;;;;; +A8F0;COMBINING DEVANAGARI LETTER VI;Mn;230;NSM;;;;;N;;;;; +A8F1;COMBINING DEVANAGARI SIGN AVAGRAHA;Mn;230;NSM;;;;;N;;;;; +A8F2;DEVANAGARI SIGN SPACING CANDRABINDU;Lo;0;L;;;;;N;;;;; +A8F3;DEVANAGARI SIGN CANDRABINDU VIRAMA;Lo;0;L;;;;;N;;;;; +A8F4;DEVANAGARI SIGN DOUBLE CANDRABINDU VIRAMA;Lo;0;L;;;;;N;;;;; +A8F5;DEVANAGARI SIGN CANDRABINDU TWO;Lo;0;L;;;;;N;;;;; +A8F6;DEVANAGARI SIGN CANDRABINDU THREE;Lo;0;L;;;;;N;;;;; +A8F7;DEVANAGARI SIGN CANDRABINDU AVAGRAHA;Lo;0;L;;;;;N;;;;; +A8F8;DEVANAGARI SIGN PUSHPIKA;Po;0;L;;;;;N;;;;; +A8F9;DEVANAGARI GAP FILLER;Po;0;L;;;;;N;;;;; +A8FA;DEVANAGARI CARET;Po;0;L;;;;;N;;;;; +A8FB;DEVANAGARI HEADSTROKE;Lo;0;L;;;;;N;;;;; +A900;KAYAH LI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +A901;KAYAH LI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +A902;KAYAH LI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +A903;KAYAH LI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +A904;KAYAH LI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +A905;KAYAH LI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +A906;KAYAH LI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +A907;KAYAH LI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +A908;KAYAH LI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +A909;KAYAH LI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +A90A;KAYAH LI LETTER KA;Lo;0;L;;;;;N;;;;; +A90B;KAYAH LI LETTER KHA;Lo;0;L;;;;;N;;;;; +A90C;KAYAH LI LETTER GA;Lo;0;L;;;;;N;;;;; +A90D;KAYAH LI LETTER NGA;Lo;0;L;;;;;N;;;;; +A90E;KAYAH LI LETTER SA;Lo;0;L;;;;;N;;;;; +A90F;KAYAH LI LETTER SHA;Lo;0;L;;;;;N;;;;; +A910;KAYAH LI LETTER ZA;Lo;0;L;;;;;N;;;;; +A911;KAYAH LI LETTER NYA;Lo;0;L;;;;;N;;;;; +A912;KAYAH LI LETTER TA;Lo;0;L;;;;;N;;;;; +A913;KAYAH LI LETTER HTA;Lo;0;L;;;;;N;;;;; +A914;KAYAH LI LETTER NA;Lo;0;L;;;;;N;;;;; +A915;KAYAH LI LETTER PA;Lo;0;L;;;;;N;;;;; +A916;KAYAH LI LETTER PHA;Lo;0;L;;;;;N;;;;; +A917;KAYAH LI LETTER MA;Lo;0;L;;;;;N;;;;; +A918;KAYAH LI LETTER DA;Lo;0;L;;;;;N;;;;; +A919;KAYAH LI LETTER BA;Lo;0;L;;;;;N;;;;; +A91A;KAYAH LI LETTER RA;Lo;0;L;;;;;N;;;;; +A91B;KAYAH LI LETTER YA;Lo;0;L;;;;;N;;;;; +A91C;KAYAH LI LETTER LA;Lo;0;L;;;;;N;;;;; +A91D;KAYAH LI LETTER WA;Lo;0;L;;;;;N;;;;; +A91E;KAYAH LI LETTER THA;Lo;0;L;;;;;N;;;;; +A91F;KAYAH LI LETTER HA;Lo;0;L;;;;;N;;;;; +A920;KAYAH LI LETTER VA;Lo;0;L;;;;;N;;;;; +A921;KAYAH LI LETTER CA;Lo;0;L;;;;;N;;;;; +A922;KAYAH LI LETTER A;Lo;0;L;;;;;N;;;;; +A923;KAYAH LI LETTER OE;Lo;0;L;;;;;N;;;;; +A924;KAYAH LI LETTER I;Lo;0;L;;;;;N;;;;; +A925;KAYAH LI LETTER OO;Lo;0;L;;;;;N;;;;; +A926;KAYAH LI VOWEL UE;Mn;0;NSM;;;;;N;;;;; +A927;KAYAH LI VOWEL E;Mn;0;NSM;;;;;N;;;;; +A928;KAYAH LI VOWEL U;Mn;0;NSM;;;;;N;;;;; +A929;KAYAH LI VOWEL EE;Mn;0;NSM;;;;;N;;;;; +A92A;KAYAH LI VOWEL O;Mn;0;NSM;;;;;N;;;;; +A92B;KAYAH LI TONE PLOPHU;Mn;220;NSM;;;;;N;;;;; +A92C;KAYAH LI TONE CALYA;Mn;220;NSM;;;;;N;;;;; +A92D;KAYAH LI TONE CALYA PLOPHU;Mn;220;NSM;;;;;N;;;;; +A92E;KAYAH LI SIGN CWI;Po;0;L;;;;;N;;;;; +A92F;KAYAH LI SIGN SHYA;Po;0;L;;;;;N;;;;; +A930;REJANG LETTER KA;Lo;0;L;;;;;N;;;;; +A931;REJANG LETTER GA;Lo;0;L;;;;;N;;;;; +A932;REJANG LETTER NGA;Lo;0;L;;;;;N;;;;; +A933;REJANG LETTER TA;Lo;0;L;;;;;N;;;;; +A934;REJANG LETTER DA;Lo;0;L;;;;;N;;;;; +A935;REJANG LETTER NA;Lo;0;L;;;;;N;;;;; +A936;REJANG LETTER PA;Lo;0;L;;;;;N;;;;; +A937;REJANG LETTER BA;Lo;0;L;;;;;N;;;;; +A938;REJANG LETTER MA;Lo;0;L;;;;;N;;;;; +A939;REJANG LETTER CA;Lo;0;L;;;;;N;;;;; +A93A;REJANG LETTER JA;Lo;0;L;;;;;N;;;;; +A93B;REJANG LETTER NYA;Lo;0;L;;;;;N;;;;; +A93C;REJANG LETTER SA;Lo;0;L;;;;;N;;;;; +A93D;REJANG LETTER RA;Lo;0;L;;;;;N;;;;; +A93E;REJANG LETTER LA;Lo;0;L;;;;;N;;;;; +A93F;REJANG LETTER YA;Lo;0;L;;;;;N;;;;; +A940;REJANG LETTER WA;Lo;0;L;;;;;N;;;;; +A941;REJANG LETTER HA;Lo;0;L;;;;;N;;;;; +A942;REJANG LETTER MBA;Lo;0;L;;;;;N;;;;; +A943;REJANG LETTER NGGA;Lo;0;L;;;;;N;;;;; +A944;REJANG LETTER NDA;Lo;0;L;;;;;N;;;;; +A945;REJANG LETTER NYJA;Lo;0;L;;;;;N;;;;; +A946;REJANG LETTER A;Lo;0;L;;;;;N;;;;; +A947;REJANG VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +A948;REJANG VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +A949;REJANG VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +A94A;REJANG VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +A94B;REJANG VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +A94C;REJANG VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +A94D;REJANG VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;; +A94E;REJANG VOWEL SIGN EA;Mn;0;NSM;;;;;N;;;;; +A94F;REJANG CONSONANT SIGN NG;Mn;0;NSM;;;;;N;;;;; +A950;REJANG CONSONANT SIGN N;Mn;0;NSM;;;;;N;;;;; +A951;REJANG CONSONANT SIGN R;Mn;0;NSM;;;;;N;;;;; +A952;REJANG CONSONANT SIGN H;Mc;0;L;;;;;N;;;;; +A953;REJANG VIRAMA;Mc;9;L;;;;;N;;;;; +A95F;REJANG SECTION MARK;Po;0;L;;;;;N;;;;; +A960;HANGUL CHOSEONG TIKEUT-MIEUM;Lo;0;L;;;;;N;;;;; +A961;HANGUL CHOSEONG TIKEUT-PIEUP;Lo;0;L;;;;;N;;;;; +A962;HANGUL CHOSEONG TIKEUT-SIOS;Lo;0;L;;;;;N;;;;; +A963;HANGUL CHOSEONG TIKEUT-CIEUC;Lo;0;L;;;;;N;;;;; +A964;HANGUL CHOSEONG RIEUL-KIYEOK;Lo;0;L;;;;;N;;;;; +A965;HANGUL CHOSEONG RIEUL-SSANGKIYEOK;Lo;0;L;;;;;N;;;;; +A966;HANGUL CHOSEONG RIEUL-TIKEUT;Lo;0;L;;;;;N;;;;; +A967;HANGUL CHOSEONG RIEUL-SSANGTIKEUT;Lo;0;L;;;;;N;;;;; +A968;HANGUL CHOSEONG RIEUL-MIEUM;Lo;0;L;;;;;N;;;;; +A969;HANGUL CHOSEONG RIEUL-PIEUP;Lo;0;L;;;;;N;;;;; +A96A;HANGUL CHOSEONG RIEUL-SSANGPIEUP;Lo;0;L;;;;;N;;;;; +A96B;HANGUL CHOSEONG RIEUL-KAPYEOUNPIEUP;Lo;0;L;;;;;N;;;;; +A96C;HANGUL CHOSEONG RIEUL-SIOS;Lo;0;L;;;;;N;;;;; +A96D;HANGUL CHOSEONG RIEUL-CIEUC;Lo;0;L;;;;;N;;;;; +A96E;HANGUL CHOSEONG RIEUL-KHIEUKH;Lo;0;L;;;;;N;;;;; +A96F;HANGUL CHOSEONG MIEUM-KIYEOK;Lo;0;L;;;;;N;;;;; +A970;HANGUL CHOSEONG MIEUM-TIKEUT;Lo;0;L;;;;;N;;;;; +A971;HANGUL CHOSEONG MIEUM-SIOS;Lo;0;L;;;;;N;;;;; +A972;HANGUL CHOSEONG PIEUP-SIOS-THIEUTH;Lo;0;L;;;;;N;;;;; +A973;HANGUL CHOSEONG PIEUP-KHIEUKH;Lo;0;L;;;;;N;;;;; +A974;HANGUL CHOSEONG PIEUP-HIEUH;Lo;0;L;;;;;N;;;;; +A975;HANGUL CHOSEONG SSANGSIOS-PIEUP;Lo;0;L;;;;;N;;;;; +A976;HANGUL CHOSEONG IEUNG-RIEUL;Lo;0;L;;;;;N;;;;; +A977;HANGUL CHOSEONG IEUNG-HIEUH;Lo;0;L;;;;;N;;;;; +A978;HANGUL CHOSEONG SSANGCIEUC-HIEUH;Lo;0;L;;;;;N;;;;; +A979;HANGUL CHOSEONG SSANGTHIEUTH;Lo;0;L;;;;;N;;;;; +A97A;HANGUL CHOSEONG PHIEUPH-HIEUH;Lo;0;L;;;;;N;;;;; +A97B;HANGUL CHOSEONG HIEUH-SIOS;Lo;0;L;;;;;N;;;;; +A97C;HANGUL CHOSEONG SSANGYEORINHIEUH;Lo;0;L;;;;;N;;;;; +A980;JAVANESE SIGN PANYANGGA;Mn;0;NSM;;;;;N;;;;; +A981;JAVANESE SIGN CECAK;Mn;0;NSM;;;;;N;;;;; +A982;JAVANESE SIGN LAYAR;Mn;0;NSM;;;;;N;;;;; +A983;JAVANESE SIGN WIGNYAN;Mc;0;L;;;;;N;;;;; +A984;JAVANESE LETTER A;Lo;0;L;;;;;N;;;;; +A985;JAVANESE LETTER I KAWI;Lo;0;L;;;;;N;;;;; +A986;JAVANESE LETTER I;Lo;0;L;;;;;N;;;;; +A987;JAVANESE LETTER II;Lo;0;L;;;;;N;;;;; +A988;JAVANESE LETTER U;Lo;0;L;;;;;N;;;;; +A989;JAVANESE LETTER PA CEREK;Lo;0;L;;;;;N;;;;; +A98A;JAVANESE LETTER NGA LELET;Lo;0;L;;;;;N;;;;; +A98B;JAVANESE LETTER NGA LELET RASWADI;Lo;0;L;;;;;N;;;;; +A98C;JAVANESE LETTER E;Lo;0;L;;;;;N;;;;; +A98D;JAVANESE LETTER AI;Lo;0;L;;;;;N;;;;; +A98E;JAVANESE LETTER O;Lo;0;L;;;;;N;;;;; +A98F;JAVANESE LETTER KA;Lo;0;L;;;;;N;;;;; +A990;JAVANESE LETTER KA SASAK;Lo;0;L;;;;;N;;;;; +A991;JAVANESE LETTER KA MURDA;Lo;0;L;;;;;N;;;;; +A992;JAVANESE LETTER GA;Lo;0;L;;;;;N;;;;; +A993;JAVANESE LETTER GA MURDA;Lo;0;L;;;;;N;;;;; +A994;JAVANESE LETTER NGA;Lo;0;L;;;;;N;;;;; +A995;JAVANESE LETTER CA;Lo;0;L;;;;;N;;;;; +A996;JAVANESE LETTER CA MURDA;Lo;0;L;;;;;N;;;;; +A997;JAVANESE LETTER JA;Lo;0;L;;;;;N;;;;; +A998;JAVANESE LETTER NYA MURDA;Lo;0;L;;;;;N;;;;; +A999;JAVANESE LETTER JA MAHAPRANA;Lo;0;L;;;;;N;;;;; +A99A;JAVANESE LETTER NYA;Lo;0;L;;;;;N;;;;; +A99B;JAVANESE LETTER TTA;Lo;0;L;;;;;N;;;;; +A99C;JAVANESE LETTER TTA MAHAPRANA;Lo;0;L;;;;;N;;;;; +A99D;JAVANESE LETTER DDA;Lo;0;L;;;;;N;;;;; +A99E;JAVANESE LETTER DDA MAHAPRANA;Lo;0;L;;;;;N;;;;; +A99F;JAVANESE LETTER NA MURDA;Lo;0;L;;;;;N;;;;; +A9A0;JAVANESE LETTER TA;Lo;0;L;;;;;N;;;;; +A9A1;JAVANESE LETTER TA MURDA;Lo;0;L;;;;;N;;;;; +A9A2;JAVANESE LETTER DA;Lo;0;L;;;;;N;;;;; +A9A3;JAVANESE LETTER DA MAHAPRANA;Lo;0;L;;;;;N;;;;; +A9A4;JAVANESE LETTER NA;Lo;0;L;;;;;N;;;;; +A9A5;JAVANESE LETTER PA;Lo;0;L;;;;;N;;;;; +A9A6;JAVANESE LETTER PA MURDA;Lo;0;L;;;;;N;;;;; +A9A7;JAVANESE LETTER BA;Lo;0;L;;;;;N;;;;; +A9A8;JAVANESE LETTER BA MURDA;Lo;0;L;;;;;N;;;;; +A9A9;JAVANESE LETTER MA;Lo;0;L;;;;;N;;;;; +A9AA;JAVANESE LETTER YA;Lo;0;L;;;;;N;;;;; +A9AB;JAVANESE LETTER RA;Lo;0;L;;;;;N;;;;; +A9AC;JAVANESE LETTER RA AGUNG;Lo;0;L;;;;;N;;;;; +A9AD;JAVANESE LETTER LA;Lo;0;L;;;;;N;;;;; +A9AE;JAVANESE LETTER WA;Lo;0;L;;;;;N;;;;; +A9AF;JAVANESE LETTER SA MURDA;Lo;0;L;;;;;N;;;;; +A9B0;JAVANESE LETTER SA MAHAPRANA;Lo;0;L;;;;;N;;;;; +A9B1;JAVANESE LETTER SA;Lo;0;L;;;;;N;;;;; +A9B2;JAVANESE LETTER HA;Lo;0;L;;;;;N;;;;; +A9B3;JAVANESE SIGN CECAK TELU;Mn;7;NSM;;;;;N;;;;; +A9B4;JAVANESE VOWEL SIGN TARUNG;Mc;0;L;;;;;N;;;;; +A9B5;JAVANESE VOWEL SIGN TOLONG;Mc;0;L;;;;;N;;;;; +A9B6;JAVANESE VOWEL SIGN WULU;Mn;0;NSM;;;;;N;;;;; +A9B7;JAVANESE VOWEL SIGN WULU MELIK;Mn;0;NSM;;;;;N;;;;; +A9B8;JAVANESE VOWEL SIGN SUKU;Mn;0;NSM;;;;;N;;;;; +A9B9;JAVANESE VOWEL SIGN SUKU MENDUT;Mn;0;NSM;;;;;N;;;;; +A9BA;JAVANESE VOWEL SIGN TALING;Mc;0;L;;;;;N;;;;; +A9BB;JAVANESE VOWEL SIGN DIRGA MURE;Mc;0;L;;;;;N;;;;; +A9BC;JAVANESE VOWEL SIGN PEPET;Mn;0;NSM;;;;;N;;;;; +A9BD;JAVANESE CONSONANT SIGN KERET;Mc;0;L;;;;;N;;;;; +A9BE;JAVANESE CONSONANT SIGN PENGKAL;Mc;0;L;;;;;N;;;;; +A9BF;JAVANESE CONSONANT SIGN CAKRA;Mc;0;L;;;;;N;;;;; +A9C0;JAVANESE PANGKON;Mc;9;L;;;;;N;;;;; +A9C1;JAVANESE LEFT RERENGGAN;Po;0;L;;;;;N;;;;; +A9C2;JAVANESE RIGHT RERENGGAN;Po;0;L;;;;;N;;;;; +A9C3;JAVANESE PADA ANDAP;Po;0;L;;;;;N;;;;; +A9C4;JAVANESE PADA MADYA;Po;0;L;;;;;N;;;;; +A9C5;JAVANESE PADA LUHUR;Po;0;L;;;;;N;;;;; +A9C6;JAVANESE PADA WINDU;Po;0;L;;;;;N;;;;; +A9C7;JAVANESE PADA PANGKAT;Po;0;L;;;;;N;;;;; +A9C8;JAVANESE PADA LINGSA;Po;0;L;;;;;N;;;;; +A9C9;JAVANESE PADA LUNGSI;Po;0;L;;;;;N;;;;; +A9CA;JAVANESE PADA ADEG;Po;0;L;;;;;N;;;;; +A9CB;JAVANESE PADA ADEG ADEG;Po;0;L;;;;;N;;;;; +A9CC;JAVANESE PADA PISELEH;Po;0;L;;;;;N;;;;; +A9CD;JAVANESE TURNED PADA PISELEH;Po;0;L;;;;;N;;;;; +A9CF;JAVANESE PANGRANGKEP;Lm;0;L;;;;;N;;;;; +A9D0;JAVANESE DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +A9D1;JAVANESE DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +A9D2;JAVANESE DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +A9D3;JAVANESE DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +A9D4;JAVANESE DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +A9D5;JAVANESE DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +A9D6;JAVANESE DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +A9D7;JAVANESE DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +A9D8;JAVANESE DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +A9D9;JAVANESE DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +A9DE;JAVANESE PADA TIRTA TUMETES;Po;0;L;;;;;N;;;;; +A9DF;JAVANESE PADA ISEN-ISEN;Po;0;L;;;;;N;;;;; +AA00;CHAM LETTER A;Lo;0;L;;;;;N;;;;; +AA01;CHAM LETTER I;Lo;0;L;;;;;N;;;;; +AA02;CHAM LETTER U;Lo;0;L;;;;;N;;;;; +AA03;CHAM LETTER E;Lo;0;L;;;;;N;;;;; +AA04;CHAM LETTER AI;Lo;0;L;;;;;N;;;;; +AA05;CHAM LETTER O;Lo;0;L;;;;;N;;;;; +AA06;CHAM LETTER KA;Lo;0;L;;;;;N;;;;; +AA07;CHAM LETTER KHA;Lo;0;L;;;;;N;;;;; +AA08;CHAM LETTER GA;Lo;0;L;;;;;N;;;;; +AA09;CHAM LETTER GHA;Lo;0;L;;;;;N;;;;; +AA0A;CHAM LETTER NGUE;Lo;0;L;;;;;N;;;;; +AA0B;CHAM LETTER NGA;Lo;0;L;;;;;N;;;;; +AA0C;CHAM LETTER CHA;Lo;0;L;;;;;N;;;;; +AA0D;CHAM LETTER CHHA;Lo;0;L;;;;;N;;;;; +AA0E;CHAM LETTER JA;Lo;0;L;;;;;N;;;;; +AA0F;CHAM LETTER JHA;Lo;0;L;;;;;N;;;;; +AA10;CHAM LETTER NHUE;Lo;0;L;;;;;N;;;;; +AA11;CHAM LETTER NHA;Lo;0;L;;;;;N;;;;; +AA12;CHAM LETTER NHJA;Lo;0;L;;;;;N;;;;; +AA13;CHAM LETTER TA;Lo;0;L;;;;;N;;;;; +AA14;CHAM LETTER THA;Lo;0;L;;;;;N;;;;; +AA15;CHAM LETTER DA;Lo;0;L;;;;;N;;;;; +AA16;CHAM LETTER DHA;Lo;0;L;;;;;N;;;;; +AA17;CHAM LETTER NUE;Lo;0;L;;;;;N;;;;; +AA18;CHAM LETTER NA;Lo;0;L;;;;;N;;;;; +AA19;CHAM LETTER DDA;Lo;0;L;;;;;N;;;;; +AA1A;CHAM LETTER PA;Lo;0;L;;;;;N;;;;; +AA1B;CHAM LETTER PPA;Lo;0;L;;;;;N;;;;; +AA1C;CHAM LETTER PHA;Lo;0;L;;;;;N;;;;; +AA1D;CHAM LETTER BA;Lo;0;L;;;;;N;;;;; +AA1E;CHAM LETTER BHA;Lo;0;L;;;;;N;;;;; +AA1F;CHAM LETTER MUE;Lo;0;L;;;;;N;;;;; +AA20;CHAM LETTER MA;Lo;0;L;;;;;N;;;;; +AA21;CHAM LETTER BBA;Lo;0;L;;;;;N;;;;; +AA22;CHAM LETTER YA;Lo;0;L;;;;;N;;;;; +AA23;CHAM LETTER RA;Lo;0;L;;;;;N;;;;; +AA24;CHAM LETTER LA;Lo;0;L;;;;;N;;;;; +AA25;CHAM LETTER VA;Lo;0;L;;;;;N;;;;; +AA26;CHAM LETTER SSA;Lo;0;L;;;;;N;;;;; +AA27;CHAM LETTER SA;Lo;0;L;;;;;N;;;;; +AA28;CHAM LETTER HA;Lo;0;L;;;;;N;;;;; +AA29;CHAM VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +AA2A;CHAM VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +AA2B;CHAM VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +AA2C;CHAM VOWEL SIGN EI;Mn;0;NSM;;;;;N;;;;; +AA2D;CHAM VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +AA2E;CHAM VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; +AA2F;CHAM VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +AA30;CHAM VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +AA31;CHAM VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +AA32;CHAM VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +AA33;CHAM CONSONANT SIGN YA;Mc;0;L;;;;;N;;;;; +AA34;CHAM CONSONANT SIGN RA;Mc;0;L;;;;;N;;;;; +AA35;CHAM CONSONANT SIGN LA;Mn;0;NSM;;;;;N;;;;; +AA36;CHAM CONSONANT SIGN WA;Mn;0;NSM;;;;;N;;;;; +AA40;CHAM LETTER FINAL K;Lo;0;L;;;;;N;;;;; +AA41;CHAM LETTER FINAL G;Lo;0;L;;;;;N;;;;; +AA42;CHAM LETTER FINAL NG;Lo;0;L;;;;;N;;;;; +AA43;CHAM CONSONANT SIGN FINAL NG;Mn;0;NSM;;;;;N;;;;; +AA44;CHAM LETTER FINAL CH;Lo;0;L;;;;;N;;;;; +AA45;CHAM LETTER FINAL T;Lo;0;L;;;;;N;;;;; +AA46;CHAM LETTER FINAL N;Lo;0;L;;;;;N;;;;; +AA47;CHAM LETTER FINAL P;Lo;0;L;;;;;N;;;;; +AA48;CHAM LETTER FINAL Y;Lo;0;L;;;;;N;;;;; +AA49;CHAM LETTER FINAL R;Lo;0;L;;;;;N;;;;; +AA4A;CHAM LETTER FINAL L;Lo;0;L;;;;;N;;;;; +AA4B;CHAM LETTER FINAL SS;Lo;0;L;;;;;N;;;;; +AA4C;CHAM CONSONANT SIGN FINAL M;Mn;0;NSM;;;;;N;;;;; +AA4D;CHAM CONSONANT SIGN FINAL H;Mc;0;L;;;;;N;;;;; +AA50;CHAM DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +AA51;CHAM DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +AA52;CHAM DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +AA53;CHAM DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +AA54;CHAM DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +AA55;CHAM DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +AA56;CHAM DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +AA57;CHAM DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +AA58;CHAM DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +AA59;CHAM DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +AA5C;CHAM PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;; +AA5D;CHAM PUNCTUATION DANDA;Po;0;L;;;;;N;;;;; +AA5E;CHAM PUNCTUATION DOUBLE DANDA;Po;0;L;;;;;N;;;;; +AA5F;CHAM PUNCTUATION TRIPLE DANDA;Po;0;L;;;;;N;;;;; +AA60;MYANMAR LETTER KHAMTI GA;Lo;0;L;;;;;N;;;;; +AA61;MYANMAR LETTER KHAMTI CA;Lo;0;L;;;;;N;;;;; +AA62;MYANMAR LETTER KHAMTI CHA;Lo;0;L;;;;;N;;;;; +AA63;MYANMAR LETTER KHAMTI JA;Lo;0;L;;;;;N;;;;; +AA64;MYANMAR LETTER KHAMTI JHA;Lo;0;L;;;;;N;;;;; +AA65;MYANMAR LETTER KHAMTI NYA;Lo;0;L;;;;;N;;;;; +AA66;MYANMAR LETTER KHAMTI TTA;Lo;0;L;;;;;N;;;;; +AA67;MYANMAR LETTER KHAMTI TTHA;Lo;0;L;;;;;N;;;;; +AA68;MYANMAR LETTER KHAMTI DDA;Lo;0;L;;;;;N;;;;; +AA69;MYANMAR LETTER KHAMTI DDHA;Lo;0;L;;;;;N;;;;; +AA6A;MYANMAR LETTER KHAMTI DHA;Lo;0;L;;;;;N;;;;; +AA6B;MYANMAR LETTER KHAMTI NA;Lo;0;L;;;;;N;;;;; +AA6C;MYANMAR LETTER KHAMTI SA;Lo;0;L;;;;;N;;;;; +AA6D;MYANMAR LETTER KHAMTI HA;Lo;0;L;;;;;N;;;;; +AA6E;MYANMAR LETTER KHAMTI HHA;Lo;0;L;;;;;N;;;;; +AA6F;MYANMAR LETTER KHAMTI FA;Lo;0;L;;;;;N;;;;; +AA70;MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION;Lm;0;L;;;;;N;;;;; +AA71;MYANMAR LETTER KHAMTI XA;Lo;0;L;;;;;N;;;;; +AA72;MYANMAR LETTER KHAMTI ZA;Lo;0;L;;;;;N;;;;; +AA73;MYANMAR LETTER KHAMTI RA;Lo;0;L;;;;;N;;;;; +AA74;MYANMAR LOGOGRAM KHAMTI OAY;Lo;0;L;;;;;N;;;;; +AA75;MYANMAR LOGOGRAM KHAMTI QN;Lo;0;L;;;;;N;;;;; +AA76;MYANMAR LOGOGRAM KHAMTI HM;Lo;0;L;;;;;N;;;;; +AA77;MYANMAR SYMBOL AITON EXCLAMATION;So;0;L;;;;;N;;;;; +AA78;MYANMAR SYMBOL AITON ONE;So;0;L;;;;;N;;;;; +AA79;MYANMAR SYMBOL AITON TWO;So;0;L;;;;;N;;;;; +AA7A;MYANMAR LETTER AITON RA;Lo;0;L;;;;;N;;;;; +AA7B;MYANMAR SIGN PAO KAREN TONE;Mc;0;L;;;;;N;;;;; +AA80;TAI VIET LETTER LOW KO;Lo;0;L;;;;;N;;;;; +AA81;TAI VIET LETTER HIGH KO;Lo;0;L;;;;;N;;;;; +AA82;TAI VIET LETTER LOW KHO;Lo;0;L;;;;;N;;;;; +AA83;TAI VIET LETTER HIGH KHO;Lo;0;L;;;;;N;;;;; +AA84;TAI VIET LETTER LOW KHHO;Lo;0;L;;;;;N;;;;; +AA85;TAI VIET LETTER HIGH KHHO;Lo;0;L;;;;;N;;;;; +AA86;TAI VIET LETTER LOW GO;Lo;0;L;;;;;N;;;;; +AA87;TAI VIET LETTER HIGH GO;Lo;0;L;;;;;N;;;;; +AA88;TAI VIET LETTER LOW NGO;Lo;0;L;;;;;N;;;;; +AA89;TAI VIET LETTER HIGH NGO;Lo;0;L;;;;;N;;;;; +AA8A;TAI VIET LETTER LOW CO;Lo;0;L;;;;;N;;;;; +AA8B;TAI VIET LETTER HIGH CO;Lo;0;L;;;;;N;;;;; +AA8C;TAI VIET LETTER LOW CHO;Lo;0;L;;;;;N;;;;; +AA8D;TAI VIET LETTER HIGH CHO;Lo;0;L;;;;;N;;;;; +AA8E;TAI VIET LETTER LOW SO;Lo;0;L;;;;;N;;;;; +AA8F;TAI VIET LETTER HIGH SO;Lo;0;L;;;;;N;;;;; +AA90;TAI VIET LETTER LOW NYO;Lo;0;L;;;;;N;;;;; +AA91;TAI VIET LETTER HIGH NYO;Lo;0;L;;;;;N;;;;; +AA92;TAI VIET LETTER LOW DO;Lo;0;L;;;;;N;;;;; +AA93;TAI VIET LETTER HIGH DO;Lo;0;L;;;;;N;;;;; +AA94;TAI VIET LETTER LOW TO;Lo;0;L;;;;;N;;;;; +AA95;TAI VIET LETTER HIGH TO;Lo;0;L;;;;;N;;;;; +AA96;TAI VIET LETTER LOW THO;Lo;0;L;;;;;N;;;;; +AA97;TAI VIET LETTER HIGH THO;Lo;0;L;;;;;N;;;;; +AA98;TAI VIET LETTER LOW NO;Lo;0;L;;;;;N;;;;; +AA99;TAI VIET LETTER HIGH NO;Lo;0;L;;;;;N;;;;; +AA9A;TAI VIET LETTER LOW BO;Lo;0;L;;;;;N;;;;; +AA9B;TAI VIET LETTER HIGH BO;Lo;0;L;;;;;N;;;;; +AA9C;TAI VIET LETTER LOW PO;Lo;0;L;;;;;N;;;;; +AA9D;TAI VIET LETTER HIGH PO;Lo;0;L;;;;;N;;;;; +AA9E;TAI VIET LETTER LOW PHO;Lo;0;L;;;;;N;;;;; +AA9F;TAI VIET LETTER HIGH PHO;Lo;0;L;;;;;N;;;;; +AAA0;TAI VIET LETTER LOW FO;Lo;0;L;;;;;N;;;;; +AAA1;TAI VIET LETTER HIGH FO;Lo;0;L;;;;;N;;;;; +AAA2;TAI VIET LETTER LOW MO;Lo;0;L;;;;;N;;;;; +AAA3;TAI VIET LETTER HIGH MO;Lo;0;L;;;;;N;;;;; +AAA4;TAI VIET LETTER LOW YO;Lo;0;L;;;;;N;;;;; +AAA5;TAI VIET LETTER HIGH YO;Lo;0;L;;;;;N;;;;; +AAA6;TAI VIET LETTER LOW RO;Lo;0;L;;;;;N;;;;; +AAA7;TAI VIET LETTER HIGH RO;Lo;0;L;;;;;N;;;;; +AAA8;TAI VIET LETTER LOW LO;Lo;0;L;;;;;N;;;;; +AAA9;TAI VIET LETTER HIGH LO;Lo;0;L;;;;;N;;;;; +AAAA;TAI VIET LETTER LOW VO;Lo;0;L;;;;;N;;;;; +AAAB;TAI VIET LETTER HIGH VO;Lo;0;L;;;;;N;;;;; +AAAC;TAI VIET LETTER LOW HO;Lo;0;L;;;;;N;;;;; +AAAD;TAI VIET LETTER HIGH HO;Lo;0;L;;;;;N;;;;; +AAAE;TAI VIET LETTER LOW O;Lo;0;L;;;;;N;;;;; +AAAF;TAI VIET LETTER HIGH O;Lo;0;L;;;;;N;;;;; +AAB0;TAI VIET MAI KANG;Mn;230;NSM;;;;;N;;;;; +AAB1;TAI VIET VOWEL AA;Lo;0;L;;;;;N;;;;; +AAB2;TAI VIET VOWEL I;Mn;230;NSM;;;;;N;;;;; +AAB3;TAI VIET VOWEL UE;Mn;230;NSM;;;;;N;;;;; +AAB4;TAI VIET VOWEL U;Mn;220;NSM;;;;;N;;;;; +AAB5;TAI VIET VOWEL E;Lo;0;L;;;;;N;;;;; +AAB6;TAI VIET VOWEL O;Lo;0;L;;;;;N;;;;; +AAB7;TAI VIET MAI KHIT;Mn;230;NSM;;;;;N;;;;; +AAB8;TAI VIET VOWEL IA;Mn;230;NSM;;;;;N;;;;; +AAB9;TAI VIET VOWEL UEA;Lo;0;L;;;;;N;;;;; +AABA;TAI VIET VOWEL UA;Lo;0;L;;;;;N;;;;; +AABB;TAI VIET VOWEL AUE;Lo;0;L;;;;;N;;;;; +AABC;TAI VIET VOWEL AY;Lo;0;L;;;;;N;;;;; +AABD;TAI VIET VOWEL AN;Lo;0;L;;;;;N;;;;; +AABE;TAI VIET VOWEL AM;Mn;230;NSM;;;;;N;;;;; +AABF;TAI VIET TONE MAI EK;Mn;230;NSM;;;;;N;;;;; +AAC0;TAI VIET TONE MAI NUENG;Lo;0;L;;;;;N;;;;; +AAC1;TAI VIET TONE MAI THO;Mn;230;NSM;;;;;N;;;;; +AAC2;TAI VIET TONE MAI SONG;Lo;0;L;;;;;N;;;;; +AADB;TAI VIET SYMBOL KON;Lo;0;L;;;;;N;;;;; +AADC;TAI VIET SYMBOL NUENG;Lo;0;L;;;;;N;;;;; +AADD;TAI VIET SYMBOL SAM;Lm;0;L;;;;;N;;;;; +AADE;TAI VIET SYMBOL HO HOI;Po;0;L;;;;;N;;;;; +AADF;TAI VIET SYMBOL KOI KOI;Po;0;L;;;;;N;;;;; +AAE0;MEETEI MAYEK LETTER E;Lo;0;L;;;;;N;;;;; +AAE1;MEETEI MAYEK LETTER O;Lo;0;L;;;;;N;;;;; +AAE2;MEETEI MAYEK LETTER CHA;Lo;0;L;;;;;N;;;;; +AAE3;MEETEI MAYEK LETTER NYA;Lo;0;L;;;;;N;;;;; +AAE4;MEETEI MAYEK LETTER TTA;Lo;0;L;;;;;N;;;;; +AAE5;MEETEI MAYEK LETTER TTHA;Lo;0;L;;;;;N;;;;; +AAE6;MEETEI MAYEK LETTER DDA;Lo;0;L;;;;;N;;;;; +AAE7;MEETEI MAYEK LETTER DDHA;Lo;0;L;;;;;N;;;;; +AAE8;MEETEI MAYEK LETTER NNA;Lo;0;L;;;;;N;;;;; +AAE9;MEETEI MAYEK LETTER SHA;Lo;0;L;;;;;N;;;;; +AAEA;MEETEI MAYEK LETTER SSA;Lo;0;L;;;;;N;;;;; +AAEB;MEETEI MAYEK VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +AAEC;MEETEI MAYEK VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +AAED;MEETEI MAYEK VOWEL SIGN AAI;Mn;0;NSM;;;;;N;;;;; +AAEE;MEETEI MAYEK VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +AAEF;MEETEI MAYEK VOWEL SIGN AAU;Mc;0;L;;;;;N;;;;; +AAF0;MEETEI MAYEK CHEIKHAN;Po;0;L;;;;;N;;;;; +AAF1;MEETEI MAYEK AHANG KHUDAM;Po;0;L;;;;;N;;;;; +AAF2;MEETEI MAYEK ANJI;Lo;0;L;;;;;N;;;;; +AAF3;MEETEI MAYEK SYLLABLE REPETITION MARK;Lm;0;L;;;;;N;;;;; +AAF4;MEETEI MAYEK WORD REPETITION MARK;Lm;0;L;;;;;N;;;;; +AAF5;MEETEI MAYEK VOWEL SIGN VISARGA;Mc;0;L;;;;;N;;;;; +AAF6;MEETEI MAYEK VIRAMA;Mn;9;NSM;;;;;N;;;;; +AB01;ETHIOPIC SYLLABLE TTHU;Lo;0;L;;;;;N;;;;; +AB02;ETHIOPIC SYLLABLE TTHI;Lo;0;L;;;;;N;;;;; +AB03;ETHIOPIC SYLLABLE TTHAA;Lo;0;L;;;;;N;;;;; +AB04;ETHIOPIC SYLLABLE TTHEE;Lo;0;L;;;;;N;;;;; +AB05;ETHIOPIC SYLLABLE TTHE;Lo;0;L;;;;;N;;;;; +AB06;ETHIOPIC SYLLABLE TTHO;Lo;0;L;;;;;N;;;;; +AB09;ETHIOPIC SYLLABLE DDHU;Lo;0;L;;;;;N;;;;; +AB0A;ETHIOPIC SYLLABLE DDHI;Lo;0;L;;;;;N;;;;; +AB0B;ETHIOPIC SYLLABLE DDHAA;Lo;0;L;;;;;N;;;;; +AB0C;ETHIOPIC SYLLABLE DDHEE;Lo;0;L;;;;;N;;;;; +AB0D;ETHIOPIC SYLLABLE DDHE;Lo;0;L;;;;;N;;;;; +AB0E;ETHIOPIC SYLLABLE DDHO;Lo;0;L;;;;;N;;;;; +AB11;ETHIOPIC SYLLABLE DZU;Lo;0;L;;;;;N;;;;; +AB12;ETHIOPIC SYLLABLE DZI;Lo;0;L;;;;;N;;;;; +AB13;ETHIOPIC SYLLABLE DZAA;Lo;0;L;;;;;N;;;;; +AB14;ETHIOPIC SYLLABLE DZEE;Lo;0;L;;;;;N;;;;; +AB15;ETHIOPIC SYLLABLE DZE;Lo;0;L;;;;;N;;;;; +AB16;ETHIOPIC SYLLABLE DZO;Lo;0;L;;;;;N;;;;; +AB20;ETHIOPIC SYLLABLE CCHHA;Lo;0;L;;;;;N;;;;; +AB21;ETHIOPIC SYLLABLE CCHHU;Lo;0;L;;;;;N;;;;; +AB22;ETHIOPIC SYLLABLE CCHHI;Lo;0;L;;;;;N;;;;; +AB23;ETHIOPIC SYLLABLE CCHHAA;Lo;0;L;;;;;N;;;;; +AB24;ETHIOPIC SYLLABLE CCHHEE;Lo;0;L;;;;;N;;;;; +AB25;ETHIOPIC SYLLABLE CCHHE;Lo;0;L;;;;;N;;;;; +AB26;ETHIOPIC SYLLABLE CCHHO;Lo;0;L;;;;;N;;;;; +AB28;ETHIOPIC SYLLABLE BBA;Lo;0;L;;;;;N;;;;; +AB29;ETHIOPIC SYLLABLE BBU;Lo;0;L;;;;;N;;;;; +AB2A;ETHIOPIC SYLLABLE BBI;Lo;0;L;;;;;N;;;;; +AB2B;ETHIOPIC SYLLABLE BBAA;Lo;0;L;;;;;N;;;;; +AB2C;ETHIOPIC SYLLABLE BBEE;Lo;0;L;;;;;N;;;;; +AB2D;ETHIOPIC SYLLABLE BBE;Lo;0;L;;;;;N;;;;; +AB2E;ETHIOPIC SYLLABLE BBO;Lo;0;L;;;;;N;;;;; +ABC0;MEETEI MAYEK LETTER KOK;Lo;0;L;;;;;N;;;;; +ABC1;MEETEI MAYEK LETTER SAM;Lo;0;L;;;;;N;;;;; +ABC2;MEETEI MAYEK LETTER LAI;Lo;0;L;;;;;N;;;;; +ABC3;MEETEI MAYEK LETTER MIT;Lo;0;L;;;;;N;;;;; +ABC4;MEETEI MAYEK LETTER PA;Lo;0;L;;;;;N;;;;; +ABC5;MEETEI MAYEK LETTER NA;Lo;0;L;;;;;N;;;;; +ABC6;MEETEI MAYEK LETTER CHIL;Lo;0;L;;;;;N;;;;; +ABC7;MEETEI MAYEK LETTER TIL;Lo;0;L;;;;;N;;;;; +ABC8;MEETEI MAYEK LETTER KHOU;Lo;0;L;;;;;N;;;;; +ABC9;MEETEI MAYEK LETTER NGOU;Lo;0;L;;;;;N;;;;; +ABCA;MEETEI MAYEK LETTER THOU;Lo;0;L;;;;;N;;;;; +ABCB;MEETEI MAYEK LETTER WAI;Lo;0;L;;;;;N;;;;; +ABCC;MEETEI MAYEK LETTER YANG;Lo;0;L;;;;;N;;;;; +ABCD;MEETEI MAYEK LETTER HUK;Lo;0;L;;;;;N;;;;; +ABCE;MEETEI MAYEK LETTER UN;Lo;0;L;;;;;N;;;;; +ABCF;MEETEI MAYEK LETTER I;Lo;0;L;;;;;N;;;;; +ABD0;MEETEI MAYEK LETTER PHAM;Lo;0;L;;;;;N;;;;; +ABD1;MEETEI MAYEK LETTER ATIYA;Lo;0;L;;;;;N;;;;; +ABD2;MEETEI MAYEK LETTER GOK;Lo;0;L;;;;;N;;;;; +ABD3;MEETEI MAYEK LETTER JHAM;Lo;0;L;;;;;N;;;;; +ABD4;MEETEI MAYEK LETTER RAI;Lo;0;L;;;;;N;;;;; +ABD5;MEETEI MAYEK LETTER BA;Lo;0;L;;;;;N;;;;; +ABD6;MEETEI MAYEK LETTER JIL;Lo;0;L;;;;;N;;;;; +ABD7;MEETEI MAYEK LETTER DIL;Lo;0;L;;;;;N;;;;; +ABD8;MEETEI MAYEK LETTER GHOU;Lo;0;L;;;;;N;;;;; +ABD9;MEETEI MAYEK LETTER DHOU;Lo;0;L;;;;;N;;;;; +ABDA;MEETEI MAYEK LETTER BHAM;Lo;0;L;;;;;N;;;;; +ABDB;MEETEI MAYEK LETTER KOK LONSUM;Lo;0;L;;;;;N;;;;; +ABDC;MEETEI MAYEK LETTER LAI LONSUM;Lo;0;L;;;;;N;;;;; +ABDD;MEETEI MAYEK LETTER MIT LONSUM;Lo;0;L;;;;;N;;;;; +ABDE;MEETEI MAYEK LETTER PA LONSUM;Lo;0;L;;;;;N;;;;; +ABDF;MEETEI MAYEK LETTER NA LONSUM;Lo;0;L;;;;;N;;;;; +ABE0;MEETEI MAYEK LETTER TIL LONSUM;Lo;0;L;;;;;N;;;;; +ABE1;MEETEI MAYEK LETTER NGOU LONSUM;Lo;0;L;;;;;N;;;;; +ABE2;MEETEI MAYEK LETTER I LONSUM;Lo;0;L;;;;;N;;;;; +ABE3;MEETEI MAYEK VOWEL SIGN ONAP;Mc;0;L;;;;;N;;;;; +ABE4;MEETEI MAYEK VOWEL SIGN INAP;Mc;0;L;;;;;N;;;;; +ABE5;MEETEI MAYEK VOWEL SIGN ANAP;Mn;0;NSM;;;;;N;;;;; +ABE6;MEETEI MAYEK VOWEL SIGN YENAP;Mc;0;L;;;;;N;;;;; +ABE7;MEETEI MAYEK VOWEL SIGN SOUNAP;Mc;0;L;;;;;N;;;;; +ABE8;MEETEI MAYEK VOWEL SIGN UNAP;Mn;0;NSM;;;;;N;;;;; +ABE9;MEETEI MAYEK VOWEL SIGN CHEINAP;Mc;0;L;;;;;N;;;;; +ABEA;MEETEI MAYEK VOWEL SIGN NUNG;Mc;0;L;;;;;N;;;;; +ABEB;MEETEI MAYEK CHEIKHEI;Po;0;L;;;;;N;;;;; +ABEC;MEETEI MAYEK LUM IYEK;Mc;0;L;;;;;N;;;;; +ABED;MEETEI MAYEK APUN IYEK;Mn;9;NSM;;;;;N;;;;; +ABF0;MEETEI MAYEK DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +ABF1;MEETEI MAYEK DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +ABF2;MEETEI MAYEK DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +ABF3;MEETEI MAYEK DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +ABF4;MEETEI MAYEK DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +ABF5;MEETEI MAYEK DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +ABF6;MEETEI MAYEK DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +ABF7;MEETEI MAYEK DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +ABF8;MEETEI MAYEK DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +ABF9;MEETEI MAYEK DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +AC00;;Lo;0;L;;;;;N;;;;; +D7A3;;Lo;0;L;;;;;N;;;;; +D7B0;HANGUL JUNGSEONG O-YEO;Lo;0;L;;;;;N;;;;; +D7B1;HANGUL JUNGSEONG O-O-I;Lo;0;L;;;;;N;;;;; +D7B2;HANGUL JUNGSEONG YO-A;Lo;0;L;;;;;N;;;;; +D7B3;HANGUL JUNGSEONG YO-AE;Lo;0;L;;;;;N;;;;; +D7B4;HANGUL JUNGSEONG YO-EO;Lo;0;L;;;;;N;;;;; +D7B5;HANGUL JUNGSEONG U-YEO;Lo;0;L;;;;;N;;;;; +D7B6;HANGUL JUNGSEONG U-I-I;Lo;0;L;;;;;N;;;;; +D7B7;HANGUL JUNGSEONG YU-AE;Lo;0;L;;;;;N;;;;; +D7B8;HANGUL JUNGSEONG YU-O;Lo;0;L;;;;;N;;;;; +D7B9;HANGUL JUNGSEONG EU-A;Lo;0;L;;;;;N;;;;; +D7BA;HANGUL JUNGSEONG EU-EO;Lo;0;L;;;;;N;;;;; +D7BB;HANGUL JUNGSEONG EU-E;Lo;0;L;;;;;N;;;;; +D7BC;HANGUL JUNGSEONG EU-O;Lo;0;L;;;;;N;;;;; +D7BD;HANGUL JUNGSEONG I-YA-O;Lo;0;L;;;;;N;;;;; +D7BE;HANGUL JUNGSEONG I-YAE;Lo;0;L;;;;;N;;;;; +D7BF;HANGUL JUNGSEONG I-YEO;Lo;0;L;;;;;N;;;;; +D7C0;HANGUL JUNGSEONG I-YE;Lo;0;L;;;;;N;;;;; +D7C1;HANGUL JUNGSEONG I-O-I;Lo;0;L;;;;;N;;;;; +D7C2;HANGUL JUNGSEONG I-YO;Lo;0;L;;;;;N;;;;; +D7C3;HANGUL JUNGSEONG I-YU;Lo;0;L;;;;;N;;;;; +D7C4;HANGUL JUNGSEONG I-I;Lo;0;L;;;;;N;;;;; +D7C5;HANGUL JUNGSEONG ARAEA-A;Lo;0;L;;;;;N;;;;; +D7C6;HANGUL JUNGSEONG ARAEA-E;Lo;0;L;;;;;N;;;;; +D7CB;HANGUL JONGSEONG NIEUN-RIEUL;Lo;0;L;;;;;N;;;;; +D7CC;HANGUL JONGSEONG NIEUN-CHIEUCH;Lo;0;L;;;;;N;;;;; +D7CD;HANGUL JONGSEONG SSANGTIKEUT;Lo;0;L;;;;;N;;;;; +D7CE;HANGUL JONGSEONG SSANGTIKEUT-PIEUP;Lo;0;L;;;;;N;;;;; +D7CF;HANGUL JONGSEONG TIKEUT-PIEUP;Lo;0;L;;;;;N;;;;; +D7D0;HANGUL JONGSEONG TIKEUT-SIOS;Lo;0;L;;;;;N;;;;; +D7D1;HANGUL JONGSEONG TIKEUT-SIOS-KIYEOK;Lo;0;L;;;;;N;;;;; +D7D2;HANGUL JONGSEONG TIKEUT-CIEUC;Lo;0;L;;;;;N;;;;; +D7D3;HANGUL JONGSEONG TIKEUT-CHIEUCH;Lo;0;L;;;;;N;;;;; +D7D4;HANGUL JONGSEONG TIKEUT-THIEUTH;Lo;0;L;;;;;N;;;;; +D7D5;HANGUL JONGSEONG RIEUL-SSANGKIYEOK;Lo;0;L;;;;;N;;;;; +D7D6;HANGUL JONGSEONG RIEUL-KIYEOK-HIEUH;Lo;0;L;;;;;N;;;;; +D7D7;HANGUL JONGSEONG SSANGRIEUL-KHIEUKH;Lo;0;L;;;;;N;;;;; +D7D8;HANGUL JONGSEONG RIEUL-MIEUM-HIEUH;Lo;0;L;;;;;N;;;;; +D7D9;HANGUL JONGSEONG RIEUL-PIEUP-TIKEUT;Lo;0;L;;;;;N;;;;; +D7DA;HANGUL JONGSEONG RIEUL-PIEUP-PHIEUPH;Lo;0;L;;;;;N;;;;; +D7DB;HANGUL JONGSEONG RIEUL-YESIEUNG;Lo;0;L;;;;;N;;;;; +D7DC;HANGUL JONGSEONG RIEUL-YEORINHIEUH-HIEUH;Lo;0;L;;;;;N;;;;; +D7DD;HANGUL JONGSEONG KAPYEOUNRIEUL;Lo;0;L;;;;;N;;;;; +D7DE;HANGUL JONGSEONG MIEUM-NIEUN;Lo;0;L;;;;;N;;;;; +D7DF;HANGUL JONGSEONG MIEUM-SSANGNIEUN;Lo;0;L;;;;;N;;;;; +D7E0;HANGUL JONGSEONG SSANGMIEUM;Lo;0;L;;;;;N;;;;; +D7E1;HANGUL JONGSEONG MIEUM-PIEUP-SIOS;Lo;0;L;;;;;N;;;;; +D7E2;HANGUL JONGSEONG MIEUM-CIEUC;Lo;0;L;;;;;N;;;;; +D7E3;HANGUL JONGSEONG PIEUP-TIKEUT;Lo;0;L;;;;;N;;;;; +D7E4;HANGUL JONGSEONG PIEUP-RIEUL-PHIEUPH;Lo;0;L;;;;;N;;;;; +D7E5;HANGUL JONGSEONG PIEUP-MIEUM;Lo;0;L;;;;;N;;;;; +D7E6;HANGUL JONGSEONG SSANGPIEUP;Lo;0;L;;;;;N;;;;; +D7E7;HANGUL JONGSEONG PIEUP-SIOS-TIKEUT;Lo;0;L;;;;;N;;;;; +D7E8;HANGUL JONGSEONG PIEUP-CIEUC;Lo;0;L;;;;;N;;;;; +D7E9;HANGUL JONGSEONG PIEUP-CHIEUCH;Lo;0;L;;;;;N;;;;; +D7EA;HANGUL JONGSEONG SIOS-MIEUM;Lo;0;L;;;;;N;;;;; +D7EB;HANGUL JONGSEONG SIOS-KAPYEOUNPIEUP;Lo;0;L;;;;;N;;;;; +D7EC;HANGUL JONGSEONG SSANGSIOS-KIYEOK;Lo;0;L;;;;;N;;;;; +D7ED;HANGUL JONGSEONG SSANGSIOS-TIKEUT;Lo;0;L;;;;;N;;;;; +D7EE;HANGUL JONGSEONG SIOS-PANSIOS;Lo;0;L;;;;;N;;;;; +D7EF;HANGUL JONGSEONG SIOS-CIEUC;Lo;0;L;;;;;N;;;;; +D7F0;HANGUL JONGSEONG SIOS-CHIEUCH;Lo;0;L;;;;;N;;;;; +D7F1;HANGUL JONGSEONG SIOS-THIEUTH;Lo;0;L;;;;;N;;;;; +D7F2;HANGUL JONGSEONG SIOS-HIEUH;Lo;0;L;;;;;N;;;;; +D7F3;HANGUL JONGSEONG PANSIOS-PIEUP;Lo;0;L;;;;;N;;;;; +D7F4;HANGUL JONGSEONG PANSIOS-KAPYEOUNPIEUP;Lo;0;L;;;;;N;;;;; +D7F5;HANGUL JONGSEONG YESIEUNG-MIEUM;Lo;0;L;;;;;N;;;;; +D7F6;HANGUL JONGSEONG YESIEUNG-HIEUH;Lo;0;L;;;;;N;;;;; +D7F7;HANGUL JONGSEONG CIEUC-PIEUP;Lo;0;L;;;;;N;;;;; +D7F8;HANGUL JONGSEONG CIEUC-SSANGPIEUP;Lo;0;L;;;;;N;;;;; +D7F9;HANGUL JONGSEONG SSANGCIEUC;Lo;0;L;;;;;N;;;;; +D7FA;HANGUL JONGSEONG PHIEUPH-SIOS;Lo;0;L;;;;;N;;;;; +D7FB;HANGUL JONGSEONG PHIEUPH-THIEUTH;Lo;0;L;;;;;N;;;;; +D800;;Cs;0;L;;;;;N;;;;; +DB7F;;Cs;0;L;;;;;N;;;;; +DB80;;Cs;0;L;;;;;N;;;;; +DBFF;;Cs;0;L;;;;;N;;;;; +DC00;;Cs;0;L;;;;;N;;;;; +DFFF;;Cs;0;L;;;;;N;;;;; +E000;;Co;0;L;;;;;N;;;;; +F8FF;;Co;0;L;;;;;N;;;;; +F900;CJK COMPATIBILITY IDEOGRAPH-F900;Lo;0;L;8C48;;;;N;;;;; +F901;CJK COMPATIBILITY IDEOGRAPH-F901;Lo;0;L;66F4;;;;N;;;;; +F902;CJK COMPATIBILITY IDEOGRAPH-F902;Lo;0;L;8ECA;;;;N;;;;; +F903;CJK COMPATIBILITY IDEOGRAPH-F903;Lo;0;L;8CC8;;;;N;;;;; +F904;CJK COMPATIBILITY IDEOGRAPH-F904;Lo;0;L;6ED1;;;;N;;;;; +F905;CJK COMPATIBILITY IDEOGRAPH-F905;Lo;0;L;4E32;;;;N;;;;; +F906;CJK COMPATIBILITY IDEOGRAPH-F906;Lo;0;L;53E5;;;;N;;;;; +F907;CJK COMPATIBILITY IDEOGRAPH-F907;Lo;0;L;9F9C;;;;N;;;;; +F908;CJK COMPATIBILITY IDEOGRAPH-F908;Lo;0;L;9F9C;;;;N;;;;; +F909;CJK COMPATIBILITY IDEOGRAPH-F909;Lo;0;L;5951;;;;N;;;;; +F90A;CJK COMPATIBILITY IDEOGRAPH-F90A;Lo;0;L;91D1;;;;N;;;;; +F90B;CJK COMPATIBILITY IDEOGRAPH-F90B;Lo;0;L;5587;;;;N;;;;; +F90C;CJK COMPATIBILITY IDEOGRAPH-F90C;Lo;0;L;5948;;;;N;;;;; +F90D;CJK COMPATIBILITY IDEOGRAPH-F90D;Lo;0;L;61F6;;;;N;;;;; +F90E;CJK COMPATIBILITY IDEOGRAPH-F90E;Lo;0;L;7669;;;;N;;;;; +F90F;CJK COMPATIBILITY IDEOGRAPH-F90F;Lo;0;L;7F85;;;;N;;;;; +F910;CJK COMPATIBILITY IDEOGRAPH-F910;Lo;0;L;863F;;;;N;;;;; +F911;CJK COMPATIBILITY IDEOGRAPH-F911;Lo;0;L;87BA;;;;N;;;;; +F912;CJK COMPATIBILITY IDEOGRAPH-F912;Lo;0;L;88F8;;;;N;;;;; +F913;CJK COMPATIBILITY IDEOGRAPH-F913;Lo;0;L;908F;;;;N;;;;; +F914;CJK COMPATIBILITY IDEOGRAPH-F914;Lo;0;L;6A02;;;;N;;;;; +F915;CJK COMPATIBILITY IDEOGRAPH-F915;Lo;0;L;6D1B;;;;N;;;;; +F916;CJK COMPATIBILITY IDEOGRAPH-F916;Lo;0;L;70D9;;;;N;;;;; +F917;CJK COMPATIBILITY IDEOGRAPH-F917;Lo;0;L;73DE;;;;N;;;;; +F918;CJK COMPATIBILITY IDEOGRAPH-F918;Lo;0;L;843D;;;;N;;;;; +F919;CJK COMPATIBILITY IDEOGRAPH-F919;Lo;0;L;916A;;;;N;;;;; +F91A;CJK COMPATIBILITY IDEOGRAPH-F91A;Lo;0;L;99F1;;;;N;;;;; +F91B;CJK COMPATIBILITY IDEOGRAPH-F91B;Lo;0;L;4E82;;;;N;;;;; +F91C;CJK COMPATIBILITY IDEOGRAPH-F91C;Lo;0;L;5375;;;;N;;;;; +F91D;CJK COMPATIBILITY IDEOGRAPH-F91D;Lo;0;L;6B04;;;;N;;;;; +F91E;CJK COMPATIBILITY IDEOGRAPH-F91E;Lo;0;L;721B;;;;N;;;;; +F91F;CJK COMPATIBILITY IDEOGRAPH-F91F;Lo;0;L;862D;;;;N;;;;; +F920;CJK COMPATIBILITY IDEOGRAPH-F920;Lo;0;L;9E1E;;;;N;;;;; +F921;CJK COMPATIBILITY IDEOGRAPH-F921;Lo;0;L;5D50;;;;N;;;;; +F922;CJK COMPATIBILITY IDEOGRAPH-F922;Lo;0;L;6FEB;;;;N;;;;; +F923;CJK COMPATIBILITY IDEOGRAPH-F923;Lo;0;L;85CD;;;;N;;;;; +F924;CJK COMPATIBILITY IDEOGRAPH-F924;Lo;0;L;8964;;;;N;;;;; +F925;CJK COMPATIBILITY IDEOGRAPH-F925;Lo;0;L;62C9;;;;N;;;;; +F926;CJK COMPATIBILITY IDEOGRAPH-F926;Lo;0;L;81D8;;;;N;;;;; +F927;CJK COMPATIBILITY IDEOGRAPH-F927;Lo;0;L;881F;;;;N;;;;; +F928;CJK COMPATIBILITY IDEOGRAPH-F928;Lo;0;L;5ECA;;;;N;;;;; +F929;CJK COMPATIBILITY IDEOGRAPH-F929;Lo;0;L;6717;;;;N;;;;; +F92A;CJK COMPATIBILITY IDEOGRAPH-F92A;Lo;0;L;6D6A;;;;N;;;;; +F92B;CJK COMPATIBILITY IDEOGRAPH-F92B;Lo;0;L;72FC;;;;N;;;;; +F92C;CJK COMPATIBILITY IDEOGRAPH-F92C;Lo;0;L;90CE;;;;N;;;;; +F92D;CJK COMPATIBILITY IDEOGRAPH-F92D;Lo;0;L;4F86;;;;N;;;;; +F92E;CJK COMPATIBILITY IDEOGRAPH-F92E;Lo;0;L;51B7;;;;N;;;;; +F92F;CJK COMPATIBILITY IDEOGRAPH-F92F;Lo;0;L;52DE;;;;N;;;;; +F930;CJK COMPATIBILITY IDEOGRAPH-F930;Lo;0;L;64C4;;;;N;;;;; +F931;CJK COMPATIBILITY IDEOGRAPH-F931;Lo;0;L;6AD3;;;;N;;;;; +F932;CJK COMPATIBILITY IDEOGRAPH-F932;Lo;0;L;7210;;;;N;;;;; +F933;CJK COMPATIBILITY IDEOGRAPH-F933;Lo;0;L;76E7;;;;N;;;;; +F934;CJK COMPATIBILITY IDEOGRAPH-F934;Lo;0;L;8001;;;;N;;;;; +F935;CJK COMPATIBILITY IDEOGRAPH-F935;Lo;0;L;8606;;;;N;;;;; +F936;CJK COMPATIBILITY IDEOGRAPH-F936;Lo;0;L;865C;;;;N;;;;; +F937;CJK COMPATIBILITY IDEOGRAPH-F937;Lo;0;L;8DEF;;;;N;;;;; +F938;CJK COMPATIBILITY IDEOGRAPH-F938;Lo;0;L;9732;;;;N;;;;; +F939;CJK COMPATIBILITY IDEOGRAPH-F939;Lo;0;L;9B6F;;;;N;;;;; +F93A;CJK COMPATIBILITY IDEOGRAPH-F93A;Lo;0;L;9DFA;;;;N;;;;; +F93B;CJK COMPATIBILITY IDEOGRAPH-F93B;Lo;0;L;788C;;;;N;;;;; +F93C;CJK COMPATIBILITY IDEOGRAPH-F93C;Lo;0;L;797F;;;;N;;;;; +F93D;CJK COMPATIBILITY IDEOGRAPH-F93D;Lo;0;L;7DA0;;;;N;;;;; +F93E;CJK COMPATIBILITY IDEOGRAPH-F93E;Lo;0;L;83C9;;;;N;;;;; +F93F;CJK COMPATIBILITY IDEOGRAPH-F93F;Lo;0;L;9304;;;;N;;;;; +F940;CJK COMPATIBILITY IDEOGRAPH-F940;Lo;0;L;9E7F;;;;N;;;;; +F941;CJK COMPATIBILITY IDEOGRAPH-F941;Lo;0;L;8AD6;;;;N;;;;; +F942;CJK COMPATIBILITY IDEOGRAPH-F942;Lo;0;L;58DF;;;;N;;;;; +F943;CJK COMPATIBILITY IDEOGRAPH-F943;Lo;0;L;5F04;;;;N;;;;; +F944;CJK COMPATIBILITY IDEOGRAPH-F944;Lo;0;L;7C60;;;;N;;;;; +F945;CJK COMPATIBILITY IDEOGRAPH-F945;Lo;0;L;807E;;;;N;;;;; +F946;CJK COMPATIBILITY IDEOGRAPH-F946;Lo;0;L;7262;;;;N;;;;; +F947;CJK COMPATIBILITY IDEOGRAPH-F947;Lo;0;L;78CA;;;;N;;;;; +F948;CJK COMPATIBILITY IDEOGRAPH-F948;Lo;0;L;8CC2;;;;N;;;;; +F949;CJK COMPATIBILITY IDEOGRAPH-F949;Lo;0;L;96F7;;;;N;;;;; +F94A;CJK COMPATIBILITY IDEOGRAPH-F94A;Lo;0;L;58D8;;;;N;;;;; +F94B;CJK COMPATIBILITY IDEOGRAPH-F94B;Lo;0;L;5C62;;;;N;;;;; +F94C;CJK COMPATIBILITY IDEOGRAPH-F94C;Lo;0;L;6A13;;;;N;;;;; +F94D;CJK COMPATIBILITY IDEOGRAPH-F94D;Lo;0;L;6DDA;;;;N;;;;; +F94E;CJK COMPATIBILITY IDEOGRAPH-F94E;Lo;0;L;6F0F;;;;N;;;;; +F94F;CJK COMPATIBILITY IDEOGRAPH-F94F;Lo;0;L;7D2F;;;;N;;;;; +F950;CJK COMPATIBILITY IDEOGRAPH-F950;Lo;0;L;7E37;;;;N;;;;; +F951;CJK COMPATIBILITY IDEOGRAPH-F951;Lo;0;L;964B;;;;N;;;;; +F952;CJK COMPATIBILITY IDEOGRAPH-F952;Lo;0;L;52D2;;;;N;;;;; +F953;CJK COMPATIBILITY IDEOGRAPH-F953;Lo;0;L;808B;;;;N;;;;; +F954;CJK COMPATIBILITY IDEOGRAPH-F954;Lo;0;L;51DC;;;;N;;;;; +F955;CJK COMPATIBILITY IDEOGRAPH-F955;Lo;0;L;51CC;;;;N;;;;; +F956;CJK COMPATIBILITY IDEOGRAPH-F956;Lo;0;L;7A1C;;;;N;;;;; +F957;CJK COMPATIBILITY IDEOGRAPH-F957;Lo;0;L;7DBE;;;;N;;;;; +F958;CJK COMPATIBILITY IDEOGRAPH-F958;Lo;0;L;83F1;;;;N;;;;; +F959;CJK COMPATIBILITY IDEOGRAPH-F959;Lo;0;L;9675;;;;N;;;;; +F95A;CJK COMPATIBILITY IDEOGRAPH-F95A;Lo;0;L;8B80;;;;N;;;;; +F95B;CJK COMPATIBILITY IDEOGRAPH-F95B;Lo;0;L;62CF;;;;N;;;;; +F95C;CJK COMPATIBILITY IDEOGRAPH-F95C;Lo;0;L;6A02;;;;N;;;;; +F95D;CJK COMPATIBILITY IDEOGRAPH-F95D;Lo;0;L;8AFE;;;;N;;;;; +F95E;CJK COMPATIBILITY IDEOGRAPH-F95E;Lo;0;L;4E39;;;;N;;;;; +F95F;CJK COMPATIBILITY IDEOGRAPH-F95F;Lo;0;L;5BE7;;;;N;;;;; +F960;CJK COMPATIBILITY IDEOGRAPH-F960;Lo;0;L;6012;;;;N;;;;; +F961;CJK COMPATIBILITY IDEOGRAPH-F961;Lo;0;L;7387;;;;N;;;;; +F962;CJK COMPATIBILITY IDEOGRAPH-F962;Lo;0;L;7570;;;;N;;;;; +F963;CJK COMPATIBILITY IDEOGRAPH-F963;Lo;0;L;5317;;;;N;;;;; +F964;CJK COMPATIBILITY IDEOGRAPH-F964;Lo;0;L;78FB;;;;N;;;;; +F965;CJK COMPATIBILITY IDEOGRAPH-F965;Lo;0;L;4FBF;;;;N;;;;; +F966;CJK COMPATIBILITY IDEOGRAPH-F966;Lo;0;L;5FA9;;;;N;;;;; +F967;CJK COMPATIBILITY IDEOGRAPH-F967;Lo;0;L;4E0D;;;;N;;;;; +F968;CJK COMPATIBILITY IDEOGRAPH-F968;Lo;0;L;6CCC;;;;N;;;;; +F969;CJK COMPATIBILITY IDEOGRAPH-F969;Lo;0;L;6578;;;;N;;;;; +F96A;CJK COMPATIBILITY IDEOGRAPH-F96A;Lo;0;L;7D22;;;;N;;;;; +F96B;CJK COMPATIBILITY IDEOGRAPH-F96B;Lo;0;L;53C3;;;3;N;;;;; +F96C;CJK COMPATIBILITY IDEOGRAPH-F96C;Lo;0;L;585E;;;;N;;;;; +F96D;CJK COMPATIBILITY IDEOGRAPH-F96D;Lo;0;L;7701;;;;N;;;;; +F96E;CJK COMPATIBILITY IDEOGRAPH-F96E;Lo;0;L;8449;;;;N;;;;; +F96F;CJK COMPATIBILITY IDEOGRAPH-F96F;Lo;0;L;8AAA;;;;N;;;;; +F970;CJK COMPATIBILITY IDEOGRAPH-F970;Lo;0;L;6BBA;;;;N;;;;; +F971;CJK COMPATIBILITY IDEOGRAPH-F971;Lo;0;L;8FB0;;;;N;;;;; +F972;CJK COMPATIBILITY IDEOGRAPH-F972;Lo;0;L;6C88;;;;N;;;;; +F973;CJK COMPATIBILITY IDEOGRAPH-F973;Lo;0;L;62FE;;;10;N;;;;; +F974;CJK COMPATIBILITY IDEOGRAPH-F974;Lo;0;L;82E5;;;;N;;;;; +F975;CJK COMPATIBILITY IDEOGRAPH-F975;Lo;0;L;63A0;;;;N;;;;; +F976;CJK COMPATIBILITY IDEOGRAPH-F976;Lo;0;L;7565;;;;N;;;;; +F977;CJK COMPATIBILITY IDEOGRAPH-F977;Lo;0;L;4EAE;;;;N;;;;; +F978;CJK COMPATIBILITY IDEOGRAPH-F978;Lo;0;L;5169;;;2;N;;;;; +F979;CJK COMPATIBILITY IDEOGRAPH-F979;Lo;0;L;51C9;;;;N;;;;; +F97A;CJK COMPATIBILITY IDEOGRAPH-F97A;Lo;0;L;6881;;;;N;;;;; +F97B;CJK COMPATIBILITY IDEOGRAPH-F97B;Lo;0;L;7CE7;;;;N;;;;; +F97C;CJK COMPATIBILITY IDEOGRAPH-F97C;Lo;0;L;826F;;;;N;;;;; +F97D;CJK COMPATIBILITY IDEOGRAPH-F97D;Lo;0;L;8AD2;;;;N;;;;; +F97E;CJK COMPATIBILITY IDEOGRAPH-F97E;Lo;0;L;91CF;;;;N;;;;; +F97F;CJK COMPATIBILITY IDEOGRAPH-F97F;Lo;0;L;52F5;;;;N;;;;; +F980;CJK COMPATIBILITY IDEOGRAPH-F980;Lo;0;L;5442;;;;N;;;;; +F981;CJK COMPATIBILITY IDEOGRAPH-F981;Lo;0;L;5973;;;;N;;;;; +F982;CJK COMPATIBILITY IDEOGRAPH-F982;Lo;0;L;5EEC;;;;N;;;;; +F983;CJK COMPATIBILITY IDEOGRAPH-F983;Lo;0;L;65C5;;;;N;;;;; +F984;CJK COMPATIBILITY IDEOGRAPH-F984;Lo;0;L;6FFE;;;;N;;;;; +F985;CJK COMPATIBILITY IDEOGRAPH-F985;Lo;0;L;792A;;;;N;;;;; +F986;CJK COMPATIBILITY IDEOGRAPH-F986;Lo;0;L;95AD;;;;N;;;;; +F987;CJK COMPATIBILITY IDEOGRAPH-F987;Lo;0;L;9A6A;;;;N;;;;; +F988;CJK COMPATIBILITY IDEOGRAPH-F988;Lo;0;L;9E97;;;;N;;;;; +F989;CJK COMPATIBILITY IDEOGRAPH-F989;Lo;0;L;9ECE;;;;N;;;;; +F98A;CJK COMPATIBILITY IDEOGRAPH-F98A;Lo;0;L;529B;;;;N;;;;; +F98B;CJK COMPATIBILITY IDEOGRAPH-F98B;Lo;0;L;66C6;;;;N;;;;; +F98C;CJK COMPATIBILITY IDEOGRAPH-F98C;Lo;0;L;6B77;;;;N;;;;; +F98D;CJK COMPATIBILITY IDEOGRAPH-F98D;Lo;0;L;8F62;;;;N;;;;; +F98E;CJK COMPATIBILITY IDEOGRAPH-F98E;Lo;0;L;5E74;;;;N;;;;; +F98F;CJK COMPATIBILITY IDEOGRAPH-F98F;Lo;0;L;6190;;;;N;;;;; +F990;CJK COMPATIBILITY IDEOGRAPH-F990;Lo;0;L;6200;;;;N;;;;; +F991;CJK COMPATIBILITY IDEOGRAPH-F991;Lo;0;L;649A;;;;N;;;;; +F992;CJK COMPATIBILITY IDEOGRAPH-F992;Lo;0;L;6F23;;;;N;;;;; +F993;CJK COMPATIBILITY IDEOGRAPH-F993;Lo;0;L;7149;;;;N;;;;; +F994;CJK COMPATIBILITY IDEOGRAPH-F994;Lo;0;L;7489;;;;N;;;;; +F995;CJK COMPATIBILITY IDEOGRAPH-F995;Lo;0;L;79CA;;;;N;;;;; +F996;CJK COMPATIBILITY IDEOGRAPH-F996;Lo;0;L;7DF4;;;;N;;;;; +F997;CJK COMPATIBILITY IDEOGRAPH-F997;Lo;0;L;806F;;;;N;;;;; +F998;CJK COMPATIBILITY IDEOGRAPH-F998;Lo;0;L;8F26;;;;N;;;;; +F999;CJK COMPATIBILITY IDEOGRAPH-F999;Lo;0;L;84EE;;;;N;;;;; +F99A;CJK COMPATIBILITY IDEOGRAPH-F99A;Lo;0;L;9023;;;;N;;;;; +F99B;CJK COMPATIBILITY IDEOGRAPH-F99B;Lo;0;L;934A;;;;N;;;;; +F99C;CJK COMPATIBILITY IDEOGRAPH-F99C;Lo;0;L;5217;;;;N;;;;; +F99D;CJK COMPATIBILITY IDEOGRAPH-F99D;Lo;0;L;52A3;;;;N;;;;; +F99E;CJK COMPATIBILITY IDEOGRAPH-F99E;Lo;0;L;54BD;;;;N;;;;; +F99F;CJK COMPATIBILITY IDEOGRAPH-F99F;Lo;0;L;70C8;;;;N;;;;; +F9A0;CJK COMPATIBILITY IDEOGRAPH-F9A0;Lo;0;L;88C2;;;;N;;;;; +F9A1;CJK COMPATIBILITY IDEOGRAPH-F9A1;Lo;0;L;8AAA;;;;N;;;;; +F9A2;CJK COMPATIBILITY IDEOGRAPH-F9A2;Lo;0;L;5EC9;;;;N;;;;; +F9A3;CJK COMPATIBILITY IDEOGRAPH-F9A3;Lo;0;L;5FF5;;;;N;;;;; +F9A4;CJK COMPATIBILITY IDEOGRAPH-F9A4;Lo;0;L;637B;;;;N;;;;; +F9A5;CJK COMPATIBILITY IDEOGRAPH-F9A5;Lo;0;L;6BAE;;;;N;;;;; +F9A6;CJK COMPATIBILITY IDEOGRAPH-F9A6;Lo;0;L;7C3E;;;;N;;;;; +F9A7;CJK COMPATIBILITY IDEOGRAPH-F9A7;Lo;0;L;7375;;;;N;;;;; +F9A8;CJK COMPATIBILITY IDEOGRAPH-F9A8;Lo;0;L;4EE4;;;;N;;;;; +F9A9;CJK COMPATIBILITY IDEOGRAPH-F9A9;Lo;0;L;56F9;;;;N;;;;; +F9AA;CJK COMPATIBILITY IDEOGRAPH-F9AA;Lo;0;L;5BE7;;;;N;;;;; +F9AB;CJK COMPATIBILITY IDEOGRAPH-F9AB;Lo;0;L;5DBA;;;;N;;;;; +F9AC;CJK COMPATIBILITY IDEOGRAPH-F9AC;Lo;0;L;601C;;;;N;;;;; +F9AD;CJK COMPATIBILITY IDEOGRAPH-F9AD;Lo;0;L;73B2;;;;N;;;;; +F9AE;CJK COMPATIBILITY IDEOGRAPH-F9AE;Lo;0;L;7469;;;;N;;;;; +F9AF;CJK COMPATIBILITY IDEOGRAPH-F9AF;Lo;0;L;7F9A;;;;N;;;;; +F9B0;CJK COMPATIBILITY IDEOGRAPH-F9B0;Lo;0;L;8046;;;;N;;;;; +F9B1;CJK COMPATIBILITY IDEOGRAPH-F9B1;Lo;0;L;9234;;;;N;;;;; +F9B2;CJK COMPATIBILITY IDEOGRAPH-F9B2;Lo;0;L;96F6;;;0;N;;;;; +F9B3;CJK COMPATIBILITY IDEOGRAPH-F9B3;Lo;0;L;9748;;;;N;;;;; +F9B4;CJK COMPATIBILITY IDEOGRAPH-F9B4;Lo;0;L;9818;;;;N;;;;; +F9B5;CJK COMPATIBILITY IDEOGRAPH-F9B5;Lo;0;L;4F8B;;;;N;;;;; +F9B6;CJK COMPATIBILITY IDEOGRAPH-F9B6;Lo;0;L;79AE;;;;N;;;;; +F9B7;CJK COMPATIBILITY IDEOGRAPH-F9B7;Lo;0;L;91B4;;;;N;;;;; +F9B8;CJK COMPATIBILITY IDEOGRAPH-F9B8;Lo;0;L;96B8;;;;N;;;;; +F9B9;CJK COMPATIBILITY IDEOGRAPH-F9B9;Lo;0;L;60E1;;;;N;;;;; +F9BA;CJK COMPATIBILITY IDEOGRAPH-F9BA;Lo;0;L;4E86;;;;N;;;;; +F9BB;CJK COMPATIBILITY IDEOGRAPH-F9BB;Lo;0;L;50DA;;;;N;;;;; +F9BC;CJK COMPATIBILITY IDEOGRAPH-F9BC;Lo;0;L;5BEE;;;;N;;;;; +F9BD;CJK COMPATIBILITY IDEOGRAPH-F9BD;Lo;0;L;5C3F;;;;N;;;;; +F9BE;CJK COMPATIBILITY IDEOGRAPH-F9BE;Lo;0;L;6599;;;;N;;;;; +F9BF;CJK COMPATIBILITY IDEOGRAPH-F9BF;Lo;0;L;6A02;;;;N;;;;; +F9C0;CJK COMPATIBILITY IDEOGRAPH-F9C0;Lo;0;L;71CE;;;;N;;;;; +F9C1;CJK COMPATIBILITY IDEOGRAPH-F9C1;Lo;0;L;7642;;;;N;;;;; +F9C2;CJK COMPATIBILITY IDEOGRAPH-F9C2;Lo;0;L;84FC;;;;N;;;;; +F9C3;CJK COMPATIBILITY IDEOGRAPH-F9C3;Lo;0;L;907C;;;;N;;;;; +F9C4;CJK COMPATIBILITY IDEOGRAPH-F9C4;Lo;0;L;9F8D;;;;N;;;;; +F9C5;CJK COMPATIBILITY IDEOGRAPH-F9C5;Lo;0;L;6688;;;;N;;;;; +F9C6;CJK COMPATIBILITY IDEOGRAPH-F9C6;Lo;0;L;962E;;;;N;;;;; +F9C7;CJK COMPATIBILITY IDEOGRAPH-F9C7;Lo;0;L;5289;;;;N;;;;; +F9C8;CJK COMPATIBILITY IDEOGRAPH-F9C8;Lo;0;L;677B;;;;N;;;;; +F9C9;CJK COMPATIBILITY IDEOGRAPH-F9C9;Lo;0;L;67F3;;;;N;;;;; +F9CA;CJK COMPATIBILITY IDEOGRAPH-F9CA;Lo;0;L;6D41;;;;N;;;;; +F9CB;CJK COMPATIBILITY IDEOGRAPH-F9CB;Lo;0;L;6E9C;;;;N;;;;; +F9CC;CJK COMPATIBILITY IDEOGRAPH-F9CC;Lo;0;L;7409;;;;N;;;;; +F9CD;CJK COMPATIBILITY IDEOGRAPH-F9CD;Lo;0;L;7559;;;;N;;;;; +F9CE;CJK COMPATIBILITY IDEOGRAPH-F9CE;Lo;0;L;786B;;;;N;;;;; +F9CF;CJK COMPATIBILITY IDEOGRAPH-F9CF;Lo;0;L;7D10;;;;N;;;;; +F9D0;CJK COMPATIBILITY IDEOGRAPH-F9D0;Lo;0;L;985E;;;;N;;;;; +F9D1;CJK COMPATIBILITY IDEOGRAPH-F9D1;Lo;0;L;516D;;;6;N;;;;; +F9D2;CJK COMPATIBILITY IDEOGRAPH-F9D2;Lo;0;L;622E;;;;N;;;;; +F9D3;CJK COMPATIBILITY IDEOGRAPH-F9D3;Lo;0;L;9678;;;6;N;;;;; +F9D4;CJK COMPATIBILITY IDEOGRAPH-F9D4;Lo;0;L;502B;;;;N;;;;; +F9D5;CJK COMPATIBILITY IDEOGRAPH-F9D5;Lo;0;L;5D19;;;;N;;;;; +F9D6;CJK COMPATIBILITY IDEOGRAPH-F9D6;Lo;0;L;6DEA;;;;N;;;;; +F9D7;CJK COMPATIBILITY IDEOGRAPH-F9D7;Lo;0;L;8F2A;;;;N;;;;; +F9D8;CJK COMPATIBILITY IDEOGRAPH-F9D8;Lo;0;L;5F8B;;;;N;;;;; +F9D9;CJK COMPATIBILITY IDEOGRAPH-F9D9;Lo;0;L;6144;;;;N;;;;; +F9DA;CJK COMPATIBILITY IDEOGRAPH-F9DA;Lo;0;L;6817;;;;N;;;;; +F9DB;CJK COMPATIBILITY IDEOGRAPH-F9DB;Lo;0;L;7387;;;;N;;;;; +F9DC;CJK COMPATIBILITY IDEOGRAPH-F9DC;Lo;0;L;9686;;;;N;;;;; +F9DD;CJK COMPATIBILITY IDEOGRAPH-F9DD;Lo;0;L;5229;;;;N;;;;; +F9DE;CJK COMPATIBILITY IDEOGRAPH-F9DE;Lo;0;L;540F;;;;N;;;;; +F9DF;CJK COMPATIBILITY IDEOGRAPH-F9DF;Lo;0;L;5C65;;;;N;;;;; +F9E0;CJK COMPATIBILITY IDEOGRAPH-F9E0;Lo;0;L;6613;;;;N;;;;; +F9E1;CJK COMPATIBILITY IDEOGRAPH-F9E1;Lo;0;L;674E;;;;N;;;;; +F9E2;CJK COMPATIBILITY IDEOGRAPH-F9E2;Lo;0;L;68A8;;;;N;;;;; +F9E3;CJK COMPATIBILITY IDEOGRAPH-F9E3;Lo;0;L;6CE5;;;;N;;;;; +F9E4;CJK COMPATIBILITY IDEOGRAPH-F9E4;Lo;0;L;7406;;;;N;;;;; +F9E5;CJK COMPATIBILITY IDEOGRAPH-F9E5;Lo;0;L;75E2;;;;N;;;;; +F9E6;CJK COMPATIBILITY IDEOGRAPH-F9E6;Lo;0;L;7F79;;;;N;;;;; +F9E7;CJK COMPATIBILITY IDEOGRAPH-F9E7;Lo;0;L;88CF;;;;N;;;;; +F9E8;CJK COMPATIBILITY IDEOGRAPH-F9E8;Lo;0;L;88E1;;;;N;;;;; +F9E9;CJK COMPATIBILITY IDEOGRAPH-F9E9;Lo;0;L;91CC;;;;N;;;;; +F9EA;CJK COMPATIBILITY IDEOGRAPH-F9EA;Lo;0;L;96E2;;;;N;;;;; +F9EB;CJK COMPATIBILITY IDEOGRAPH-F9EB;Lo;0;L;533F;;;;N;;;;; +F9EC;CJK COMPATIBILITY IDEOGRAPH-F9EC;Lo;0;L;6EBA;;;;N;;;;; +F9ED;CJK COMPATIBILITY IDEOGRAPH-F9ED;Lo;0;L;541D;;;;N;;;;; +F9EE;CJK COMPATIBILITY IDEOGRAPH-F9EE;Lo;0;L;71D0;;;;N;;;;; +F9EF;CJK COMPATIBILITY IDEOGRAPH-F9EF;Lo;0;L;7498;;;;N;;;;; +F9F0;CJK COMPATIBILITY IDEOGRAPH-F9F0;Lo;0;L;85FA;;;;N;;;;; +F9F1;CJK COMPATIBILITY IDEOGRAPH-F9F1;Lo;0;L;96A3;;;;N;;;;; +F9F2;CJK COMPATIBILITY IDEOGRAPH-F9F2;Lo;0;L;9C57;;;;N;;;;; +F9F3;CJK COMPATIBILITY IDEOGRAPH-F9F3;Lo;0;L;9E9F;;;;N;;;;; +F9F4;CJK COMPATIBILITY IDEOGRAPH-F9F4;Lo;0;L;6797;;;;N;;;;; +F9F5;CJK COMPATIBILITY IDEOGRAPH-F9F5;Lo;0;L;6DCB;;;;N;;;;; +F9F6;CJK COMPATIBILITY IDEOGRAPH-F9F6;Lo;0;L;81E8;;;;N;;;;; +F9F7;CJK COMPATIBILITY IDEOGRAPH-F9F7;Lo;0;L;7ACB;;;;N;;;;; +F9F8;CJK COMPATIBILITY IDEOGRAPH-F9F8;Lo;0;L;7B20;;;;N;;;;; +F9F9;CJK COMPATIBILITY IDEOGRAPH-F9F9;Lo;0;L;7C92;;;;N;;;;; +F9FA;CJK COMPATIBILITY IDEOGRAPH-F9FA;Lo;0;L;72C0;;;;N;;;;; +F9FB;CJK COMPATIBILITY IDEOGRAPH-F9FB;Lo;0;L;7099;;;;N;;;;; +F9FC;CJK COMPATIBILITY IDEOGRAPH-F9FC;Lo;0;L;8B58;;;;N;;;;; +F9FD;CJK COMPATIBILITY IDEOGRAPH-F9FD;Lo;0;L;4EC0;;;10;N;;;;; +F9FE;CJK COMPATIBILITY IDEOGRAPH-F9FE;Lo;0;L;8336;;;;N;;;;; +F9FF;CJK COMPATIBILITY IDEOGRAPH-F9FF;Lo;0;L;523A;;;;N;;;;; +FA00;CJK COMPATIBILITY IDEOGRAPH-FA00;Lo;0;L;5207;;;;N;;;;; +FA01;CJK COMPATIBILITY IDEOGRAPH-FA01;Lo;0;L;5EA6;;;;N;;;;; +FA02;CJK COMPATIBILITY IDEOGRAPH-FA02;Lo;0;L;62D3;;;;N;;;;; +FA03;CJK COMPATIBILITY IDEOGRAPH-FA03;Lo;0;L;7CD6;;;;N;;;;; +FA04;CJK COMPATIBILITY IDEOGRAPH-FA04;Lo;0;L;5B85;;;;N;;;;; +FA05;CJK COMPATIBILITY IDEOGRAPH-FA05;Lo;0;L;6D1E;;;;N;;;;; +FA06;CJK COMPATIBILITY IDEOGRAPH-FA06;Lo;0;L;66B4;;;;N;;;;; +FA07;CJK COMPATIBILITY IDEOGRAPH-FA07;Lo;0;L;8F3B;;;;N;;;;; +FA08;CJK COMPATIBILITY IDEOGRAPH-FA08;Lo;0;L;884C;;;;N;;;;; +FA09;CJK COMPATIBILITY IDEOGRAPH-FA09;Lo;0;L;964D;;;;N;;;;; +FA0A;CJK COMPATIBILITY IDEOGRAPH-FA0A;Lo;0;L;898B;;;;N;;;;; +FA0B;CJK COMPATIBILITY IDEOGRAPH-FA0B;Lo;0;L;5ED3;;;;N;;;;; +FA0C;CJK COMPATIBILITY IDEOGRAPH-FA0C;Lo;0;L;5140;;;;N;;;;; +FA0D;CJK COMPATIBILITY IDEOGRAPH-FA0D;Lo;0;L;55C0;;;;N;;;;; +FA0E;CJK COMPATIBILITY IDEOGRAPH-FA0E;Lo;0;L;;;;;N;;;;; +FA0F;CJK COMPATIBILITY IDEOGRAPH-FA0F;Lo;0;L;;;;;N;;;;; +FA10;CJK COMPATIBILITY IDEOGRAPH-FA10;Lo;0;L;585A;;;;N;;;;; +FA11;CJK COMPATIBILITY IDEOGRAPH-FA11;Lo;0;L;;;;;N;;;;; +FA12;CJK COMPATIBILITY IDEOGRAPH-FA12;Lo;0;L;6674;;;;N;;;;; +FA13;CJK COMPATIBILITY IDEOGRAPH-FA13;Lo;0;L;;;;;N;;;;; +FA14;CJK COMPATIBILITY IDEOGRAPH-FA14;Lo;0;L;;;;;N;;;;; +FA15;CJK COMPATIBILITY IDEOGRAPH-FA15;Lo;0;L;51DE;;;;N;;;;; +FA16;CJK COMPATIBILITY IDEOGRAPH-FA16;Lo;0;L;732A;;;;N;;;;; +FA17;CJK COMPATIBILITY IDEOGRAPH-FA17;Lo;0;L;76CA;;;;N;;;;; +FA18;CJK COMPATIBILITY IDEOGRAPH-FA18;Lo;0;L;793C;;;;N;;;;; +FA19;CJK COMPATIBILITY IDEOGRAPH-FA19;Lo;0;L;795E;;;;N;;;;; +FA1A;CJK COMPATIBILITY IDEOGRAPH-FA1A;Lo;0;L;7965;;;;N;;;;; +FA1B;CJK COMPATIBILITY IDEOGRAPH-FA1B;Lo;0;L;798F;;;;N;;;;; +FA1C;CJK COMPATIBILITY IDEOGRAPH-FA1C;Lo;0;L;9756;;;;N;;;;; +FA1D;CJK COMPATIBILITY IDEOGRAPH-FA1D;Lo;0;L;7CBE;;;;N;;;;; +FA1E;CJK COMPATIBILITY IDEOGRAPH-FA1E;Lo;0;L;7FBD;;;;N;;;;; +FA1F;CJK COMPATIBILITY IDEOGRAPH-FA1F;Lo;0;L;;;;;N;;;;; +FA20;CJK COMPATIBILITY IDEOGRAPH-FA20;Lo;0;L;8612;;;;N;;;;; +FA21;CJK COMPATIBILITY IDEOGRAPH-FA21;Lo;0;L;;;;;N;;;;; +FA22;CJK COMPATIBILITY IDEOGRAPH-FA22;Lo;0;L;8AF8;;;;N;;;;; +FA23;CJK COMPATIBILITY IDEOGRAPH-FA23;Lo;0;L;;;;;N;;;;; +FA24;CJK COMPATIBILITY IDEOGRAPH-FA24;Lo;0;L;;;;;N;;;;; +FA25;CJK COMPATIBILITY IDEOGRAPH-FA25;Lo;0;L;9038;;;;N;;;;; +FA26;CJK COMPATIBILITY IDEOGRAPH-FA26;Lo;0;L;90FD;;;;N;;;;; +FA27;CJK COMPATIBILITY IDEOGRAPH-FA27;Lo;0;L;;;;;N;;;;; +FA28;CJK COMPATIBILITY IDEOGRAPH-FA28;Lo;0;L;;;;;N;;;;; +FA29;CJK COMPATIBILITY IDEOGRAPH-FA29;Lo;0;L;;;;;N;;;;; +FA2A;CJK COMPATIBILITY IDEOGRAPH-FA2A;Lo;0;L;98EF;;;;N;;;;; +FA2B;CJK COMPATIBILITY IDEOGRAPH-FA2B;Lo;0;L;98FC;;;;N;;;;; +FA2C;CJK COMPATIBILITY IDEOGRAPH-FA2C;Lo;0;L;9928;;;;N;;;;; +FA2D;CJK COMPATIBILITY IDEOGRAPH-FA2D;Lo;0;L;9DB4;;;;N;;;;; +FA2E;CJK COMPATIBILITY IDEOGRAPH-FA2E;Lo;0;L;90DE;;;;N;;;;; +FA2F;CJK COMPATIBILITY IDEOGRAPH-FA2F;Lo;0;L;96B7;;;;N;;;;; +FA30;CJK COMPATIBILITY IDEOGRAPH-FA30;Lo;0;L;4FAE;;;;N;;;;; +FA31;CJK COMPATIBILITY IDEOGRAPH-FA31;Lo;0;L;50E7;;;;N;;;;; +FA32;CJK COMPATIBILITY IDEOGRAPH-FA32;Lo;0;L;514D;;;;N;;;;; +FA33;CJK COMPATIBILITY IDEOGRAPH-FA33;Lo;0;L;52C9;;;;N;;;;; +FA34;CJK COMPATIBILITY IDEOGRAPH-FA34;Lo;0;L;52E4;;;;N;;;;; +FA35;CJK COMPATIBILITY IDEOGRAPH-FA35;Lo;0;L;5351;;;;N;;;;; +FA36;CJK COMPATIBILITY IDEOGRAPH-FA36;Lo;0;L;559D;;;;N;;;;; +FA37;CJK COMPATIBILITY IDEOGRAPH-FA37;Lo;0;L;5606;;;;N;;;;; +FA38;CJK COMPATIBILITY IDEOGRAPH-FA38;Lo;0;L;5668;;;;N;;;;; +FA39;CJK COMPATIBILITY IDEOGRAPH-FA39;Lo;0;L;5840;;;;N;;;;; +FA3A;CJK COMPATIBILITY IDEOGRAPH-FA3A;Lo;0;L;58A8;;;;N;;;;; +FA3B;CJK COMPATIBILITY IDEOGRAPH-FA3B;Lo;0;L;5C64;;;;N;;;;; +FA3C;CJK COMPATIBILITY IDEOGRAPH-FA3C;Lo;0;L;5C6E;;;;N;;;;; +FA3D;CJK COMPATIBILITY IDEOGRAPH-FA3D;Lo;0;L;6094;;;;N;;;;; +FA3E;CJK COMPATIBILITY IDEOGRAPH-FA3E;Lo;0;L;6168;;;;N;;;;; +FA3F;CJK COMPATIBILITY IDEOGRAPH-FA3F;Lo;0;L;618E;;;;N;;;;; +FA40;CJK COMPATIBILITY IDEOGRAPH-FA40;Lo;0;L;61F2;;;;N;;;;; +FA41;CJK COMPATIBILITY IDEOGRAPH-FA41;Lo;0;L;654F;;;;N;;;;; +FA42;CJK COMPATIBILITY IDEOGRAPH-FA42;Lo;0;L;65E2;;;;N;;;;; +FA43;CJK COMPATIBILITY IDEOGRAPH-FA43;Lo;0;L;6691;;;;N;;;;; +FA44;CJK COMPATIBILITY IDEOGRAPH-FA44;Lo;0;L;6885;;;;N;;;;; +FA45;CJK COMPATIBILITY IDEOGRAPH-FA45;Lo;0;L;6D77;;;;N;;;;; +FA46;CJK COMPATIBILITY IDEOGRAPH-FA46;Lo;0;L;6E1A;;;;N;;;;; +FA47;CJK COMPATIBILITY IDEOGRAPH-FA47;Lo;0;L;6F22;;;;N;;;;; +FA48;CJK COMPATIBILITY IDEOGRAPH-FA48;Lo;0;L;716E;;;;N;;;;; +FA49;CJK COMPATIBILITY IDEOGRAPH-FA49;Lo;0;L;722B;;;;N;;;;; +FA4A;CJK COMPATIBILITY IDEOGRAPH-FA4A;Lo;0;L;7422;;;;N;;;;; +FA4B;CJK COMPATIBILITY IDEOGRAPH-FA4B;Lo;0;L;7891;;;;N;;;;; +FA4C;CJK COMPATIBILITY IDEOGRAPH-FA4C;Lo;0;L;793E;;;;N;;;;; +FA4D;CJK COMPATIBILITY IDEOGRAPH-FA4D;Lo;0;L;7949;;;;N;;;;; +FA4E;CJK COMPATIBILITY IDEOGRAPH-FA4E;Lo;0;L;7948;;;;N;;;;; +FA4F;CJK COMPATIBILITY IDEOGRAPH-FA4F;Lo;0;L;7950;;;;N;;;;; +FA50;CJK COMPATIBILITY IDEOGRAPH-FA50;Lo;0;L;7956;;;;N;;;;; +FA51;CJK COMPATIBILITY IDEOGRAPH-FA51;Lo;0;L;795D;;;;N;;;;; +FA52;CJK COMPATIBILITY IDEOGRAPH-FA52;Lo;0;L;798D;;;;N;;;;; +FA53;CJK COMPATIBILITY IDEOGRAPH-FA53;Lo;0;L;798E;;;;N;;;;; +FA54;CJK COMPATIBILITY IDEOGRAPH-FA54;Lo;0;L;7A40;;;;N;;;;; +FA55;CJK COMPATIBILITY IDEOGRAPH-FA55;Lo;0;L;7A81;;;;N;;;;; +FA56;CJK COMPATIBILITY IDEOGRAPH-FA56;Lo;0;L;7BC0;;;;N;;;;; +FA57;CJK COMPATIBILITY IDEOGRAPH-FA57;Lo;0;L;7DF4;;;;N;;;;; +FA58;CJK COMPATIBILITY IDEOGRAPH-FA58;Lo;0;L;7E09;;;;N;;;;; +FA59;CJK COMPATIBILITY IDEOGRAPH-FA59;Lo;0;L;7E41;;;;N;;;;; +FA5A;CJK COMPATIBILITY IDEOGRAPH-FA5A;Lo;0;L;7F72;;;;N;;;;; +FA5B;CJK COMPATIBILITY IDEOGRAPH-FA5B;Lo;0;L;8005;;;;N;;;;; +FA5C;CJK COMPATIBILITY IDEOGRAPH-FA5C;Lo;0;L;81ED;;;;N;;;;; +FA5D;CJK COMPATIBILITY IDEOGRAPH-FA5D;Lo;0;L;8279;;;;N;;;;; +FA5E;CJK COMPATIBILITY IDEOGRAPH-FA5E;Lo;0;L;8279;;;;N;;;;; +FA5F;CJK COMPATIBILITY IDEOGRAPH-FA5F;Lo;0;L;8457;;;;N;;;;; +FA60;CJK COMPATIBILITY IDEOGRAPH-FA60;Lo;0;L;8910;;;;N;;;;; +FA61;CJK COMPATIBILITY IDEOGRAPH-FA61;Lo;0;L;8996;;;;N;;;;; +FA62;CJK COMPATIBILITY IDEOGRAPH-FA62;Lo;0;L;8B01;;;;N;;;;; +FA63;CJK COMPATIBILITY IDEOGRAPH-FA63;Lo;0;L;8B39;;;;N;;;;; +FA64;CJK COMPATIBILITY IDEOGRAPH-FA64;Lo;0;L;8CD3;;;;N;;;;; +FA65;CJK COMPATIBILITY IDEOGRAPH-FA65;Lo;0;L;8D08;;;;N;;;;; +FA66;CJK COMPATIBILITY IDEOGRAPH-FA66;Lo;0;L;8FB6;;;;N;;;;; +FA67;CJK COMPATIBILITY IDEOGRAPH-FA67;Lo;0;L;9038;;;;N;;;;; +FA68;CJK COMPATIBILITY IDEOGRAPH-FA68;Lo;0;L;96E3;;;;N;;;;; +FA69;CJK COMPATIBILITY IDEOGRAPH-FA69;Lo;0;L;97FF;;;;N;;;;; +FA6A;CJK COMPATIBILITY IDEOGRAPH-FA6A;Lo;0;L;983B;;;;N;;;;; +FA6B;CJK COMPATIBILITY IDEOGRAPH-FA6B;Lo;0;L;6075;;;;N;;;;; +FA6C;CJK COMPATIBILITY IDEOGRAPH-FA6C;Lo;0;L;242EE;;;;N;;;;; +FA6D;CJK COMPATIBILITY IDEOGRAPH-FA6D;Lo;0;L;8218;;;;N;;;;; +FA70;CJK COMPATIBILITY IDEOGRAPH-FA70;Lo;0;L;4E26;;;;N;;;;; +FA71;CJK COMPATIBILITY IDEOGRAPH-FA71;Lo;0;L;51B5;;;;N;;;;; +FA72;CJK COMPATIBILITY IDEOGRAPH-FA72;Lo;0;L;5168;;;;N;;;;; +FA73;CJK COMPATIBILITY IDEOGRAPH-FA73;Lo;0;L;4F80;;;;N;;;;; +FA74;CJK COMPATIBILITY IDEOGRAPH-FA74;Lo;0;L;5145;;;;N;;;;; +FA75;CJK COMPATIBILITY IDEOGRAPH-FA75;Lo;0;L;5180;;;;N;;;;; +FA76;CJK COMPATIBILITY IDEOGRAPH-FA76;Lo;0;L;52C7;;;;N;;;;; +FA77;CJK COMPATIBILITY IDEOGRAPH-FA77;Lo;0;L;52FA;;;;N;;;;; +FA78;CJK COMPATIBILITY IDEOGRAPH-FA78;Lo;0;L;559D;;;;N;;;;; +FA79;CJK COMPATIBILITY IDEOGRAPH-FA79;Lo;0;L;5555;;;;N;;;;; +FA7A;CJK COMPATIBILITY IDEOGRAPH-FA7A;Lo;0;L;5599;;;;N;;;;; +FA7B;CJK COMPATIBILITY IDEOGRAPH-FA7B;Lo;0;L;55E2;;;;N;;;;; +FA7C;CJK COMPATIBILITY IDEOGRAPH-FA7C;Lo;0;L;585A;;;;N;;;;; +FA7D;CJK COMPATIBILITY IDEOGRAPH-FA7D;Lo;0;L;58B3;;;;N;;;;; +FA7E;CJK COMPATIBILITY IDEOGRAPH-FA7E;Lo;0;L;5944;;;;N;;;;; +FA7F;CJK COMPATIBILITY IDEOGRAPH-FA7F;Lo;0;L;5954;;;;N;;;;; +FA80;CJK COMPATIBILITY IDEOGRAPH-FA80;Lo;0;L;5A62;;;;N;;;;; +FA81;CJK COMPATIBILITY IDEOGRAPH-FA81;Lo;0;L;5B28;;;;N;;;;; +FA82;CJK COMPATIBILITY IDEOGRAPH-FA82;Lo;0;L;5ED2;;;;N;;;;; +FA83;CJK COMPATIBILITY IDEOGRAPH-FA83;Lo;0;L;5ED9;;;;N;;;;; +FA84;CJK COMPATIBILITY IDEOGRAPH-FA84;Lo;0;L;5F69;;;;N;;;;; +FA85;CJK COMPATIBILITY IDEOGRAPH-FA85;Lo;0;L;5FAD;;;;N;;;;; +FA86;CJK COMPATIBILITY IDEOGRAPH-FA86;Lo;0;L;60D8;;;;N;;;;; +FA87;CJK COMPATIBILITY IDEOGRAPH-FA87;Lo;0;L;614E;;;;N;;;;; +FA88;CJK COMPATIBILITY IDEOGRAPH-FA88;Lo;0;L;6108;;;;N;;;;; +FA89;CJK COMPATIBILITY IDEOGRAPH-FA89;Lo;0;L;618E;;;;N;;;;; +FA8A;CJK COMPATIBILITY IDEOGRAPH-FA8A;Lo;0;L;6160;;;;N;;;;; +FA8B;CJK COMPATIBILITY IDEOGRAPH-FA8B;Lo;0;L;61F2;;;;N;;;;; +FA8C;CJK COMPATIBILITY IDEOGRAPH-FA8C;Lo;0;L;6234;;;;N;;;;; +FA8D;CJK COMPATIBILITY IDEOGRAPH-FA8D;Lo;0;L;63C4;;;;N;;;;; +FA8E;CJK COMPATIBILITY IDEOGRAPH-FA8E;Lo;0;L;641C;;;;N;;;;; +FA8F;CJK COMPATIBILITY IDEOGRAPH-FA8F;Lo;0;L;6452;;;;N;;;;; +FA90;CJK COMPATIBILITY IDEOGRAPH-FA90;Lo;0;L;6556;;;;N;;;;; +FA91;CJK COMPATIBILITY IDEOGRAPH-FA91;Lo;0;L;6674;;;;N;;;;; +FA92;CJK COMPATIBILITY IDEOGRAPH-FA92;Lo;0;L;6717;;;;N;;;;; +FA93;CJK COMPATIBILITY IDEOGRAPH-FA93;Lo;0;L;671B;;;;N;;;;; +FA94;CJK COMPATIBILITY IDEOGRAPH-FA94;Lo;0;L;6756;;;;N;;;;; +FA95;CJK COMPATIBILITY IDEOGRAPH-FA95;Lo;0;L;6B79;;;;N;;;;; +FA96;CJK COMPATIBILITY IDEOGRAPH-FA96;Lo;0;L;6BBA;;;;N;;;;; +FA97;CJK COMPATIBILITY IDEOGRAPH-FA97;Lo;0;L;6D41;;;;N;;;;; +FA98;CJK COMPATIBILITY IDEOGRAPH-FA98;Lo;0;L;6EDB;;;;N;;;;; +FA99;CJK COMPATIBILITY IDEOGRAPH-FA99;Lo;0;L;6ECB;;;;N;;;;; +FA9A;CJK COMPATIBILITY IDEOGRAPH-FA9A;Lo;0;L;6F22;;;;N;;;;; +FA9B;CJK COMPATIBILITY IDEOGRAPH-FA9B;Lo;0;L;701E;;;;N;;;;; +FA9C;CJK COMPATIBILITY IDEOGRAPH-FA9C;Lo;0;L;716E;;;;N;;;;; +FA9D;CJK COMPATIBILITY IDEOGRAPH-FA9D;Lo;0;L;77A7;;;;N;;;;; +FA9E;CJK COMPATIBILITY IDEOGRAPH-FA9E;Lo;0;L;7235;;;;N;;;;; +FA9F;CJK COMPATIBILITY IDEOGRAPH-FA9F;Lo;0;L;72AF;;;;N;;;;; +FAA0;CJK COMPATIBILITY IDEOGRAPH-FAA0;Lo;0;L;732A;;;;N;;;;; +FAA1;CJK COMPATIBILITY IDEOGRAPH-FAA1;Lo;0;L;7471;;;;N;;;;; +FAA2;CJK COMPATIBILITY IDEOGRAPH-FAA2;Lo;0;L;7506;;;;N;;;;; +FAA3;CJK COMPATIBILITY IDEOGRAPH-FAA3;Lo;0;L;753B;;;;N;;;;; +FAA4;CJK COMPATIBILITY IDEOGRAPH-FAA4;Lo;0;L;761D;;;;N;;;;; +FAA5;CJK COMPATIBILITY IDEOGRAPH-FAA5;Lo;0;L;761F;;;;N;;;;; +FAA6;CJK COMPATIBILITY IDEOGRAPH-FAA6;Lo;0;L;76CA;;;;N;;;;; +FAA7;CJK COMPATIBILITY IDEOGRAPH-FAA7;Lo;0;L;76DB;;;;N;;;;; +FAA8;CJK COMPATIBILITY IDEOGRAPH-FAA8;Lo;0;L;76F4;;;;N;;;;; +FAA9;CJK COMPATIBILITY IDEOGRAPH-FAA9;Lo;0;L;774A;;;;N;;;;; +FAAA;CJK COMPATIBILITY IDEOGRAPH-FAAA;Lo;0;L;7740;;;;N;;;;; +FAAB;CJK COMPATIBILITY IDEOGRAPH-FAAB;Lo;0;L;78CC;;;;N;;;;; +FAAC;CJK COMPATIBILITY IDEOGRAPH-FAAC;Lo;0;L;7AB1;;;;N;;;;; +FAAD;CJK COMPATIBILITY IDEOGRAPH-FAAD;Lo;0;L;7BC0;;;;N;;;;; +FAAE;CJK COMPATIBILITY IDEOGRAPH-FAAE;Lo;0;L;7C7B;;;;N;;;;; +FAAF;CJK COMPATIBILITY IDEOGRAPH-FAAF;Lo;0;L;7D5B;;;;N;;;;; +FAB0;CJK COMPATIBILITY IDEOGRAPH-FAB0;Lo;0;L;7DF4;;;;N;;;;; +FAB1;CJK COMPATIBILITY IDEOGRAPH-FAB1;Lo;0;L;7F3E;;;;N;;;;; +FAB2;CJK COMPATIBILITY IDEOGRAPH-FAB2;Lo;0;L;8005;;;;N;;;;; +FAB3;CJK COMPATIBILITY IDEOGRAPH-FAB3;Lo;0;L;8352;;;;N;;;;; +FAB4;CJK COMPATIBILITY IDEOGRAPH-FAB4;Lo;0;L;83EF;;;;N;;;;; +FAB5;CJK COMPATIBILITY IDEOGRAPH-FAB5;Lo;0;L;8779;;;;N;;;;; +FAB6;CJK COMPATIBILITY IDEOGRAPH-FAB6;Lo;0;L;8941;;;;N;;;;; +FAB7;CJK COMPATIBILITY IDEOGRAPH-FAB7;Lo;0;L;8986;;;;N;;;;; +FAB8;CJK COMPATIBILITY IDEOGRAPH-FAB8;Lo;0;L;8996;;;;N;;;;; +FAB9;CJK COMPATIBILITY IDEOGRAPH-FAB9;Lo;0;L;8ABF;;;;N;;;;; +FABA;CJK COMPATIBILITY IDEOGRAPH-FABA;Lo;0;L;8AF8;;;;N;;;;; +FABB;CJK COMPATIBILITY IDEOGRAPH-FABB;Lo;0;L;8ACB;;;;N;;;;; +FABC;CJK COMPATIBILITY IDEOGRAPH-FABC;Lo;0;L;8B01;;;;N;;;;; +FABD;CJK COMPATIBILITY IDEOGRAPH-FABD;Lo;0;L;8AFE;;;;N;;;;; +FABE;CJK COMPATIBILITY IDEOGRAPH-FABE;Lo;0;L;8AED;;;;N;;;;; +FABF;CJK COMPATIBILITY IDEOGRAPH-FABF;Lo;0;L;8B39;;;;N;;;;; +FAC0;CJK COMPATIBILITY IDEOGRAPH-FAC0;Lo;0;L;8B8A;;;;N;;;;; +FAC1;CJK COMPATIBILITY IDEOGRAPH-FAC1;Lo;0;L;8D08;;;;N;;;;; +FAC2;CJK COMPATIBILITY IDEOGRAPH-FAC2;Lo;0;L;8F38;;;;N;;;;; +FAC3;CJK COMPATIBILITY IDEOGRAPH-FAC3;Lo;0;L;9072;;;;N;;;;; +FAC4;CJK COMPATIBILITY IDEOGRAPH-FAC4;Lo;0;L;9199;;;;N;;;;; +FAC5;CJK COMPATIBILITY IDEOGRAPH-FAC5;Lo;0;L;9276;;;;N;;;;; +FAC6;CJK COMPATIBILITY IDEOGRAPH-FAC6;Lo;0;L;967C;;;;N;;;;; +FAC7;CJK COMPATIBILITY IDEOGRAPH-FAC7;Lo;0;L;96E3;;;;N;;;;; +FAC8;CJK COMPATIBILITY IDEOGRAPH-FAC8;Lo;0;L;9756;;;;N;;;;; +FAC9;CJK COMPATIBILITY IDEOGRAPH-FAC9;Lo;0;L;97DB;;;;N;;;;; +FACA;CJK COMPATIBILITY IDEOGRAPH-FACA;Lo;0;L;97FF;;;;N;;;;; +FACB;CJK COMPATIBILITY IDEOGRAPH-FACB;Lo;0;L;980B;;;;N;;;;; +FACC;CJK COMPATIBILITY IDEOGRAPH-FACC;Lo;0;L;983B;;;;N;;;;; +FACD;CJK COMPATIBILITY IDEOGRAPH-FACD;Lo;0;L;9B12;;;;N;;;;; +FACE;CJK COMPATIBILITY IDEOGRAPH-FACE;Lo;0;L;9F9C;;;;N;;;;; +FACF;CJK COMPATIBILITY IDEOGRAPH-FACF;Lo;0;L;2284A;;;;N;;;;; +FAD0;CJK COMPATIBILITY IDEOGRAPH-FAD0;Lo;0;L;22844;;;;N;;;;; +FAD1;CJK COMPATIBILITY IDEOGRAPH-FAD1;Lo;0;L;233D5;;;;N;;;;; +FAD2;CJK COMPATIBILITY IDEOGRAPH-FAD2;Lo;0;L;3B9D;;;;N;;;;; +FAD3;CJK COMPATIBILITY IDEOGRAPH-FAD3;Lo;0;L;4018;;;;N;;;;; +FAD4;CJK COMPATIBILITY IDEOGRAPH-FAD4;Lo;0;L;4039;;;;N;;;;; +FAD5;CJK COMPATIBILITY IDEOGRAPH-FAD5;Lo;0;L;25249;;;;N;;;;; +FAD6;CJK COMPATIBILITY IDEOGRAPH-FAD6;Lo;0;L;25CD0;;;;N;;;;; +FAD7;CJK COMPATIBILITY IDEOGRAPH-FAD7;Lo;0;L;27ED3;;;;N;;;;; +FAD8;CJK COMPATIBILITY IDEOGRAPH-FAD8;Lo;0;L;9F43;;;;N;;;;; +FAD9;CJK COMPATIBILITY IDEOGRAPH-FAD9;Lo;0;L;9F8E;;;;N;;;;; +FB00;LATIN SMALL LIGATURE FF;Ll;0;L; 0066 0066;;;;N;;;;; +FB01;LATIN SMALL LIGATURE FI;Ll;0;L; 0066 0069;;;;N;;;;; +FB02;LATIN SMALL LIGATURE FL;Ll;0;L; 0066 006C;;;;N;;;;; +FB03;LATIN SMALL LIGATURE FFI;Ll;0;L; 0066 0066 0069;;;;N;;;;; +FB04;LATIN SMALL LIGATURE FFL;Ll;0;L; 0066 0066 006C;;;;N;;;;; +FB05;LATIN SMALL LIGATURE LONG S T;Ll;0;L; 017F 0074;;;;N;;;;; +FB06;LATIN SMALL LIGATURE ST;Ll;0;L; 0073 0074;;;;N;;;;; +FB13;ARMENIAN SMALL LIGATURE MEN NOW;Ll;0;L; 0574 0576;;;;N;;;;; +FB14;ARMENIAN SMALL LIGATURE MEN ECH;Ll;0;L; 0574 0565;;;;N;;;;; +FB15;ARMENIAN SMALL LIGATURE MEN INI;Ll;0;L; 0574 056B;;;;N;;;;; +FB16;ARMENIAN SMALL LIGATURE VEW NOW;Ll;0;L; 057E 0576;;;;N;;;;; +FB17;ARMENIAN SMALL LIGATURE MEN XEH;Ll;0;L; 0574 056D;;;;N;;;;; +FB1D;HEBREW LETTER YOD WITH HIRIQ;Lo;0;R;05D9 05B4;;;;N;;;;; +FB1E;HEBREW POINT JUDEO-SPANISH VARIKA;Mn;26;NSM;;;;;N;HEBREW POINT VARIKA;;;; +FB1F;HEBREW LIGATURE YIDDISH YOD YOD PATAH;Lo;0;R;05F2 05B7;;;;N;;;;; +FB20;HEBREW LETTER ALTERNATIVE AYIN;Lo;0;R; 05E2;;;;N;;;;; +FB21;HEBREW LETTER WIDE ALEF;Lo;0;R; 05D0;;;;N;;;;; +FB22;HEBREW LETTER WIDE DALET;Lo;0;R; 05D3;;;;N;;;;; +FB23;HEBREW LETTER WIDE HE;Lo;0;R; 05D4;;;;N;;;;; +FB24;HEBREW LETTER WIDE KAF;Lo;0;R; 05DB;;;;N;;;;; +FB25;HEBREW LETTER WIDE LAMED;Lo;0;R; 05DC;;;;N;;;;; +FB26;HEBREW LETTER WIDE FINAL MEM;Lo;0;R; 05DD;;;;N;;;;; +FB27;HEBREW LETTER WIDE RESH;Lo;0;R; 05E8;;;;N;;;;; +FB28;HEBREW LETTER WIDE TAV;Lo;0;R; 05EA;;;;N;;;;; +FB29;HEBREW LETTER ALTERNATIVE PLUS SIGN;Sm;0;ES; 002B;;;;N;;;;; +FB2A;HEBREW LETTER SHIN WITH SHIN DOT;Lo;0;R;05E9 05C1;;;;N;;;;; +FB2B;HEBREW LETTER SHIN WITH SIN DOT;Lo;0;R;05E9 05C2;;;;N;;;;; +FB2C;HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT;Lo;0;R;FB49 05C1;;;;N;;;;; +FB2D;HEBREW LETTER SHIN WITH DAGESH AND SIN DOT;Lo;0;R;FB49 05C2;;;;N;;;;; +FB2E;HEBREW LETTER ALEF WITH PATAH;Lo;0;R;05D0 05B7;;;;N;;;;; +FB2F;HEBREW LETTER ALEF WITH QAMATS;Lo;0;R;05D0 05B8;;;;N;;;;; +FB30;HEBREW LETTER ALEF WITH MAPIQ;Lo;0;R;05D0 05BC;;;;N;;;;; +FB31;HEBREW LETTER BET WITH DAGESH;Lo;0;R;05D1 05BC;;;;N;;;;; +FB32;HEBREW LETTER GIMEL WITH DAGESH;Lo;0;R;05D2 05BC;;;;N;;;;; +FB33;HEBREW LETTER DALET WITH DAGESH;Lo;0;R;05D3 05BC;;;;N;;;;; +FB34;HEBREW LETTER HE WITH MAPIQ;Lo;0;R;05D4 05BC;;;;N;;;;; +FB35;HEBREW LETTER VAV WITH DAGESH;Lo;0;R;05D5 05BC;;;;N;;;;; +FB36;HEBREW LETTER ZAYIN WITH DAGESH;Lo;0;R;05D6 05BC;;;;N;;;;; +FB38;HEBREW LETTER TET WITH DAGESH;Lo;0;R;05D8 05BC;;;;N;;;;; +FB39;HEBREW LETTER YOD WITH DAGESH;Lo;0;R;05D9 05BC;;;;N;;;;; +FB3A;HEBREW LETTER FINAL KAF WITH DAGESH;Lo;0;R;05DA 05BC;;;;N;;;;; +FB3B;HEBREW LETTER KAF WITH DAGESH;Lo;0;R;05DB 05BC;;;;N;;;;; +FB3C;HEBREW LETTER LAMED WITH DAGESH;Lo;0;R;05DC 05BC;;;;N;;;;; +FB3E;HEBREW LETTER MEM WITH DAGESH;Lo;0;R;05DE 05BC;;;;N;;;;; +FB40;HEBREW LETTER NUN WITH DAGESH;Lo;0;R;05E0 05BC;;;;N;;;;; +FB41;HEBREW LETTER SAMEKH WITH DAGESH;Lo;0;R;05E1 05BC;;;;N;;;;; +FB43;HEBREW LETTER FINAL PE WITH DAGESH;Lo;0;R;05E3 05BC;;;;N;;;;; +FB44;HEBREW LETTER PE WITH DAGESH;Lo;0;R;05E4 05BC;;;;N;;;;; +FB46;HEBREW LETTER TSADI WITH DAGESH;Lo;0;R;05E6 05BC;;;;N;;;;; +FB47;HEBREW LETTER QOF WITH DAGESH;Lo;0;R;05E7 05BC;;;;N;;;;; +FB48;HEBREW LETTER RESH WITH DAGESH;Lo;0;R;05E8 05BC;;;;N;;;;; +FB49;HEBREW LETTER SHIN WITH DAGESH;Lo;0;R;05E9 05BC;;;;N;;;;; +FB4A;HEBREW LETTER TAV WITH DAGESH;Lo;0;R;05EA 05BC;;;;N;;;;; +FB4B;HEBREW LETTER VAV WITH HOLAM;Lo;0;R;05D5 05B9;;;;N;;;;; +FB4C;HEBREW LETTER BET WITH RAFE;Lo;0;R;05D1 05BF;;;;N;;;;; +FB4D;HEBREW LETTER KAF WITH RAFE;Lo;0;R;05DB 05BF;;;;N;;;;; +FB4E;HEBREW LETTER PE WITH RAFE;Lo;0;R;05E4 05BF;;;;N;;;;; +FB4F;HEBREW LIGATURE ALEF LAMED;Lo;0;R; 05D0 05DC;;;;N;;;;; +FB50;ARABIC LETTER ALEF WASLA ISOLATED FORM;Lo;0;AL; 0671;;;;N;;;;; +FB51;ARABIC LETTER ALEF WASLA FINAL FORM;Lo;0;AL; 0671;;;;N;;;;; +FB52;ARABIC LETTER BEEH ISOLATED FORM;Lo;0;AL; 067B;;;;N;;;;; +FB53;ARABIC LETTER BEEH FINAL FORM;Lo;0;AL; 067B;;;;N;;;;; +FB54;ARABIC LETTER BEEH INITIAL FORM;Lo;0;AL; 067B;;;;N;;;;; +FB55;ARABIC LETTER BEEH MEDIAL FORM;Lo;0;AL; 067B;;;;N;;;;; +FB56;ARABIC LETTER PEH ISOLATED FORM;Lo;0;AL; 067E;;;;N;;;;; +FB57;ARABIC LETTER PEH FINAL FORM;Lo;0;AL; 067E;;;;N;;;;; +FB58;ARABIC LETTER PEH INITIAL FORM;Lo;0;AL; 067E;;;;N;;;;; +FB59;ARABIC LETTER PEH MEDIAL FORM;Lo;0;AL; 067E;;;;N;;;;; +FB5A;ARABIC LETTER BEHEH ISOLATED FORM;Lo;0;AL; 0680;;;;N;;;;; +FB5B;ARABIC LETTER BEHEH FINAL FORM;Lo;0;AL; 0680;;;;N;;;;; +FB5C;ARABIC LETTER BEHEH INITIAL FORM;Lo;0;AL; 0680;;;;N;;;;; +FB5D;ARABIC LETTER BEHEH MEDIAL FORM;Lo;0;AL; 0680;;;;N;;;;; +FB5E;ARABIC LETTER TTEHEH ISOLATED FORM;Lo;0;AL; 067A;;;;N;;;;; +FB5F;ARABIC LETTER TTEHEH FINAL FORM;Lo;0;AL; 067A;;;;N;;;;; +FB60;ARABIC LETTER TTEHEH INITIAL FORM;Lo;0;AL; 067A;;;;N;;;;; +FB61;ARABIC LETTER TTEHEH MEDIAL FORM;Lo;0;AL; 067A;;;;N;;;;; +FB62;ARABIC LETTER TEHEH ISOLATED FORM;Lo;0;AL; 067F;;;;N;;;;; +FB63;ARABIC LETTER TEHEH FINAL FORM;Lo;0;AL; 067F;;;;N;;;;; +FB64;ARABIC LETTER TEHEH INITIAL FORM;Lo;0;AL; 067F;;;;N;;;;; +FB65;ARABIC LETTER TEHEH MEDIAL FORM;Lo;0;AL; 067F;;;;N;;;;; +FB66;ARABIC LETTER TTEH ISOLATED FORM;Lo;0;AL; 0679;;;;N;;;;; +FB67;ARABIC LETTER TTEH FINAL FORM;Lo;0;AL; 0679;;;;N;;;;; +FB68;ARABIC LETTER TTEH INITIAL FORM;Lo;0;AL; 0679;;;;N;;;;; +FB69;ARABIC LETTER TTEH MEDIAL FORM;Lo;0;AL; 0679;;;;N;;;;; +FB6A;ARABIC LETTER VEH ISOLATED FORM;Lo;0;AL; 06A4;;;;N;;;;; +FB6B;ARABIC LETTER VEH FINAL FORM;Lo;0;AL; 06A4;;;;N;;;;; +FB6C;ARABIC LETTER VEH INITIAL FORM;Lo;0;AL; 06A4;;;;N;;;;; +FB6D;ARABIC LETTER VEH MEDIAL FORM;Lo;0;AL; 06A4;;;;N;;;;; +FB6E;ARABIC LETTER PEHEH ISOLATED FORM;Lo;0;AL; 06A6;;;;N;;;;; +FB6F;ARABIC LETTER PEHEH FINAL FORM;Lo;0;AL; 06A6;;;;N;;;;; +FB70;ARABIC LETTER PEHEH INITIAL FORM;Lo;0;AL; 06A6;;;;N;;;;; +FB71;ARABIC LETTER PEHEH MEDIAL FORM;Lo;0;AL; 06A6;;;;N;;;;; +FB72;ARABIC LETTER DYEH ISOLATED FORM;Lo;0;AL; 0684;;;;N;;;;; +FB73;ARABIC LETTER DYEH FINAL FORM;Lo;0;AL; 0684;;;;N;;;;; +FB74;ARABIC LETTER DYEH INITIAL FORM;Lo;0;AL; 0684;;;;N;;;;; +FB75;ARABIC LETTER DYEH MEDIAL FORM;Lo;0;AL; 0684;;;;N;;;;; +FB76;ARABIC LETTER NYEH ISOLATED FORM;Lo;0;AL; 0683;;;;N;;;;; +FB77;ARABIC LETTER NYEH FINAL FORM;Lo;0;AL; 0683;;;;N;;;;; +FB78;ARABIC LETTER NYEH INITIAL FORM;Lo;0;AL; 0683;;;;N;;;;; +FB79;ARABIC LETTER NYEH MEDIAL FORM;Lo;0;AL; 0683;;;;N;;;;; +FB7A;ARABIC LETTER TCHEH ISOLATED FORM;Lo;0;AL; 0686;;;;N;;;;; +FB7B;ARABIC LETTER TCHEH FINAL FORM;Lo;0;AL; 0686;;;;N;;;;; +FB7C;ARABIC LETTER TCHEH INITIAL FORM;Lo;0;AL; 0686;;;;N;;;;; +FB7D;ARABIC LETTER TCHEH MEDIAL FORM;Lo;0;AL; 0686;;;;N;;;;; +FB7E;ARABIC LETTER TCHEHEH ISOLATED FORM;Lo;0;AL; 0687;;;;N;;;;; +FB7F;ARABIC LETTER TCHEHEH FINAL FORM;Lo;0;AL; 0687;;;;N;;;;; +FB80;ARABIC LETTER TCHEHEH INITIAL FORM;Lo;0;AL; 0687;;;;N;;;;; +FB81;ARABIC LETTER TCHEHEH MEDIAL FORM;Lo;0;AL; 0687;;;;N;;;;; +FB82;ARABIC LETTER DDAHAL ISOLATED FORM;Lo;0;AL; 068D;;;;N;;;;; +FB83;ARABIC LETTER DDAHAL FINAL FORM;Lo;0;AL; 068D;;;;N;;;;; +FB84;ARABIC LETTER DAHAL ISOLATED FORM;Lo;0;AL; 068C;;;;N;;;;; +FB85;ARABIC LETTER DAHAL FINAL FORM;Lo;0;AL; 068C;;;;N;;;;; +FB86;ARABIC LETTER DUL ISOLATED FORM;Lo;0;AL; 068E;;;;N;;;;; +FB87;ARABIC LETTER DUL FINAL FORM;Lo;0;AL; 068E;;;;N;;;;; +FB88;ARABIC LETTER DDAL ISOLATED FORM;Lo;0;AL; 0688;;;;N;;;;; +FB89;ARABIC LETTER DDAL FINAL FORM;Lo;0;AL; 0688;;;;N;;;;; +FB8A;ARABIC LETTER JEH ISOLATED FORM;Lo;0;AL; 0698;;;;N;;;;; +FB8B;ARABIC LETTER JEH FINAL FORM;Lo;0;AL; 0698;;;;N;;;;; +FB8C;ARABIC LETTER RREH ISOLATED FORM;Lo;0;AL; 0691;;;;N;;;;; +FB8D;ARABIC LETTER RREH FINAL FORM;Lo;0;AL; 0691;;;;N;;;;; +FB8E;ARABIC LETTER KEHEH ISOLATED FORM;Lo;0;AL; 06A9;;;;N;;;;; +FB8F;ARABIC LETTER KEHEH FINAL FORM;Lo;0;AL; 06A9;;;;N;;;;; +FB90;ARABIC LETTER KEHEH INITIAL FORM;Lo;0;AL; 06A9;;;;N;;;;; +FB91;ARABIC LETTER KEHEH MEDIAL FORM;Lo;0;AL; 06A9;;;;N;;;;; +FB92;ARABIC LETTER GAF ISOLATED FORM;Lo;0;AL; 06AF;;;;N;;;;; +FB93;ARABIC LETTER GAF FINAL FORM;Lo;0;AL; 06AF;;;;N;;;;; +FB94;ARABIC LETTER GAF INITIAL FORM;Lo;0;AL; 06AF;;;;N;;;;; +FB95;ARABIC LETTER GAF MEDIAL FORM;Lo;0;AL; 06AF;;;;N;;;;; +FB96;ARABIC LETTER GUEH ISOLATED FORM;Lo;0;AL; 06B3;;;;N;;;;; +FB97;ARABIC LETTER GUEH FINAL FORM;Lo;0;AL; 06B3;;;;N;;;;; +FB98;ARABIC LETTER GUEH INITIAL FORM;Lo;0;AL; 06B3;;;;N;;;;; +FB99;ARABIC LETTER GUEH MEDIAL FORM;Lo;0;AL; 06B3;;;;N;;;;; +FB9A;ARABIC LETTER NGOEH ISOLATED FORM;Lo;0;AL; 06B1;;;;N;;;;; +FB9B;ARABIC LETTER NGOEH FINAL FORM;Lo;0;AL; 06B1;;;;N;;;;; +FB9C;ARABIC LETTER NGOEH INITIAL FORM;Lo;0;AL; 06B1;;;;N;;;;; +FB9D;ARABIC LETTER NGOEH MEDIAL FORM;Lo;0;AL; 06B1;;;;N;;;;; +FB9E;ARABIC LETTER NOON GHUNNA ISOLATED FORM;Lo;0;AL; 06BA;;;;N;;;;; +FB9F;ARABIC LETTER NOON GHUNNA FINAL FORM;Lo;0;AL; 06BA;;;;N;;;;; +FBA0;ARABIC LETTER RNOON ISOLATED FORM;Lo;0;AL; 06BB;;;;N;;;;; +FBA1;ARABIC LETTER RNOON FINAL FORM;Lo;0;AL; 06BB;;;;N;;;;; +FBA2;ARABIC LETTER RNOON INITIAL FORM;Lo;0;AL; 06BB;;;;N;;;;; +FBA3;ARABIC LETTER RNOON MEDIAL FORM;Lo;0;AL; 06BB;;;;N;;;;; +FBA4;ARABIC LETTER HEH WITH YEH ABOVE ISOLATED FORM;Lo;0;AL; 06C0;;;;N;;;;; +FBA5;ARABIC LETTER HEH WITH YEH ABOVE FINAL FORM;Lo;0;AL; 06C0;;;;N;;;;; +FBA6;ARABIC LETTER HEH GOAL ISOLATED FORM;Lo;0;AL; 06C1;;;;N;;;;; +FBA7;ARABIC LETTER HEH GOAL FINAL FORM;Lo;0;AL; 06C1;;;;N;;;;; +FBA8;ARABIC LETTER HEH GOAL INITIAL FORM;Lo;0;AL; 06C1;;;;N;;;;; +FBA9;ARABIC LETTER HEH GOAL MEDIAL FORM;Lo;0;AL; 06C1;;;;N;;;;; +FBAA;ARABIC LETTER HEH DOACHASHMEE ISOLATED FORM;Lo;0;AL; 06BE;;;;N;;;;; +FBAB;ARABIC LETTER HEH DOACHASHMEE FINAL FORM;Lo;0;AL; 06BE;;;;N;;;;; +FBAC;ARABIC LETTER HEH DOACHASHMEE INITIAL FORM;Lo;0;AL; 06BE;;;;N;;;;; +FBAD;ARABIC LETTER HEH DOACHASHMEE MEDIAL FORM;Lo;0;AL; 06BE;;;;N;;;;; +FBAE;ARABIC LETTER YEH BARREE ISOLATED FORM;Lo;0;AL; 06D2;;;;N;;;;; +FBAF;ARABIC LETTER YEH BARREE FINAL FORM;Lo;0;AL; 06D2;;;;N;;;;; +FBB0;ARABIC LETTER YEH BARREE WITH HAMZA ABOVE ISOLATED FORM;Lo;0;AL; 06D3;;;;N;;;;; +FBB1;ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM;Lo;0;AL; 06D3;;;;N;;;;; +FBB2;ARABIC SYMBOL DOT ABOVE;Sk;0;AL;;;;;N;;;;; +FBB3;ARABIC SYMBOL DOT BELOW;Sk;0;AL;;;;;N;;;;; +FBB4;ARABIC SYMBOL TWO DOTS ABOVE;Sk;0;AL;;;;;N;;;;; +FBB5;ARABIC SYMBOL TWO DOTS BELOW;Sk;0;AL;;;;;N;;;;; +FBB6;ARABIC SYMBOL THREE DOTS ABOVE;Sk;0;AL;;;;;N;;;;; +FBB7;ARABIC SYMBOL THREE DOTS BELOW;Sk;0;AL;;;;;N;;;;; +FBB8;ARABIC SYMBOL THREE DOTS POINTING DOWNWARDS ABOVE;Sk;0;AL;;;;;N;;;;; +FBB9;ARABIC SYMBOL THREE DOTS POINTING DOWNWARDS BELOW;Sk;0;AL;;;;;N;;;;; +FBBA;ARABIC SYMBOL FOUR DOTS ABOVE;Sk;0;AL;;;;;N;;;;; +FBBB;ARABIC SYMBOL FOUR DOTS BELOW;Sk;0;AL;;;;;N;;;;; +FBBC;ARABIC SYMBOL DOUBLE VERTICAL BAR BELOW;Sk;0;AL;;;;;N;;;;; +FBBD;ARABIC SYMBOL TWO DOTS VERTICALLY ABOVE;Sk;0;AL;;;;;N;;;;; +FBBE;ARABIC SYMBOL TWO DOTS VERTICALLY BELOW;Sk;0;AL;;;;;N;;;;; +FBBF;ARABIC SYMBOL RING;Sk;0;AL;;;;;N;;;;; +FBC0;ARABIC SYMBOL SMALL TAH ABOVE;Sk;0;AL;;;;;N;;;;; +FBC1;ARABIC SYMBOL SMALL TAH BELOW;Sk;0;AL;;;;;N;;;;; +FBD3;ARABIC LETTER NG ISOLATED FORM;Lo;0;AL; 06AD;;;;N;;;;; +FBD4;ARABIC LETTER NG FINAL FORM;Lo;0;AL; 06AD;;;;N;;;;; +FBD5;ARABIC LETTER NG INITIAL FORM;Lo;0;AL; 06AD;;;;N;;;;; +FBD6;ARABIC LETTER NG MEDIAL FORM;Lo;0;AL; 06AD;;;;N;;;;; +FBD7;ARABIC LETTER U ISOLATED FORM;Lo;0;AL; 06C7;;;;N;;;;; +FBD8;ARABIC LETTER U FINAL FORM;Lo;0;AL; 06C7;;;;N;;;;; +FBD9;ARABIC LETTER OE ISOLATED FORM;Lo;0;AL; 06C6;;;;N;;;;; +FBDA;ARABIC LETTER OE FINAL FORM;Lo;0;AL; 06C6;;;;N;;;;; +FBDB;ARABIC LETTER YU ISOLATED FORM;Lo;0;AL; 06C8;;;;N;;;;; +FBDC;ARABIC LETTER YU FINAL FORM;Lo;0;AL; 06C8;;;;N;;;;; +FBDD;ARABIC LETTER U WITH HAMZA ABOVE ISOLATED FORM;Lo;0;AL; 0677;;;;N;;;;; +FBDE;ARABIC LETTER VE ISOLATED FORM;Lo;0;AL; 06CB;;;;N;;;;; +FBDF;ARABIC LETTER VE FINAL FORM;Lo;0;AL; 06CB;;;;N;;;;; +FBE0;ARABIC LETTER KIRGHIZ OE ISOLATED FORM;Lo;0;AL; 06C5;;;;N;;;;; +FBE1;ARABIC LETTER KIRGHIZ OE FINAL FORM;Lo;0;AL; 06C5;;;;N;;;;; +FBE2;ARABIC LETTER KIRGHIZ YU ISOLATED FORM;Lo;0;AL; 06C9;;;;N;;;;; +FBE3;ARABIC LETTER KIRGHIZ YU FINAL FORM;Lo;0;AL; 06C9;;;;N;;;;; +FBE4;ARABIC LETTER E ISOLATED FORM;Lo;0;AL; 06D0;;;;N;;;;; +FBE5;ARABIC LETTER E FINAL FORM;Lo;0;AL; 06D0;;;;N;;;;; +FBE6;ARABIC LETTER E INITIAL FORM;Lo;0;AL; 06D0;;;;N;;;;; +FBE7;ARABIC LETTER E MEDIAL FORM;Lo;0;AL; 06D0;;;;N;;;;; +FBE8;ARABIC LETTER UIGHUR KAZAKH KIRGHIZ ALEF MAKSURA INITIAL FORM;Lo;0;AL; 0649;;;;N;;;;; +FBE9;ARABIC LETTER UIGHUR KAZAKH KIRGHIZ ALEF MAKSURA MEDIAL FORM;Lo;0;AL; 0649;;;;N;;;;; +FBEA;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF ISOLATED FORM;Lo;0;AL; 0626 0627;;;;N;;;;; +FBEB;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF FINAL FORM;Lo;0;AL; 0626 0627;;;;N;;;;; +FBEC;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH AE ISOLATED FORM;Lo;0;AL; 0626 06D5;;;;N;;;;; +FBED;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH AE FINAL FORM;Lo;0;AL; 0626 06D5;;;;N;;;;; +FBEE;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH WAW ISOLATED FORM;Lo;0;AL; 0626 0648;;;;N;;;;; +FBEF;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH WAW FINAL FORM;Lo;0;AL; 0626 0648;;;;N;;;;; +FBF0;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH U ISOLATED FORM;Lo;0;AL; 0626 06C7;;;;N;;;;; +FBF1;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH U FINAL FORM;Lo;0;AL; 0626 06C7;;;;N;;;;; +FBF2;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH OE ISOLATED FORM;Lo;0;AL; 0626 06C6;;;;N;;;;; +FBF3;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH OE FINAL FORM;Lo;0;AL; 0626 06C6;;;;N;;;;; +FBF4;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YU ISOLATED FORM;Lo;0;AL; 0626 06C8;;;;N;;;;; +FBF5;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YU FINAL FORM;Lo;0;AL; 0626 06C8;;;;N;;;;; +FBF6;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH E ISOLATED FORM;Lo;0;AL; 0626 06D0;;;;N;;;;; +FBF7;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH E FINAL FORM;Lo;0;AL; 0626 06D0;;;;N;;;;; +FBF8;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH E INITIAL FORM;Lo;0;AL; 0626 06D0;;;;N;;;;; +FBF9;ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0626 0649;;;;N;;;;; +FBFA;ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0626 0649;;;;N;;;;; +FBFB;ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA INITIAL FORM;Lo;0;AL; 0626 0649;;;;N;;;;; +FBFC;ARABIC LETTER FARSI YEH ISOLATED FORM;Lo;0;AL; 06CC;;;;N;;;;; +FBFD;ARABIC LETTER FARSI YEH FINAL FORM;Lo;0;AL; 06CC;;;;N;;;;; +FBFE;ARABIC LETTER FARSI YEH INITIAL FORM;Lo;0;AL; 06CC;;;;N;;;;; +FBFF;ARABIC LETTER FARSI YEH MEDIAL FORM;Lo;0;AL; 06CC;;;;N;;;;; +FC00;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH JEEM ISOLATED FORM;Lo;0;AL; 0626 062C;;;;N;;;;; +FC01;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HAH ISOLATED FORM;Lo;0;AL; 0626 062D;;;;N;;;;; +FC02;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM ISOLATED FORM;Lo;0;AL; 0626 0645;;;;N;;;;; +FC03;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0626 0649;;;;N;;;;; +FC04;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YEH ISOLATED FORM;Lo;0;AL; 0626 064A;;;;N;;;;; +FC05;ARABIC LIGATURE BEH WITH JEEM ISOLATED FORM;Lo;0;AL; 0628 062C;;;;N;;;;; +FC06;ARABIC LIGATURE BEH WITH HAH ISOLATED FORM;Lo;0;AL; 0628 062D;;;;N;;;;; +FC07;ARABIC LIGATURE BEH WITH KHAH ISOLATED FORM;Lo;0;AL; 0628 062E;;;;N;;;;; +FC08;ARABIC LIGATURE BEH WITH MEEM ISOLATED FORM;Lo;0;AL; 0628 0645;;;;N;;;;; +FC09;ARABIC LIGATURE BEH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0628 0649;;;;N;;;;; +FC0A;ARABIC LIGATURE BEH WITH YEH ISOLATED FORM;Lo;0;AL; 0628 064A;;;;N;;;;; +FC0B;ARABIC LIGATURE TEH WITH JEEM ISOLATED FORM;Lo;0;AL; 062A 062C;;;;N;;;;; +FC0C;ARABIC LIGATURE TEH WITH HAH ISOLATED FORM;Lo;0;AL; 062A 062D;;;;N;;;;; +FC0D;ARABIC LIGATURE TEH WITH KHAH ISOLATED FORM;Lo;0;AL; 062A 062E;;;;N;;;;; +FC0E;ARABIC LIGATURE TEH WITH MEEM ISOLATED FORM;Lo;0;AL; 062A 0645;;;;N;;;;; +FC0F;ARABIC LIGATURE TEH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 062A 0649;;;;N;;;;; +FC10;ARABIC LIGATURE TEH WITH YEH ISOLATED FORM;Lo;0;AL; 062A 064A;;;;N;;;;; +FC11;ARABIC LIGATURE THEH WITH JEEM ISOLATED FORM;Lo;0;AL; 062B 062C;;;;N;;;;; +FC12;ARABIC LIGATURE THEH WITH MEEM ISOLATED FORM;Lo;0;AL; 062B 0645;;;;N;;;;; +FC13;ARABIC LIGATURE THEH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 062B 0649;;;;N;;;;; +FC14;ARABIC LIGATURE THEH WITH YEH ISOLATED FORM;Lo;0;AL; 062B 064A;;;;N;;;;; +FC15;ARABIC LIGATURE JEEM WITH HAH ISOLATED FORM;Lo;0;AL; 062C 062D;;;;N;;;;; +FC16;ARABIC LIGATURE JEEM WITH MEEM ISOLATED FORM;Lo;0;AL; 062C 0645;;;;N;;;;; +FC17;ARABIC LIGATURE HAH WITH JEEM ISOLATED FORM;Lo;0;AL; 062D 062C;;;;N;;;;; +FC18;ARABIC LIGATURE HAH WITH MEEM ISOLATED FORM;Lo;0;AL; 062D 0645;;;;N;;;;; +FC19;ARABIC LIGATURE KHAH WITH JEEM ISOLATED FORM;Lo;0;AL; 062E 062C;;;;N;;;;; +FC1A;ARABIC LIGATURE KHAH WITH HAH ISOLATED FORM;Lo;0;AL; 062E 062D;;;;N;;;;; +FC1B;ARABIC LIGATURE KHAH WITH MEEM ISOLATED FORM;Lo;0;AL; 062E 0645;;;;N;;;;; +FC1C;ARABIC LIGATURE SEEN WITH JEEM ISOLATED FORM;Lo;0;AL; 0633 062C;;;;N;;;;; +FC1D;ARABIC LIGATURE SEEN WITH HAH ISOLATED FORM;Lo;0;AL; 0633 062D;;;;N;;;;; +FC1E;ARABIC LIGATURE SEEN WITH KHAH ISOLATED FORM;Lo;0;AL; 0633 062E;;;;N;;;;; +FC1F;ARABIC LIGATURE SEEN WITH MEEM ISOLATED FORM;Lo;0;AL; 0633 0645;;;;N;;;;; +FC20;ARABIC LIGATURE SAD WITH HAH ISOLATED FORM;Lo;0;AL; 0635 062D;;;;N;;;;; +FC21;ARABIC LIGATURE SAD WITH MEEM ISOLATED FORM;Lo;0;AL; 0635 0645;;;;N;;;;; +FC22;ARABIC LIGATURE DAD WITH JEEM ISOLATED FORM;Lo;0;AL; 0636 062C;;;;N;;;;; +FC23;ARABIC LIGATURE DAD WITH HAH ISOLATED FORM;Lo;0;AL; 0636 062D;;;;N;;;;; +FC24;ARABIC LIGATURE DAD WITH KHAH ISOLATED FORM;Lo;0;AL; 0636 062E;;;;N;;;;; +FC25;ARABIC LIGATURE DAD WITH MEEM ISOLATED FORM;Lo;0;AL; 0636 0645;;;;N;;;;; +FC26;ARABIC LIGATURE TAH WITH HAH ISOLATED FORM;Lo;0;AL; 0637 062D;;;;N;;;;; +FC27;ARABIC LIGATURE TAH WITH MEEM ISOLATED FORM;Lo;0;AL; 0637 0645;;;;N;;;;; +FC28;ARABIC LIGATURE ZAH WITH MEEM ISOLATED FORM;Lo;0;AL; 0638 0645;;;;N;;;;; +FC29;ARABIC LIGATURE AIN WITH JEEM ISOLATED FORM;Lo;0;AL; 0639 062C;;;;N;;;;; +FC2A;ARABIC LIGATURE AIN WITH MEEM ISOLATED FORM;Lo;0;AL; 0639 0645;;;;N;;;;; +FC2B;ARABIC LIGATURE GHAIN WITH JEEM ISOLATED FORM;Lo;0;AL; 063A 062C;;;;N;;;;; +FC2C;ARABIC LIGATURE GHAIN WITH MEEM ISOLATED FORM;Lo;0;AL; 063A 0645;;;;N;;;;; +FC2D;ARABIC LIGATURE FEH WITH JEEM ISOLATED FORM;Lo;0;AL; 0641 062C;;;;N;;;;; +FC2E;ARABIC LIGATURE FEH WITH HAH ISOLATED FORM;Lo;0;AL; 0641 062D;;;;N;;;;; +FC2F;ARABIC LIGATURE FEH WITH KHAH ISOLATED FORM;Lo;0;AL; 0641 062E;;;;N;;;;; +FC30;ARABIC LIGATURE FEH WITH MEEM ISOLATED FORM;Lo;0;AL; 0641 0645;;;;N;;;;; +FC31;ARABIC LIGATURE FEH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0641 0649;;;;N;;;;; +FC32;ARABIC LIGATURE FEH WITH YEH ISOLATED FORM;Lo;0;AL; 0641 064A;;;;N;;;;; +FC33;ARABIC LIGATURE QAF WITH HAH ISOLATED FORM;Lo;0;AL; 0642 062D;;;;N;;;;; +FC34;ARABIC LIGATURE QAF WITH MEEM ISOLATED FORM;Lo;0;AL; 0642 0645;;;;N;;;;; +FC35;ARABIC LIGATURE QAF WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0642 0649;;;;N;;;;; +FC36;ARABIC LIGATURE QAF WITH YEH ISOLATED FORM;Lo;0;AL; 0642 064A;;;;N;;;;; +FC37;ARABIC LIGATURE KAF WITH ALEF ISOLATED FORM;Lo;0;AL; 0643 0627;;;;N;;;;; +FC38;ARABIC LIGATURE KAF WITH JEEM ISOLATED FORM;Lo;0;AL; 0643 062C;;;;N;;;;; +FC39;ARABIC LIGATURE KAF WITH HAH ISOLATED FORM;Lo;0;AL; 0643 062D;;;;N;;;;; +FC3A;ARABIC LIGATURE KAF WITH KHAH ISOLATED FORM;Lo;0;AL; 0643 062E;;;;N;;;;; +FC3B;ARABIC LIGATURE KAF WITH LAM ISOLATED FORM;Lo;0;AL; 0643 0644;;;;N;;;;; +FC3C;ARABIC LIGATURE KAF WITH MEEM ISOLATED FORM;Lo;0;AL; 0643 0645;;;;N;;;;; +FC3D;ARABIC LIGATURE KAF WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0643 0649;;;;N;;;;; +FC3E;ARABIC LIGATURE KAF WITH YEH ISOLATED FORM;Lo;0;AL; 0643 064A;;;;N;;;;; +FC3F;ARABIC LIGATURE LAM WITH JEEM ISOLATED FORM;Lo;0;AL; 0644 062C;;;;N;;;;; +FC40;ARABIC LIGATURE LAM WITH HAH ISOLATED FORM;Lo;0;AL; 0644 062D;;;;N;;;;; +FC41;ARABIC LIGATURE LAM WITH KHAH ISOLATED FORM;Lo;0;AL; 0644 062E;;;;N;;;;; +FC42;ARABIC LIGATURE LAM WITH MEEM ISOLATED FORM;Lo;0;AL; 0644 0645;;;;N;;;;; +FC43;ARABIC LIGATURE LAM WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0644 0649;;;;N;;;;; +FC44;ARABIC LIGATURE LAM WITH YEH ISOLATED FORM;Lo;0;AL; 0644 064A;;;;N;;;;; +FC45;ARABIC LIGATURE MEEM WITH JEEM ISOLATED FORM;Lo;0;AL; 0645 062C;;;;N;;;;; +FC46;ARABIC LIGATURE MEEM WITH HAH ISOLATED FORM;Lo;0;AL; 0645 062D;;;;N;;;;; +FC47;ARABIC LIGATURE MEEM WITH KHAH ISOLATED FORM;Lo;0;AL; 0645 062E;;;;N;;;;; +FC48;ARABIC LIGATURE MEEM WITH MEEM ISOLATED FORM;Lo;0;AL; 0645 0645;;;;N;;;;; +FC49;ARABIC LIGATURE MEEM WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0645 0649;;;;N;;;;; +FC4A;ARABIC LIGATURE MEEM WITH YEH ISOLATED FORM;Lo;0;AL; 0645 064A;;;;N;;;;; +FC4B;ARABIC LIGATURE NOON WITH JEEM ISOLATED FORM;Lo;0;AL; 0646 062C;;;;N;;;;; +FC4C;ARABIC LIGATURE NOON WITH HAH ISOLATED FORM;Lo;0;AL; 0646 062D;;;;N;;;;; +FC4D;ARABIC LIGATURE NOON WITH KHAH ISOLATED FORM;Lo;0;AL; 0646 062E;;;;N;;;;; +FC4E;ARABIC LIGATURE NOON WITH MEEM ISOLATED FORM;Lo;0;AL; 0646 0645;;;;N;;;;; +FC4F;ARABIC LIGATURE NOON WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0646 0649;;;;N;;;;; +FC50;ARABIC LIGATURE NOON WITH YEH ISOLATED FORM;Lo;0;AL; 0646 064A;;;;N;;;;; +FC51;ARABIC LIGATURE HEH WITH JEEM ISOLATED FORM;Lo;0;AL; 0647 062C;;;;N;;;;; +FC52;ARABIC LIGATURE HEH WITH MEEM ISOLATED FORM;Lo;0;AL; 0647 0645;;;;N;;;;; +FC53;ARABIC LIGATURE HEH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0647 0649;;;;N;;;;; +FC54;ARABIC LIGATURE HEH WITH YEH ISOLATED FORM;Lo;0;AL; 0647 064A;;;;N;;;;; +FC55;ARABIC LIGATURE YEH WITH JEEM ISOLATED FORM;Lo;0;AL; 064A 062C;;;;N;;;;; +FC56;ARABIC LIGATURE YEH WITH HAH ISOLATED FORM;Lo;0;AL; 064A 062D;;;;N;;;;; +FC57;ARABIC LIGATURE YEH WITH KHAH ISOLATED FORM;Lo;0;AL; 064A 062E;;;;N;;;;; +FC58;ARABIC LIGATURE YEH WITH MEEM ISOLATED FORM;Lo;0;AL; 064A 0645;;;;N;;;;; +FC59;ARABIC LIGATURE YEH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 064A 0649;;;;N;;;;; +FC5A;ARABIC LIGATURE YEH WITH YEH ISOLATED FORM;Lo;0;AL; 064A 064A;;;;N;;;;; +FC5B;ARABIC LIGATURE THAL WITH SUPERSCRIPT ALEF ISOLATED FORM;Lo;0;AL; 0630 0670;;;;N;;;;; +FC5C;ARABIC LIGATURE REH WITH SUPERSCRIPT ALEF ISOLATED FORM;Lo;0;AL; 0631 0670;;;;N;;;;; +FC5D;ARABIC LIGATURE ALEF MAKSURA WITH SUPERSCRIPT ALEF ISOLATED FORM;Lo;0;AL; 0649 0670;;;;N;;;;; +FC5E;ARABIC LIGATURE SHADDA WITH DAMMATAN ISOLATED FORM;Lo;0;AL; 0020 064C 0651;;;;N;;;;; +FC5F;ARABIC LIGATURE SHADDA WITH KASRATAN ISOLATED FORM;Lo;0;AL; 0020 064D 0651;;;;N;;;;; +FC60;ARABIC LIGATURE SHADDA WITH FATHA ISOLATED FORM;Lo;0;AL; 0020 064E 0651;;;;N;;;;; +FC61;ARABIC LIGATURE SHADDA WITH DAMMA ISOLATED FORM;Lo;0;AL; 0020 064F 0651;;;;N;;;;; +FC62;ARABIC LIGATURE SHADDA WITH KASRA ISOLATED FORM;Lo;0;AL; 0020 0650 0651;;;;N;;;;; +FC63;ARABIC LIGATURE SHADDA WITH SUPERSCRIPT ALEF ISOLATED FORM;Lo;0;AL; 0020 0651 0670;;;;N;;;;; +FC64;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH REH FINAL FORM;Lo;0;AL; 0626 0631;;;;N;;;;; +FC65;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ZAIN FINAL FORM;Lo;0;AL; 0626 0632;;;;N;;;;; +FC66;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM FINAL FORM;Lo;0;AL; 0626 0645;;;;N;;;;; +FC67;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH NOON FINAL FORM;Lo;0;AL; 0626 0646;;;;N;;;;; +FC68;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0626 0649;;;;N;;;;; +FC69;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YEH FINAL FORM;Lo;0;AL; 0626 064A;;;;N;;;;; +FC6A;ARABIC LIGATURE BEH WITH REH FINAL FORM;Lo;0;AL; 0628 0631;;;;N;;;;; +FC6B;ARABIC LIGATURE BEH WITH ZAIN FINAL FORM;Lo;0;AL; 0628 0632;;;;N;;;;; +FC6C;ARABIC LIGATURE BEH WITH MEEM FINAL FORM;Lo;0;AL; 0628 0645;;;;N;;;;; +FC6D;ARABIC LIGATURE BEH WITH NOON FINAL FORM;Lo;0;AL; 0628 0646;;;;N;;;;; +FC6E;ARABIC LIGATURE BEH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0628 0649;;;;N;;;;; +FC6F;ARABIC LIGATURE BEH WITH YEH FINAL FORM;Lo;0;AL; 0628 064A;;;;N;;;;; +FC70;ARABIC LIGATURE TEH WITH REH FINAL FORM;Lo;0;AL; 062A 0631;;;;N;;;;; +FC71;ARABIC LIGATURE TEH WITH ZAIN FINAL FORM;Lo;0;AL; 062A 0632;;;;N;;;;; +FC72;ARABIC LIGATURE TEH WITH MEEM FINAL FORM;Lo;0;AL; 062A 0645;;;;N;;;;; +FC73;ARABIC LIGATURE TEH WITH NOON FINAL FORM;Lo;0;AL; 062A 0646;;;;N;;;;; +FC74;ARABIC LIGATURE TEH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062A 0649;;;;N;;;;; +FC75;ARABIC LIGATURE TEH WITH YEH FINAL FORM;Lo;0;AL; 062A 064A;;;;N;;;;; +FC76;ARABIC LIGATURE THEH WITH REH FINAL FORM;Lo;0;AL; 062B 0631;;;;N;;;;; +FC77;ARABIC LIGATURE THEH WITH ZAIN FINAL FORM;Lo;0;AL; 062B 0632;;;;N;;;;; +FC78;ARABIC LIGATURE THEH WITH MEEM FINAL FORM;Lo;0;AL; 062B 0645;;;;N;;;;; +FC79;ARABIC LIGATURE THEH WITH NOON FINAL FORM;Lo;0;AL; 062B 0646;;;;N;;;;; +FC7A;ARABIC LIGATURE THEH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062B 0649;;;;N;;;;; +FC7B;ARABIC LIGATURE THEH WITH YEH FINAL FORM;Lo;0;AL; 062B 064A;;;;N;;;;; +FC7C;ARABIC LIGATURE FEH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0641 0649;;;;N;;;;; +FC7D;ARABIC LIGATURE FEH WITH YEH FINAL FORM;Lo;0;AL; 0641 064A;;;;N;;;;; +FC7E;ARABIC LIGATURE QAF WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0642 0649;;;;N;;;;; +FC7F;ARABIC LIGATURE QAF WITH YEH FINAL FORM;Lo;0;AL; 0642 064A;;;;N;;;;; +FC80;ARABIC LIGATURE KAF WITH ALEF FINAL FORM;Lo;0;AL; 0643 0627;;;;N;;;;; +FC81;ARABIC LIGATURE KAF WITH LAM FINAL FORM;Lo;0;AL; 0643 0644;;;;N;;;;; +FC82;ARABIC LIGATURE KAF WITH MEEM FINAL FORM;Lo;0;AL; 0643 0645;;;;N;;;;; +FC83;ARABIC LIGATURE KAF WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0643 0649;;;;N;;;;; +FC84;ARABIC LIGATURE KAF WITH YEH FINAL FORM;Lo;0;AL; 0643 064A;;;;N;;;;; +FC85;ARABIC LIGATURE LAM WITH MEEM FINAL FORM;Lo;0;AL; 0644 0645;;;;N;;;;; +FC86;ARABIC LIGATURE LAM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0644 0649;;;;N;;;;; +FC87;ARABIC LIGATURE LAM WITH YEH FINAL FORM;Lo;0;AL; 0644 064A;;;;N;;;;; +FC88;ARABIC LIGATURE MEEM WITH ALEF FINAL FORM;Lo;0;AL; 0645 0627;;;;N;;;;; +FC89;ARABIC LIGATURE MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0645 0645;;;;N;;;;; +FC8A;ARABIC LIGATURE NOON WITH REH FINAL FORM;Lo;0;AL; 0646 0631;;;;N;;;;; +FC8B;ARABIC LIGATURE NOON WITH ZAIN FINAL FORM;Lo;0;AL; 0646 0632;;;;N;;;;; +FC8C;ARABIC LIGATURE NOON WITH MEEM FINAL FORM;Lo;0;AL; 0646 0645;;;;N;;;;; +FC8D;ARABIC LIGATURE NOON WITH NOON FINAL FORM;Lo;0;AL; 0646 0646;;;;N;;;;; +FC8E;ARABIC LIGATURE NOON WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0646 0649;;;;N;;;;; +FC8F;ARABIC LIGATURE NOON WITH YEH FINAL FORM;Lo;0;AL; 0646 064A;;;;N;;;;; +FC90;ARABIC LIGATURE ALEF MAKSURA WITH SUPERSCRIPT ALEF FINAL FORM;Lo;0;AL; 0649 0670;;;;N;;;;; +FC91;ARABIC LIGATURE YEH WITH REH FINAL FORM;Lo;0;AL; 064A 0631;;;;N;;;;; +FC92;ARABIC LIGATURE YEH WITH ZAIN FINAL FORM;Lo;0;AL; 064A 0632;;;;N;;;;; +FC93;ARABIC LIGATURE YEH WITH MEEM FINAL FORM;Lo;0;AL; 064A 0645;;;;N;;;;; +FC94;ARABIC LIGATURE YEH WITH NOON FINAL FORM;Lo;0;AL; 064A 0646;;;;N;;;;; +FC95;ARABIC LIGATURE YEH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 064A 0649;;;;N;;;;; +FC96;ARABIC LIGATURE YEH WITH YEH FINAL FORM;Lo;0;AL; 064A 064A;;;;N;;;;; +FC97;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH JEEM INITIAL FORM;Lo;0;AL; 0626 062C;;;;N;;;;; +FC98;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HAH INITIAL FORM;Lo;0;AL; 0626 062D;;;;N;;;;; +FC99;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH KHAH INITIAL FORM;Lo;0;AL; 0626 062E;;;;N;;;;; +FC9A;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM INITIAL FORM;Lo;0;AL; 0626 0645;;;;N;;;;; +FC9B;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HEH INITIAL FORM;Lo;0;AL; 0626 0647;;;;N;;;;; +FC9C;ARABIC LIGATURE BEH WITH JEEM INITIAL FORM;Lo;0;AL; 0628 062C;;;;N;;;;; +FC9D;ARABIC LIGATURE BEH WITH HAH INITIAL FORM;Lo;0;AL; 0628 062D;;;;N;;;;; +FC9E;ARABIC LIGATURE BEH WITH KHAH INITIAL FORM;Lo;0;AL; 0628 062E;;;;N;;;;; +FC9F;ARABIC LIGATURE BEH WITH MEEM INITIAL FORM;Lo;0;AL; 0628 0645;;;;N;;;;; +FCA0;ARABIC LIGATURE BEH WITH HEH INITIAL FORM;Lo;0;AL; 0628 0647;;;;N;;;;; +FCA1;ARABIC LIGATURE TEH WITH JEEM INITIAL FORM;Lo;0;AL; 062A 062C;;;;N;;;;; +FCA2;ARABIC LIGATURE TEH WITH HAH INITIAL FORM;Lo;0;AL; 062A 062D;;;;N;;;;; +FCA3;ARABIC LIGATURE TEH WITH KHAH INITIAL FORM;Lo;0;AL; 062A 062E;;;;N;;;;; +FCA4;ARABIC LIGATURE TEH WITH MEEM INITIAL FORM;Lo;0;AL; 062A 0645;;;;N;;;;; +FCA5;ARABIC LIGATURE TEH WITH HEH INITIAL FORM;Lo;0;AL; 062A 0647;;;;N;;;;; +FCA6;ARABIC LIGATURE THEH WITH MEEM INITIAL FORM;Lo;0;AL; 062B 0645;;;;N;;;;; +FCA7;ARABIC LIGATURE JEEM WITH HAH INITIAL FORM;Lo;0;AL; 062C 062D;;;;N;;;;; +FCA8;ARABIC LIGATURE JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 062C 0645;;;;N;;;;; +FCA9;ARABIC LIGATURE HAH WITH JEEM INITIAL FORM;Lo;0;AL; 062D 062C;;;;N;;;;; +FCAA;ARABIC LIGATURE HAH WITH MEEM INITIAL FORM;Lo;0;AL; 062D 0645;;;;N;;;;; +FCAB;ARABIC LIGATURE KHAH WITH JEEM INITIAL FORM;Lo;0;AL; 062E 062C;;;;N;;;;; +FCAC;ARABIC LIGATURE KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 062E 0645;;;;N;;;;; +FCAD;ARABIC LIGATURE SEEN WITH JEEM INITIAL FORM;Lo;0;AL; 0633 062C;;;;N;;;;; +FCAE;ARABIC LIGATURE SEEN WITH HAH INITIAL FORM;Lo;0;AL; 0633 062D;;;;N;;;;; +FCAF;ARABIC LIGATURE SEEN WITH KHAH INITIAL FORM;Lo;0;AL; 0633 062E;;;;N;;;;; +FCB0;ARABIC LIGATURE SEEN WITH MEEM INITIAL FORM;Lo;0;AL; 0633 0645;;;;N;;;;; +FCB1;ARABIC LIGATURE SAD WITH HAH INITIAL FORM;Lo;0;AL; 0635 062D;;;;N;;;;; +FCB2;ARABIC LIGATURE SAD WITH KHAH INITIAL FORM;Lo;0;AL; 0635 062E;;;;N;;;;; +FCB3;ARABIC LIGATURE SAD WITH MEEM INITIAL FORM;Lo;0;AL; 0635 0645;;;;N;;;;; +FCB4;ARABIC LIGATURE DAD WITH JEEM INITIAL FORM;Lo;0;AL; 0636 062C;;;;N;;;;; +FCB5;ARABIC LIGATURE DAD WITH HAH INITIAL FORM;Lo;0;AL; 0636 062D;;;;N;;;;; +FCB6;ARABIC LIGATURE DAD WITH KHAH INITIAL FORM;Lo;0;AL; 0636 062E;;;;N;;;;; +FCB7;ARABIC LIGATURE DAD WITH MEEM INITIAL FORM;Lo;0;AL; 0636 0645;;;;N;;;;; +FCB8;ARABIC LIGATURE TAH WITH HAH INITIAL FORM;Lo;0;AL; 0637 062D;;;;N;;;;; +FCB9;ARABIC LIGATURE ZAH WITH MEEM INITIAL FORM;Lo;0;AL; 0638 0645;;;;N;;;;; +FCBA;ARABIC LIGATURE AIN WITH JEEM INITIAL FORM;Lo;0;AL; 0639 062C;;;;N;;;;; +FCBB;ARABIC LIGATURE AIN WITH MEEM INITIAL FORM;Lo;0;AL; 0639 0645;;;;N;;;;; +FCBC;ARABIC LIGATURE GHAIN WITH JEEM INITIAL FORM;Lo;0;AL; 063A 062C;;;;N;;;;; +FCBD;ARABIC LIGATURE GHAIN WITH MEEM INITIAL FORM;Lo;0;AL; 063A 0645;;;;N;;;;; +FCBE;ARABIC LIGATURE FEH WITH JEEM INITIAL FORM;Lo;0;AL; 0641 062C;;;;N;;;;; +FCBF;ARABIC LIGATURE FEH WITH HAH INITIAL FORM;Lo;0;AL; 0641 062D;;;;N;;;;; +FCC0;ARABIC LIGATURE FEH WITH KHAH INITIAL FORM;Lo;0;AL; 0641 062E;;;;N;;;;; +FCC1;ARABIC LIGATURE FEH WITH MEEM INITIAL FORM;Lo;0;AL; 0641 0645;;;;N;;;;; +FCC2;ARABIC LIGATURE QAF WITH HAH INITIAL FORM;Lo;0;AL; 0642 062D;;;;N;;;;; +FCC3;ARABIC LIGATURE QAF WITH MEEM INITIAL FORM;Lo;0;AL; 0642 0645;;;;N;;;;; +FCC4;ARABIC LIGATURE KAF WITH JEEM INITIAL FORM;Lo;0;AL; 0643 062C;;;;N;;;;; +FCC5;ARABIC LIGATURE KAF WITH HAH INITIAL FORM;Lo;0;AL; 0643 062D;;;;N;;;;; +FCC6;ARABIC LIGATURE KAF WITH KHAH INITIAL FORM;Lo;0;AL; 0643 062E;;;;N;;;;; +FCC7;ARABIC LIGATURE KAF WITH LAM INITIAL FORM;Lo;0;AL; 0643 0644;;;;N;;;;; +FCC8;ARABIC LIGATURE KAF WITH MEEM INITIAL FORM;Lo;0;AL; 0643 0645;;;;N;;;;; +FCC9;ARABIC LIGATURE LAM WITH JEEM INITIAL FORM;Lo;0;AL; 0644 062C;;;;N;;;;; +FCCA;ARABIC LIGATURE LAM WITH HAH INITIAL FORM;Lo;0;AL; 0644 062D;;;;N;;;;; +FCCB;ARABIC LIGATURE LAM WITH KHAH INITIAL FORM;Lo;0;AL; 0644 062E;;;;N;;;;; +FCCC;ARABIC LIGATURE LAM WITH MEEM INITIAL FORM;Lo;0;AL; 0644 0645;;;;N;;;;; +FCCD;ARABIC LIGATURE LAM WITH HEH INITIAL FORM;Lo;0;AL; 0644 0647;;;;N;;;;; +FCCE;ARABIC LIGATURE MEEM WITH JEEM INITIAL FORM;Lo;0;AL; 0645 062C;;;;N;;;;; +FCCF;ARABIC LIGATURE MEEM WITH HAH INITIAL FORM;Lo;0;AL; 0645 062D;;;;N;;;;; +FCD0;ARABIC LIGATURE MEEM WITH KHAH INITIAL FORM;Lo;0;AL; 0645 062E;;;;N;;;;; +FCD1;ARABIC LIGATURE MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0645 0645;;;;N;;;;; +FCD2;ARABIC LIGATURE NOON WITH JEEM INITIAL FORM;Lo;0;AL; 0646 062C;;;;N;;;;; +FCD3;ARABIC LIGATURE NOON WITH HAH INITIAL FORM;Lo;0;AL; 0646 062D;;;;N;;;;; +FCD4;ARABIC LIGATURE NOON WITH KHAH INITIAL FORM;Lo;0;AL; 0646 062E;;;;N;;;;; +FCD5;ARABIC LIGATURE NOON WITH MEEM INITIAL FORM;Lo;0;AL; 0646 0645;;;;N;;;;; +FCD6;ARABIC LIGATURE NOON WITH HEH INITIAL FORM;Lo;0;AL; 0646 0647;;;;N;;;;; +FCD7;ARABIC LIGATURE HEH WITH JEEM INITIAL FORM;Lo;0;AL; 0647 062C;;;;N;;;;; +FCD8;ARABIC LIGATURE HEH WITH MEEM INITIAL FORM;Lo;0;AL; 0647 0645;;;;N;;;;; +FCD9;ARABIC LIGATURE HEH WITH SUPERSCRIPT ALEF INITIAL FORM;Lo;0;AL; 0647 0670;;;;N;;;;; +FCDA;ARABIC LIGATURE YEH WITH JEEM INITIAL FORM;Lo;0;AL; 064A 062C;;;;N;;;;; +FCDB;ARABIC LIGATURE YEH WITH HAH INITIAL FORM;Lo;0;AL; 064A 062D;;;;N;;;;; +FCDC;ARABIC LIGATURE YEH WITH KHAH INITIAL FORM;Lo;0;AL; 064A 062E;;;;N;;;;; +FCDD;ARABIC LIGATURE YEH WITH MEEM INITIAL FORM;Lo;0;AL; 064A 0645;;;;N;;;;; +FCDE;ARABIC LIGATURE YEH WITH HEH INITIAL FORM;Lo;0;AL; 064A 0647;;;;N;;;;; +FCDF;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM MEDIAL FORM;Lo;0;AL; 0626 0645;;;;N;;;;; +FCE0;ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HEH MEDIAL FORM;Lo;0;AL; 0626 0647;;;;N;;;;; +FCE1;ARABIC LIGATURE BEH WITH MEEM MEDIAL FORM;Lo;0;AL; 0628 0645;;;;N;;;;; +FCE2;ARABIC LIGATURE BEH WITH HEH MEDIAL FORM;Lo;0;AL; 0628 0647;;;;N;;;;; +FCE3;ARABIC LIGATURE TEH WITH MEEM MEDIAL FORM;Lo;0;AL; 062A 0645;;;;N;;;;; +FCE4;ARABIC LIGATURE TEH WITH HEH MEDIAL FORM;Lo;0;AL; 062A 0647;;;;N;;;;; +FCE5;ARABIC LIGATURE THEH WITH MEEM MEDIAL FORM;Lo;0;AL; 062B 0645;;;;N;;;;; +FCE6;ARABIC LIGATURE THEH WITH HEH MEDIAL FORM;Lo;0;AL; 062B 0647;;;;N;;;;; +FCE7;ARABIC LIGATURE SEEN WITH MEEM MEDIAL FORM;Lo;0;AL; 0633 0645;;;;N;;;;; +FCE8;ARABIC LIGATURE SEEN WITH HEH MEDIAL FORM;Lo;0;AL; 0633 0647;;;;N;;;;; +FCE9;ARABIC LIGATURE SHEEN WITH MEEM MEDIAL FORM;Lo;0;AL; 0634 0645;;;;N;;;;; +FCEA;ARABIC LIGATURE SHEEN WITH HEH MEDIAL FORM;Lo;0;AL; 0634 0647;;;;N;;;;; +FCEB;ARABIC LIGATURE KAF WITH LAM MEDIAL FORM;Lo;0;AL; 0643 0644;;;;N;;;;; +FCEC;ARABIC LIGATURE KAF WITH MEEM MEDIAL FORM;Lo;0;AL; 0643 0645;;;;N;;;;; +FCED;ARABIC LIGATURE LAM WITH MEEM MEDIAL FORM;Lo;0;AL; 0644 0645;;;;N;;;;; +FCEE;ARABIC LIGATURE NOON WITH MEEM MEDIAL FORM;Lo;0;AL; 0646 0645;;;;N;;;;; +FCEF;ARABIC LIGATURE NOON WITH HEH MEDIAL FORM;Lo;0;AL; 0646 0647;;;;N;;;;; +FCF0;ARABIC LIGATURE YEH WITH MEEM MEDIAL FORM;Lo;0;AL; 064A 0645;;;;N;;;;; +FCF1;ARABIC LIGATURE YEH WITH HEH MEDIAL FORM;Lo;0;AL; 064A 0647;;;;N;;;;; +FCF2;ARABIC LIGATURE SHADDA WITH FATHA MEDIAL FORM;Lo;0;AL; 0640 064E 0651;;;;N;;;;; +FCF3;ARABIC LIGATURE SHADDA WITH DAMMA MEDIAL FORM;Lo;0;AL; 0640 064F 0651;;;;N;;;;; +FCF4;ARABIC LIGATURE SHADDA WITH KASRA MEDIAL FORM;Lo;0;AL; 0640 0650 0651;;;;N;;;;; +FCF5;ARABIC LIGATURE TAH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0637 0649;;;;N;;;;; +FCF6;ARABIC LIGATURE TAH WITH YEH ISOLATED FORM;Lo;0;AL; 0637 064A;;;;N;;;;; +FCF7;ARABIC LIGATURE AIN WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0639 0649;;;;N;;;;; +FCF8;ARABIC LIGATURE AIN WITH YEH ISOLATED FORM;Lo;0;AL; 0639 064A;;;;N;;;;; +FCF9;ARABIC LIGATURE GHAIN WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 063A 0649;;;;N;;;;; +FCFA;ARABIC LIGATURE GHAIN WITH YEH ISOLATED FORM;Lo;0;AL; 063A 064A;;;;N;;;;; +FCFB;ARABIC LIGATURE SEEN WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0633 0649;;;;N;;;;; +FCFC;ARABIC LIGATURE SEEN WITH YEH ISOLATED FORM;Lo;0;AL; 0633 064A;;;;N;;;;; +FCFD;ARABIC LIGATURE SHEEN WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0634 0649;;;;N;;;;; +FCFE;ARABIC LIGATURE SHEEN WITH YEH ISOLATED FORM;Lo;0;AL; 0634 064A;;;;N;;;;; +FCFF;ARABIC LIGATURE HAH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 062D 0649;;;;N;;;;; +FD00;ARABIC LIGATURE HAH WITH YEH ISOLATED FORM;Lo;0;AL; 062D 064A;;;;N;;;;; +FD01;ARABIC LIGATURE JEEM WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 062C 0649;;;;N;;;;; +FD02;ARABIC LIGATURE JEEM WITH YEH ISOLATED FORM;Lo;0;AL; 062C 064A;;;;N;;;;; +FD03;ARABIC LIGATURE KHAH WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 062E 0649;;;;N;;;;; +FD04;ARABIC LIGATURE KHAH WITH YEH ISOLATED FORM;Lo;0;AL; 062E 064A;;;;N;;;;; +FD05;ARABIC LIGATURE SAD WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0635 0649;;;;N;;;;; +FD06;ARABIC LIGATURE SAD WITH YEH ISOLATED FORM;Lo;0;AL; 0635 064A;;;;N;;;;; +FD07;ARABIC LIGATURE DAD WITH ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0636 0649;;;;N;;;;; +FD08;ARABIC LIGATURE DAD WITH YEH ISOLATED FORM;Lo;0;AL; 0636 064A;;;;N;;;;; +FD09;ARABIC LIGATURE SHEEN WITH JEEM ISOLATED FORM;Lo;0;AL; 0634 062C;;;;N;;;;; +FD0A;ARABIC LIGATURE SHEEN WITH HAH ISOLATED FORM;Lo;0;AL; 0634 062D;;;;N;;;;; +FD0B;ARABIC LIGATURE SHEEN WITH KHAH ISOLATED FORM;Lo;0;AL; 0634 062E;;;;N;;;;; +FD0C;ARABIC LIGATURE SHEEN WITH MEEM ISOLATED FORM;Lo;0;AL; 0634 0645;;;;N;;;;; +FD0D;ARABIC LIGATURE SHEEN WITH REH ISOLATED FORM;Lo;0;AL; 0634 0631;;;;N;;;;; +FD0E;ARABIC LIGATURE SEEN WITH REH ISOLATED FORM;Lo;0;AL; 0633 0631;;;;N;;;;; +FD0F;ARABIC LIGATURE SAD WITH REH ISOLATED FORM;Lo;0;AL; 0635 0631;;;;N;;;;; +FD10;ARABIC LIGATURE DAD WITH REH ISOLATED FORM;Lo;0;AL; 0636 0631;;;;N;;;;; +FD11;ARABIC LIGATURE TAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0637 0649;;;;N;;;;; +FD12;ARABIC LIGATURE TAH WITH YEH FINAL FORM;Lo;0;AL; 0637 064A;;;;N;;;;; +FD13;ARABIC LIGATURE AIN WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0639 0649;;;;N;;;;; +FD14;ARABIC LIGATURE AIN WITH YEH FINAL FORM;Lo;0;AL; 0639 064A;;;;N;;;;; +FD15;ARABIC LIGATURE GHAIN WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 063A 0649;;;;N;;;;; +FD16;ARABIC LIGATURE GHAIN WITH YEH FINAL FORM;Lo;0;AL; 063A 064A;;;;N;;;;; +FD17;ARABIC LIGATURE SEEN WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0633 0649;;;;N;;;;; +FD18;ARABIC LIGATURE SEEN WITH YEH FINAL FORM;Lo;0;AL; 0633 064A;;;;N;;;;; +FD19;ARABIC LIGATURE SHEEN WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0634 0649;;;;N;;;;; +FD1A;ARABIC LIGATURE SHEEN WITH YEH FINAL FORM;Lo;0;AL; 0634 064A;;;;N;;;;; +FD1B;ARABIC LIGATURE HAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062D 0649;;;;N;;;;; +FD1C;ARABIC LIGATURE HAH WITH YEH FINAL FORM;Lo;0;AL; 062D 064A;;;;N;;;;; +FD1D;ARABIC LIGATURE JEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062C 0649;;;;N;;;;; +FD1E;ARABIC LIGATURE JEEM WITH YEH FINAL FORM;Lo;0;AL; 062C 064A;;;;N;;;;; +FD1F;ARABIC LIGATURE KHAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062E 0649;;;;N;;;;; +FD20;ARABIC LIGATURE KHAH WITH YEH FINAL FORM;Lo;0;AL; 062E 064A;;;;N;;;;; +FD21;ARABIC LIGATURE SAD WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0635 0649;;;;N;;;;; +FD22;ARABIC LIGATURE SAD WITH YEH FINAL FORM;Lo;0;AL; 0635 064A;;;;N;;;;; +FD23;ARABIC LIGATURE DAD WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0636 0649;;;;N;;;;; +FD24;ARABIC LIGATURE DAD WITH YEH FINAL FORM;Lo;0;AL; 0636 064A;;;;N;;;;; +FD25;ARABIC LIGATURE SHEEN WITH JEEM FINAL FORM;Lo;0;AL; 0634 062C;;;;N;;;;; +FD26;ARABIC LIGATURE SHEEN WITH HAH FINAL FORM;Lo;0;AL; 0634 062D;;;;N;;;;; +FD27;ARABIC LIGATURE SHEEN WITH KHAH FINAL FORM;Lo;0;AL; 0634 062E;;;;N;;;;; +FD28;ARABIC LIGATURE SHEEN WITH MEEM FINAL FORM;Lo;0;AL; 0634 0645;;;;N;;;;; +FD29;ARABIC LIGATURE SHEEN WITH REH FINAL FORM;Lo;0;AL; 0634 0631;;;;N;;;;; +FD2A;ARABIC LIGATURE SEEN WITH REH FINAL FORM;Lo;0;AL; 0633 0631;;;;N;;;;; +FD2B;ARABIC LIGATURE SAD WITH REH FINAL FORM;Lo;0;AL; 0635 0631;;;;N;;;;; +FD2C;ARABIC LIGATURE DAD WITH REH FINAL FORM;Lo;0;AL; 0636 0631;;;;N;;;;; +FD2D;ARABIC LIGATURE SHEEN WITH JEEM INITIAL FORM;Lo;0;AL; 0634 062C;;;;N;;;;; +FD2E;ARABIC LIGATURE SHEEN WITH HAH INITIAL FORM;Lo;0;AL; 0634 062D;;;;N;;;;; +FD2F;ARABIC LIGATURE SHEEN WITH KHAH INITIAL FORM;Lo;0;AL; 0634 062E;;;;N;;;;; +FD30;ARABIC LIGATURE SHEEN WITH MEEM INITIAL FORM;Lo;0;AL; 0634 0645;;;;N;;;;; +FD31;ARABIC LIGATURE SEEN WITH HEH INITIAL FORM;Lo;0;AL; 0633 0647;;;;N;;;;; +FD32;ARABIC LIGATURE SHEEN WITH HEH INITIAL FORM;Lo;0;AL; 0634 0647;;;;N;;;;; +FD33;ARABIC LIGATURE TAH WITH MEEM INITIAL FORM;Lo;0;AL; 0637 0645;;;;N;;;;; +FD34;ARABIC LIGATURE SEEN WITH JEEM MEDIAL FORM;Lo;0;AL; 0633 062C;;;;N;;;;; +FD35;ARABIC LIGATURE SEEN WITH HAH MEDIAL FORM;Lo;0;AL; 0633 062D;;;;N;;;;; +FD36;ARABIC LIGATURE SEEN WITH KHAH MEDIAL FORM;Lo;0;AL; 0633 062E;;;;N;;;;; +FD37;ARABIC LIGATURE SHEEN WITH JEEM MEDIAL FORM;Lo;0;AL; 0634 062C;;;;N;;;;; +FD38;ARABIC LIGATURE SHEEN WITH HAH MEDIAL FORM;Lo;0;AL; 0634 062D;;;;N;;;;; +FD39;ARABIC LIGATURE SHEEN WITH KHAH MEDIAL FORM;Lo;0;AL; 0634 062E;;;;N;;;;; +FD3A;ARABIC LIGATURE TAH WITH MEEM MEDIAL FORM;Lo;0;AL; 0637 0645;;;;N;;;;; +FD3B;ARABIC LIGATURE ZAH WITH MEEM MEDIAL FORM;Lo;0;AL; 0638 0645;;;;N;;;;; +FD3C;ARABIC LIGATURE ALEF WITH FATHATAN FINAL FORM;Lo;0;AL; 0627 064B;;;;N;;;;; +FD3D;ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM;Lo;0;AL; 0627 064B;;;;N;;;;; +FD3E;ORNATE LEFT PARENTHESIS;Ps;0;ON;;;;;N;;;;; +FD3F;ORNATE RIGHT PARENTHESIS;Pe;0;ON;;;;;N;;;;; +FD50;ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 062A 062C 0645;;;;N;;;;; +FD51;ARABIC LIGATURE TEH WITH HAH WITH JEEM FINAL FORM;Lo;0;AL; 062A 062D 062C;;;;N;;;;; +FD52;ARABIC LIGATURE TEH WITH HAH WITH JEEM INITIAL FORM;Lo;0;AL; 062A 062D 062C;;;;N;;;;; +FD53;ARABIC LIGATURE TEH WITH HAH WITH MEEM INITIAL FORM;Lo;0;AL; 062A 062D 0645;;;;N;;;;; +FD54;ARABIC LIGATURE TEH WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 062A 062E 0645;;;;N;;;;; +FD55;ARABIC LIGATURE TEH WITH MEEM WITH JEEM INITIAL FORM;Lo;0;AL; 062A 0645 062C;;;;N;;;;; +FD56;ARABIC LIGATURE TEH WITH MEEM WITH HAH INITIAL FORM;Lo;0;AL; 062A 0645 062D;;;;N;;;;; +FD57;ARABIC LIGATURE TEH WITH MEEM WITH KHAH INITIAL FORM;Lo;0;AL; 062A 0645 062E;;;;N;;;;; +FD58;ARABIC LIGATURE JEEM WITH MEEM WITH HAH FINAL FORM;Lo;0;AL; 062C 0645 062D;;;;N;;;;; +FD59;ARABIC LIGATURE JEEM WITH MEEM WITH HAH INITIAL FORM;Lo;0;AL; 062C 0645 062D;;;;N;;;;; +FD5A;ARABIC LIGATURE HAH WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 062D 0645 064A;;;;N;;;;; +FD5B;ARABIC LIGATURE HAH WITH MEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062D 0645 0649;;;;N;;;;; +FD5C;ARABIC LIGATURE SEEN WITH HAH WITH JEEM INITIAL FORM;Lo;0;AL; 0633 062D 062C;;;;N;;;;; +FD5D;ARABIC LIGATURE SEEN WITH JEEM WITH HAH INITIAL FORM;Lo;0;AL; 0633 062C 062D;;;;N;;;;; +FD5E;ARABIC LIGATURE SEEN WITH JEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0633 062C 0649;;;;N;;;;; +FD5F;ARABIC LIGATURE SEEN WITH MEEM WITH HAH FINAL FORM;Lo;0;AL; 0633 0645 062D;;;;N;;;;; +FD60;ARABIC LIGATURE SEEN WITH MEEM WITH HAH INITIAL FORM;Lo;0;AL; 0633 0645 062D;;;;N;;;;; +FD61;ARABIC LIGATURE SEEN WITH MEEM WITH JEEM INITIAL FORM;Lo;0;AL; 0633 0645 062C;;;;N;;;;; +FD62;ARABIC LIGATURE SEEN WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0633 0645 0645;;;;N;;;;; +FD63;ARABIC LIGATURE SEEN WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0633 0645 0645;;;;N;;;;; +FD64;ARABIC LIGATURE SAD WITH HAH WITH HAH FINAL FORM;Lo;0;AL; 0635 062D 062D;;;;N;;;;; +FD65;ARABIC LIGATURE SAD WITH HAH WITH HAH INITIAL FORM;Lo;0;AL; 0635 062D 062D;;;;N;;;;; +FD66;ARABIC LIGATURE SAD WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0635 0645 0645;;;;N;;;;; +FD67;ARABIC LIGATURE SHEEN WITH HAH WITH MEEM FINAL FORM;Lo;0;AL; 0634 062D 0645;;;;N;;;;; +FD68;ARABIC LIGATURE SHEEN WITH HAH WITH MEEM INITIAL FORM;Lo;0;AL; 0634 062D 0645;;;;N;;;;; +FD69;ARABIC LIGATURE SHEEN WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 0634 062C 064A;;;;N;;;;; +FD6A;ARABIC LIGATURE SHEEN WITH MEEM WITH KHAH FINAL FORM;Lo;0;AL; 0634 0645 062E;;;;N;;;;; +FD6B;ARABIC LIGATURE SHEEN WITH MEEM WITH KHAH INITIAL FORM;Lo;0;AL; 0634 0645 062E;;;;N;;;;; +FD6C;ARABIC LIGATURE SHEEN WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0634 0645 0645;;;;N;;;;; +FD6D;ARABIC LIGATURE SHEEN WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0634 0645 0645;;;;N;;;;; +FD6E;ARABIC LIGATURE DAD WITH HAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0636 062D 0649;;;;N;;;;; +FD6F;ARABIC LIGATURE DAD WITH KHAH WITH MEEM FINAL FORM;Lo;0;AL; 0636 062E 0645;;;;N;;;;; +FD70;ARABIC LIGATURE DAD WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 0636 062E 0645;;;;N;;;;; +FD71;ARABIC LIGATURE TAH WITH MEEM WITH HAH FINAL FORM;Lo;0;AL; 0637 0645 062D;;;;N;;;;; +FD72;ARABIC LIGATURE TAH WITH MEEM WITH HAH INITIAL FORM;Lo;0;AL; 0637 0645 062D;;;;N;;;;; +FD73;ARABIC LIGATURE TAH WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0637 0645 0645;;;;N;;;;; +FD74;ARABIC LIGATURE TAH WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0637 0645 064A;;;;N;;;;; +FD75;ARABIC LIGATURE AIN WITH JEEM WITH MEEM FINAL FORM;Lo;0;AL; 0639 062C 0645;;;;N;;;;; +FD76;ARABIC LIGATURE AIN WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0639 0645 0645;;;;N;;;;; +FD77;ARABIC LIGATURE AIN WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0639 0645 0645;;;;N;;;;; +FD78;ARABIC LIGATURE AIN WITH MEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0639 0645 0649;;;;N;;;;; +FD79;ARABIC LIGATURE GHAIN WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 063A 0645 0645;;;;N;;;;; +FD7A;ARABIC LIGATURE GHAIN WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 063A 0645 064A;;;;N;;;;; +FD7B;ARABIC LIGATURE GHAIN WITH MEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 063A 0645 0649;;;;N;;;;; +FD7C;ARABIC LIGATURE FEH WITH KHAH WITH MEEM FINAL FORM;Lo;0;AL; 0641 062E 0645;;;;N;;;;; +FD7D;ARABIC LIGATURE FEH WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 0641 062E 0645;;;;N;;;;; +FD7E;ARABIC LIGATURE QAF WITH MEEM WITH HAH FINAL FORM;Lo;0;AL; 0642 0645 062D;;;;N;;;;; +FD7F;ARABIC LIGATURE QAF WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0642 0645 0645;;;;N;;;;; +FD80;ARABIC LIGATURE LAM WITH HAH WITH MEEM FINAL FORM;Lo;0;AL; 0644 062D 0645;;;;N;;;;; +FD81;ARABIC LIGATURE LAM WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0644 062D 064A;;;;N;;;;; +FD82;ARABIC LIGATURE LAM WITH HAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0644 062D 0649;;;;N;;;;; +FD83;ARABIC LIGATURE LAM WITH JEEM WITH JEEM INITIAL FORM;Lo;0;AL; 0644 062C 062C;;;;N;;;;; +FD84;ARABIC LIGATURE LAM WITH JEEM WITH JEEM FINAL FORM;Lo;0;AL; 0644 062C 062C;;;;N;;;;; +FD85;ARABIC LIGATURE LAM WITH KHAH WITH MEEM FINAL FORM;Lo;0;AL; 0644 062E 0645;;;;N;;;;; +FD86;ARABIC LIGATURE LAM WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 0644 062E 0645;;;;N;;;;; +FD87;ARABIC LIGATURE LAM WITH MEEM WITH HAH FINAL FORM;Lo;0;AL; 0644 0645 062D;;;;N;;;;; +FD88;ARABIC LIGATURE LAM WITH MEEM WITH HAH INITIAL FORM;Lo;0;AL; 0644 0645 062D;;;;N;;;;; +FD89;ARABIC LIGATURE MEEM WITH HAH WITH JEEM INITIAL FORM;Lo;0;AL; 0645 062D 062C;;;;N;;;;; +FD8A;ARABIC LIGATURE MEEM WITH HAH WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062D 0645;;;;N;;;;; +FD8B;ARABIC LIGATURE MEEM WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0645 062D 064A;;;;N;;;;; +FD8C;ARABIC LIGATURE MEEM WITH JEEM WITH HAH INITIAL FORM;Lo;0;AL; 0645 062C 062D;;;;N;;;;; +FD8D;ARABIC LIGATURE MEEM WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062C 0645;;;;N;;;;; +FD8E;ARABIC LIGATURE MEEM WITH KHAH WITH JEEM INITIAL FORM;Lo;0;AL; 0645 062E 062C;;;;N;;;;; +FD8F;ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062E 0645;;;;N;;;;; +FD92;ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM;Lo;0;AL; 0645 062C 062E;;;;N;;;;; +FD93;ARABIC LIGATURE HEH WITH MEEM WITH JEEM INITIAL FORM;Lo;0;AL; 0647 0645 062C;;;;N;;;;; +FD94;ARABIC LIGATURE HEH WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0647 0645 0645;;;;N;;;;; +FD95;ARABIC LIGATURE NOON WITH HAH WITH MEEM INITIAL FORM;Lo;0;AL; 0646 062D 0645;;;;N;;;;; +FD96;ARABIC LIGATURE NOON WITH HAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0646 062D 0649;;;;N;;;;; +FD97;ARABIC LIGATURE NOON WITH JEEM WITH MEEM FINAL FORM;Lo;0;AL; 0646 062C 0645;;;;N;;;;; +FD98;ARABIC LIGATURE NOON WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0646 062C 0645;;;;N;;;;; +FD99;ARABIC LIGATURE NOON WITH JEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0646 062C 0649;;;;N;;;;; +FD9A;ARABIC LIGATURE NOON WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0646 0645 064A;;;;N;;;;; +FD9B;ARABIC LIGATURE NOON WITH MEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0646 0645 0649;;;;N;;;;; +FD9C;ARABIC LIGATURE YEH WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 064A 0645 0645;;;;N;;;;; +FD9D;ARABIC LIGATURE YEH WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 064A 0645 0645;;;;N;;;;; +FD9E;ARABIC LIGATURE BEH WITH KHAH WITH YEH FINAL FORM;Lo;0;AL; 0628 062E 064A;;;;N;;;;; +FD9F;ARABIC LIGATURE TEH WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 062A 062C 064A;;;;N;;;;; +FDA0;ARABIC LIGATURE TEH WITH JEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062A 062C 0649;;;;N;;;;; +FDA1;ARABIC LIGATURE TEH WITH KHAH WITH YEH FINAL FORM;Lo;0;AL; 062A 062E 064A;;;;N;;;;; +FDA2;ARABIC LIGATURE TEH WITH KHAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062A 062E 0649;;;;N;;;;; +FDA3;ARABIC LIGATURE TEH WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 062A 0645 064A;;;;N;;;;; +FDA4;ARABIC LIGATURE TEH WITH MEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062A 0645 0649;;;;N;;;;; +FDA5;ARABIC LIGATURE JEEM WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 062C 0645 064A;;;;N;;;;; +FDA6;ARABIC LIGATURE JEEM WITH HAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062C 062D 0649;;;;N;;;;; +FDA7;ARABIC LIGATURE JEEM WITH MEEM WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 062C 0645 0649;;;;N;;;;; +FDA8;ARABIC LIGATURE SEEN WITH KHAH WITH ALEF MAKSURA FINAL FORM;Lo;0;AL; 0633 062E 0649;;;;N;;;;; +FDA9;ARABIC LIGATURE SAD WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0635 062D 064A;;;;N;;;;; +FDAA;ARABIC LIGATURE SHEEN WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0634 062D 064A;;;;N;;;;; +FDAB;ARABIC LIGATURE DAD WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0636 062D 064A;;;;N;;;;; +FDAC;ARABIC LIGATURE LAM WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 0644 062C 064A;;;;N;;;;; +FDAD;ARABIC LIGATURE LAM WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0644 0645 064A;;;;N;;;;; +FDAE;ARABIC LIGATURE YEH WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 064A 062D 064A;;;;N;;;;; +FDAF;ARABIC LIGATURE YEH WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 064A 062C 064A;;;;N;;;;; +FDB0;ARABIC LIGATURE YEH WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 064A 0645 064A;;;;N;;;;; +FDB1;ARABIC LIGATURE MEEM WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0645 0645 064A;;;;N;;;;; +FDB2;ARABIC LIGATURE QAF WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0642 0645 064A;;;;N;;;;; +FDB3;ARABIC LIGATURE NOON WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0646 062D 064A;;;;N;;;;; +FDB4;ARABIC LIGATURE QAF WITH MEEM WITH HAH INITIAL FORM;Lo;0;AL; 0642 0645 062D;;;;N;;;;; +FDB5;ARABIC LIGATURE LAM WITH HAH WITH MEEM INITIAL FORM;Lo;0;AL; 0644 062D 0645;;;;N;;;;; +FDB6;ARABIC LIGATURE AIN WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0639 0645 064A;;;;N;;;;; +FDB7;ARABIC LIGATURE KAF WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0643 0645 064A;;;;N;;;;; +FDB8;ARABIC LIGATURE NOON WITH JEEM WITH HAH INITIAL FORM;Lo;0;AL; 0646 062C 062D;;;;N;;;;; +FDB9;ARABIC LIGATURE MEEM WITH KHAH WITH YEH FINAL FORM;Lo;0;AL; 0645 062E 064A;;;;N;;;;; +FDBA;ARABIC LIGATURE LAM WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0644 062C 0645;;;;N;;;;; +FDBB;ARABIC LIGATURE KAF WITH MEEM WITH MEEM FINAL FORM;Lo;0;AL; 0643 0645 0645;;;;N;;;;; +FDBC;ARABIC LIGATURE LAM WITH JEEM WITH MEEM FINAL FORM;Lo;0;AL; 0644 062C 0645;;;;N;;;;; +FDBD;ARABIC LIGATURE NOON WITH JEEM WITH HAH FINAL FORM;Lo;0;AL; 0646 062C 062D;;;;N;;;;; +FDBE;ARABIC LIGATURE JEEM WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 062C 062D 064A;;;;N;;;;; +FDBF;ARABIC LIGATURE HAH WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 062D 062C 064A;;;;N;;;;; +FDC0;ARABIC LIGATURE MEEM WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 0645 062C 064A;;;;N;;;;; +FDC1;ARABIC LIGATURE FEH WITH MEEM WITH YEH FINAL FORM;Lo;0;AL; 0641 0645 064A;;;;N;;;;; +FDC2;ARABIC LIGATURE BEH WITH HAH WITH YEH FINAL FORM;Lo;0;AL; 0628 062D 064A;;;;N;;;;; +FDC3;ARABIC LIGATURE KAF WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0643 0645 0645;;;;N;;;;; +FDC4;ARABIC LIGATURE AIN WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0639 062C 0645;;;;N;;;;; +FDC5;ARABIC LIGATURE SAD WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0635 0645 0645;;;;N;;;;; +FDC6;ARABIC LIGATURE SEEN WITH KHAH WITH YEH FINAL FORM;Lo;0;AL; 0633 062E 064A;;;;N;;;;; +FDC7;ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 0646 062C 064A;;;;N;;;;; +FDF0;ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM;Lo;0;AL; 0635 0644 06D2;;;;N;;;;; +FDF1;ARABIC LIGATURE QALA USED AS KORANIC STOP SIGN ISOLATED FORM;Lo;0;AL; 0642 0644 06D2;;;;N;;;;; +FDF2;ARABIC LIGATURE ALLAH ISOLATED FORM;Lo;0;AL; 0627 0644 0644 0647;;;;N;;;;; +FDF3;ARABIC LIGATURE AKBAR ISOLATED FORM;Lo;0;AL; 0627 0643 0628 0631;;;;N;;;;; +FDF4;ARABIC LIGATURE MOHAMMAD ISOLATED FORM;Lo;0;AL; 0645 062D 0645 062F;;;;N;;;;; +FDF5;ARABIC LIGATURE SALAM ISOLATED FORM;Lo;0;AL; 0635 0644 0639 0645;;;;N;;;;; +FDF6;ARABIC LIGATURE RASOUL ISOLATED FORM;Lo;0;AL; 0631 0633 0648 0644;;;;N;;;;; +FDF7;ARABIC LIGATURE ALAYHE ISOLATED FORM;Lo;0;AL; 0639 0644 064A 0647;;;;N;;;;; +FDF8;ARABIC LIGATURE WASALLAM ISOLATED FORM;Lo;0;AL; 0648 0633 0644 0645;;;;N;;;;; +FDF9;ARABIC LIGATURE SALLA ISOLATED FORM;Lo;0;AL; 0635 0644 0649;;;;N;;;;; +FDFA;ARABIC LIGATURE SALLALLAHOU ALAYHE WASALLAM;Lo;0;AL; 0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645;;;;N;ARABIC LETTER SALLALLAHOU ALAYHE WASALLAM;;;; +FDFB;ARABIC LIGATURE JALLAJALALOUHOU;Lo;0;AL; 062C 0644 0020 062C 0644 0627 0644 0647;;;;N;ARABIC LETTER JALLAJALALOUHOU;;;; +FDFC;RIAL SIGN;Sc;0;AL; 0631 06CC 0627 0644;;;;N;;;;; +FDFD;ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM;So;0;ON;;;;;N;;;;; +FE00;VARIATION SELECTOR-1;Mn;0;NSM;;;;;N;;;;; +FE01;VARIATION SELECTOR-2;Mn;0;NSM;;;;;N;;;;; +FE02;VARIATION SELECTOR-3;Mn;0;NSM;;;;;N;;;;; +FE03;VARIATION SELECTOR-4;Mn;0;NSM;;;;;N;;;;; +FE04;VARIATION SELECTOR-5;Mn;0;NSM;;;;;N;;;;; +FE05;VARIATION SELECTOR-6;Mn;0;NSM;;;;;N;;;;; +FE06;VARIATION SELECTOR-7;Mn;0;NSM;;;;;N;;;;; +FE07;VARIATION SELECTOR-8;Mn;0;NSM;;;;;N;;;;; +FE08;VARIATION SELECTOR-9;Mn;0;NSM;;;;;N;;;;; +FE09;VARIATION SELECTOR-10;Mn;0;NSM;;;;;N;;;;; +FE0A;VARIATION SELECTOR-11;Mn;0;NSM;;;;;N;;;;; +FE0B;VARIATION SELECTOR-12;Mn;0;NSM;;;;;N;;;;; +FE0C;VARIATION SELECTOR-13;Mn;0;NSM;;;;;N;;;;; +FE0D;VARIATION SELECTOR-14;Mn;0;NSM;;;;;N;;;;; +FE0E;VARIATION SELECTOR-15;Mn;0;NSM;;;;;N;;;;; +FE0F;VARIATION SELECTOR-16;Mn;0;NSM;;;;;N;;;;; +FE10;PRESENTATION FORM FOR VERTICAL COMMA;Po;0;ON; 002C;;;;N;;;;; +FE11;PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC COMMA;Po;0;ON; 3001;;;;N;;;;; +FE12;PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC FULL STOP;Po;0;ON; 3002;;;;N;;;;; +FE13;PRESENTATION FORM FOR VERTICAL COLON;Po;0;ON; 003A;;;;N;;;;; +FE14;PRESENTATION FORM FOR VERTICAL SEMICOLON;Po;0;ON; 003B;;;;N;;;;; +FE15;PRESENTATION FORM FOR VERTICAL EXCLAMATION MARK;Po;0;ON; 0021;;;;N;;;;; +FE16;PRESENTATION FORM FOR VERTICAL QUESTION MARK;Po;0;ON; 003F;;;;N;;;;; +FE17;PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET;Ps;0;ON; 3016;;;;N;;;;; +FE18;PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET;Pe;0;ON; 3017;;;;N;;;;; +FE19;PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS;Po;0;ON; 2026;;;;N;;;;; +FE20;COMBINING LIGATURE LEFT HALF;Mn;230;NSM;;;;;N;;;;; +FE21;COMBINING LIGATURE RIGHT HALF;Mn;230;NSM;;;;;N;;;;; +FE22;COMBINING DOUBLE TILDE LEFT HALF;Mn;230;NSM;;;;;N;;;;; +FE23;COMBINING DOUBLE TILDE RIGHT HALF;Mn;230;NSM;;;;;N;;;;; +FE24;COMBINING MACRON LEFT HALF;Mn;230;NSM;;;;;N;;;;; +FE25;COMBINING MACRON RIGHT HALF;Mn;230;NSM;;;;;N;;;;; +FE26;COMBINING CONJOINING MACRON;Mn;230;NSM;;;;;N;;;;; +FE30;PRESENTATION FORM FOR VERTICAL TWO DOT LEADER;Po;0;ON; 2025;;;;N;GLYPH FOR VERTICAL TWO DOT LEADER;;;; +FE31;PRESENTATION FORM FOR VERTICAL EM DASH;Pd;0;ON; 2014;;;;N;GLYPH FOR VERTICAL EM DASH;;;; +FE32;PRESENTATION FORM FOR VERTICAL EN DASH;Pd;0;ON; 2013;;;;N;GLYPH FOR VERTICAL EN DASH;;;; +FE33;PRESENTATION FORM FOR VERTICAL LOW LINE;Pc;0;ON; 005F;;;;N;GLYPH FOR VERTICAL SPACING UNDERSCORE;;;; +FE34;PRESENTATION FORM FOR VERTICAL WAVY LOW LINE;Pc;0;ON; 005F;;;;N;GLYPH FOR VERTICAL SPACING WAVY UNDERSCORE;;;; +FE35;PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS;Ps;0;ON; 0028;;;;N;GLYPH FOR VERTICAL OPENING PARENTHESIS;;;; +FE36;PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS;Pe;0;ON; 0029;;;;N;GLYPH FOR VERTICAL CLOSING PARENTHESIS;;;; +FE37;PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET;Ps;0;ON; 007B;;;;N;GLYPH FOR VERTICAL OPENING CURLY BRACKET;;;; +FE38;PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET;Pe;0;ON; 007D;;;;N;GLYPH FOR VERTICAL CLOSING CURLY BRACKET;;;; +FE39;PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET;Ps;0;ON; 3014;;;;N;GLYPH FOR VERTICAL OPENING TORTOISE SHELL BRACKET;;;; +FE3A;PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET;Pe;0;ON; 3015;;;;N;GLYPH FOR VERTICAL CLOSING TORTOISE SHELL BRACKET;;;; +FE3B;PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET;Ps;0;ON; 3010;;;;N;GLYPH FOR VERTICAL OPENING BLACK LENTICULAR BRACKET;;;; +FE3C;PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET;Pe;0;ON; 3011;;;;N;GLYPH FOR VERTICAL CLOSING BLACK LENTICULAR BRACKET;;;; +FE3D;PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET;Ps;0;ON; 300A;;;;N;GLYPH FOR VERTICAL OPENING DOUBLE ANGLE BRACKET;;;; +FE3E;PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET;Pe;0;ON; 300B;;;;N;GLYPH FOR VERTICAL CLOSING DOUBLE ANGLE BRACKET;;;; +FE3F;PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET;Ps;0;ON; 3008;;;;N;GLYPH FOR VERTICAL OPENING ANGLE BRACKET;;;; +FE40;PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET;Pe;0;ON; 3009;;;;N;GLYPH FOR VERTICAL CLOSING ANGLE BRACKET;;;; +FE41;PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET;Ps;0;ON; 300C;;;;N;GLYPH FOR VERTICAL OPENING CORNER BRACKET;;;; +FE42;PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET;Pe;0;ON; 300D;;;;N;GLYPH FOR VERTICAL CLOSING CORNER BRACKET;;;; +FE43;PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET;Ps;0;ON; 300E;;;;N;GLYPH FOR VERTICAL OPENING WHITE CORNER BRACKET;;;; +FE44;PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET;Pe;0;ON; 300F;;;;N;GLYPH FOR VERTICAL CLOSING WHITE CORNER BRACKET;;;; +FE45;SESAME DOT;Po;0;ON;;;;;N;;;;; +FE46;WHITE SESAME DOT;Po;0;ON;;;;;N;;;;; +FE47;PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET;Ps;0;ON; 005B;;;;N;;;;; +FE48;PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET;Pe;0;ON; 005D;;;;N;;;;; +FE49;DASHED OVERLINE;Po;0;ON; 203E;;;;N;SPACING DASHED OVERSCORE;;;; +FE4A;CENTRELINE OVERLINE;Po;0;ON; 203E;;;;N;SPACING CENTERLINE OVERSCORE;;;; +FE4B;WAVY OVERLINE;Po;0;ON; 203E;;;;N;SPACING WAVY OVERSCORE;;;; +FE4C;DOUBLE WAVY OVERLINE;Po;0;ON; 203E;;;;N;SPACING DOUBLE WAVY OVERSCORE;;;; +FE4D;DASHED LOW LINE;Pc;0;ON; 005F;;;;N;SPACING DASHED UNDERSCORE;;;; +FE4E;CENTRELINE LOW LINE;Pc;0;ON; 005F;;;;N;SPACING CENTERLINE UNDERSCORE;;;; +FE4F;WAVY LOW LINE;Pc;0;ON; 005F;;;;N;SPACING WAVY UNDERSCORE;;;; +FE50;SMALL COMMA;Po;0;CS; 002C;;;;N;;;;; +FE51;SMALL IDEOGRAPHIC COMMA;Po;0;ON; 3001;;;;N;;;;; +FE52;SMALL FULL STOP;Po;0;CS; 002E;;;;N;SMALL PERIOD;;;; +FE54;SMALL SEMICOLON;Po;0;ON; 003B;;;;N;;;;; +FE55;SMALL COLON;Po;0;CS; 003A;;;;N;;;;; +FE56;SMALL QUESTION MARK;Po;0;ON; 003F;;;;N;;;;; +FE57;SMALL EXCLAMATION MARK;Po;0;ON; 0021;;;;N;;;;; +FE58;SMALL EM DASH;Pd;0;ON; 2014;;;;N;;;;; +FE59;SMALL LEFT PARENTHESIS;Ps;0;ON; 0028;;;;Y;SMALL OPENING PARENTHESIS;;;; +FE5A;SMALL RIGHT PARENTHESIS;Pe;0;ON; 0029;;;;Y;SMALL CLOSING PARENTHESIS;;;; +FE5B;SMALL LEFT CURLY BRACKET;Ps;0;ON; 007B;;;;Y;SMALL OPENING CURLY BRACKET;;;; +FE5C;SMALL RIGHT CURLY BRACKET;Pe;0;ON; 007D;;;;Y;SMALL CLOSING CURLY BRACKET;;;; +FE5D;SMALL LEFT TORTOISE SHELL BRACKET;Ps;0;ON; 3014;;;;Y;SMALL OPENING TORTOISE SHELL BRACKET;;;; +FE5E;SMALL RIGHT TORTOISE SHELL BRACKET;Pe;0;ON; 3015;;;;Y;SMALL CLOSING TORTOISE SHELL BRACKET;;;; +FE5F;SMALL NUMBER SIGN;Po;0;ET; 0023;;;;N;;;;; +FE60;SMALL AMPERSAND;Po;0;ON; 0026;;;;N;;;;; +FE61;SMALL ASTERISK;Po;0;ON; 002A;;;;N;;;;; +FE62;SMALL PLUS SIGN;Sm;0;ES; 002B;;;;N;;;;; +FE63;SMALL HYPHEN-MINUS;Pd;0;ES; 002D;;;;N;;;;; +FE64;SMALL LESS-THAN SIGN;Sm;0;ON; 003C;;;;Y;;;;; +FE65;SMALL GREATER-THAN SIGN;Sm;0;ON; 003E;;;;Y;;;;; +FE66;SMALL EQUALS SIGN;Sm;0;ON; 003D;;;;N;;;;; +FE68;SMALL REVERSE SOLIDUS;Po;0;ON; 005C;;;;N;SMALL BACKSLASH;;;; +FE69;SMALL DOLLAR SIGN;Sc;0;ET; 0024;;;;N;;;;; +FE6A;SMALL PERCENT SIGN;Po;0;ET; 0025;;;;N;;;;; +FE6B;SMALL COMMERCIAL AT;Po;0;ON; 0040;;;;N;;;;; +FE70;ARABIC FATHATAN ISOLATED FORM;Lo;0;AL; 0020 064B;;;;N;ARABIC SPACING FATHATAN;;;; +FE71;ARABIC TATWEEL WITH FATHATAN ABOVE;Lo;0;AL; 0640 064B;;;;N;ARABIC FATHATAN ON TATWEEL;;;; +FE72;ARABIC DAMMATAN ISOLATED FORM;Lo;0;AL; 0020 064C;;;;N;ARABIC SPACING DAMMATAN;;;; +FE73;ARABIC TAIL FRAGMENT;Lo;0;AL;;;;;N;;;;; +FE74;ARABIC KASRATAN ISOLATED FORM;Lo;0;AL; 0020 064D;;;;N;ARABIC SPACING KASRATAN;;;; +FE76;ARABIC FATHA ISOLATED FORM;Lo;0;AL; 0020 064E;;;;N;ARABIC SPACING FATHAH;;;; +FE77;ARABIC FATHA MEDIAL FORM;Lo;0;AL; 0640 064E;;;;N;ARABIC FATHAH ON TATWEEL;;;; +FE78;ARABIC DAMMA ISOLATED FORM;Lo;0;AL; 0020 064F;;;;N;ARABIC SPACING DAMMAH;;;; +FE79;ARABIC DAMMA MEDIAL FORM;Lo;0;AL; 0640 064F;;;;N;ARABIC DAMMAH ON TATWEEL;;;; +FE7A;ARABIC KASRA ISOLATED FORM;Lo;0;AL; 0020 0650;;;;N;ARABIC SPACING KASRAH;;;; +FE7B;ARABIC KASRA MEDIAL FORM;Lo;0;AL; 0640 0650;;;;N;ARABIC KASRAH ON TATWEEL;;;; +FE7C;ARABIC SHADDA ISOLATED FORM;Lo;0;AL; 0020 0651;;;;N;ARABIC SPACING SHADDAH;;;; +FE7D;ARABIC SHADDA MEDIAL FORM;Lo;0;AL; 0640 0651;;;;N;ARABIC SHADDAH ON TATWEEL;;;; +FE7E;ARABIC SUKUN ISOLATED FORM;Lo;0;AL; 0020 0652;;;;N;ARABIC SPACING SUKUN;;;; +FE7F;ARABIC SUKUN MEDIAL FORM;Lo;0;AL; 0640 0652;;;;N;ARABIC SUKUN ON TATWEEL;;;; +FE80;ARABIC LETTER HAMZA ISOLATED FORM;Lo;0;AL; 0621;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH;;;; +FE81;ARABIC LETTER ALEF WITH MADDA ABOVE ISOLATED FORM;Lo;0;AL; 0622;;;;N;GLYPH FOR ISOLATE ARABIC MADDAH ON ALEF;;;; +FE82;ARABIC LETTER ALEF WITH MADDA ABOVE FINAL FORM;Lo;0;AL; 0622;;;;N;GLYPH FOR FINAL ARABIC MADDAH ON ALEF;;;; +FE83;ARABIC LETTER ALEF WITH HAMZA ABOVE ISOLATED FORM;Lo;0;AL; 0623;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH ON ALEF;;;; +FE84;ARABIC LETTER ALEF WITH HAMZA ABOVE FINAL FORM;Lo;0;AL; 0623;;;;N;GLYPH FOR FINAL ARABIC HAMZAH ON ALEF;;;; +FE85;ARABIC LETTER WAW WITH HAMZA ABOVE ISOLATED FORM;Lo;0;AL; 0624;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH ON WAW;;;; +FE86;ARABIC LETTER WAW WITH HAMZA ABOVE FINAL FORM;Lo;0;AL; 0624;;;;N;GLYPH FOR FINAL ARABIC HAMZAH ON WAW;;;; +FE87;ARABIC LETTER ALEF WITH HAMZA BELOW ISOLATED FORM;Lo;0;AL; 0625;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH UNDER ALEF;;;; +FE88;ARABIC LETTER ALEF WITH HAMZA BELOW FINAL FORM;Lo;0;AL; 0625;;;;N;GLYPH FOR FINAL ARABIC HAMZAH UNDER ALEF;;;; +FE89;ARABIC LETTER YEH WITH HAMZA ABOVE ISOLATED FORM;Lo;0;AL; 0626;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH ON YA;;;; +FE8A;ARABIC LETTER YEH WITH HAMZA ABOVE FINAL FORM;Lo;0;AL; 0626;;;;N;GLYPH FOR FINAL ARABIC HAMZAH ON YA;;;; +FE8B;ARABIC LETTER YEH WITH HAMZA ABOVE INITIAL FORM;Lo;0;AL; 0626;;;;N;GLYPH FOR INITIAL ARABIC HAMZAH ON YA;;;; +FE8C;ARABIC LETTER YEH WITH HAMZA ABOVE MEDIAL FORM;Lo;0;AL; 0626;;;;N;GLYPH FOR MEDIAL ARABIC HAMZAH ON YA;;;; +FE8D;ARABIC LETTER ALEF ISOLATED FORM;Lo;0;AL; 0627;;;;N;GLYPH FOR ISOLATE ARABIC ALEF;;;; +FE8E;ARABIC LETTER ALEF FINAL FORM;Lo;0;AL; 0627;;;;N;GLYPH FOR FINAL ARABIC ALEF;;;; +FE8F;ARABIC LETTER BEH ISOLATED FORM;Lo;0;AL; 0628;;;;N;GLYPH FOR ISOLATE ARABIC BAA;;;; +FE90;ARABIC LETTER BEH FINAL FORM;Lo;0;AL; 0628;;;;N;GLYPH FOR FINAL ARABIC BAA;;;; +FE91;ARABIC LETTER BEH INITIAL FORM;Lo;0;AL; 0628;;;;N;GLYPH FOR INITIAL ARABIC BAA;;;; +FE92;ARABIC LETTER BEH MEDIAL FORM;Lo;0;AL; 0628;;;;N;GLYPH FOR MEDIAL ARABIC BAA;;;; +FE93;ARABIC LETTER TEH MARBUTA ISOLATED FORM;Lo;0;AL; 0629;;;;N;GLYPH FOR ISOLATE ARABIC TAA MARBUTAH;;;; +FE94;ARABIC LETTER TEH MARBUTA FINAL FORM;Lo;0;AL; 0629;;;;N;GLYPH FOR FINAL ARABIC TAA MARBUTAH;;;; +FE95;ARABIC LETTER TEH ISOLATED FORM;Lo;0;AL; 062A;;;;N;GLYPH FOR ISOLATE ARABIC TAA;;;; +FE96;ARABIC LETTER TEH FINAL FORM;Lo;0;AL; 062A;;;;N;GLYPH FOR FINAL ARABIC TAA;;;; +FE97;ARABIC LETTER TEH INITIAL FORM;Lo;0;AL; 062A;;;;N;GLYPH FOR INITIAL ARABIC TAA;;;; +FE98;ARABIC LETTER TEH MEDIAL FORM;Lo;0;AL; 062A;;;;N;GLYPH FOR MEDIAL ARABIC TAA;;;; +FE99;ARABIC LETTER THEH ISOLATED FORM;Lo;0;AL; 062B;;;;N;GLYPH FOR ISOLATE ARABIC THAA;;;; +FE9A;ARABIC LETTER THEH FINAL FORM;Lo;0;AL; 062B;;;;N;GLYPH FOR FINAL ARABIC THAA;;;; +FE9B;ARABIC LETTER THEH INITIAL FORM;Lo;0;AL; 062B;;;;N;GLYPH FOR INITIAL ARABIC THAA;;;; +FE9C;ARABIC LETTER THEH MEDIAL FORM;Lo;0;AL; 062B;;;;N;GLYPH FOR MEDIAL ARABIC THAA;;;; +FE9D;ARABIC LETTER JEEM ISOLATED FORM;Lo;0;AL; 062C;;;;N;GLYPH FOR ISOLATE ARABIC JEEM;;;; +FE9E;ARABIC LETTER JEEM FINAL FORM;Lo;0;AL; 062C;;;;N;GLYPH FOR FINAL ARABIC JEEM;;;; +FE9F;ARABIC LETTER JEEM INITIAL FORM;Lo;0;AL; 062C;;;;N;GLYPH FOR INITIAL ARABIC JEEM;;;; +FEA0;ARABIC LETTER JEEM MEDIAL FORM;Lo;0;AL; 062C;;;;N;GLYPH FOR MEDIAL ARABIC JEEM;;;; +FEA1;ARABIC LETTER HAH ISOLATED FORM;Lo;0;AL; 062D;;;;N;GLYPH FOR ISOLATE ARABIC HAA;;;; +FEA2;ARABIC LETTER HAH FINAL FORM;Lo;0;AL; 062D;;;;N;GLYPH FOR FINAL ARABIC HAA;;;; +FEA3;ARABIC LETTER HAH INITIAL FORM;Lo;0;AL; 062D;;;;N;GLYPH FOR INITIAL ARABIC HAA;;;; +FEA4;ARABIC LETTER HAH MEDIAL FORM;Lo;0;AL; 062D;;;;N;GLYPH FOR MEDIAL ARABIC HAA;;;; +FEA5;ARABIC LETTER KHAH ISOLATED FORM;Lo;0;AL; 062E;;;;N;GLYPH FOR ISOLATE ARABIC KHAA;;;; +FEA6;ARABIC LETTER KHAH FINAL FORM;Lo;0;AL; 062E;;;;N;GLYPH FOR FINAL ARABIC KHAA;;;; +FEA7;ARABIC LETTER KHAH INITIAL FORM;Lo;0;AL; 062E;;;;N;GLYPH FOR INITIAL ARABIC KHAA;;;; +FEA8;ARABIC LETTER KHAH MEDIAL FORM;Lo;0;AL; 062E;;;;N;GLYPH FOR MEDIAL ARABIC KHAA;;;; +FEA9;ARABIC LETTER DAL ISOLATED FORM;Lo;0;AL; 062F;;;;N;GLYPH FOR ISOLATE ARABIC DAL;;;; +FEAA;ARABIC LETTER DAL FINAL FORM;Lo;0;AL; 062F;;;;N;GLYPH FOR FINAL ARABIC DAL;;;; +FEAB;ARABIC LETTER THAL ISOLATED FORM;Lo;0;AL; 0630;;;;N;GLYPH FOR ISOLATE ARABIC THAL;;;; +FEAC;ARABIC LETTER THAL FINAL FORM;Lo;0;AL; 0630;;;;N;GLYPH FOR FINAL ARABIC THAL;;;; +FEAD;ARABIC LETTER REH ISOLATED FORM;Lo;0;AL; 0631;;;;N;GLYPH FOR ISOLATE ARABIC RA;;;; +FEAE;ARABIC LETTER REH FINAL FORM;Lo;0;AL; 0631;;;;N;GLYPH FOR FINAL ARABIC RA;;;; +FEAF;ARABIC LETTER ZAIN ISOLATED FORM;Lo;0;AL; 0632;;;;N;GLYPH FOR ISOLATE ARABIC ZAIN;;;; +FEB0;ARABIC LETTER ZAIN FINAL FORM;Lo;0;AL; 0632;;;;N;GLYPH FOR FINAL ARABIC ZAIN;;;; +FEB1;ARABIC LETTER SEEN ISOLATED FORM;Lo;0;AL; 0633;;;;N;GLYPH FOR ISOLATE ARABIC SEEN;;;; +FEB2;ARABIC LETTER SEEN FINAL FORM;Lo;0;AL; 0633;;;;N;GLYPH FOR FINAL ARABIC SEEN;;;; +FEB3;ARABIC LETTER SEEN INITIAL FORM;Lo;0;AL; 0633;;;;N;GLYPH FOR INITIAL ARABIC SEEN;;;; +FEB4;ARABIC LETTER SEEN MEDIAL FORM;Lo;0;AL; 0633;;;;N;GLYPH FOR MEDIAL ARABIC SEEN;;;; +FEB5;ARABIC LETTER SHEEN ISOLATED FORM;Lo;0;AL; 0634;;;;N;GLYPH FOR ISOLATE ARABIC SHEEN;;;; +FEB6;ARABIC LETTER SHEEN FINAL FORM;Lo;0;AL; 0634;;;;N;GLYPH FOR FINAL ARABIC SHEEN;;;; +FEB7;ARABIC LETTER SHEEN INITIAL FORM;Lo;0;AL; 0634;;;;N;GLYPH FOR INITIAL ARABIC SHEEN;;;; +FEB8;ARABIC LETTER SHEEN MEDIAL FORM;Lo;0;AL; 0634;;;;N;GLYPH FOR MEDIAL ARABIC SHEEN;;;; +FEB9;ARABIC LETTER SAD ISOLATED FORM;Lo;0;AL; 0635;;;;N;GLYPH FOR ISOLATE ARABIC SAD;;;; +FEBA;ARABIC LETTER SAD FINAL FORM;Lo;0;AL; 0635;;;;N;GLYPH FOR FINAL ARABIC SAD;;;; +FEBB;ARABIC LETTER SAD INITIAL FORM;Lo;0;AL; 0635;;;;N;GLYPH FOR INITIAL ARABIC SAD;;;; +FEBC;ARABIC LETTER SAD MEDIAL FORM;Lo;0;AL; 0635;;;;N;GLYPH FOR MEDIAL ARABIC SAD;;;; +FEBD;ARABIC LETTER DAD ISOLATED FORM;Lo;0;AL; 0636;;;;N;GLYPH FOR ISOLATE ARABIC DAD;;;; +FEBE;ARABIC LETTER DAD FINAL FORM;Lo;0;AL; 0636;;;;N;GLYPH FOR FINAL ARABIC DAD;;;; +FEBF;ARABIC LETTER DAD INITIAL FORM;Lo;0;AL; 0636;;;;N;GLYPH FOR INITIAL ARABIC DAD;;;; +FEC0;ARABIC LETTER DAD MEDIAL FORM;Lo;0;AL; 0636;;;;N;GLYPH FOR MEDIAL ARABIC DAD;;;; +FEC1;ARABIC LETTER TAH ISOLATED FORM;Lo;0;AL; 0637;;;;N;GLYPH FOR ISOLATE ARABIC TAH;;;; +FEC2;ARABIC LETTER TAH FINAL FORM;Lo;0;AL; 0637;;;;N;GLYPH FOR FINAL ARABIC TAH;;;; +FEC3;ARABIC LETTER TAH INITIAL FORM;Lo;0;AL; 0637;;;;N;GLYPH FOR INITIAL ARABIC TAH;;;; +FEC4;ARABIC LETTER TAH MEDIAL FORM;Lo;0;AL; 0637;;;;N;GLYPH FOR MEDIAL ARABIC TAH;;;; +FEC5;ARABIC LETTER ZAH ISOLATED FORM;Lo;0;AL; 0638;;;;N;GLYPH FOR ISOLATE ARABIC DHAH;;;; +FEC6;ARABIC LETTER ZAH FINAL FORM;Lo;0;AL; 0638;;;;N;GLYPH FOR FINAL ARABIC DHAH;;;; +FEC7;ARABIC LETTER ZAH INITIAL FORM;Lo;0;AL; 0638;;;;N;GLYPH FOR INITIAL ARABIC DHAH;;;; +FEC8;ARABIC LETTER ZAH MEDIAL FORM;Lo;0;AL; 0638;;;;N;GLYPH FOR MEDIAL ARABIC DHAH;;;; +FEC9;ARABIC LETTER AIN ISOLATED FORM;Lo;0;AL; 0639;;;;N;GLYPH FOR ISOLATE ARABIC AIN;;;; +FECA;ARABIC LETTER AIN FINAL FORM;Lo;0;AL; 0639;;;;N;GLYPH FOR FINAL ARABIC AIN;;;; +FECB;ARABIC LETTER AIN INITIAL FORM;Lo;0;AL; 0639;;;;N;GLYPH FOR INITIAL ARABIC AIN;;;; +FECC;ARABIC LETTER AIN MEDIAL FORM;Lo;0;AL; 0639;;;;N;GLYPH FOR MEDIAL ARABIC AIN;;;; +FECD;ARABIC LETTER GHAIN ISOLATED FORM;Lo;0;AL; 063A;;;;N;GLYPH FOR ISOLATE ARABIC GHAIN;;;; +FECE;ARABIC LETTER GHAIN FINAL FORM;Lo;0;AL; 063A;;;;N;GLYPH FOR FINAL ARABIC GHAIN;;;; +FECF;ARABIC LETTER GHAIN INITIAL FORM;Lo;0;AL; 063A;;;;N;GLYPH FOR INITIAL ARABIC GHAIN;;;; +FED0;ARABIC LETTER GHAIN MEDIAL FORM;Lo;0;AL; 063A;;;;N;GLYPH FOR MEDIAL ARABIC GHAIN;;;; +FED1;ARABIC LETTER FEH ISOLATED FORM;Lo;0;AL; 0641;;;;N;GLYPH FOR ISOLATE ARABIC FA;;;; +FED2;ARABIC LETTER FEH FINAL FORM;Lo;0;AL; 0641;;;;N;GLYPH FOR FINAL ARABIC FA;;;; +FED3;ARABIC LETTER FEH INITIAL FORM;Lo;0;AL; 0641;;;;N;GLYPH FOR INITIAL ARABIC FA;;;; +FED4;ARABIC LETTER FEH MEDIAL FORM;Lo;0;AL; 0641;;;;N;GLYPH FOR MEDIAL ARABIC FA;;;; +FED5;ARABIC LETTER QAF ISOLATED FORM;Lo;0;AL; 0642;;;;N;GLYPH FOR ISOLATE ARABIC QAF;;;; +FED6;ARABIC LETTER QAF FINAL FORM;Lo;0;AL; 0642;;;;N;GLYPH FOR FINAL ARABIC QAF;;;; +FED7;ARABIC LETTER QAF INITIAL FORM;Lo;0;AL; 0642;;;;N;GLYPH FOR INITIAL ARABIC QAF;;;; +FED8;ARABIC LETTER QAF MEDIAL FORM;Lo;0;AL; 0642;;;;N;GLYPH FOR MEDIAL ARABIC QAF;;;; +FED9;ARABIC LETTER KAF ISOLATED FORM;Lo;0;AL; 0643;;;;N;GLYPH FOR ISOLATE ARABIC CAF;;;; +FEDA;ARABIC LETTER KAF FINAL FORM;Lo;0;AL; 0643;;;;N;GLYPH FOR FINAL ARABIC CAF;;;; +FEDB;ARABIC LETTER KAF INITIAL FORM;Lo;0;AL; 0643;;;;N;GLYPH FOR INITIAL ARABIC CAF;;;; +FEDC;ARABIC LETTER KAF MEDIAL FORM;Lo;0;AL; 0643;;;;N;GLYPH FOR MEDIAL ARABIC CAF;;;; +FEDD;ARABIC LETTER LAM ISOLATED FORM;Lo;0;AL; 0644;;;;N;GLYPH FOR ISOLATE ARABIC LAM;;;; +FEDE;ARABIC LETTER LAM FINAL FORM;Lo;0;AL; 0644;;;;N;GLYPH FOR FINAL ARABIC LAM;;;; +FEDF;ARABIC LETTER LAM INITIAL FORM;Lo;0;AL; 0644;;;;N;GLYPH FOR INITIAL ARABIC LAM;;;; +FEE0;ARABIC LETTER LAM MEDIAL FORM;Lo;0;AL; 0644;;;;N;GLYPH FOR MEDIAL ARABIC LAM;;;; +FEE1;ARABIC LETTER MEEM ISOLATED FORM;Lo;0;AL; 0645;;;;N;GLYPH FOR ISOLATE ARABIC MEEM;;;; +FEE2;ARABIC LETTER MEEM FINAL FORM;Lo;0;AL; 0645;;;;N;GLYPH FOR FINAL ARABIC MEEM;;;; +FEE3;ARABIC LETTER MEEM INITIAL FORM;Lo;0;AL; 0645;;;;N;GLYPH FOR INITIAL ARABIC MEEM;;;; +FEE4;ARABIC LETTER MEEM MEDIAL FORM;Lo;0;AL; 0645;;;;N;GLYPH FOR MEDIAL ARABIC MEEM;;;; +FEE5;ARABIC LETTER NOON ISOLATED FORM;Lo;0;AL; 0646;;;;N;GLYPH FOR ISOLATE ARABIC NOON;;;; +FEE6;ARABIC LETTER NOON FINAL FORM;Lo;0;AL; 0646;;;;N;GLYPH FOR FINAL ARABIC NOON;;;; +FEE7;ARABIC LETTER NOON INITIAL FORM;Lo;0;AL; 0646;;;;N;GLYPH FOR INITIAL ARABIC NOON;;;; +FEE8;ARABIC LETTER NOON MEDIAL FORM;Lo;0;AL; 0646;;;;N;GLYPH FOR MEDIAL ARABIC NOON;;;; +FEE9;ARABIC LETTER HEH ISOLATED FORM;Lo;0;AL; 0647;;;;N;GLYPH FOR ISOLATE ARABIC HA;;;; +FEEA;ARABIC LETTER HEH FINAL FORM;Lo;0;AL; 0647;;;;N;GLYPH FOR FINAL ARABIC HA;;;; +FEEB;ARABIC LETTER HEH INITIAL FORM;Lo;0;AL; 0647;;;;N;GLYPH FOR INITIAL ARABIC HA;;;; +FEEC;ARABIC LETTER HEH MEDIAL FORM;Lo;0;AL; 0647;;;;N;GLYPH FOR MEDIAL ARABIC HA;;;; +FEED;ARABIC LETTER WAW ISOLATED FORM;Lo;0;AL; 0648;;;;N;GLYPH FOR ISOLATE ARABIC WAW;;;; +FEEE;ARABIC LETTER WAW FINAL FORM;Lo;0;AL; 0648;;;;N;GLYPH FOR FINAL ARABIC WAW;;;; +FEEF;ARABIC LETTER ALEF MAKSURA ISOLATED FORM;Lo;0;AL; 0649;;;;N;GLYPH FOR ISOLATE ARABIC ALEF MAQSURAH;;;; +FEF0;ARABIC LETTER ALEF MAKSURA FINAL FORM;Lo;0;AL; 0649;;;;N;GLYPH FOR FINAL ARABIC ALEF MAQSURAH;;;; +FEF1;ARABIC LETTER YEH ISOLATED FORM;Lo;0;AL; 064A;;;;N;GLYPH FOR ISOLATE ARABIC YA;;;; +FEF2;ARABIC LETTER YEH FINAL FORM;Lo;0;AL; 064A;;;;N;GLYPH FOR FINAL ARABIC YA;;;; +FEF3;ARABIC LETTER YEH INITIAL FORM;Lo;0;AL; 064A;;;;N;GLYPH FOR INITIAL ARABIC YA;;;; +FEF4;ARABIC LETTER YEH MEDIAL FORM;Lo;0;AL; 064A;;;;N;GLYPH FOR MEDIAL ARABIC YA;;;; +FEF5;ARABIC LIGATURE LAM WITH ALEF WITH MADDA ABOVE ISOLATED FORM;Lo;0;AL; 0644 0622;;;;N;GLYPH FOR ISOLATE ARABIC MADDAH ON LIGATURE LAM ALEF;;;; +FEF6;ARABIC LIGATURE LAM WITH ALEF WITH MADDA ABOVE FINAL FORM;Lo;0;AL; 0644 0622;;;;N;GLYPH FOR FINAL ARABIC MADDAH ON LIGATURE LAM ALEF;;;; +FEF7;ARABIC LIGATURE LAM WITH ALEF WITH HAMZA ABOVE ISOLATED FORM;Lo;0;AL; 0644 0623;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH ON LIGATURE LAM ALEF;;;; +FEF8;ARABIC LIGATURE LAM WITH ALEF WITH HAMZA ABOVE FINAL FORM;Lo;0;AL; 0644 0623;;;;N;GLYPH FOR FINAL ARABIC HAMZAH ON LIGATURE LAM ALEF;;;; +FEF9;ARABIC LIGATURE LAM WITH ALEF WITH HAMZA BELOW ISOLATED FORM;Lo;0;AL; 0644 0625;;;;N;GLYPH FOR ISOLATE ARABIC HAMZAH UNDER LIGATURE LAM ALEF;;;; +FEFA;ARABIC LIGATURE LAM WITH ALEF WITH HAMZA BELOW FINAL FORM;Lo;0;AL; 0644 0625;;;;N;GLYPH FOR FINAL ARABIC HAMZAH UNDER LIGATURE LAM ALEF;;;; +FEFB;ARABIC LIGATURE LAM WITH ALEF ISOLATED FORM;Lo;0;AL; 0644 0627;;;;N;GLYPH FOR ISOLATE ARABIC LIGATURE LAM ALEF;;;; +FEFC;ARABIC LIGATURE LAM WITH ALEF FINAL FORM;Lo;0;AL; 0644 0627;;;;N;GLYPH FOR FINAL ARABIC LIGATURE LAM ALEF;;;; +FEFF;ZERO WIDTH NO-BREAK SPACE;Cf;0;BN;;;;;N;BYTE ORDER MARK;;;; +FF01;FULLWIDTH EXCLAMATION MARK;Po;0;ON; 0021;;;;N;;;;; +FF02;FULLWIDTH QUOTATION MARK;Po;0;ON; 0022;;;;N;;;;; +FF03;FULLWIDTH NUMBER SIGN;Po;0;ET; 0023;;;;N;;;;; +FF04;FULLWIDTH DOLLAR SIGN;Sc;0;ET; 0024;;;;N;;;;; +FF05;FULLWIDTH PERCENT SIGN;Po;0;ET; 0025;;;;N;;;;; +FF06;FULLWIDTH AMPERSAND;Po;0;ON; 0026;;;;N;;;;; +FF07;FULLWIDTH APOSTROPHE;Po;0;ON; 0027;;;;N;;;;; +FF08;FULLWIDTH LEFT PARENTHESIS;Ps;0;ON; 0028;;;;Y;FULLWIDTH OPENING PARENTHESIS;;;; +FF09;FULLWIDTH RIGHT PARENTHESIS;Pe;0;ON; 0029;;;;Y;FULLWIDTH CLOSING PARENTHESIS;;;; +FF0A;FULLWIDTH ASTERISK;Po;0;ON; 002A;;;;N;;;;; +FF0B;FULLWIDTH PLUS SIGN;Sm;0;ES; 002B;;;;N;;;;; +FF0C;FULLWIDTH COMMA;Po;0;CS; 002C;;;;N;;;;; +FF0D;FULLWIDTH HYPHEN-MINUS;Pd;0;ES; 002D;;;;N;;;;; +FF0E;FULLWIDTH FULL STOP;Po;0;CS; 002E;;;;N;FULLWIDTH PERIOD;;;; +FF0F;FULLWIDTH SOLIDUS;Po;0;CS; 002F;;;;N;FULLWIDTH SLASH;;;; +FF10;FULLWIDTH DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +FF11;FULLWIDTH DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +FF12;FULLWIDTH DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +FF13;FULLWIDTH DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +FF14;FULLWIDTH DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +FF15;FULLWIDTH DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +FF16;FULLWIDTH DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +FF17;FULLWIDTH DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +FF18;FULLWIDTH DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +FF19;FULLWIDTH DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +FF1A;FULLWIDTH COLON;Po;0;CS; 003A;;;;N;;;;; +FF1B;FULLWIDTH SEMICOLON;Po;0;ON; 003B;;;;N;;;;; +FF1C;FULLWIDTH LESS-THAN SIGN;Sm;0;ON; 003C;;;;Y;;;;; +FF1D;FULLWIDTH EQUALS SIGN;Sm;0;ON; 003D;;;;N;;;;; +FF1E;FULLWIDTH GREATER-THAN SIGN;Sm;0;ON; 003E;;;;Y;;;;; +FF1F;FULLWIDTH QUESTION MARK;Po;0;ON; 003F;;;;N;;;;; +FF20;FULLWIDTH COMMERCIAL AT;Po;0;ON; 0040;;;;N;;;;; +FF21;FULLWIDTH LATIN CAPITAL LETTER A;Lu;0;L; 0041;;;;N;;;;FF41; +FF22;FULLWIDTH LATIN CAPITAL LETTER B;Lu;0;L; 0042;;;;N;;;;FF42; +FF23;FULLWIDTH LATIN CAPITAL LETTER C;Lu;0;L; 0043;;;;N;;;;FF43; +FF24;FULLWIDTH LATIN CAPITAL LETTER D;Lu;0;L; 0044;;;;N;;;;FF44; +FF25;FULLWIDTH LATIN CAPITAL LETTER E;Lu;0;L; 0045;;;;N;;;;FF45; +FF26;FULLWIDTH LATIN CAPITAL LETTER F;Lu;0;L; 0046;;;;N;;;;FF46; +FF27;FULLWIDTH LATIN CAPITAL LETTER G;Lu;0;L; 0047;;;;N;;;;FF47; +FF28;FULLWIDTH LATIN CAPITAL LETTER H;Lu;0;L; 0048;;;;N;;;;FF48; +FF29;FULLWIDTH LATIN CAPITAL LETTER I;Lu;0;L; 0049;;;;N;;;;FF49; +FF2A;FULLWIDTH LATIN CAPITAL LETTER J;Lu;0;L; 004A;;;;N;;;;FF4A; +FF2B;FULLWIDTH LATIN CAPITAL LETTER K;Lu;0;L; 004B;;;;N;;;;FF4B; +FF2C;FULLWIDTH LATIN CAPITAL LETTER L;Lu;0;L; 004C;;;;N;;;;FF4C; +FF2D;FULLWIDTH LATIN CAPITAL LETTER M;Lu;0;L; 004D;;;;N;;;;FF4D; +FF2E;FULLWIDTH LATIN CAPITAL LETTER N;Lu;0;L; 004E;;;;N;;;;FF4E; +FF2F;FULLWIDTH LATIN CAPITAL LETTER O;Lu;0;L; 004F;;;;N;;;;FF4F; +FF30;FULLWIDTH LATIN CAPITAL LETTER P;Lu;0;L; 0050;;;;N;;;;FF50; +FF31;FULLWIDTH LATIN CAPITAL LETTER Q;Lu;0;L; 0051;;;;N;;;;FF51; +FF32;FULLWIDTH LATIN CAPITAL LETTER R;Lu;0;L; 0052;;;;N;;;;FF52; +FF33;FULLWIDTH LATIN CAPITAL LETTER S;Lu;0;L; 0053;;;;N;;;;FF53; +FF34;FULLWIDTH LATIN CAPITAL LETTER T;Lu;0;L; 0054;;;;N;;;;FF54; +FF35;FULLWIDTH LATIN CAPITAL LETTER U;Lu;0;L; 0055;;;;N;;;;FF55; +FF36;FULLWIDTH LATIN CAPITAL LETTER V;Lu;0;L; 0056;;;;N;;;;FF56; +FF37;FULLWIDTH LATIN CAPITAL LETTER W;Lu;0;L; 0057;;;;N;;;;FF57; +FF38;FULLWIDTH LATIN CAPITAL LETTER X;Lu;0;L; 0058;;;;N;;;;FF58; +FF39;FULLWIDTH LATIN CAPITAL LETTER Y;Lu;0;L; 0059;;;;N;;;;FF59; +FF3A;FULLWIDTH LATIN CAPITAL LETTER Z;Lu;0;L; 005A;;;;N;;;;FF5A; +FF3B;FULLWIDTH LEFT SQUARE BRACKET;Ps;0;ON; 005B;;;;Y;FULLWIDTH OPENING SQUARE BRACKET;;;; +FF3C;FULLWIDTH REVERSE SOLIDUS;Po;0;ON; 005C;;;;N;FULLWIDTH BACKSLASH;;;; +FF3D;FULLWIDTH RIGHT SQUARE BRACKET;Pe;0;ON; 005D;;;;Y;FULLWIDTH CLOSING SQUARE BRACKET;;;; +FF3E;FULLWIDTH CIRCUMFLEX ACCENT;Sk;0;ON; 005E;;;;N;FULLWIDTH SPACING CIRCUMFLEX;;;; +FF3F;FULLWIDTH LOW LINE;Pc;0;ON; 005F;;;;N;FULLWIDTH SPACING UNDERSCORE;;;; +FF40;FULLWIDTH GRAVE ACCENT;Sk;0;ON; 0060;;;;N;FULLWIDTH SPACING GRAVE;;;; +FF41;FULLWIDTH LATIN SMALL LETTER A;Ll;0;L; 0061;;;;N;;;FF21;;FF21 +FF42;FULLWIDTH LATIN SMALL LETTER B;Ll;0;L; 0062;;;;N;;;FF22;;FF22 +FF43;FULLWIDTH LATIN SMALL LETTER C;Ll;0;L; 0063;;;;N;;;FF23;;FF23 +FF44;FULLWIDTH LATIN SMALL LETTER D;Ll;0;L; 0064;;;;N;;;FF24;;FF24 +FF45;FULLWIDTH LATIN SMALL LETTER E;Ll;0;L; 0065;;;;N;;;FF25;;FF25 +FF46;FULLWIDTH LATIN SMALL LETTER F;Ll;0;L; 0066;;;;N;;;FF26;;FF26 +FF47;FULLWIDTH LATIN SMALL LETTER G;Ll;0;L; 0067;;;;N;;;FF27;;FF27 +FF48;FULLWIDTH LATIN SMALL LETTER H;Ll;0;L; 0068;;;;N;;;FF28;;FF28 +FF49;FULLWIDTH LATIN SMALL LETTER I;Ll;0;L; 0069;;;;N;;;FF29;;FF29 +FF4A;FULLWIDTH LATIN SMALL LETTER J;Ll;0;L; 006A;;;;N;;;FF2A;;FF2A +FF4B;FULLWIDTH LATIN SMALL LETTER K;Ll;0;L; 006B;;;;N;;;FF2B;;FF2B +FF4C;FULLWIDTH LATIN SMALL LETTER L;Ll;0;L; 006C;;;;N;;;FF2C;;FF2C +FF4D;FULLWIDTH LATIN SMALL LETTER M;Ll;0;L; 006D;;;;N;;;FF2D;;FF2D +FF4E;FULLWIDTH LATIN SMALL LETTER N;Ll;0;L; 006E;;;;N;;;FF2E;;FF2E +FF4F;FULLWIDTH LATIN SMALL LETTER O;Ll;0;L; 006F;;;;N;;;FF2F;;FF2F +FF50;FULLWIDTH LATIN SMALL LETTER P;Ll;0;L; 0070;;;;N;;;FF30;;FF30 +FF51;FULLWIDTH LATIN SMALL LETTER Q;Ll;0;L; 0071;;;;N;;;FF31;;FF31 +FF52;FULLWIDTH LATIN SMALL LETTER R;Ll;0;L; 0072;;;;N;;;FF32;;FF32 +FF53;FULLWIDTH LATIN SMALL LETTER S;Ll;0;L; 0073;;;;N;;;FF33;;FF33 +FF54;FULLWIDTH LATIN SMALL LETTER T;Ll;0;L; 0074;;;;N;;;FF34;;FF34 +FF55;FULLWIDTH LATIN SMALL LETTER U;Ll;0;L; 0075;;;;N;;;FF35;;FF35 +FF56;FULLWIDTH LATIN SMALL LETTER V;Ll;0;L; 0076;;;;N;;;FF36;;FF36 +FF57;FULLWIDTH LATIN SMALL LETTER W;Ll;0;L; 0077;;;;N;;;FF37;;FF37 +FF58;FULLWIDTH LATIN SMALL LETTER X;Ll;0;L; 0078;;;;N;;;FF38;;FF38 +FF59;FULLWIDTH LATIN SMALL LETTER Y;Ll;0;L; 0079;;;;N;;;FF39;;FF39 +FF5A;FULLWIDTH LATIN SMALL LETTER Z;Ll;0;L; 007A;;;;N;;;FF3A;;FF3A +FF5B;FULLWIDTH LEFT CURLY BRACKET;Ps;0;ON; 007B;;;;Y;FULLWIDTH OPENING CURLY BRACKET;;;; +FF5C;FULLWIDTH VERTICAL LINE;Sm;0;ON; 007C;;;;N;FULLWIDTH VERTICAL BAR;;;; +FF5D;FULLWIDTH RIGHT CURLY BRACKET;Pe;0;ON; 007D;;;;Y;FULLWIDTH CLOSING CURLY BRACKET;;;; +FF5E;FULLWIDTH TILDE;Sm;0;ON; 007E;;;;N;FULLWIDTH SPACING TILDE;;;; +FF5F;FULLWIDTH LEFT WHITE PARENTHESIS;Ps;0;ON; 2985;;;;Y;;;;; +FF60;FULLWIDTH RIGHT WHITE PARENTHESIS;Pe;0;ON; 2986;;;;Y;;;;; +FF61;HALFWIDTH IDEOGRAPHIC FULL STOP;Po;0;ON; 3002;;;;N;HALFWIDTH IDEOGRAPHIC PERIOD;;;; +FF62;HALFWIDTH LEFT CORNER BRACKET;Ps;0;ON; 300C;;;;Y;HALFWIDTH OPENING CORNER BRACKET;;;; +FF63;HALFWIDTH RIGHT CORNER BRACKET;Pe;0;ON; 300D;;;;Y;HALFWIDTH CLOSING CORNER BRACKET;;;; +FF64;HALFWIDTH IDEOGRAPHIC COMMA;Po;0;ON; 3001;;;;N;;;;; +FF65;HALFWIDTH KATAKANA MIDDLE DOT;Po;0;ON; 30FB;;;;N;;;;; +FF66;HALFWIDTH KATAKANA LETTER WO;Lo;0;L; 30F2;;;;N;;;;; +FF67;HALFWIDTH KATAKANA LETTER SMALL A;Lo;0;L; 30A1;;;;N;;;;; +FF68;HALFWIDTH KATAKANA LETTER SMALL I;Lo;0;L; 30A3;;;;N;;;;; +FF69;HALFWIDTH KATAKANA LETTER SMALL U;Lo;0;L; 30A5;;;;N;;;;; +FF6A;HALFWIDTH KATAKANA LETTER SMALL E;Lo;0;L; 30A7;;;;N;;;;; +FF6B;HALFWIDTH KATAKANA LETTER SMALL O;Lo;0;L; 30A9;;;;N;;;;; +FF6C;HALFWIDTH KATAKANA LETTER SMALL YA;Lo;0;L; 30E3;;;;N;;;;; +FF6D;HALFWIDTH KATAKANA LETTER SMALL YU;Lo;0;L; 30E5;;;;N;;;;; +FF6E;HALFWIDTH KATAKANA LETTER SMALL YO;Lo;0;L; 30E7;;;;N;;;;; +FF6F;HALFWIDTH KATAKANA LETTER SMALL TU;Lo;0;L; 30C3;;;;N;;;;; +FF70;HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK;Lm;0;L; 30FC;;;;N;;;;; +FF71;HALFWIDTH KATAKANA LETTER A;Lo;0;L; 30A2;;;;N;;;;; +FF72;HALFWIDTH KATAKANA LETTER I;Lo;0;L; 30A4;;;;N;;;;; +FF73;HALFWIDTH KATAKANA LETTER U;Lo;0;L; 30A6;;;;N;;;;; +FF74;HALFWIDTH KATAKANA LETTER E;Lo;0;L; 30A8;;;;N;;;;; +FF75;HALFWIDTH KATAKANA LETTER O;Lo;0;L; 30AA;;;;N;;;;; +FF76;HALFWIDTH KATAKANA LETTER KA;Lo;0;L; 30AB;;;;N;;;;; +FF77;HALFWIDTH KATAKANA LETTER KI;Lo;0;L; 30AD;;;;N;;;;; +FF78;HALFWIDTH KATAKANA LETTER KU;Lo;0;L; 30AF;;;;N;;;;; +FF79;HALFWIDTH KATAKANA LETTER KE;Lo;0;L; 30B1;;;;N;;;;; +FF7A;HALFWIDTH KATAKANA LETTER KO;Lo;0;L; 30B3;;;;N;;;;; +FF7B;HALFWIDTH KATAKANA LETTER SA;Lo;0;L; 30B5;;;;N;;;;; +FF7C;HALFWIDTH KATAKANA LETTER SI;Lo;0;L; 30B7;;;;N;;;;; +FF7D;HALFWIDTH KATAKANA LETTER SU;Lo;0;L; 30B9;;;;N;;;;; +FF7E;HALFWIDTH KATAKANA LETTER SE;Lo;0;L; 30BB;;;;N;;;;; +FF7F;HALFWIDTH KATAKANA LETTER SO;Lo;0;L; 30BD;;;;N;;;;; +FF80;HALFWIDTH KATAKANA LETTER TA;Lo;0;L; 30BF;;;;N;;;;; +FF81;HALFWIDTH KATAKANA LETTER TI;Lo;0;L; 30C1;;;;N;;;;; +FF82;HALFWIDTH KATAKANA LETTER TU;Lo;0;L; 30C4;;;;N;;;;; +FF83;HALFWIDTH KATAKANA LETTER TE;Lo;0;L; 30C6;;;;N;;;;; +FF84;HALFWIDTH KATAKANA LETTER TO;Lo;0;L; 30C8;;;;N;;;;; +FF85;HALFWIDTH KATAKANA LETTER NA;Lo;0;L; 30CA;;;;N;;;;; +FF86;HALFWIDTH KATAKANA LETTER NI;Lo;0;L; 30CB;;;;N;;;;; +FF87;HALFWIDTH KATAKANA LETTER NU;Lo;0;L; 30CC;;;;N;;;;; +FF88;HALFWIDTH KATAKANA LETTER NE;Lo;0;L; 30CD;;;;N;;;;; +FF89;HALFWIDTH KATAKANA LETTER NO;Lo;0;L; 30CE;;;;N;;;;; +FF8A;HALFWIDTH KATAKANA LETTER HA;Lo;0;L; 30CF;;;;N;;;;; +FF8B;HALFWIDTH KATAKANA LETTER HI;Lo;0;L; 30D2;;;;N;;;;; +FF8C;HALFWIDTH KATAKANA LETTER HU;Lo;0;L; 30D5;;;;N;;;;; +FF8D;HALFWIDTH KATAKANA LETTER HE;Lo;0;L; 30D8;;;;N;;;;; +FF8E;HALFWIDTH KATAKANA LETTER HO;Lo;0;L; 30DB;;;;N;;;;; +FF8F;HALFWIDTH KATAKANA LETTER MA;Lo;0;L; 30DE;;;;N;;;;; +FF90;HALFWIDTH KATAKANA LETTER MI;Lo;0;L; 30DF;;;;N;;;;; +FF91;HALFWIDTH KATAKANA LETTER MU;Lo;0;L; 30E0;;;;N;;;;; +FF92;HALFWIDTH KATAKANA LETTER ME;Lo;0;L; 30E1;;;;N;;;;; +FF93;HALFWIDTH KATAKANA LETTER MO;Lo;0;L; 30E2;;;;N;;;;; +FF94;HALFWIDTH KATAKANA LETTER YA;Lo;0;L; 30E4;;;;N;;;;; +FF95;HALFWIDTH KATAKANA LETTER YU;Lo;0;L; 30E6;;;;N;;;;; +FF96;HALFWIDTH KATAKANA LETTER YO;Lo;0;L; 30E8;;;;N;;;;; +FF97;HALFWIDTH KATAKANA LETTER RA;Lo;0;L; 30E9;;;;N;;;;; +FF98;HALFWIDTH KATAKANA LETTER RI;Lo;0;L; 30EA;;;;N;;;;; +FF99;HALFWIDTH KATAKANA LETTER RU;Lo;0;L; 30EB;;;;N;;;;; +FF9A;HALFWIDTH KATAKANA LETTER RE;Lo;0;L; 30EC;;;;N;;;;; +FF9B;HALFWIDTH KATAKANA LETTER RO;Lo;0;L; 30ED;;;;N;;;;; +FF9C;HALFWIDTH KATAKANA LETTER WA;Lo;0;L; 30EF;;;;N;;;;; +FF9D;HALFWIDTH KATAKANA LETTER N;Lo;0;L; 30F3;;;;N;;;;; +FF9E;HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L; 3099;;;;N;;;;; +FF9F;HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L; 309A;;;;N;;;;; +FFA0;HALFWIDTH HANGUL FILLER;Lo;0;L; 3164;;;;N;HALFWIDTH HANGUL CAE OM;;;; +FFA1;HALFWIDTH HANGUL LETTER KIYEOK;Lo;0;L; 3131;;;;N;HALFWIDTH HANGUL LETTER GIYEOG;;;; +FFA2;HALFWIDTH HANGUL LETTER SSANGKIYEOK;Lo;0;L; 3132;;;;N;HALFWIDTH HANGUL LETTER SSANG GIYEOG;;;; +FFA3;HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;; +FFA4;HALFWIDTH HANGUL LETTER NIEUN;Lo;0;L; 3134;;;;N;;;;; +FFA5;HALFWIDTH HANGUL LETTER NIEUN-CIEUC;Lo;0;L; 3135;;;;N;HALFWIDTH HANGUL LETTER NIEUN JIEUJ;;;; +FFA6;HALFWIDTH HANGUL LETTER NIEUN-HIEUH;Lo;0;L; 3136;;;;N;HALFWIDTH HANGUL LETTER NIEUN HIEUH;;;; +FFA7;HALFWIDTH HANGUL LETTER TIKEUT;Lo;0;L; 3137;;;;N;HALFWIDTH HANGUL LETTER DIGEUD;;;; +FFA8;HALFWIDTH HANGUL LETTER SSANGTIKEUT;Lo;0;L; 3138;;;;N;HALFWIDTH HANGUL LETTER SSANG DIGEUD;;;; +FFA9;HALFWIDTH HANGUL LETTER RIEUL;Lo;0;L; 3139;;;;N;HALFWIDTH HANGUL LETTER LIEUL;;;; +FFAA;HALFWIDTH HANGUL LETTER RIEUL-KIYEOK;Lo;0;L; 313A;;;;N;HALFWIDTH HANGUL LETTER LIEUL GIYEOG;;;; +FFAB;HALFWIDTH HANGUL LETTER RIEUL-MIEUM;Lo;0;L; 313B;;;;N;HALFWIDTH HANGUL LETTER LIEUL MIEUM;;;; +FFAC;HALFWIDTH HANGUL LETTER RIEUL-PIEUP;Lo;0;L; 313C;;;;N;HALFWIDTH HANGUL LETTER LIEUL BIEUB;;;; +FFAD;HALFWIDTH HANGUL LETTER RIEUL-SIOS;Lo;0;L; 313D;;;;N;HALFWIDTH HANGUL LETTER LIEUL SIOS;;;; +FFAE;HALFWIDTH HANGUL LETTER RIEUL-THIEUTH;Lo;0;L; 313E;;;;N;HALFWIDTH HANGUL LETTER LIEUL TIEUT;;;; +FFAF;HALFWIDTH HANGUL LETTER RIEUL-PHIEUPH;Lo;0;L; 313F;;;;N;HALFWIDTH HANGUL LETTER LIEUL PIEUP;;;; +FFB0;HALFWIDTH HANGUL LETTER RIEUL-HIEUH;Lo;0;L; 3140;;;;N;HALFWIDTH HANGUL LETTER LIEUL HIEUH;;;; +FFB1;HALFWIDTH HANGUL LETTER MIEUM;Lo;0;L; 3141;;;;N;;;;; +FFB2;HALFWIDTH HANGUL LETTER PIEUP;Lo;0;L; 3142;;;;N;HALFWIDTH HANGUL LETTER BIEUB;;;; +FFB3;HALFWIDTH HANGUL LETTER SSANGPIEUP;Lo;0;L; 3143;;;;N;HALFWIDTH HANGUL LETTER SSANG BIEUB;;;; +FFB4;HALFWIDTH HANGUL LETTER PIEUP-SIOS;Lo;0;L; 3144;;;;N;HALFWIDTH HANGUL LETTER BIEUB SIOS;;;; +FFB5;HALFWIDTH HANGUL LETTER SIOS;Lo;0;L; 3145;;;;N;;;;; +FFB6;HALFWIDTH HANGUL LETTER SSANGSIOS;Lo;0;L; 3146;;;;N;HALFWIDTH HANGUL LETTER SSANG SIOS;;;; +FFB7;HALFWIDTH HANGUL LETTER IEUNG;Lo;0;L; 3147;;;;N;;;;; +FFB8;HALFWIDTH HANGUL LETTER CIEUC;Lo;0;L; 3148;;;;N;HALFWIDTH HANGUL LETTER JIEUJ;;;; +FFB9;HALFWIDTH HANGUL LETTER SSANGCIEUC;Lo;0;L; 3149;;;;N;HALFWIDTH HANGUL LETTER SSANG JIEUJ;;;; +FFBA;HALFWIDTH HANGUL LETTER CHIEUCH;Lo;0;L; 314A;;;;N;HALFWIDTH HANGUL LETTER CIEUC;;;; +FFBB;HALFWIDTH HANGUL LETTER KHIEUKH;Lo;0;L; 314B;;;;N;HALFWIDTH HANGUL LETTER KIYEOK;;;; +FFBC;HALFWIDTH HANGUL LETTER THIEUTH;Lo;0;L; 314C;;;;N;HALFWIDTH HANGUL LETTER TIEUT;;;; +FFBD;HALFWIDTH HANGUL LETTER PHIEUPH;Lo;0;L; 314D;;;;N;HALFWIDTH HANGUL LETTER PIEUP;;;; +FFBE;HALFWIDTH HANGUL LETTER HIEUH;Lo;0;L; 314E;;;;N;;;;; +FFC2;HALFWIDTH HANGUL LETTER A;Lo;0;L; 314F;;;;N;;;;; +FFC3;HALFWIDTH HANGUL LETTER AE;Lo;0;L; 3150;;;;N;;;;; +FFC4;HALFWIDTH HANGUL LETTER YA;Lo;0;L; 3151;;;;N;;;;; +FFC5;HALFWIDTH HANGUL LETTER YAE;Lo;0;L; 3152;;;;N;;;;; +FFC6;HALFWIDTH HANGUL LETTER EO;Lo;0;L; 3153;;;;N;;;;; +FFC7;HALFWIDTH HANGUL LETTER E;Lo;0;L; 3154;;;;N;;;;; +FFCA;HALFWIDTH HANGUL LETTER YEO;Lo;0;L; 3155;;;;N;;;;; +FFCB;HALFWIDTH HANGUL LETTER YE;Lo;0;L; 3156;;;;N;;;;; +FFCC;HALFWIDTH HANGUL LETTER O;Lo;0;L; 3157;;;;N;;;;; +FFCD;HALFWIDTH HANGUL LETTER WA;Lo;0;L; 3158;;;;N;;;;; +FFCE;HALFWIDTH HANGUL LETTER WAE;Lo;0;L; 3159;;;;N;;;;; +FFCF;HALFWIDTH HANGUL LETTER OE;Lo;0;L; 315A;;;;N;;;;; +FFD2;HALFWIDTH HANGUL LETTER YO;Lo;0;L; 315B;;;;N;;;;; +FFD3;HALFWIDTH HANGUL LETTER U;Lo;0;L; 315C;;;;N;;;;; +FFD4;HALFWIDTH HANGUL LETTER WEO;Lo;0;L; 315D;;;;N;;;;; +FFD5;HALFWIDTH HANGUL LETTER WE;Lo;0;L; 315E;;;;N;;;;; +FFD6;HALFWIDTH HANGUL LETTER WI;Lo;0;L; 315F;;;;N;;;;; +FFD7;HALFWIDTH HANGUL LETTER YU;Lo;0;L; 3160;;;;N;;;;; +FFDA;HALFWIDTH HANGUL LETTER EU;Lo;0;L; 3161;;;;N;;;;; +FFDB;HALFWIDTH HANGUL LETTER YI;Lo;0;L; 3162;;;;N;;;;; +FFDC;HALFWIDTH HANGUL LETTER I;Lo;0;L; 3163;;;;N;;;;; +FFE0;FULLWIDTH CENT SIGN;Sc;0;ET; 00A2;;;;N;;;;; +FFE1;FULLWIDTH POUND SIGN;Sc;0;ET; 00A3;;;;N;;;;; +FFE2;FULLWIDTH NOT SIGN;Sm;0;ON; 00AC;;;;N;;;;; +FFE3;FULLWIDTH MACRON;Sk;0;ON; 00AF;;;;N;FULLWIDTH SPACING MACRON;;;; +FFE4;FULLWIDTH BROKEN BAR;So;0;ON; 00A6;;;;N;FULLWIDTH BROKEN VERTICAL BAR;;;; +FFE5;FULLWIDTH YEN SIGN;Sc;0;ET; 00A5;;;;N;;;;; +FFE6;FULLWIDTH WON SIGN;Sc;0;ET; 20A9;;;;N;;;;; +FFE8;HALFWIDTH FORMS LIGHT VERTICAL;So;0;ON; 2502;;;;N;;;;; +FFE9;HALFWIDTH LEFTWARDS ARROW;Sm;0;ON; 2190;;;;N;;;;; +FFEA;HALFWIDTH UPWARDS ARROW;Sm;0;ON; 2191;;;;N;;;;; +FFEB;HALFWIDTH RIGHTWARDS ARROW;Sm;0;ON; 2192;;;;N;;;;; +FFEC;HALFWIDTH DOWNWARDS ARROW;Sm;0;ON; 2193;;;;N;;;;; +FFED;HALFWIDTH BLACK SQUARE;So;0;ON; 25A0;;;;N;;;;; +FFEE;HALFWIDTH WHITE CIRCLE;So;0;ON; 25CB;;;;N;;;;; +FFF9;INTERLINEAR ANNOTATION ANCHOR;Cf;0;ON;;;;;N;;;;; +FFFA;INTERLINEAR ANNOTATION SEPARATOR;Cf;0;ON;;;;;N;;;;; +FFFB;INTERLINEAR ANNOTATION TERMINATOR;Cf;0;ON;;;;;N;;;;; +FFFC;OBJECT REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; +FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; +10000;LINEAR B SYLLABLE B008 A;Lo;0;L;;;;;N;;;;; +10001;LINEAR B SYLLABLE B038 E;Lo;0;L;;;;;N;;;;; +10002;LINEAR B SYLLABLE B028 I;Lo;0;L;;;;;N;;;;; +10003;LINEAR B SYLLABLE B061 O;Lo;0;L;;;;;N;;;;; +10004;LINEAR B SYLLABLE B010 U;Lo;0;L;;;;;N;;;;; +10005;LINEAR B SYLLABLE B001 DA;Lo;0;L;;;;;N;;;;; +10006;LINEAR B SYLLABLE B045 DE;Lo;0;L;;;;;N;;;;; +10007;LINEAR B SYLLABLE B007 DI;Lo;0;L;;;;;N;;;;; +10008;LINEAR B SYLLABLE B014 DO;Lo;0;L;;;;;N;;;;; +10009;LINEAR B SYLLABLE B051 DU;Lo;0;L;;;;;N;;;;; +1000A;LINEAR B SYLLABLE B057 JA;Lo;0;L;;;;;N;;;;; +1000B;LINEAR B SYLLABLE B046 JE;Lo;0;L;;;;;N;;;;; +1000D;LINEAR B SYLLABLE B036 JO;Lo;0;L;;;;;N;;;;; +1000E;LINEAR B SYLLABLE B065 JU;Lo;0;L;;;;;N;;;;; +1000F;LINEAR B SYLLABLE B077 KA;Lo;0;L;;;;;N;;;;; +10010;LINEAR B SYLLABLE B044 KE;Lo;0;L;;;;;N;;;;; +10011;LINEAR B SYLLABLE B067 KI;Lo;0;L;;;;;N;;;;; +10012;LINEAR B SYLLABLE B070 KO;Lo;0;L;;;;;N;;;;; +10013;LINEAR B SYLLABLE B081 KU;Lo;0;L;;;;;N;;;;; +10014;LINEAR B SYLLABLE B080 MA;Lo;0;L;;;;;N;;;;; +10015;LINEAR B SYLLABLE B013 ME;Lo;0;L;;;;;N;;;;; +10016;LINEAR B SYLLABLE B073 MI;Lo;0;L;;;;;N;;;;; +10017;LINEAR B SYLLABLE B015 MO;Lo;0;L;;;;;N;;;;; +10018;LINEAR B SYLLABLE B023 MU;Lo;0;L;;;;;N;;;;; +10019;LINEAR B SYLLABLE B006 NA;Lo;0;L;;;;;N;;;;; +1001A;LINEAR B SYLLABLE B024 NE;Lo;0;L;;;;;N;;;;; +1001B;LINEAR B SYLLABLE B030 NI;Lo;0;L;;;;;N;;;;; +1001C;LINEAR B SYLLABLE B052 NO;Lo;0;L;;;;;N;;;;; +1001D;LINEAR B SYLLABLE B055 NU;Lo;0;L;;;;;N;;;;; +1001E;LINEAR B SYLLABLE B003 PA;Lo;0;L;;;;;N;;;;; +1001F;LINEAR B SYLLABLE B072 PE;Lo;0;L;;;;;N;;;;; +10020;LINEAR B SYLLABLE B039 PI;Lo;0;L;;;;;N;;;;; +10021;LINEAR B SYLLABLE B011 PO;Lo;0;L;;;;;N;;;;; +10022;LINEAR B SYLLABLE B050 PU;Lo;0;L;;;;;N;;;;; +10023;LINEAR B SYLLABLE B016 QA;Lo;0;L;;;;;N;;;;; +10024;LINEAR B SYLLABLE B078 QE;Lo;0;L;;;;;N;;;;; +10025;LINEAR B SYLLABLE B021 QI;Lo;0;L;;;;;N;;;;; +10026;LINEAR B SYLLABLE B032 QO;Lo;0;L;;;;;N;;;;; +10028;LINEAR B SYLLABLE B060 RA;Lo;0;L;;;;;N;;;;; +10029;LINEAR B SYLLABLE B027 RE;Lo;0;L;;;;;N;;;;; +1002A;LINEAR B SYLLABLE B053 RI;Lo;0;L;;;;;N;;;;; +1002B;LINEAR B SYLLABLE B002 RO;Lo;0;L;;;;;N;;;;; +1002C;LINEAR B SYLLABLE B026 RU;Lo;0;L;;;;;N;;;;; +1002D;LINEAR B SYLLABLE B031 SA;Lo;0;L;;;;;N;;;;; +1002E;LINEAR B SYLLABLE B009 SE;Lo;0;L;;;;;N;;;;; +1002F;LINEAR B SYLLABLE B041 SI;Lo;0;L;;;;;N;;;;; +10030;LINEAR B SYLLABLE B012 SO;Lo;0;L;;;;;N;;;;; +10031;LINEAR B SYLLABLE B058 SU;Lo;0;L;;;;;N;;;;; +10032;LINEAR B SYLLABLE B059 TA;Lo;0;L;;;;;N;;;;; +10033;LINEAR B SYLLABLE B004 TE;Lo;0;L;;;;;N;;;;; +10034;LINEAR B SYLLABLE B037 TI;Lo;0;L;;;;;N;;;;; +10035;LINEAR B SYLLABLE B005 TO;Lo;0;L;;;;;N;;;;; +10036;LINEAR B SYLLABLE B069 TU;Lo;0;L;;;;;N;;;;; +10037;LINEAR B SYLLABLE B054 WA;Lo;0;L;;;;;N;;;;; +10038;LINEAR B SYLLABLE B075 WE;Lo;0;L;;;;;N;;;;; +10039;LINEAR B SYLLABLE B040 WI;Lo;0;L;;;;;N;;;;; +1003A;LINEAR B SYLLABLE B042 WO;Lo;0;L;;;;;N;;;;; +1003C;LINEAR B SYLLABLE B017 ZA;Lo;0;L;;;;;N;;;;; +1003D;LINEAR B SYLLABLE B074 ZE;Lo;0;L;;;;;N;;;;; +1003F;LINEAR B SYLLABLE B020 ZO;Lo;0;L;;;;;N;;;;; +10040;LINEAR B SYLLABLE B025 A2;Lo;0;L;;;;;N;;;;; +10041;LINEAR B SYLLABLE B043 A3;Lo;0;L;;;;;N;;;;; +10042;LINEAR B SYLLABLE B085 AU;Lo;0;L;;;;;N;;;;; +10043;LINEAR B SYLLABLE B071 DWE;Lo;0;L;;;;;N;;;;; +10044;LINEAR B SYLLABLE B090 DWO;Lo;0;L;;;;;N;;;;; +10045;LINEAR B SYLLABLE B048 NWA;Lo;0;L;;;;;N;;;;; +10046;LINEAR B SYLLABLE B029 PU2;Lo;0;L;;;;;N;;;;; +10047;LINEAR B SYLLABLE B062 PTE;Lo;0;L;;;;;N;;;;; +10048;LINEAR B SYLLABLE B076 RA2;Lo;0;L;;;;;N;;;;; +10049;LINEAR B SYLLABLE B033 RA3;Lo;0;L;;;;;N;;;;; +1004A;LINEAR B SYLLABLE B068 RO2;Lo;0;L;;;;;N;;;;; +1004B;LINEAR B SYLLABLE B066 TA2;Lo;0;L;;;;;N;;;;; +1004C;LINEAR B SYLLABLE B087 TWE;Lo;0;L;;;;;N;;;;; +1004D;LINEAR B SYLLABLE B091 TWO;Lo;0;L;;;;;N;;;;; +10050;LINEAR B SYMBOL B018;Lo;0;L;;;;;N;;;;; +10051;LINEAR B SYMBOL B019;Lo;0;L;;;;;N;;;;; +10052;LINEAR B SYMBOL B022;Lo;0;L;;;;;N;;;;; +10053;LINEAR B SYMBOL B034;Lo;0;L;;;;;N;;;;; +10054;LINEAR B SYMBOL B047;Lo;0;L;;;;;N;;;;; +10055;LINEAR B SYMBOL B049;Lo;0;L;;;;;N;;;;; +10056;LINEAR B SYMBOL B056;Lo;0;L;;;;;N;;;;; +10057;LINEAR B SYMBOL B063;Lo;0;L;;;;;N;;;;; +10058;LINEAR B SYMBOL B064;Lo;0;L;;;;;N;;;;; +10059;LINEAR B SYMBOL B079;Lo;0;L;;;;;N;;;;; +1005A;LINEAR B SYMBOL B082;Lo;0;L;;;;;N;;;;; +1005B;LINEAR B SYMBOL B083;Lo;0;L;;;;;N;;;;; +1005C;LINEAR B SYMBOL B086;Lo;0;L;;;;;N;;;;; +1005D;LINEAR B SYMBOL B089;Lo;0;L;;;;;N;;;;; +10080;LINEAR B IDEOGRAM B100 MAN;Lo;0;L;;;;;N;;;;; +10081;LINEAR B IDEOGRAM B102 WOMAN;Lo;0;L;;;;;N;;;;; +10082;LINEAR B IDEOGRAM B104 DEER;Lo;0;L;;;;;N;;;;; +10083;LINEAR B IDEOGRAM B105 EQUID;Lo;0;L;;;;;N;;;;; +10084;LINEAR B IDEOGRAM B105F MARE;Lo;0;L;;;;;N;;;;; +10085;LINEAR B IDEOGRAM B105M STALLION;Lo;0;L;;;;;N;;;;; +10086;LINEAR B IDEOGRAM B106F EWE;Lo;0;L;;;;;N;;;;; +10087;LINEAR B IDEOGRAM B106M RAM;Lo;0;L;;;;;N;;;;; +10088;LINEAR B IDEOGRAM B107F SHE-GOAT;Lo;0;L;;;;;N;;;;; +10089;LINEAR B IDEOGRAM B107M HE-GOAT;Lo;0;L;;;;;N;;;;; +1008A;LINEAR B IDEOGRAM B108F SOW;Lo;0;L;;;;;N;;;;; +1008B;LINEAR B IDEOGRAM B108M BOAR;Lo;0;L;;;;;N;;;;; +1008C;LINEAR B IDEOGRAM B109F COW;Lo;0;L;;;;;N;;;;; +1008D;LINEAR B IDEOGRAM B109M BULL;Lo;0;L;;;;;N;;;;; +1008E;LINEAR B IDEOGRAM B120 WHEAT;Lo;0;L;;;;;N;;;;; +1008F;LINEAR B IDEOGRAM B121 BARLEY;Lo;0;L;;;;;N;;;;; +10090;LINEAR B IDEOGRAM B122 OLIVE;Lo;0;L;;;;;N;;;;; +10091;LINEAR B IDEOGRAM B123 SPICE;Lo;0;L;;;;;N;;;;; +10092;LINEAR B IDEOGRAM B125 CYPERUS;Lo;0;L;;;;;N;;;;; +10093;LINEAR B MONOGRAM B127 KAPO;Lo;0;L;;;;;N;;;;; +10094;LINEAR B MONOGRAM B128 KANAKO;Lo;0;L;;;;;N;;;;; +10095;LINEAR B IDEOGRAM B130 OIL;Lo;0;L;;;;;N;;;;; +10096;LINEAR B IDEOGRAM B131 WINE;Lo;0;L;;;;;N;;;;; +10097;LINEAR B IDEOGRAM B132;Lo;0;L;;;;;N;;;;; +10098;LINEAR B MONOGRAM B133 AREPA;Lo;0;L;;;;;N;;;;; +10099;LINEAR B MONOGRAM B135 MERI;Lo;0;L;;;;;N;;;;; +1009A;LINEAR B IDEOGRAM B140 BRONZE;Lo;0;L;;;;;N;;;;; +1009B;LINEAR B IDEOGRAM B141 GOLD;Lo;0;L;;;;;N;;;;; +1009C;LINEAR B IDEOGRAM B142;Lo;0;L;;;;;N;;;;; +1009D;LINEAR B IDEOGRAM B145 WOOL;Lo;0;L;;;;;N;;;;; +1009E;LINEAR B IDEOGRAM B146;Lo;0;L;;;;;N;;;;; +1009F;LINEAR B IDEOGRAM B150;Lo;0;L;;;;;N;;;;; +100A0;LINEAR B IDEOGRAM B151 HORN;Lo;0;L;;;;;N;;;;; +100A1;LINEAR B IDEOGRAM B152;Lo;0;L;;;;;N;;;;; +100A2;LINEAR B IDEOGRAM B153;Lo;0;L;;;;;N;;;;; +100A3;LINEAR B IDEOGRAM B154;Lo;0;L;;;;;N;;;;; +100A4;LINEAR B MONOGRAM B156 TURO2;Lo;0;L;;;;;N;;;;; +100A5;LINEAR B IDEOGRAM B157;Lo;0;L;;;;;N;;;;; +100A6;LINEAR B IDEOGRAM B158;Lo;0;L;;;;;N;;;;; +100A7;LINEAR B IDEOGRAM B159 CLOTH;Lo;0;L;;;;;N;;;;; +100A8;LINEAR B IDEOGRAM B160;Lo;0;L;;;;;N;;;;; +100A9;LINEAR B IDEOGRAM B161;Lo;0;L;;;;;N;;;;; +100AA;LINEAR B IDEOGRAM B162 GARMENT;Lo;0;L;;;;;N;;;;; +100AB;LINEAR B IDEOGRAM B163 ARMOUR;Lo;0;L;;;;;N;;;;; +100AC;LINEAR B IDEOGRAM B164;Lo;0;L;;;;;N;;;;; +100AD;LINEAR B IDEOGRAM B165;Lo;0;L;;;;;N;;;;; +100AE;LINEAR B IDEOGRAM B166;Lo;0;L;;;;;N;;;;; +100AF;LINEAR B IDEOGRAM B167;Lo;0;L;;;;;N;;;;; +100B0;LINEAR B IDEOGRAM B168;Lo;0;L;;;;;N;;;;; +100B1;LINEAR B IDEOGRAM B169;Lo;0;L;;;;;N;;;;; +100B2;LINEAR B IDEOGRAM B170;Lo;0;L;;;;;N;;;;; +100B3;LINEAR B IDEOGRAM B171;Lo;0;L;;;;;N;;;;; +100B4;LINEAR B IDEOGRAM B172;Lo;0;L;;;;;N;;;;; +100B5;LINEAR B IDEOGRAM B173 MONTH;Lo;0;L;;;;;N;;;;; +100B6;LINEAR B IDEOGRAM B174;Lo;0;L;;;;;N;;;;; +100B7;LINEAR B IDEOGRAM B176 TREE;Lo;0;L;;;;;N;;;;; +100B8;LINEAR B IDEOGRAM B177;Lo;0;L;;;;;N;;;;; +100B9;LINEAR B IDEOGRAM B178;Lo;0;L;;;;;N;;;;; +100BA;LINEAR B IDEOGRAM B179;Lo;0;L;;;;;N;;;;; +100BB;LINEAR B IDEOGRAM B180;Lo;0;L;;;;;N;;;;; +100BC;LINEAR B IDEOGRAM B181;Lo;0;L;;;;;N;;;;; +100BD;LINEAR B IDEOGRAM B182;Lo;0;L;;;;;N;;;;; +100BE;LINEAR B IDEOGRAM B183;Lo;0;L;;;;;N;;;;; +100BF;LINEAR B IDEOGRAM B184;Lo;0;L;;;;;N;;;;; +100C0;LINEAR B IDEOGRAM B185;Lo;0;L;;;;;N;;;;; +100C1;LINEAR B IDEOGRAM B189;Lo;0;L;;;;;N;;;;; +100C2;LINEAR B IDEOGRAM B190;Lo;0;L;;;;;N;;;;; +100C3;LINEAR B IDEOGRAM B191 HELMET;Lo;0;L;;;;;N;;;;; +100C4;LINEAR B IDEOGRAM B220 FOOTSTOOL;Lo;0;L;;;;;N;;;;; +100C5;LINEAR B IDEOGRAM B225 BATHTUB;Lo;0;L;;;;;N;;;;; +100C6;LINEAR B IDEOGRAM B230 SPEAR;Lo;0;L;;;;;N;;;;; +100C7;LINEAR B IDEOGRAM B231 ARROW;Lo;0;L;;;;;N;;;;; +100C8;LINEAR B IDEOGRAM B232;Lo;0;L;;;;;N;;;;; +100C9;LINEAR B IDEOGRAM B233 SWORD;Lo;0;L;;;;;N;;;;; +100CA;LINEAR B IDEOGRAM B234;Lo;0;L;;;;;N;;;;; +100CB;LINEAR B IDEOGRAM B236;Lo;0;L;;;;;N;;;;; +100CC;LINEAR B IDEOGRAM B240 WHEELED CHARIOT;Lo;0;L;;;;;N;;;;; +100CD;LINEAR B IDEOGRAM B241 CHARIOT;Lo;0;L;;;;;N;;;;; +100CE;LINEAR B IDEOGRAM B242 CHARIOT FRAME;Lo;0;L;;;;;N;;;;; +100CF;LINEAR B IDEOGRAM B243 WHEEL;Lo;0;L;;;;;N;;;;; +100D0;LINEAR B IDEOGRAM B245;Lo;0;L;;;;;N;;;;; +100D1;LINEAR B IDEOGRAM B246;Lo;0;L;;;;;N;;;;; +100D2;LINEAR B MONOGRAM B247 DIPTE;Lo;0;L;;;;;N;;;;; +100D3;LINEAR B IDEOGRAM B248;Lo;0;L;;;;;N;;;;; +100D4;LINEAR B IDEOGRAM B249;Lo;0;L;;;;;N;;;;; +100D5;LINEAR B IDEOGRAM B251;Lo;0;L;;;;;N;;;;; +100D6;LINEAR B IDEOGRAM B252;Lo;0;L;;;;;N;;;;; +100D7;LINEAR B IDEOGRAM B253;Lo;0;L;;;;;N;;;;; +100D8;LINEAR B IDEOGRAM B254 DART;Lo;0;L;;;;;N;;;;; +100D9;LINEAR B IDEOGRAM B255;Lo;0;L;;;;;N;;;;; +100DA;LINEAR B IDEOGRAM B256;Lo;0;L;;;;;N;;;;; +100DB;LINEAR B IDEOGRAM B257;Lo;0;L;;;;;N;;;;; +100DC;LINEAR B IDEOGRAM B258;Lo;0;L;;;;;N;;;;; +100DD;LINEAR B IDEOGRAM B259;Lo;0;L;;;;;N;;;;; +100DE;LINEAR B IDEOGRAM VESSEL B155;Lo;0;L;;;;;N;;;;; +100DF;LINEAR B IDEOGRAM VESSEL B200;Lo;0;L;;;;;N;;;;; +100E0;LINEAR B IDEOGRAM VESSEL B201;Lo;0;L;;;;;N;;;;; +100E1;LINEAR B IDEOGRAM VESSEL B202;Lo;0;L;;;;;N;;;;; +100E2;LINEAR B IDEOGRAM VESSEL B203;Lo;0;L;;;;;N;;;;; +100E3;LINEAR B IDEOGRAM VESSEL B204;Lo;0;L;;;;;N;;;;; +100E4;LINEAR B IDEOGRAM VESSEL B205;Lo;0;L;;;;;N;;;;; +100E5;LINEAR B IDEOGRAM VESSEL B206;Lo;0;L;;;;;N;;;;; +100E6;LINEAR B IDEOGRAM VESSEL B207;Lo;0;L;;;;;N;;;;; +100E7;LINEAR B IDEOGRAM VESSEL B208;Lo;0;L;;;;;N;;;;; +100E8;LINEAR B IDEOGRAM VESSEL B209;Lo;0;L;;;;;N;;;;; +100E9;LINEAR B IDEOGRAM VESSEL B210;Lo;0;L;;;;;N;;;;; +100EA;LINEAR B IDEOGRAM VESSEL B211;Lo;0;L;;;;;N;;;;; +100EB;LINEAR B IDEOGRAM VESSEL B212;Lo;0;L;;;;;N;;;;; +100EC;LINEAR B IDEOGRAM VESSEL B213;Lo;0;L;;;;;N;;;;; +100ED;LINEAR B IDEOGRAM VESSEL B214;Lo;0;L;;;;;N;;;;; +100EE;LINEAR B IDEOGRAM VESSEL B215;Lo;0;L;;;;;N;;;;; +100EF;LINEAR B IDEOGRAM VESSEL B216;Lo;0;L;;;;;N;;;;; +100F0;LINEAR B IDEOGRAM VESSEL B217;Lo;0;L;;;;;N;;;;; +100F1;LINEAR B IDEOGRAM VESSEL B218;Lo;0;L;;;;;N;;;;; +100F2;LINEAR B IDEOGRAM VESSEL B219;Lo;0;L;;;;;N;;;;; +100F3;LINEAR B IDEOGRAM VESSEL B221;Lo;0;L;;;;;N;;;;; +100F4;LINEAR B IDEOGRAM VESSEL B222;Lo;0;L;;;;;N;;;;; +100F5;LINEAR B IDEOGRAM VESSEL B226;Lo;0;L;;;;;N;;;;; +100F6;LINEAR B IDEOGRAM VESSEL B227;Lo;0;L;;;;;N;;;;; +100F7;LINEAR B IDEOGRAM VESSEL B228;Lo;0;L;;;;;N;;;;; +100F8;LINEAR B IDEOGRAM VESSEL B229;Lo;0;L;;;;;N;;;;; +100F9;LINEAR B IDEOGRAM VESSEL B250;Lo;0;L;;;;;N;;;;; +100FA;LINEAR B IDEOGRAM VESSEL B305;Lo;0;L;;;;;N;;;;; +10100;AEGEAN WORD SEPARATOR LINE;Po;0;L;;;;;N;;;;; +10101;AEGEAN WORD SEPARATOR DOT;Po;0;ON;;;;;N;;;;; +10102;AEGEAN CHECK MARK;Po;0;L;;;;;N;;;;; +10107;AEGEAN NUMBER ONE;No;0;L;;;;1;N;;;;; +10108;AEGEAN NUMBER TWO;No;0;L;;;;2;N;;;;; +10109;AEGEAN NUMBER THREE;No;0;L;;;;3;N;;;;; +1010A;AEGEAN NUMBER FOUR;No;0;L;;;;4;N;;;;; +1010B;AEGEAN NUMBER FIVE;No;0;L;;;;5;N;;;;; +1010C;AEGEAN NUMBER SIX;No;0;L;;;;6;N;;;;; +1010D;AEGEAN NUMBER SEVEN;No;0;L;;;;7;N;;;;; +1010E;AEGEAN NUMBER EIGHT;No;0;L;;;;8;N;;;;; +1010F;AEGEAN NUMBER NINE;No;0;L;;;;9;N;;;;; +10110;AEGEAN NUMBER TEN;No;0;L;;;;10;N;;;;; +10111;AEGEAN NUMBER TWENTY;No;0;L;;;;20;N;;;;; +10112;AEGEAN NUMBER THIRTY;No;0;L;;;;30;N;;;;; +10113;AEGEAN NUMBER FORTY;No;0;L;;;;40;N;;;;; +10114;AEGEAN NUMBER FIFTY;No;0;L;;;;50;N;;;;; +10115;AEGEAN NUMBER SIXTY;No;0;L;;;;60;N;;;;; +10116;AEGEAN NUMBER SEVENTY;No;0;L;;;;70;N;;;;; +10117;AEGEAN NUMBER EIGHTY;No;0;L;;;;80;N;;;;; +10118;AEGEAN NUMBER NINETY;No;0;L;;;;90;N;;;;; +10119;AEGEAN NUMBER ONE HUNDRED;No;0;L;;;;100;N;;;;; +1011A;AEGEAN NUMBER TWO HUNDRED;No;0;L;;;;200;N;;;;; +1011B;AEGEAN NUMBER THREE HUNDRED;No;0;L;;;;300;N;;;;; +1011C;AEGEAN NUMBER FOUR HUNDRED;No;0;L;;;;400;N;;;;; +1011D;AEGEAN NUMBER FIVE HUNDRED;No;0;L;;;;500;N;;;;; +1011E;AEGEAN NUMBER SIX HUNDRED;No;0;L;;;;600;N;;;;; +1011F;AEGEAN NUMBER SEVEN HUNDRED;No;0;L;;;;700;N;;;;; +10120;AEGEAN NUMBER EIGHT HUNDRED;No;0;L;;;;800;N;;;;; +10121;AEGEAN NUMBER NINE HUNDRED;No;0;L;;;;900;N;;;;; +10122;AEGEAN NUMBER ONE THOUSAND;No;0;L;;;;1000;N;;;;; +10123;AEGEAN NUMBER TWO THOUSAND;No;0;L;;;;2000;N;;;;; +10124;AEGEAN NUMBER THREE THOUSAND;No;0;L;;;;3000;N;;;;; +10125;AEGEAN NUMBER FOUR THOUSAND;No;0;L;;;;4000;N;;;;; +10126;AEGEAN NUMBER FIVE THOUSAND;No;0;L;;;;5000;N;;;;; +10127;AEGEAN NUMBER SIX THOUSAND;No;0;L;;;;6000;N;;;;; +10128;AEGEAN NUMBER SEVEN THOUSAND;No;0;L;;;;7000;N;;;;; +10129;AEGEAN NUMBER EIGHT THOUSAND;No;0;L;;;;8000;N;;;;; +1012A;AEGEAN NUMBER NINE THOUSAND;No;0;L;;;;9000;N;;;;; +1012B;AEGEAN NUMBER TEN THOUSAND;No;0;L;;;;10000;N;;;;; +1012C;AEGEAN NUMBER TWENTY THOUSAND;No;0;L;;;;20000;N;;;;; +1012D;AEGEAN NUMBER THIRTY THOUSAND;No;0;L;;;;30000;N;;;;; +1012E;AEGEAN NUMBER FORTY THOUSAND;No;0;L;;;;40000;N;;;;; +1012F;AEGEAN NUMBER FIFTY THOUSAND;No;0;L;;;;50000;N;;;;; +10130;AEGEAN NUMBER SIXTY THOUSAND;No;0;L;;;;60000;N;;;;; +10131;AEGEAN NUMBER SEVENTY THOUSAND;No;0;L;;;;70000;N;;;;; +10132;AEGEAN NUMBER EIGHTY THOUSAND;No;0;L;;;;80000;N;;;;; +10133;AEGEAN NUMBER NINETY THOUSAND;No;0;L;;;;90000;N;;;;; +10137;AEGEAN WEIGHT BASE UNIT;So;0;L;;;;;N;;;;; +10138;AEGEAN WEIGHT FIRST SUBUNIT;So;0;L;;;;;N;;;;; +10139;AEGEAN WEIGHT SECOND SUBUNIT;So;0;L;;;;;N;;;;; +1013A;AEGEAN WEIGHT THIRD SUBUNIT;So;0;L;;;;;N;;;;; +1013B;AEGEAN WEIGHT FOURTH SUBUNIT;So;0;L;;;;;N;;;;; +1013C;AEGEAN DRY MEASURE FIRST SUBUNIT;So;0;L;;;;;N;;;;; +1013D;AEGEAN LIQUID MEASURE FIRST SUBUNIT;So;0;L;;;;;N;;;;; +1013E;AEGEAN MEASURE SECOND SUBUNIT;So;0;L;;;;;N;;;;; +1013F;AEGEAN MEASURE THIRD SUBUNIT;So;0;L;;;;;N;;;;; +10140;GREEK ACROPHONIC ATTIC ONE QUARTER;Nl;0;ON;;;;1/4;N;;;;; +10141;GREEK ACROPHONIC ATTIC ONE HALF;Nl;0;ON;;;;1/2;N;;;;; +10142;GREEK ACROPHONIC ATTIC ONE DRACHMA;Nl;0;ON;;;;1;N;;;;; +10143;GREEK ACROPHONIC ATTIC FIVE;Nl;0;ON;;;;5;N;;;;; +10144;GREEK ACROPHONIC ATTIC FIFTY;Nl;0;ON;;;;50;N;;;;; +10145;GREEK ACROPHONIC ATTIC FIVE HUNDRED;Nl;0;ON;;;;500;N;;;;; +10146;GREEK ACROPHONIC ATTIC FIVE THOUSAND;Nl;0;ON;;;;5000;N;;;;; +10147;GREEK ACROPHONIC ATTIC FIFTY THOUSAND;Nl;0;ON;;;;50000;N;;;;; +10148;GREEK ACROPHONIC ATTIC FIVE TALENTS;Nl;0;ON;;;;5;N;;;;; +10149;GREEK ACROPHONIC ATTIC TEN TALENTS;Nl;0;ON;;;;10;N;;;;; +1014A;GREEK ACROPHONIC ATTIC FIFTY TALENTS;Nl;0;ON;;;;50;N;;;;; +1014B;GREEK ACROPHONIC ATTIC ONE HUNDRED TALENTS;Nl;0;ON;;;;100;N;;;;; +1014C;GREEK ACROPHONIC ATTIC FIVE HUNDRED TALENTS;Nl;0;ON;;;;500;N;;;;; +1014D;GREEK ACROPHONIC ATTIC ONE THOUSAND TALENTS;Nl;0;ON;;;;1000;N;;;;; +1014E;GREEK ACROPHONIC ATTIC FIVE THOUSAND TALENTS;Nl;0;ON;;;;5000;N;;;;; +1014F;GREEK ACROPHONIC ATTIC FIVE STATERS;Nl;0;ON;;;;5;N;;;;; +10150;GREEK ACROPHONIC ATTIC TEN STATERS;Nl;0;ON;;;;10;N;;;;; +10151;GREEK ACROPHONIC ATTIC FIFTY STATERS;Nl;0;ON;;;;50;N;;;;; +10152;GREEK ACROPHONIC ATTIC ONE HUNDRED STATERS;Nl;0;ON;;;;100;N;;;;; +10153;GREEK ACROPHONIC ATTIC FIVE HUNDRED STATERS;Nl;0;ON;;;;500;N;;;;; +10154;GREEK ACROPHONIC ATTIC ONE THOUSAND STATERS;Nl;0;ON;;;;1000;N;;;;; +10155;GREEK ACROPHONIC ATTIC TEN THOUSAND STATERS;Nl;0;ON;;;;10000;N;;;;; +10156;GREEK ACROPHONIC ATTIC FIFTY THOUSAND STATERS;Nl;0;ON;;;;50000;N;;;;; +10157;GREEK ACROPHONIC ATTIC TEN MNAS;Nl;0;ON;;;;10;N;;;;; +10158;GREEK ACROPHONIC HERAEUM ONE PLETHRON;Nl;0;ON;;;;1;N;;;;; +10159;GREEK ACROPHONIC THESPIAN ONE;Nl;0;ON;;;;1;N;;;;; +1015A;GREEK ACROPHONIC HERMIONIAN ONE;Nl;0;ON;;;;1;N;;;;; +1015B;GREEK ACROPHONIC EPIDAUREAN TWO;Nl;0;ON;;;;2;N;;;;; +1015C;GREEK ACROPHONIC THESPIAN TWO;Nl;0;ON;;;;2;N;;;;; +1015D;GREEK ACROPHONIC CYRENAIC TWO DRACHMAS;Nl;0;ON;;;;2;N;;;;; +1015E;GREEK ACROPHONIC EPIDAUREAN TWO DRACHMAS;Nl;0;ON;;;;2;N;;;;; +1015F;GREEK ACROPHONIC TROEZENIAN FIVE;Nl;0;ON;;;;5;N;;;;; +10160;GREEK ACROPHONIC TROEZENIAN TEN;Nl;0;ON;;;;10;N;;;;; +10161;GREEK ACROPHONIC TROEZENIAN TEN ALTERNATE FORM;Nl;0;ON;;;;10;N;;;;; +10162;GREEK ACROPHONIC HERMIONIAN TEN;Nl;0;ON;;;;10;N;;;;; +10163;GREEK ACROPHONIC MESSENIAN TEN;Nl;0;ON;;;;10;N;;;;; +10164;GREEK ACROPHONIC THESPIAN TEN;Nl;0;ON;;;;10;N;;;;; +10165;GREEK ACROPHONIC THESPIAN THIRTY;Nl;0;ON;;;;30;N;;;;; +10166;GREEK ACROPHONIC TROEZENIAN FIFTY;Nl;0;ON;;;;50;N;;;;; +10167;GREEK ACROPHONIC TROEZENIAN FIFTY ALTERNATE FORM;Nl;0;ON;;;;50;N;;;;; +10168;GREEK ACROPHONIC HERMIONIAN FIFTY;Nl;0;ON;;;;50;N;;;;; +10169;GREEK ACROPHONIC THESPIAN FIFTY;Nl;0;ON;;;;50;N;;;;; +1016A;GREEK ACROPHONIC THESPIAN ONE HUNDRED;Nl;0;ON;;;;100;N;;;;; +1016B;GREEK ACROPHONIC THESPIAN THREE HUNDRED;Nl;0;ON;;;;300;N;;;;; +1016C;GREEK ACROPHONIC EPIDAUREAN FIVE HUNDRED;Nl;0;ON;;;;500;N;;;;; +1016D;GREEK ACROPHONIC TROEZENIAN FIVE HUNDRED;Nl;0;ON;;;;500;N;;;;; +1016E;GREEK ACROPHONIC THESPIAN FIVE HUNDRED;Nl;0;ON;;;;500;N;;;;; +1016F;GREEK ACROPHONIC CARYSTIAN FIVE HUNDRED;Nl;0;ON;;;;500;N;;;;; +10170;GREEK ACROPHONIC NAXIAN FIVE HUNDRED;Nl;0;ON;;;;500;N;;;;; +10171;GREEK ACROPHONIC THESPIAN ONE THOUSAND;Nl;0;ON;;;;1000;N;;;;; +10172;GREEK ACROPHONIC THESPIAN FIVE THOUSAND;Nl;0;ON;;;;5000;N;;;;; +10173;GREEK ACROPHONIC DELPHIC FIVE MNAS;Nl;0;ON;;;;5;N;;;;; +10174;GREEK ACROPHONIC STRATIAN FIFTY MNAS;Nl;0;ON;;;;50;N;;;;; +10175;GREEK ONE HALF SIGN;No;0;ON;;;;1/2;N;;;;; +10176;GREEK ONE HALF SIGN ALTERNATE FORM;No;0;ON;;;;1/2;N;;;;; +10177;GREEK TWO THIRDS SIGN;No;0;ON;;;;2/3;N;;;;; +10178;GREEK THREE QUARTERS SIGN;No;0;ON;;;;3/4;N;;;;; +10179;GREEK YEAR SIGN;So;0;ON;;;;;N;;;;; +1017A;GREEK TALENT SIGN;So;0;ON;;;;;N;;;;; +1017B;GREEK DRACHMA SIGN;So;0;ON;;;;;N;;;;; +1017C;GREEK OBOL SIGN;So;0;ON;;;;;N;;;;; +1017D;GREEK TWO OBOLS SIGN;So;0;ON;;;;;N;;;;; +1017E;GREEK THREE OBOLS SIGN;So;0;ON;;;;;N;;;;; +1017F;GREEK FOUR OBOLS SIGN;So;0;ON;;;;;N;;;;; +10180;GREEK FIVE OBOLS SIGN;So;0;ON;;;;;N;;;;; +10181;GREEK METRETES SIGN;So;0;ON;;;;;N;;;;; +10182;GREEK KYATHOS BASE SIGN;So;0;ON;;;;;N;;;;; +10183;GREEK LITRA SIGN;So;0;ON;;;;;N;;;;; +10184;GREEK OUNKIA SIGN;So;0;ON;;;;;N;;;;; +10185;GREEK XESTES SIGN;So;0;ON;;;;;N;;;;; +10186;GREEK ARTABE SIGN;So;0;ON;;;;;N;;;;; +10187;GREEK AROURA SIGN;So;0;ON;;;;;N;;;;; +10188;GREEK GRAMMA SIGN;So;0;ON;;;;;N;;;;; +10189;GREEK TRYBLION BASE SIGN;So;0;ON;;;;;N;;;;; +1018A;GREEK ZERO SIGN;No;0;ON;;;;0;N;;;;; +10190;ROMAN SEXTANS SIGN;So;0;ON;;;;;N;;;;; +10191;ROMAN UNCIA SIGN;So;0;ON;;;;;N;;;;; +10192;ROMAN SEMUNCIA SIGN;So;0;ON;;;;;N;;;;; +10193;ROMAN SEXTULA SIGN;So;0;ON;;;;;N;;;;; +10194;ROMAN DIMIDIA SEXTULA SIGN;So;0;ON;;;;;N;;;;; +10195;ROMAN SILIQUA SIGN;So;0;ON;;;;;N;;;;; +10196;ROMAN DENARIUS SIGN;So;0;ON;;;;;N;;;;; +10197;ROMAN QUINARIUS SIGN;So;0;ON;;;;;N;;;;; +10198;ROMAN SESTERTIUS SIGN;So;0;ON;;;;;N;;;;; +10199;ROMAN DUPONDIUS SIGN;So;0;ON;;;;;N;;;;; +1019A;ROMAN AS SIGN;So;0;ON;;;;;N;;;;; +1019B;ROMAN CENTURIAL SIGN;So;0;ON;;;;;N;;;;; +101D0;PHAISTOS DISC SIGN PEDESTRIAN;So;0;L;;;;;N;;;;; +101D1;PHAISTOS DISC SIGN PLUMED HEAD;So;0;L;;;;;N;;;;; +101D2;PHAISTOS DISC SIGN TATTOOED HEAD;So;0;L;;;;;N;;;;; +101D3;PHAISTOS DISC SIGN CAPTIVE;So;0;L;;;;;N;;;;; +101D4;PHAISTOS DISC SIGN CHILD;So;0;L;;;;;N;;;;; +101D5;PHAISTOS DISC SIGN WOMAN;So;0;L;;;;;N;;;;; +101D6;PHAISTOS DISC SIGN HELMET;So;0;L;;;;;N;;;;; +101D7;PHAISTOS DISC SIGN GAUNTLET;So;0;L;;;;;N;;;;; +101D8;PHAISTOS DISC SIGN TIARA;So;0;L;;;;;N;;;;; +101D9;PHAISTOS DISC SIGN ARROW;So;0;L;;;;;N;;;;; +101DA;PHAISTOS DISC SIGN BOW;So;0;L;;;;;N;;;;; +101DB;PHAISTOS DISC SIGN SHIELD;So;0;L;;;;;N;;;;; +101DC;PHAISTOS DISC SIGN CLUB;So;0;L;;;;;N;;;;; +101DD;PHAISTOS DISC SIGN MANACLES;So;0;L;;;;;N;;;;; +101DE;PHAISTOS DISC SIGN MATTOCK;So;0;L;;;;;N;;;;; +101DF;PHAISTOS DISC SIGN SAW;So;0;L;;;;;N;;;;; +101E0;PHAISTOS DISC SIGN LID;So;0;L;;;;;N;;;;; +101E1;PHAISTOS DISC SIGN BOOMERANG;So;0;L;;;;;N;;;;; +101E2;PHAISTOS DISC SIGN CARPENTRY PLANE;So;0;L;;;;;N;;;;; +101E3;PHAISTOS DISC SIGN DOLIUM;So;0;L;;;;;N;;;;; +101E4;PHAISTOS DISC SIGN COMB;So;0;L;;;;;N;;;;; +101E5;PHAISTOS DISC SIGN SLING;So;0;L;;;;;N;;;;; +101E6;PHAISTOS DISC SIGN COLUMN;So;0;L;;;;;N;;;;; +101E7;PHAISTOS DISC SIGN BEEHIVE;So;0;L;;;;;N;;;;; +101E8;PHAISTOS DISC SIGN SHIP;So;0;L;;;;;N;;;;; +101E9;PHAISTOS DISC SIGN HORN;So;0;L;;;;;N;;;;; +101EA;PHAISTOS DISC SIGN HIDE;So;0;L;;;;;N;;;;; +101EB;PHAISTOS DISC SIGN BULLS LEG;So;0;L;;;;;N;;;;; +101EC;PHAISTOS DISC SIGN CAT;So;0;L;;;;;N;;;;; +101ED;PHAISTOS DISC SIGN RAM;So;0;L;;;;;N;;;;; +101EE;PHAISTOS DISC SIGN EAGLE;So;0;L;;;;;N;;;;; +101EF;PHAISTOS DISC SIGN DOVE;So;0;L;;;;;N;;;;; +101F0;PHAISTOS DISC SIGN TUNNY;So;0;L;;;;;N;;;;; +101F1;PHAISTOS DISC SIGN BEE;So;0;L;;;;;N;;;;; +101F2;PHAISTOS DISC SIGN PLANE TREE;So;0;L;;;;;N;;;;; +101F3;PHAISTOS DISC SIGN VINE;So;0;L;;;;;N;;;;; +101F4;PHAISTOS DISC SIGN PAPYRUS;So;0;L;;;;;N;;;;; +101F5;PHAISTOS DISC SIGN ROSETTE;So;0;L;;;;;N;;;;; +101F6;PHAISTOS DISC SIGN LILY;So;0;L;;;;;N;;;;; +101F7;PHAISTOS DISC SIGN OX BACK;So;0;L;;;;;N;;;;; +101F8;PHAISTOS DISC SIGN FLUTE;So;0;L;;;;;N;;;;; +101F9;PHAISTOS DISC SIGN GRATER;So;0;L;;;;;N;;;;; +101FA;PHAISTOS DISC SIGN STRAINER;So;0;L;;;;;N;;;;; +101FB;PHAISTOS DISC SIGN SMALL AXE;So;0;L;;;;;N;;;;; +101FC;PHAISTOS DISC SIGN WAVY BAND;So;0;L;;;;;N;;;;; +101FD;PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE;Mn;220;NSM;;;;;N;;;;; +10280;LYCIAN LETTER A;Lo;0;L;;;;;N;;;;; +10281;LYCIAN LETTER E;Lo;0;L;;;;;N;;;;; +10282;LYCIAN LETTER B;Lo;0;L;;;;;N;;;;; +10283;LYCIAN LETTER BH;Lo;0;L;;;;;N;;;;; +10284;LYCIAN LETTER G;Lo;0;L;;;;;N;;;;; +10285;LYCIAN LETTER D;Lo;0;L;;;;;N;;;;; +10286;LYCIAN LETTER I;Lo;0;L;;;;;N;;;;; +10287;LYCIAN LETTER W;Lo;0;L;;;;;N;;;;; +10288;LYCIAN LETTER Z;Lo;0;L;;;;;N;;;;; +10289;LYCIAN LETTER TH;Lo;0;L;;;;;N;;;;; +1028A;LYCIAN LETTER J;Lo;0;L;;;;;N;;;;; +1028B;LYCIAN LETTER K;Lo;0;L;;;;;N;;;;; +1028C;LYCIAN LETTER Q;Lo;0;L;;;;;N;;;;; +1028D;LYCIAN LETTER L;Lo;0;L;;;;;N;;;;; +1028E;LYCIAN LETTER M;Lo;0;L;;;;;N;;;;; +1028F;LYCIAN LETTER N;Lo;0;L;;;;;N;;;;; +10290;LYCIAN LETTER MM;Lo;0;L;;;;;N;;;;; +10291;LYCIAN LETTER NN;Lo;0;L;;;;;N;;;;; +10292;LYCIAN LETTER U;Lo;0;L;;;;;N;;;;; +10293;LYCIAN LETTER P;Lo;0;L;;;;;N;;;;; +10294;LYCIAN LETTER KK;Lo;0;L;;;;;N;;;;; +10295;LYCIAN LETTER R;Lo;0;L;;;;;N;;;;; +10296;LYCIAN LETTER S;Lo;0;L;;;;;N;;;;; +10297;LYCIAN LETTER T;Lo;0;L;;;;;N;;;;; +10298;LYCIAN LETTER TT;Lo;0;L;;;;;N;;;;; +10299;LYCIAN LETTER AN;Lo;0;L;;;;;N;;;;; +1029A;LYCIAN LETTER EN;Lo;0;L;;;;;N;;;;; +1029B;LYCIAN LETTER H;Lo;0;L;;;;;N;;;;; +1029C;LYCIAN LETTER X;Lo;0;L;;;;;N;;;;; +102A0;CARIAN LETTER A;Lo;0;L;;;;;N;;;;; +102A1;CARIAN LETTER P2;Lo;0;L;;;;;N;;;;; +102A2;CARIAN LETTER D;Lo;0;L;;;;;N;;;;; +102A3;CARIAN LETTER L;Lo;0;L;;;;;N;;;;; +102A4;CARIAN LETTER UUU;Lo;0;L;;;;;N;;;;; +102A5;CARIAN LETTER R;Lo;0;L;;;;;N;;;;; +102A6;CARIAN LETTER LD;Lo;0;L;;;;;N;;;;; +102A7;CARIAN LETTER A2;Lo;0;L;;;;;N;;;;; +102A8;CARIAN LETTER Q;Lo;0;L;;;;;N;;;;; +102A9;CARIAN LETTER B;Lo;0;L;;;;;N;;;;; +102AA;CARIAN LETTER M;Lo;0;L;;;;;N;;;;; +102AB;CARIAN LETTER O;Lo;0;L;;;;;N;;;;; +102AC;CARIAN LETTER D2;Lo;0;L;;;;;N;;;;; +102AD;CARIAN LETTER T;Lo;0;L;;;;;N;;;;; +102AE;CARIAN LETTER SH;Lo;0;L;;;;;N;;;;; +102AF;CARIAN LETTER SH2;Lo;0;L;;;;;N;;;;; +102B0;CARIAN LETTER S;Lo;0;L;;;;;N;;;;; +102B1;CARIAN LETTER C-18;Lo;0;L;;;;;N;;;;; +102B2;CARIAN LETTER U;Lo;0;L;;;;;N;;;;; +102B3;CARIAN LETTER NN;Lo;0;L;;;;;N;;;;; +102B4;CARIAN LETTER X;Lo;0;L;;;;;N;;;;; +102B5;CARIAN LETTER N;Lo;0;L;;;;;N;;;;; +102B6;CARIAN LETTER TT2;Lo;0;L;;;;;N;;;;; +102B7;CARIAN LETTER P;Lo;0;L;;;;;N;;;;; +102B8;CARIAN LETTER SS;Lo;0;L;;;;;N;;;;; +102B9;CARIAN LETTER I;Lo;0;L;;;;;N;;;;; +102BA;CARIAN LETTER E;Lo;0;L;;;;;N;;;;; +102BB;CARIAN LETTER UUUU;Lo;0;L;;;;;N;;;;; +102BC;CARIAN LETTER K;Lo;0;L;;;;;N;;;;; +102BD;CARIAN LETTER K2;Lo;0;L;;;;;N;;;;; +102BE;CARIAN LETTER ND;Lo;0;L;;;;;N;;;;; +102BF;CARIAN LETTER UU;Lo;0;L;;;;;N;;;;; +102C0;CARIAN LETTER G;Lo;0;L;;;;;N;;;;; +102C1;CARIAN LETTER G2;Lo;0;L;;;;;N;;;;; +102C2;CARIAN LETTER ST;Lo;0;L;;;;;N;;;;; +102C3;CARIAN LETTER ST2;Lo;0;L;;;;;N;;;;; +102C4;CARIAN LETTER NG;Lo;0;L;;;;;N;;;;; +102C5;CARIAN LETTER II;Lo;0;L;;;;;N;;;;; +102C6;CARIAN LETTER C-39;Lo;0;L;;;;;N;;;;; +102C7;CARIAN LETTER TT;Lo;0;L;;;;;N;;;;; +102C8;CARIAN LETTER UUU2;Lo;0;L;;;;;N;;;;; +102C9;CARIAN LETTER RR;Lo;0;L;;;;;N;;;;; +102CA;CARIAN LETTER MB;Lo;0;L;;;;;N;;;;; +102CB;CARIAN LETTER MB2;Lo;0;L;;;;;N;;;;; +102CC;CARIAN LETTER MB3;Lo;0;L;;;;;N;;;;; +102CD;CARIAN LETTER MB4;Lo;0;L;;;;;N;;;;; +102CE;CARIAN LETTER LD2;Lo;0;L;;;;;N;;;;; +102CF;CARIAN LETTER E2;Lo;0;L;;;;;N;;;;; +102D0;CARIAN LETTER UUU3;Lo;0;L;;;;;N;;;;; +10300;OLD ITALIC LETTER A;Lo;0;L;;;;;N;;;;; +10301;OLD ITALIC LETTER BE;Lo;0;L;;;;;N;;;;; +10302;OLD ITALIC LETTER KE;Lo;0;L;;;;;N;;;;; +10303;OLD ITALIC LETTER DE;Lo;0;L;;;;;N;;;;; +10304;OLD ITALIC LETTER E;Lo;0;L;;;;;N;;;;; +10305;OLD ITALIC LETTER VE;Lo;0;L;;;;;N;;;;; +10306;OLD ITALIC LETTER ZE;Lo;0;L;;;;;N;;;;; +10307;OLD ITALIC LETTER HE;Lo;0;L;;;;;N;;;;; +10308;OLD ITALIC LETTER THE;Lo;0;L;;;;;N;;;;; +10309;OLD ITALIC LETTER I;Lo;0;L;;;;;N;;;;; +1030A;OLD ITALIC LETTER KA;Lo;0;L;;;;;N;;;;; +1030B;OLD ITALIC LETTER EL;Lo;0;L;;;;;N;;;;; +1030C;OLD ITALIC LETTER EM;Lo;0;L;;;;;N;;;;; +1030D;OLD ITALIC LETTER EN;Lo;0;L;;;;;N;;;;; +1030E;OLD ITALIC LETTER ESH;Lo;0;L;;;;;N;;;;; +1030F;OLD ITALIC LETTER O;Lo;0;L;;;;;N;;;;; +10310;OLD ITALIC LETTER PE;Lo;0;L;;;;;N;;;;; +10311;OLD ITALIC LETTER SHE;Lo;0;L;;;;;N;;;;; +10312;OLD ITALIC LETTER KU;Lo;0;L;;;;;N;;;;; +10313;OLD ITALIC LETTER ER;Lo;0;L;;;;;N;;;;; +10314;OLD ITALIC LETTER ES;Lo;0;L;;;;;N;;;;; +10315;OLD ITALIC LETTER TE;Lo;0;L;;;;;N;;;;; +10316;OLD ITALIC LETTER U;Lo;0;L;;;;;N;;;;; +10317;OLD ITALIC LETTER EKS;Lo;0;L;;;;;N;;;;; +10318;OLD ITALIC LETTER PHE;Lo;0;L;;;;;N;;;;; +10319;OLD ITALIC LETTER KHE;Lo;0;L;;;;;N;;;;; +1031A;OLD ITALIC LETTER EF;Lo;0;L;;;;;N;;;;; +1031B;OLD ITALIC LETTER ERS;Lo;0;L;;;;;N;;;;; +1031C;OLD ITALIC LETTER CHE;Lo;0;L;;;;;N;;;;; +1031D;OLD ITALIC LETTER II;Lo;0;L;;;;;N;;;;; +1031E;OLD ITALIC LETTER UU;Lo;0;L;;;;;N;;;;; +10320;OLD ITALIC NUMERAL ONE;No;0;L;;;;1;N;;;;; +10321;OLD ITALIC NUMERAL FIVE;No;0;L;;;;5;N;;;;; +10322;OLD ITALIC NUMERAL TEN;No;0;L;;;;10;N;;;;; +10323;OLD ITALIC NUMERAL FIFTY;No;0;L;;;;50;N;;;;; +10330;GOTHIC LETTER AHSA;Lo;0;L;;;;;N;;;;; +10331;GOTHIC LETTER BAIRKAN;Lo;0;L;;;;;N;;;;; +10332;GOTHIC LETTER GIBA;Lo;0;L;;;;;N;;;;; +10333;GOTHIC LETTER DAGS;Lo;0;L;;;;;N;;;;; +10334;GOTHIC LETTER AIHVUS;Lo;0;L;;;;;N;;;;; +10335;GOTHIC LETTER QAIRTHRA;Lo;0;L;;;;;N;;;;; +10336;GOTHIC LETTER IUJA;Lo;0;L;;;;;N;;;;; +10337;GOTHIC LETTER HAGL;Lo;0;L;;;;;N;;;;; +10338;GOTHIC LETTER THIUTH;Lo;0;L;;;;;N;;;;; +10339;GOTHIC LETTER EIS;Lo;0;L;;;;;N;;;;; +1033A;GOTHIC LETTER KUSMA;Lo;0;L;;;;;N;;;;; +1033B;GOTHIC LETTER LAGUS;Lo;0;L;;;;;N;;;;; +1033C;GOTHIC LETTER MANNA;Lo;0;L;;;;;N;;;;; +1033D;GOTHIC LETTER NAUTHS;Lo;0;L;;;;;N;;;;; +1033E;GOTHIC LETTER JER;Lo;0;L;;;;;N;;;;; +1033F;GOTHIC LETTER URUS;Lo;0;L;;;;;N;;;;; +10340;GOTHIC LETTER PAIRTHRA;Lo;0;L;;;;;N;;;;; +10341;GOTHIC LETTER NINETY;Nl;0;L;;;;90;N;;;;; +10342;GOTHIC LETTER RAIDA;Lo;0;L;;;;;N;;;;; +10343;GOTHIC LETTER SAUIL;Lo;0;L;;;;;N;;;;; +10344;GOTHIC LETTER TEIWS;Lo;0;L;;;;;N;;;;; +10345;GOTHIC LETTER WINJA;Lo;0;L;;;;;N;;;;; +10346;GOTHIC LETTER FAIHU;Lo;0;L;;;;;N;;;;; +10347;GOTHIC LETTER IGGWS;Lo;0;L;;;;;N;;;;; +10348;GOTHIC LETTER HWAIR;Lo;0;L;;;;;N;;;;; +10349;GOTHIC LETTER OTHAL;Lo;0;L;;;;;N;;;;; +1034A;GOTHIC LETTER NINE HUNDRED;Nl;0;L;;;;900;N;;;;; +10380;UGARITIC LETTER ALPA;Lo;0;L;;;;;N;;;;; +10381;UGARITIC LETTER BETA;Lo;0;L;;;;;N;;;;; +10382;UGARITIC LETTER GAMLA;Lo;0;L;;;;;N;;;;; +10383;UGARITIC LETTER KHA;Lo;0;L;;;;;N;;;;; +10384;UGARITIC LETTER DELTA;Lo;0;L;;;;;N;;;;; +10385;UGARITIC LETTER HO;Lo;0;L;;;;;N;;;;; +10386;UGARITIC LETTER WO;Lo;0;L;;;;;N;;;;; +10387;UGARITIC LETTER ZETA;Lo;0;L;;;;;N;;;;; +10388;UGARITIC LETTER HOTA;Lo;0;L;;;;;N;;;;; +10389;UGARITIC LETTER TET;Lo;0;L;;;;;N;;;;; +1038A;UGARITIC LETTER YOD;Lo;0;L;;;;;N;;;;; +1038B;UGARITIC LETTER KAF;Lo;0;L;;;;;N;;;;; +1038C;UGARITIC LETTER SHIN;Lo;0;L;;;;;N;;;;; +1038D;UGARITIC LETTER LAMDA;Lo;0;L;;;;;N;;;;; +1038E;UGARITIC LETTER MEM;Lo;0;L;;;;;N;;;;; +1038F;UGARITIC LETTER DHAL;Lo;0;L;;;;;N;;;;; +10390;UGARITIC LETTER NUN;Lo;0;L;;;;;N;;;;; +10391;UGARITIC LETTER ZU;Lo;0;L;;;;;N;;;;; +10392;UGARITIC LETTER SAMKA;Lo;0;L;;;;;N;;;;; +10393;UGARITIC LETTER AIN;Lo;0;L;;;;;N;;;;; +10394;UGARITIC LETTER PU;Lo;0;L;;;;;N;;;;; +10395;UGARITIC LETTER SADE;Lo;0;L;;;;;N;;;;; +10396;UGARITIC LETTER QOPA;Lo;0;L;;;;;N;;;;; +10397;UGARITIC LETTER RASHA;Lo;0;L;;;;;N;;;;; +10398;UGARITIC LETTER THANNA;Lo;0;L;;;;;N;;;;; +10399;UGARITIC LETTER GHAIN;Lo;0;L;;;;;N;;;;; +1039A;UGARITIC LETTER TO;Lo;0;L;;;;;N;;;;; +1039B;UGARITIC LETTER I;Lo;0;L;;;;;N;;;;; +1039C;UGARITIC LETTER U;Lo;0;L;;;;;N;;;;; +1039D;UGARITIC LETTER SSU;Lo;0;L;;;;;N;;;;; +1039F;UGARITIC WORD DIVIDER;Po;0;L;;;;;N;;;;; +103A0;OLD PERSIAN SIGN A;Lo;0;L;;;;;N;;;;; +103A1;OLD PERSIAN SIGN I;Lo;0;L;;;;;N;;;;; +103A2;OLD PERSIAN SIGN U;Lo;0;L;;;;;N;;;;; +103A3;OLD PERSIAN SIGN KA;Lo;0;L;;;;;N;;;;; +103A4;OLD PERSIAN SIGN KU;Lo;0;L;;;;;N;;;;; +103A5;OLD PERSIAN SIGN GA;Lo;0;L;;;;;N;;;;; +103A6;OLD PERSIAN SIGN GU;Lo;0;L;;;;;N;;;;; +103A7;OLD PERSIAN SIGN XA;Lo;0;L;;;;;N;;;;; +103A8;OLD PERSIAN SIGN CA;Lo;0;L;;;;;N;;;;; +103A9;OLD PERSIAN SIGN JA;Lo;0;L;;;;;N;;;;; +103AA;OLD PERSIAN SIGN JI;Lo;0;L;;;;;N;;;;; +103AB;OLD PERSIAN SIGN TA;Lo;0;L;;;;;N;;;;; +103AC;OLD PERSIAN SIGN TU;Lo;0;L;;;;;N;;;;; +103AD;OLD PERSIAN SIGN DA;Lo;0;L;;;;;N;;;;; +103AE;OLD PERSIAN SIGN DI;Lo;0;L;;;;;N;;;;; +103AF;OLD PERSIAN SIGN DU;Lo;0;L;;;;;N;;;;; +103B0;OLD PERSIAN SIGN THA;Lo;0;L;;;;;N;;;;; +103B1;OLD PERSIAN SIGN PA;Lo;0;L;;;;;N;;;;; +103B2;OLD PERSIAN SIGN BA;Lo;0;L;;;;;N;;;;; +103B3;OLD PERSIAN SIGN FA;Lo;0;L;;;;;N;;;;; +103B4;OLD PERSIAN SIGN NA;Lo;0;L;;;;;N;;;;; +103B5;OLD PERSIAN SIGN NU;Lo;0;L;;;;;N;;;;; +103B6;OLD PERSIAN SIGN MA;Lo;0;L;;;;;N;;;;; +103B7;OLD PERSIAN SIGN MI;Lo;0;L;;;;;N;;;;; +103B8;OLD PERSIAN SIGN MU;Lo;0;L;;;;;N;;;;; +103B9;OLD PERSIAN SIGN YA;Lo;0;L;;;;;N;;;;; +103BA;OLD PERSIAN SIGN VA;Lo;0;L;;;;;N;;;;; +103BB;OLD PERSIAN SIGN VI;Lo;0;L;;;;;N;;;;; +103BC;OLD PERSIAN SIGN RA;Lo;0;L;;;;;N;;;;; +103BD;OLD PERSIAN SIGN RU;Lo;0;L;;;;;N;;;;; +103BE;OLD PERSIAN SIGN LA;Lo;0;L;;;;;N;;;;; +103BF;OLD PERSIAN SIGN SA;Lo;0;L;;;;;N;;;;; +103C0;OLD PERSIAN SIGN ZA;Lo;0;L;;;;;N;;;;; +103C1;OLD PERSIAN SIGN SHA;Lo;0;L;;;;;N;;;;; +103C2;OLD PERSIAN SIGN SSA;Lo;0;L;;;;;N;;;;; +103C3;OLD PERSIAN SIGN HA;Lo;0;L;;;;;N;;;;; +103C8;OLD PERSIAN SIGN AURAMAZDAA;Lo;0;L;;;;;N;;;;; +103C9;OLD PERSIAN SIGN AURAMAZDAA-2;Lo;0;L;;;;;N;;;;; +103CA;OLD PERSIAN SIGN AURAMAZDAAHA;Lo;0;L;;;;;N;;;;; +103CB;OLD PERSIAN SIGN XSHAAYATHIYA;Lo;0;L;;;;;N;;;;; +103CC;OLD PERSIAN SIGN DAHYAAUSH;Lo;0;L;;;;;N;;;;; +103CD;OLD PERSIAN SIGN DAHYAAUSH-2;Lo;0;L;;;;;N;;;;; +103CE;OLD PERSIAN SIGN BAGA;Lo;0;L;;;;;N;;;;; +103CF;OLD PERSIAN SIGN BUUMISH;Lo;0;L;;;;;N;;;;; +103D0;OLD PERSIAN WORD DIVIDER;Po;0;L;;;;;N;;;;; +103D1;OLD PERSIAN NUMBER ONE;Nl;0;L;;;;1;N;;;;; +103D2;OLD PERSIAN NUMBER TWO;Nl;0;L;;;;2;N;;;;; +103D3;OLD PERSIAN NUMBER TEN;Nl;0;L;;;;10;N;;;;; +103D4;OLD PERSIAN NUMBER TWENTY;Nl;0;L;;;;20;N;;;;; +103D5;OLD PERSIAN NUMBER HUNDRED;Nl;0;L;;;;100;N;;;;; +10400;DESERET CAPITAL LETTER LONG I;Lu;0;L;;;;;N;;;;10428; +10401;DESERET CAPITAL LETTER LONG E;Lu;0;L;;;;;N;;;;10429; +10402;DESERET CAPITAL LETTER LONG A;Lu;0;L;;;;;N;;;;1042A; +10403;DESERET CAPITAL LETTER LONG AH;Lu;0;L;;;;;N;;;;1042B; +10404;DESERET CAPITAL LETTER LONG O;Lu;0;L;;;;;N;;;;1042C; +10405;DESERET CAPITAL LETTER LONG OO;Lu;0;L;;;;;N;;;;1042D; +10406;DESERET CAPITAL LETTER SHORT I;Lu;0;L;;;;;N;;;;1042E; +10407;DESERET CAPITAL LETTER SHORT E;Lu;0;L;;;;;N;;;;1042F; +10408;DESERET CAPITAL LETTER SHORT A;Lu;0;L;;;;;N;;;;10430; +10409;DESERET CAPITAL LETTER SHORT AH;Lu;0;L;;;;;N;;;;10431; +1040A;DESERET CAPITAL LETTER SHORT O;Lu;0;L;;;;;N;;;;10432; +1040B;DESERET CAPITAL LETTER SHORT OO;Lu;0;L;;;;;N;;;;10433; +1040C;DESERET CAPITAL LETTER AY;Lu;0;L;;;;;N;;;;10434; +1040D;DESERET CAPITAL LETTER OW;Lu;0;L;;;;;N;;;;10435; +1040E;DESERET CAPITAL LETTER WU;Lu;0;L;;;;;N;;;;10436; +1040F;DESERET CAPITAL LETTER YEE;Lu;0;L;;;;;N;;;;10437; +10410;DESERET CAPITAL LETTER H;Lu;0;L;;;;;N;;;;10438; +10411;DESERET CAPITAL LETTER PEE;Lu;0;L;;;;;N;;;;10439; +10412;DESERET CAPITAL LETTER BEE;Lu;0;L;;;;;N;;;;1043A; +10413;DESERET CAPITAL LETTER TEE;Lu;0;L;;;;;N;;;;1043B; +10414;DESERET CAPITAL LETTER DEE;Lu;0;L;;;;;N;;;;1043C; +10415;DESERET CAPITAL LETTER CHEE;Lu;0;L;;;;;N;;;;1043D; +10416;DESERET CAPITAL LETTER JEE;Lu;0;L;;;;;N;;;;1043E; +10417;DESERET CAPITAL LETTER KAY;Lu;0;L;;;;;N;;;;1043F; +10418;DESERET CAPITAL LETTER GAY;Lu;0;L;;;;;N;;;;10440; +10419;DESERET CAPITAL LETTER EF;Lu;0;L;;;;;N;;;;10441; +1041A;DESERET CAPITAL LETTER VEE;Lu;0;L;;;;;N;;;;10442; +1041B;DESERET CAPITAL LETTER ETH;Lu;0;L;;;;;N;;;;10443; +1041C;DESERET CAPITAL LETTER THEE;Lu;0;L;;;;;N;;;;10444; +1041D;DESERET CAPITAL LETTER ES;Lu;0;L;;;;;N;;;;10445; +1041E;DESERET CAPITAL LETTER ZEE;Lu;0;L;;;;;N;;;;10446; +1041F;DESERET CAPITAL LETTER ESH;Lu;0;L;;;;;N;;;;10447; +10420;DESERET CAPITAL LETTER ZHEE;Lu;0;L;;;;;N;;;;10448; +10421;DESERET CAPITAL LETTER ER;Lu;0;L;;;;;N;;;;10449; +10422;DESERET CAPITAL LETTER EL;Lu;0;L;;;;;N;;;;1044A; +10423;DESERET CAPITAL LETTER EM;Lu;0;L;;;;;N;;;;1044B; +10424;DESERET CAPITAL LETTER EN;Lu;0;L;;;;;N;;;;1044C; +10425;DESERET CAPITAL LETTER ENG;Lu;0;L;;;;;N;;;;1044D; +10426;DESERET CAPITAL LETTER OI;Lu;0;L;;;;;N;;;;1044E; +10427;DESERET CAPITAL LETTER EW;Lu;0;L;;;;;N;;;;1044F; +10428;DESERET SMALL LETTER LONG I;Ll;0;L;;;;;N;;;10400;;10400 +10429;DESERET SMALL LETTER LONG E;Ll;0;L;;;;;N;;;10401;;10401 +1042A;DESERET SMALL LETTER LONG A;Ll;0;L;;;;;N;;;10402;;10402 +1042B;DESERET SMALL LETTER LONG AH;Ll;0;L;;;;;N;;;10403;;10403 +1042C;DESERET SMALL LETTER LONG O;Ll;0;L;;;;;N;;;10404;;10404 +1042D;DESERET SMALL LETTER LONG OO;Ll;0;L;;;;;N;;;10405;;10405 +1042E;DESERET SMALL LETTER SHORT I;Ll;0;L;;;;;N;;;10406;;10406 +1042F;DESERET SMALL LETTER SHORT E;Ll;0;L;;;;;N;;;10407;;10407 +10430;DESERET SMALL LETTER SHORT A;Ll;0;L;;;;;N;;;10408;;10408 +10431;DESERET SMALL LETTER SHORT AH;Ll;0;L;;;;;N;;;10409;;10409 +10432;DESERET SMALL LETTER SHORT O;Ll;0;L;;;;;N;;;1040A;;1040A +10433;DESERET SMALL LETTER SHORT OO;Ll;0;L;;;;;N;;;1040B;;1040B +10434;DESERET SMALL LETTER AY;Ll;0;L;;;;;N;;;1040C;;1040C +10435;DESERET SMALL LETTER OW;Ll;0;L;;;;;N;;;1040D;;1040D +10436;DESERET SMALL LETTER WU;Ll;0;L;;;;;N;;;1040E;;1040E +10437;DESERET SMALL LETTER YEE;Ll;0;L;;;;;N;;;1040F;;1040F +10438;DESERET SMALL LETTER H;Ll;0;L;;;;;N;;;10410;;10410 +10439;DESERET SMALL LETTER PEE;Ll;0;L;;;;;N;;;10411;;10411 +1043A;DESERET SMALL LETTER BEE;Ll;0;L;;;;;N;;;10412;;10412 +1043B;DESERET SMALL LETTER TEE;Ll;0;L;;;;;N;;;10413;;10413 +1043C;DESERET SMALL LETTER DEE;Ll;0;L;;;;;N;;;10414;;10414 +1043D;DESERET SMALL LETTER CHEE;Ll;0;L;;;;;N;;;10415;;10415 +1043E;DESERET SMALL LETTER JEE;Ll;0;L;;;;;N;;;10416;;10416 +1043F;DESERET SMALL LETTER KAY;Ll;0;L;;;;;N;;;10417;;10417 +10440;DESERET SMALL LETTER GAY;Ll;0;L;;;;;N;;;10418;;10418 +10441;DESERET SMALL LETTER EF;Ll;0;L;;;;;N;;;10419;;10419 +10442;DESERET SMALL LETTER VEE;Ll;0;L;;;;;N;;;1041A;;1041A +10443;DESERET SMALL LETTER ETH;Ll;0;L;;;;;N;;;1041B;;1041B +10444;DESERET SMALL LETTER THEE;Ll;0;L;;;;;N;;;1041C;;1041C +10445;DESERET SMALL LETTER ES;Ll;0;L;;;;;N;;;1041D;;1041D +10446;DESERET SMALL LETTER ZEE;Ll;0;L;;;;;N;;;1041E;;1041E +10447;DESERET SMALL LETTER ESH;Ll;0;L;;;;;N;;;1041F;;1041F +10448;DESERET SMALL LETTER ZHEE;Ll;0;L;;;;;N;;;10420;;10420 +10449;DESERET SMALL LETTER ER;Ll;0;L;;;;;N;;;10421;;10421 +1044A;DESERET SMALL LETTER EL;Ll;0;L;;;;;N;;;10422;;10422 +1044B;DESERET SMALL LETTER EM;Ll;0;L;;;;;N;;;10423;;10423 +1044C;DESERET SMALL LETTER EN;Ll;0;L;;;;;N;;;10424;;10424 +1044D;DESERET SMALL LETTER ENG;Ll;0;L;;;;;N;;;10425;;10425 +1044E;DESERET SMALL LETTER OI;Ll;0;L;;;;;N;;;10426;;10426 +1044F;DESERET SMALL LETTER EW;Ll;0;L;;;;;N;;;10427;;10427 +10450;SHAVIAN LETTER PEEP;Lo;0;L;;;;;N;;;;; +10451;SHAVIAN LETTER TOT;Lo;0;L;;;;;N;;;;; +10452;SHAVIAN LETTER KICK;Lo;0;L;;;;;N;;;;; +10453;SHAVIAN LETTER FEE;Lo;0;L;;;;;N;;;;; +10454;SHAVIAN LETTER THIGH;Lo;0;L;;;;;N;;;;; +10455;SHAVIAN LETTER SO;Lo;0;L;;;;;N;;;;; +10456;SHAVIAN LETTER SURE;Lo;0;L;;;;;N;;;;; +10457;SHAVIAN LETTER CHURCH;Lo;0;L;;;;;N;;;;; +10458;SHAVIAN LETTER YEA;Lo;0;L;;;;;N;;;;; +10459;SHAVIAN LETTER HUNG;Lo;0;L;;;;;N;;;;; +1045A;SHAVIAN LETTER BIB;Lo;0;L;;;;;N;;;;; +1045B;SHAVIAN LETTER DEAD;Lo;0;L;;;;;N;;;;; +1045C;SHAVIAN LETTER GAG;Lo;0;L;;;;;N;;;;; +1045D;SHAVIAN LETTER VOW;Lo;0;L;;;;;N;;;;; +1045E;SHAVIAN LETTER THEY;Lo;0;L;;;;;N;;;;; +1045F;SHAVIAN LETTER ZOO;Lo;0;L;;;;;N;;;;; +10460;SHAVIAN LETTER MEASURE;Lo;0;L;;;;;N;;;;; +10461;SHAVIAN LETTER JUDGE;Lo;0;L;;;;;N;;;;; +10462;SHAVIAN LETTER WOE;Lo;0;L;;;;;N;;;;; +10463;SHAVIAN LETTER HA-HA;Lo;0;L;;;;;N;;;;; +10464;SHAVIAN LETTER LOLL;Lo;0;L;;;;;N;;;;; +10465;SHAVIAN LETTER MIME;Lo;0;L;;;;;N;;;;; +10466;SHAVIAN LETTER IF;Lo;0;L;;;;;N;;;;; +10467;SHAVIAN LETTER EGG;Lo;0;L;;;;;N;;;;; +10468;SHAVIAN LETTER ASH;Lo;0;L;;;;;N;;;;; +10469;SHAVIAN LETTER ADO;Lo;0;L;;;;;N;;;;; +1046A;SHAVIAN LETTER ON;Lo;0;L;;;;;N;;;;; +1046B;SHAVIAN LETTER WOOL;Lo;0;L;;;;;N;;;;; +1046C;SHAVIAN LETTER OUT;Lo;0;L;;;;;N;;;;; +1046D;SHAVIAN LETTER AH;Lo;0;L;;;;;N;;;;; +1046E;SHAVIAN LETTER ROAR;Lo;0;L;;;;;N;;;;; +1046F;SHAVIAN LETTER NUN;Lo;0;L;;;;;N;;;;; +10470;SHAVIAN LETTER EAT;Lo;0;L;;;;;N;;;;; +10471;SHAVIAN LETTER AGE;Lo;0;L;;;;;N;;;;; +10472;SHAVIAN LETTER ICE;Lo;0;L;;;;;N;;;;; +10473;SHAVIAN LETTER UP;Lo;0;L;;;;;N;;;;; +10474;SHAVIAN LETTER OAK;Lo;0;L;;;;;N;;;;; +10475;SHAVIAN LETTER OOZE;Lo;0;L;;;;;N;;;;; +10476;SHAVIAN LETTER OIL;Lo;0;L;;;;;N;;;;; +10477;SHAVIAN LETTER AWE;Lo;0;L;;;;;N;;;;; +10478;SHAVIAN LETTER ARE;Lo;0;L;;;;;N;;;;; +10479;SHAVIAN LETTER OR;Lo;0;L;;;;;N;;;;; +1047A;SHAVIAN LETTER AIR;Lo;0;L;;;;;N;;;;; +1047B;SHAVIAN LETTER ERR;Lo;0;L;;;;;N;;;;; +1047C;SHAVIAN LETTER ARRAY;Lo;0;L;;;;;N;;;;; +1047D;SHAVIAN LETTER EAR;Lo;0;L;;;;;N;;;;; +1047E;SHAVIAN LETTER IAN;Lo;0;L;;;;;N;;;;; +1047F;SHAVIAN LETTER YEW;Lo;0;L;;;;;N;;;;; +10480;OSMANYA LETTER ALEF;Lo;0;L;;;;;N;;;;; +10481;OSMANYA LETTER BA;Lo;0;L;;;;;N;;;;; +10482;OSMANYA LETTER TA;Lo;0;L;;;;;N;;;;; +10483;OSMANYA LETTER JA;Lo;0;L;;;;;N;;;;; +10484;OSMANYA LETTER XA;Lo;0;L;;;;;N;;;;; +10485;OSMANYA LETTER KHA;Lo;0;L;;;;;N;;;;; +10486;OSMANYA LETTER DEEL;Lo;0;L;;;;;N;;;;; +10487;OSMANYA LETTER RA;Lo;0;L;;;;;N;;;;; +10488;OSMANYA LETTER SA;Lo;0;L;;;;;N;;;;; +10489;OSMANYA LETTER SHIIN;Lo;0;L;;;;;N;;;;; +1048A;OSMANYA LETTER DHA;Lo;0;L;;;;;N;;;;; +1048B;OSMANYA LETTER CAYN;Lo;0;L;;;;;N;;;;; +1048C;OSMANYA LETTER GA;Lo;0;L;;;;;N;;;;; +1048D;OSMANYA LETTER FA;Lo;0;L;;;;;N;;;;; +1048E;OSMANYA LETTER QAAF;Lo;0;L;;;;;N;;;;; +1048F;OSMANYA LETTER KAAF;Lo;0;L;;;;;N;;;;; +10490;OSMANYA LETTER LAAN;Lo;0;L;;;;;N;;;;; +10491;OSMANYA LETTER MIIN;Lo;0;L;;;;;N;;;;; +10492;OSMANYA LETTER NUUN;Lo;0;L;;;;;N;;;;; +10493;OSMANYA LETTER WAW;Lo;0;L;;;;;N;;;;; +10494;OSMANYA LETTER HA;Lo;0;L;;;;;N;;;;; +10495;OSMANYA LETTER YA;Lo;0;L;;;;;N;;;;; +10496;OSMANYA LETTER A;Lo;0;L;;;;;N;;;;; +10497;OSMANYA LETTER E;Lo;0;L;;;;;N;;;;; +10498;OSMANYA LETTER I;Lo;0;L;;;;;N;;;;; +10499;OSMANYA LETTER O;Lo;0;L;;;;;N;;;;; +1049A;OSMANYA LETTER U;Lo;0;L;;;;;N;;;;; +1049B;OSMANYA LETTER AA;Lo;0;L;;;;;N;;;;; +1049C;OSMANYA LETTER EE;Lo;0;L;;;;;N;;;;; +1049D;OSMANYA LETTER OO;Lo;0;L;;;;;N;;;;; +104A0;OSMANYA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +104A1;OSMANYA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +104A2;OSMANYA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +104A3;OSMANYA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +104A4;OSMANYA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +104A5;OSMANYA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +104A6;OSMANYA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +104A7;OSMANYA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +104A8;OSMANYA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +104A9;OSMANYA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +10800;CYPRIOT SYLLABLE A;Lo;0;R;;;;;N;;;;; +10801;CYPRIOT SYLLABLE E;Lo;0;R;;;;;N;;;;; +10802;CYPRIOT SYLLABLE I;Lo;0;R;;;;;N;;;;; +10803;CYPRIOT SYLLABLE O;Lo;0;R;;;;;N;;;;; +10804;CYPRIOT SYLLABLE U;Lo;0;R;;;;;N;;;;; +10805;CYPRIOT SYLLABLE JA;Lo;0;R;;;;;N;;;;; +10808;CYPRIOT SYLLABLE JO;Lo;0;R;;;;;N;;;;; +1080A;CYPRIOT SYLLABLE KA;Lo;0;R;;;;;N;;;;; +1080B;CYPRIOT SYLLABLE KE;Lo;0;R;;;;;N;;;;; +1080C;CYPRIOT SYLLABLE KI;Lo;0;R;;;;;N;;;;; +1080D;CYPRIOT SYLLABLE KO;Lo;0;R;;;;;N;;;;; +1080E;CYPRIOT SYLLABLE KU;Lo;0;R;;;;;N;;;;; +1080F;CYPRIOT SYLLABLE LA;Lo;0;R;;;;;N;;;;; +10810;CYPRIOT SYLLABLE LE;Lo;0;R;;;;;N;;;;; +10811;CYPRIOT SYLLABLE LI;Lo;0;R;;;;;N;;;;; +10812;CYPRIOT SYLLABLE LO;Lo;0;R;;;;;N;;;;; +10813;CYPRIOT SYLLABLE LU;Lo;0;R;;;;;N;;;;; +10814;CYPRIOT SYLLABLE MA;Lo;0;R;;;;;N;;;;; +10815;CYPRIOT SYLLABLE ME;Lo;0;R;;;;;N;;;;; +10816;CYPRIOT SYLLABLE MI;Lo;0;R;;;;;N;;;;; +10817;CYPRIOT SYLLABLE MO;Lo;0;R;;;;;N;;;;; +10818;CYPRIOT SYLLABLE MU;Lo;0;R;;;;;N;;;;; +10819;CYPRIOT SYLLABLE NA;Lo;0;R;;;;;N;;;;; +1081A;CYPRIOT SYLLABLE NE;Lo;0;R;;;;;N;;;;; +1081B;CYPRIOT SYLLABLE NI;Lo;0;R;;;;;N;;;;; +1081C;CYPRIOT SYLLABLE NO;Lo;0;R;;;;;N;;;;; +1081D;CYPRIOT SYLLABLE NU;Lo;0;R;;;;;N;;;;; +1081E;CYPRIOT SYLLABLE PA;Lo;0;R;;;;;N;;;;; +1081F;CYPRIOT SYLLABLE PE;Lo;0;R;;;;;N;;;;; +10820;CYPRIOT SYLLABLE PI;Lo;0;R;;;;;N;;;;; +10821;CYPRIOT SYLLABLE PO;Lo;0;R;;;;;N;;;;; +10822;CYPRIOT SYLLABLE PU;Lo;0;R;;;;;N;;;;; +10823;CYPRIOT SYLLABLE RA;Lo;0;R;;;;;N;;;;; +10824;CYPRIOT SYLLABLE RE;Lo;0;R;;;;;N;;;;; +10825;CYPRIOT SYLLABLE RI;Lo;0;R;;;;;N;;;;; +10826;CYPRIOT SYLLABLE RO;Lo;0;R;;;;;N;;;;; +10827;CYPRIOT SYLLABLE RU;Lo;0;R;;;;;N;;;;; +10828;CYPRIOT SYLLABLE SA;Lo;0;R;;;;;N;;;;; +10829;CYPRIOT SYLLABLE SE;Lo;0;R;;;;;N;;;;; +1082A;CYPRIOT SYLLABLE SI;Lo;0;R;;;;;N;;;;; +1082B;CYPRIOT SYLLABLE SO;Lo;0;R;;;;;N;;;;; +1082C;CYPRIOT SYLLABLE SU;Lo;0;R;;;;;N;;;;; +1082D;CYPRIOT SYLLABLE TA;Lo;0;R;;;;;N;;;;; +1082E;CYPRIOT SYLLABLE TE;Lo;0;R;;;;;N;;;;; +1082F;CYPRIOT SYLLABLE TI;Lo;0;R;;;;;N;;;;; +10830;CYPRIOT SYLLABLE TO;Lo;0;R;;;;;N;;;;; +10831;CYPRIOT SYLLABLE TU;Lo;0;R;;;;;N;;;;; +10832;CYPRIOT SYLLABLE WA;Lo;0;R;;;;;N;;;;; +10833;CYPRIOT SYLLABLE WE;Lo;0;R;;;;;N;;;;; +10834;CYPRIOT SYLLABLE WI;Lo;0;R;;;;;N;;;;; +10835;CYPRIOT SYLLABLE WO;Lo;0;R;;;;;N;;;;; +10837;CYPRIOT SYLLABLE XA;Lo;0;R;;;;;N;;;;; +10838;CYPRIOT SYLLABLE XE;Lo;0;R;;;;;N;;;;; +1083C;CYPRIOT SYLLABLE ZA;Lo;0;R;;;;;N;;;;; +1083F;CYPRIOT SYLLABLE ZO;Lo;0;R;;;;;N;;;;; +10840;IMPERIAL ARAMAIC LETTER ALEPH;Lo;0;R;;;;;N;;;;; +10841;IMPERIAL ARAMAIC LETTER BETH;Lo;0;R;;;;;N;;;;; +10842;IMPERIAL ARAMAIC LETTER GIMEL;Lo;0;R;;;;;N;;;;; +10843;IMPERIAL ARAMAIC LETTER DALETH;Lo;0;R;;;;;N;;;;; +10844;IMPERIAL ARAMAIC LETTER HE;Lo;0;R;;;;;N;;;;; +10845;IMPERIAL ARAMAIC LETTER WAW;Lo;0;R;;;;;N;;;;; +10846;IMPERIAL ARAMAIC LETTER ZAYIN;Lo;0;R;;;;;N;;;;; +10847;IMPERIAL ARAMAIC LETTER HETH;Lo;0;R;;;;;N;;;;; +10848;IMPERIAL ARAMAIC LETTER TETH;Lo;0;R;;;;;N;;;;; +10849;IMPERIAL ARAMAIC LETTER YODH;Lo;0;R;;;;;N;;;;; +1084A;IMPERIAL ARAMAIC LETTER KAPH;Lo;0;R;;;;;N;;;;; +1084B;IMPERIAL ARAMAIC LETTER LAMEDH;Lo;0;R;;;;;N;;;;; +1084C;IMPERIAL ARAMAIC LETTER MEM;Lo;0;R;;;;;N;;;;; +1084D;IMPERIAL ARAMAIC LETTER NUN;Lo;0;R;;;;;N;;;;; +1084E;IMPERIAL ARAMAIC LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +1084F;IMPERIAL ARAMAIC LETTER AYIN;Lo;0;R;;;;;N;;;;; +10850;IMPERIAL ARAMAIC LETTER PE;Lo;0;R;;;;;N;;;;; +10851;IMPERIAL ARAMAIC LETTER SADHE;Lo;0;R;;;;;N;;;;; +10852;IMPERIAL ARAMAIC LETTER QOPH;Lo;0;R;;;;;N;;;;; +10853;IMPERIAL ARAMAIC LETTER RESH;Lo;0;R;;;;;N;;;;; +10854;IMPERIAL ARAMAIC LETTER SHIN;Lo;0;R;;;;;N;;;;; +10855;IMPERIAL ARAMAIC LETTER TAW;Lo;0;R;;;;;N;;;;; +10857;IMPERIAL ARAMAIC SECTION SIGN;Po;0;R;;;;;N;;;;; +10858;IMPERIAL ARAMAIC NUMBER ONE;No;0;R;;;;1;N;;;;; +10859;IMPERIAL ARAMAIC NUMBER TWO;No;0;R;;;;2;N;;;;; +1085A;IMPERIAL ARAMAIC NUMBER THREE;No;0;R;;;;3;N;;;;; +1085B;IMPERIAL ARAMAIC NUMBER TEN;No;0;R;;;;10;N;;;;; +1085C;IMPERIAL ARAMAIC NUMBER TWENTY;No;0;R;;;;20;N;;;;; +1085D;IMPERIAL ARAMAIC NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; +1085E;IMPERIAL ARAMAIC NUMBER ONE THOUSAND;No;0;R;;;;1000;N;;;;; +1085F;IMPERIAL ARAMAIC NUMBER TEN THOUSAND;No;0;R;;;;10000;N;;;;; +10900;PHOENICIAN LETTER ALF;Lo;0;R;;;;;N;;;;; +10901;PHOENICIAN LETTER BET;Lo;0;R;;;;;N;;;;; +10902;PHOENICIAN LETTER GAML;Lo;0;R;;;;;N;;;;; +10903;PHOENICIAN LETTER DELT;Lo;0;R;;;;;N;;;;; +10904;PHOENICIAN LETTER HE;Lo;0;R;;;;;N;;;;; +10905;PHOENICIAN LETTER WAU;Lo;0;R;;;;;N;;;;; +10906;PHOENICIAN LETTER ZAI;Lo;0;R;;;;;N;;;;; +10907;PHOENICIAN LETTER HET;Lo;0;R;;;;;N;;;;; +10908;PHOENICIAN LETTER TET;Lo;0;R;;;;;N;;;;; +10909;PHOENICIAN LETTER YOD;Lo;0;R;;;;;N;;;;; +1090A;PHOENICIAN LETTER KAF;Lo;0;R;;;;;N;;;;; +1090B;PHOENICIAN LETTER LAMD;Lo;0;R;;;;;N;;;;; +1090C;PHOENICIAN LETTER MEM;Lo;0;R;;;;;N;;;;; +1090D;PHOENICIAN LETTER NUN;Lo;0;R;;;;;N;;;;; +1090E;PHOENICIAN LETTER SEMK;Lo;0;R;;;;;N;;;;; +1090F;PHOENICIAN LETTER AIN;Lo;0;R;;;;;N;;;;; +10910;PHOENICIAN LETTER PE;Lo;0;R;;;;;N;;;;; +10911;PHOENICIAN LETTER SADE;Lo;0;R;;;;;N;;;;; +10912;PHOENICIAN LETTER QOF;Lo;0;R;;;;;N;;;;; +10913;PHOENICIAN LETTER ROSH;Lo;0;R;;;;;N;;;;; +10914;PHOENICIAN LETTER SHIN;Lo;0;R;;;;;N;;;;; +10915;PHOENICIAN LETTER TAU;Lo;0;R;;;;;N;;;;; +10916;PHOENICIAN NUMBER ONE;No;0;R;;;;1;N;;;;; +10917;PHOENICIAN NUMBER TEN;No;0;R;;;;10;N;;;;; +10918;PHOENICIAN NUMBER TWENTY;No;0;R;;;;20;N;;;;; +10919;PHOENICIAN NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; +1091A;PHOENICIAN NUMBER TWO;No;0;R;;;;2;N;;;;; +1091B;PHOENICIAN NUMBER THREE;No;0;R;;;;3;N;;;;; +1091F;PHOENICIAN WORD SEPARATOR;Po;0;ON;;;;;N;;;;; +10920;LYDIAN LETTER A;Lo;0;R;;;;;N;;;;; +10921;LYDIAN LETTER B;Lo;0;R;;;;;N;;;;; +10922;LYDIAN LETTER G;Lo;0;R;;;;;N;;;;; +10923;LYDIAN LETTER D;Lo;0;R;;;;;N;;;;; +10924;LYDIAN LETTER E;Lo;0;R;;;;;N;;;;; +10925;LYDIAN LETTER V;Lo;0;R;;;;;N;;;;; +10926;LYDIAN LETTER I;Lo;0;R;;;;;N;;;;; +10927;LYDIAN LETTER Y;Lo;0;R;;;;;N;;;;; +10928;LYDIAN LETTER K;Lo;0;R;;;;;N;;;;; +10929;LYDIAN LETTER L;Lo;0;R;;;;;N;;;;; +1092A;LYDIAN LETTER M;Lo;0;R;;;;;N;;;;; +1092B;LYDIAN LETTER N;Lo;0;R;;;;;N;;;;; +1092C;LYDIAN LETTER O;Lo;0;R;;;;;N;;;;; +1092D;LYDIAN LETTER R;Lo;0;R;;;;;N;;;;; +1092E;LYDIAN LETTER SS;Lo;0;R;;;;;N;;;;; +1092F;LYDIAN LETTER T;Lo;0;R;;;;;N;;;;; +10930;LYDIAN LETTER U;Lo;0;R;;;;;N;;;;; +10931;LYDIAN LETTER F;Lo;0;R;;;;;N;;;;; +10932;LYDIAN LETTER Q;Lo;0;R;;;;;N;;;;; +10933;LYDIAN LETTER S;Lo;0;R;;;;;N;;;;; +10934;LYDIAN LETTER TT;Lo;0;R;;;;;N;;;;; +10935;LYDIAN LETTER AN;Lo;0;R;;;;;N;;;;; +10936;LYDIAN LETTER EN;Lo;0;R;;;;;N;;;;; +10937;LYDIAN LETTER LY;Lo;0;R;;;;;N;;;;; +10938;LYDIAN LETTER NN;Lo;0;R;;;;;N;;;;; +10939;LYDIAN LETTER C;Lo;0;R;;;;;N;;;;; +1093F;LYDIAN TRIANGULAR MARK;Po;0;R;;;;;N;;;;; +10980;MEROITIC HIEROGLYPHIC LETTER A;Lo;0;R;;;;;N;;;;; +10981;MEROITIC HIEROGLYPHIC LETTER E;Lo;0;R;;;;;N;;;;; +10982;MEROITIC HIEROGLYPHIC LETTER I;Lo;0;R;;;;;N;;;;; +10983;MEROITIC HIEROGLYPHIC LETTER O;Lo;0;R;;;;;N;;;;; +10984;MEROITIC HIEROGLYPHIC LETTER YA;Lo;0;R;;;;;N;;;;; +10985;MEROITIC HIEROGLYPHIC LETTER WA;Lo;0;R;;;;;N;;;;; +10986;MEROITIC HIEROGLYPHIC LETTER BA;Lo;0;R;;;;;N;;;;; +10987;MEROITIC HIEROGLYPHIC LETTER BA-2;Lo;0;R;;;;;N;;;;; +10988;MEROITIC HIEROGLYPHIC LETTER PA;Lo;0;R;;;;;N;;;;; +10989;MEROITIC HIEROGLYPHIC LETTER MA;Lo;0;R;;;;;N;;;;; +1098A;MEROITIC HIEROGLYPHIC LETTER NA;Lo;0;R;;;;;N;;;;; +1098B;MEROITIC HIEROGLYPHIC LETTER NA-2;Lo;0;R;;;;;N;;;;; +1098C;MEROITIC HIEROGLYPHIC LETTER NE;Lo;0;R;;;;;N;;;;; +1098D;MEROITIC HIEROGLYPHIC LETTER NE-2;Lo;0;R;;;;;N;;;;; +1098E;MEROITIC HIEROGLYPHIC LETTER RA;Lo;0;R;;;;;N;;;;; +1098F;MEROITIC HIEROGLYPHIC LETTER RA-2;Lo;0;R;;;;;N;;;;; +10990;MEROITIC HIEROGLYPHIC LETTER LA;Lo;0;R;;;;;N;;;;; +10991;MEROITIC HIEROGLYPHIC LETTER KHA;Lo;0;R;;;;;N;;;;; +10992;MEROITIC HIEROGLYPHIC LETTER HHA;Lo;0;R;;;;;N;;;;; +10993;MEROITIC HIEROGLYPHIC LETTER SA;Lo;0;R;;;;;N;;;;; +10994;MEROITIC HIEROGLYPHIC LETTER SA-2;Lo;0;R;;;;;N;;;;; +10995;MEROITIC HIEROGLYPHIC LETTER SE;Lo;0;R;;;;;N;;;;; +10996;MEROITIC HIEROGLYPHIC LETTER KA;Lo;0;R;;;;;N;;;;; +10997;MEROITIC HIEROGLYPHIC LETTER QA;Lo;0;R;;;;;N;;;;; +10998;MEROITIC HIEROGLYPHIC LETTER TA;Lo;0;R;;;;;N;;;;; +10999;MEROITIC HIEROGLYPHIC LETTER TA-2;Lo;0;R;;;;;N;;;;; +1099A;MEROITIC HIEROGLYPHIC LETTER TE;Lo;0;R;;;;;N;;;;; +1099B;MEROITIC HIEROGLYPHIC LETTER TE-2;Lo;0;R;;;;;N;;;;; +1099C;MEROITIC HIEROGLYPHIC LETTER TO;Lo;0;R;;;;;N;;;;; +1099D;MEROITIC HIEROGLYPHIC LETTER DA;Lo;0;R;;;;;N;;;;; +1099E;MEROITIC HIEROGLYPHIC SYMBOL VIDJ;Lo;0;R;;;;;N;;;;; +1099F;MEROITIC HIEROGLYPHIC SYMBOL VIDJ-2;Lo;0;R;;;;;N;;;;; +109A0;MEROITIC CURSIVE LETTER A;Lo;0;R;;;;;N;;;;; +109A1;MEROITIC CURSIVE LETTER E;Lo;0;R;;;;;N;;;;; +109A2;MEROITIC CURSIVE LETTER I;Lo;0;R;;;;;N;;;;; +109A3;MEROITIC CURSIVE LETTER O;Lo;0;R;;;;;N;;;;; +109A4;MEROITIC CURSIVE LETTER YA;Lo;0;R;;;;;N;;;;; +109A5;MEROITIC CURSIVE LETTER WA;Lo;0;R;;;;;N;;;;; +109A6;MEROITIC CURSIVE LETTER BA;Lo;0;R;;;;;N;;;;; +109A7;MEROITIC CURSIVE LETTER PA;Lo;0;R;;;;;N;;;;; +109A8;MEROITIC CURSIVE LETTER MA;Lo;0;R;;;;;N;;;;; +109A9;MEROITIC CURSIVE LETTER NA;Lo;0;R;;;;;N;;;;; +109AA;MEROITIC CURSIVE LETTER NE;Lo;0;R;;;;;N;;;;; +109AB;MEROITIC CURSIVE LETTER RA;Lo;0;R;;;;;N;;;;; +109AC;MEROITIC CURSIVE LETTER LA;Lo;0;R;;;;;N;;;;; +109AD;MEROITIC CURSIVE LETTER KHA;Lo;0;R;;;;;N;;;;; +109AE;MEROITIC CURSIVE LETTER HHA;Lo;0;R;;;;;N;;;;; +109AF;MEROITIC CURSIVE LETTER SA;Lo;0;R;;;;;N;;;;; +109B0;MEROITIC CURSIVE LETTER ARCHAIC SA;Lo;0;R;;;;;N;;;;; +109B1;MEROITIC CURSIVE LETTER SE;Lo;0;R;;;;;N;;;;; +109B2;MEROITIC CURSIVE LETTER KA;Lo;0;R;;;;;N;;;;; +109B3;MEROITIC CURSIVE LETTER QA;Lo;0;R;;;;;N;;;;; +109B4;MEROITIC CURSIVE LETTER TA;Lo;0;R;;;;;N;;;;; +109B5;MEROITIC CURSIVE LETTER TE;Lo;0;R;;;;;N;;;;; +109B6;MEROITIC CURSIVE LETTER TO;Lo;0;R;;;;;N;;;;; +109B7;MEROITIC CURSIVE LETTER DA;Lo;0;R;;;;;N;;;;; +109BE;MEROITIC CURSIVE LOGOGRAM RMT;Lo;0;R;;;;;N;;;;; +109BF;MEROITIC CURSIVE LOGOGRAM IMN;Lo;0;R;;;;;N;;;;; +10A00;KHAROSHTHI LETTER A;Lo;0;R;;;;;N;;;;; +10A01;KHAROSHTHI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +10A02;KHAROSHTHI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +10A03;KHAROSHTHI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +10A05;KHAROSHTHI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +10A06;KHAROSHTHI VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +10A0C;KHAROSHTHI VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;; +10A0D;KHAROSHTHI SIGN DOUBLE RING BELOW;Mn;220;NSM;;;;;N;;;;; +10A0E;KHAROSHTHI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +10A0F;KHAROSHTHI SIGN VISARGA;Mn;230;NSM;;;;;N;;;;; +10A10;KHAROSHTHI LETTER KA;Lo;0;R;;;;;N;;;;; +10A11;KHAROSHTHI LETTER KHA;Lo;0;R;;;;;N;;;;; +10A12;KHAROSHTHI LETTER GA;Lo;0;R;;;;;N;;;;; +10A13;KHAROSHTHI LETTER GHA;Lo;0;R;;;;;N;;;;; +10A15;KHAROSHTHI LETTER CA;Lo;0;R;;;;;N;;;;; +10A16;KHAROSHTHI LETTER CHA;Lo;0;R;;;;;N;;;;; +10A17;KHAROSHTHI LETTER JA;Lo;0;R;;;;;N;;;;; +10A19;KHAROSHTHI LETTER NYA;Lo;0;R;;;;;N;;;;; +10A1A;KHAROSHTHI LETTER TTA;Lo;0;R;;;;;N;;;;; +10A1B;KHAROSHTHI LETTER TTHA;Lo;0;R;;;;;N;;;;; +10A1C;KHAROSHTHI LETTER DDA;Lo;0;R;;;;;N;;;;; +10A1D;KHAROSHTHI LETTER DDHA;Lo;0;R;;;;;N;;;;; +10A1E;KHAROSHTHI LETTER NNA;Lo;0;R;;;;;N;;;;; +10A1F;KHAROSHTHI LETTER TA;Lo;0;R;;;;;N;;;;; +10A20;KHAROSHTHI LETTER THA;Lo;0;R;;;;;N;;;;; +10A21;KHAROSHTHI LETTER DA;Lo;0;R;;;;;N;;;;; +10A22;KHAROSHTHI LETTER DHA;Lo;0;R;;;;;N;;;;; +10A23;KHAROSHTHI LETTER NA;Lo;0;R;;;;;N;;;;; +10A24;KHAROSHTHI LETTER PA;Lo;0;R;;;;;N;;;;; +10A25;KHAROSHTHI LETTER PHA;Lo;0;R;;;;;N;;;;; +10A26;KHAROSHTHI LETTER BA;Lo;0;R;;;;;N;;;;; +10A27;KHAROSHTHI LETTER BHA;Lo;0;R;;;;;N;;;;; +10A28;KHAROSHTHI LETTER MA;Lo;0;R;;;;;N;;;;; +10A29;KHAROSHTHI LETTER YA;Lo;0;R;;;;;N;;;;; +10A2A;KHAROSHTHI LETTER RA;Lo;0;R;;;;;N;;;;; +10A2B;KHAROSHTHI LETTER LA;Lo;0;R;;;;;N;;;;; +10A2C;KHAROSHTHI LETTER VA;Lo;0;R;;;;;N;;;;; +10A2D;KHAROSHTHI LETTER SHA;Lo;0;R;;;;;N;;;;; +10A2E;KHAROSHTHI LETTER SSA;Lo;0;R;;;;;N;;;;; +10A2F;KHAROSHTHI LETTER SA;Lo;0;R;;;;;N;;;;; +10A30;KHAROSHTHI LETTER ZA;Lo;0;R;;;;;N;;;;; +10A31;KHAROSHTHI LETTER HA;Lo;0;R;;;;;N;;;;; +10A32;KHAROSHTHI LETTER KKA;Lo;0;R;;;;;N;;;;; +10A33;KHAROSHTHI LETTER TTTHA;Lo;0;R;;;;;N;;;;; +10A38;KHAROSHTHI SIGN BAR ABOVE;Mn;230;NSM;;;;;N;;;;; +10A39;KHAROSHTHI SIGN CAUDA;Mn;1;NSM;;;;;N;;;;; +10A3A;KHAROSHTHI SIGN DOT BELOW;Mn;220;NSM;;;;;N;;;;; +10A3F;KHAROSHTHI VIRAMA;Mn;9;NSM;;;;;N;;;;; +10A40;KHAROSHTHI DIGIT ONE;No;0;R;;;1;1;N;;;;; +10A41;KHAROSHTHI DIGIT TWO;No;0;R;;;2;2;N;;;;; +10A42;KHAROSHTHI DIGIT THREE;No;0;R;;;3;3;N;;;;; +10A43;KHAROSHTHI DIGIT FOUR;No;0;R;;;4;4;N;;;;; +10A44;KHAROSHTHI NUMBER TEN;No;0;R;;;;10;N;;;;; +10A45;KHAROSHTHI NUMBER TWENTY;No;0;R;;;;20;N;;;;; +10A46;KHAROSHTHI NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; +10A47;KHAROSHTHI NUMBER ONE THOUSAND;No;0;R;;;;1000;N;;;;; +10A50;KHAROSHTHI PUNCTUATION DOT;Po;0;R;;;;;N;;;;; +10A51;KHAROSHTHI PUNCTUATION SMALL CIRCLE;Po;0;R;;;;;N;;;;; +10A52;KHAROSHTHI PUNCTUATION CIRCLE;Po;0;R;;;;;N;;;;; +10A53;KHAROSHTHI PUNCTUATION CRESCENT BAR;Po;0;R;;;;;N;;;;; +10A54;KHAROSHTHI PUNCTUATION MANGALAM;Po;0;R;;;;;N;;;;; +10A55;KHAROSHTHI PUNCTUATION LOTUS;Po;0;R;;;;;N;;;;; +10A56;KHAROSHTHI PUNCTUATION DANDA;Po;0;R;;;;;N;;;;; +10A57;KHAROSHTHI PUNCTUATION DOUBLE DANDA;Po;0;R;;;;;N;;;;; +10A58;KHAROSHTHI PUNCTUATION LINES;Po;0;R;;;;;N;;;;; +10A60;OLD SOUTH ARABIAN LETTER HE;Lo;0;R;;;;;N;;;;; +10A61;OLD SOUTH ARABIAN LETTER LAMEDH;Lo;0;R;;;;;N;;;;; +10A62;OLD SOUTH ARABIAN LETTER HETH;Lo;0;R;;;;;N;;;;; +10A63;OLD SOUTH ARABIAN LETTER MEM;Lo;0;R;;;;;N;;;;; +10A64;OLD SOUTH ARABIAN LETTER QOPH;Lo;0;R;;;;;N;;;;; +10A65;OLD SOUTH ARABIAN LETTER WAW;Lo;0;R;;;;;N;;;;; +10A66;OLD SOUTH ARABIAN LETTER SHIN;Lo;0;R;;;;;N;;;;; +10A67;OLD SOUTH ARABIAN LETTER RESH;Lo;0;R;;;;;N;;;;; +10A68;OLD SOUTH ARABIAN LETTER BETH;Lo;0;R;;;;;N;;;;; +10A69;OLD SOUTH ARABIAN LETTER TAW;Lo;0;R;;;;;N;;;;; +10A6A;OLD SOUTH ARABIAN LETTER SAT;Lo;0;R;;;;;N;;;;; +10A6B;OLD SOUTH ARABIAN LETTER KAPH;Lo;0;R;;;;;N;;;;; +10A6C;OLD SOUTH ARABIAN LETTER NUN;Lo;0;R;;;;;N;;;;; +10A6D;OLD SOUTH ARABIAN LETTER KHETH;Lo;0;R;;;;;N;;;;; +10A6E;OLD SOUTH ARABIAN LETTER SADHE;Lo;0;R;;;;;N;;;;; +10A6F;OLD SOUTH ARABIAN LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +10A70;OLD SOUTH ARABIAN LETTER FE;Lo;0;R;;;;;N;;;;; +10A71;OLD SOUTH ARABIAN LETTER ALEF;Lo;0;R;;;;;N;;;;; +10A72;OLD SOUTH ARABIAN LETTER AYN;Lo;0;R;;;;;N;;;;; +10A73;OLD SOUTH ARABIAN LETTER DHADHE;Lo;0;R;;;;;N;;;;; +10A74;OLD SOUTH ARABIAN LETTER GIMEL;Lo;0;R;;;;;N;;;;; +10A75;OLD SOUTH ARABIAN LETTER DALETH;Lo;0;R;;;;;N;;;;; +10A76;OLD SOUTH ARABIAN LETTER GHAYN;Lo;0;R;;;;;N;;;;; +10A77;OLD SOUTH ARABIAN LETTER TETH;Lo;0;R;;;;;N;;;;; +10A78;OLD SOUTH ARABIAN LETTER ZAYN;Lo;0;R;;;;;N;;;;; +10A79;OLD SOUTH ARABIAN LETTER DHALETH;Lo;0;R;;;;;N;;;;; +10A7A;OLD SOUTH ARABIAN LETTER YODH;Lo;0;R;;;;;N;;;;; +10A7B;OLD SOUTH ARABIAN LETTER THAW;Lo;0;R;;;;;N;;;;; +10A7C;OLD SOUTH ARABIAN LETTER THETH;Lo;0;R;;;;;N;;;;; +10A7D;OLD SOUTH ARABIAN NUMBER ONE;No;0;R;;;;1;N;;;;; +10A7E;OLD SOUTH ARABIAN NUMBER FIFTY;No;0;R;;;;50;N;;;;; +10A7F;OLD SOUTH ARABIAN NUMERIC INDICATOR;Po;0;R;;;;;N;;;;; +10B00;AVESTAN LETTER A;Lo;0;R;;;;;N;;;;; +10B01;AVESTAN LETTER AA;Lo;0;R;;;;;N;;;;; +10B02;AVESTAN LETTER AO;Lo;0;R;;;;;N;;;;; +10B03;AVESTAN LETTER AAO;Lo;0;R;;;;;N;;;;; +10B04;AVESTAN LETTER AN;Lo;0;R;;;;;N;;;;; +10B05;AVESTAN LETTER AAN;Lo;0;R;;;;;N;;;;; +10B06;AVESTAN LETTER AE;Lo;0;R;;;;;N;;;;; +10B07;AVESTAN LETTER AEE;Lo;0;R;;;;;N;;;;; +10B08;AVESTAN LETTER E;Lo;0;R;;;;;N;;;;; +10B09;AVESTAN LETTER EE;Lo;0;R;;;;;N;;;;; +10B0A;AVESTAN LETTER O;Lo;0;R;;;;;N;;;;; +10B0B;AVESTAN LETTER OO;Lo;0;R;;;;;N;;;;; +10B0C;AVESTAN LETTER I;Lo;0;R;;;;;N;;;;; +10B0D;AVESTAN LETTER II;Lo;0;R;;;;;N;;;;; +10B0E;AVESTAN LETTER U;Lo;0;R;;;;;N;;;;; +10B0F;AVESTAN LETTER UU;Lo;0;R;;;;;N;;;;; +10B10;AVESTAN LETTER KE;Lo;0;R;;;;;N;;;;; +10B11;AVESTAN LETTER XE;Lo;0;R;;;;;N;;;;; +10B12;AVESTAN LETTER XYE;Lo;0;R;;;;;N;;;;; +10B13;AVESTAN LETTER XVE;Lo;0;R;;;;;N;;;;; +10B14;AVESTAN LETTER GE;Lo;0;R;;;;;N;;;;; +10B15;AVESTAN LETTER GGE;Lo;0;R;;;;;N;;;;; +10B16;AVESTAN LETTER GHE;Lo;0;R;;;;;N;;;;; +10B17;AVESTAN LETTER CE;Lo;0;R;;;;;N;;;;; +10B18;AVESTAN LETTER JE;Lo;0;R;;;;;N;;;;; +10B19;AVESTAN LETTER TE;Lo;0;R;;;;;N;;;;; +10B1A;AVESTAN LETTER THE;Lo;0;R;;;;;N;;;;; +10B1B;AVESTAN LETTER DE;Lo;0;R;;;;;N;;;;; +10B1C;AVESTAN LETTER DHE;Lo;0;R;;;;;N;;;;; +10B1D;AVESTAN LETTER TTE;Lo;0;R;;;;;N;;;;; +10B1E;AVESTAN LETTER PE;Lo;0;R;;;;;N;;;;; +10B1F;AVESTAN LETTER FE;Lo;0;R;;;;;N;;;;; +10B20;AVESTAN LETTER BE;Lo;0;R;;;;;N;;;;; +10B21;AVESTAN LETTER BHE;Lo;0;R;;;;;N;;;;; +10B22;AVESTAN LETTER NGE;Lo;0;R;;;;;N;;;;; +10B23;AVESTAN LETTER NGYE;Lo;0;R;;;;;N;;;;; +10B24;AVESTAN LETTER NGVE;Lo;0;R;;;;;N;;;;; +10B25;AVESTAN LETTER NE;Lo;0;R;;;;;N;;;;; +10B26;AVESTAN LETTER NYE;Lo;0;R;;;;;N;;;;; +10B27;AVESTAN LETTER NNE;Lo;0;R;;;;;N;;;;; +10B28;AVESTAN LETTER ME;Lo;0;R;;;;;N;;;;; +10B29;AVESTAN LETTER HME;Lo;0;R;;;;;N;;;;; +10B2A;AVESTAN LETTER YYE;Lo;0;R;;;;;N;;;;; +10B2B;AVESTAN LETTER YE;Lo;0;R;;;;;N;;;;; +10B2C;AVESTAN LETTER VE;Lo;0;R;;;;;N;;;;; +10B2D;AVESTAN LETTER RE;Lo;0;R;;;;;N;;;;; +10B2E;AVESTAN LETTER LE;Lo;0;R;;;;;N;;;;; +10B2F;AVESTAN LETTER SE;Lo;0;R;;;;;N;;;;; +10B30;AVESTAN LETTER ZE;Lo;0;R;;;;;N;;;;; +10B31;AVESTAN LETTER SHE;Lo;0;R;;;;;N;;;;; +10B32;AVESTAN LETTER ZHE;Lo;0;R;;;;;N;;;;; +10B33;AVESTAN LETTER SHYE;Lo;0;R;;;;;N;;;;; +10B34;AVESTAN LETTER SSHE;Lo;0;R;;;;;N;;;;; +10B35;AVESTAN LETTER HE;Lo;0;R;;;;;N;;;;; +10B39;AVESTAN ABBREVIATION MARK;Po;0;ON;;;;;N;;;;; +10B3A;TINY TWO DOTS OVER ONE DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +10B3B;SMALL TWO DOTS OVER ONE DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +10B3C;LARGE TWO DOTS OVER ONE DOT PUNCTUATION;Po;0;ON;;;;;N;;;;; +10B3D;LARGE ONE DOT OVER TWO DOTS PUNCTUATION;Po;0;ON;;;;;N;;;;; +10B3E;LARGE TWO RINGS OVER ONE RING PUNCTUATION;Po;0;ON;;;;;N;;;;; +10B3F;LARGE ONE RING OVER TWO RINGS PUNCTUATION;Po;0;ON;;;;;N;;;;; +10B40;INSCRIPTIONAL PARTHIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; +10B41;INSCRIPTIONAL PARTHIAN LETTER BETH;Lo;0;R;;;;;N;;;;; +10B42;INSCRIPTIONAL PARTHIAN LETTER GIMEL;Lo;0;R;;;;;N;;;;; +10B43;INSCRIPTIONAL PARTHIAN LETTER DALETH;Lo;0;R;;;;;N;;;;; +10B44;INSCRIPTIONAL PARTHIAN LETTER HE;Lo;0;R;;;;;N;;;;; +10B45;INSCRIPTIONAL PARTHIAN LETTER WAW;Lo;0;R;;;;;N;;;;; +10B46;INSCRIPTIONAL PARTHIAN LETTER ZAYIN;Lo;0;R;;;;;N;;;;; +10B47;INSCRIPTIONAL PARTHIAN LETTER HETH;Lo;0;R;;;;;N;;;;; +10B48;INSCRIPTIONAL PARTHIAN LETTER TETH;Lo;0;R;;;;;N;;;;; +10B49;INSCRIPTIONAL PARTHIAN LETTER YODH;Lo;0;R;;;;;N;;;;; +10B4A;INSCRIPTIONAL PARTHIAN LETTER KAPH;Lo;0;R;;;;;N;;;;; +10B4B;INSCRIPTIONAL PARTHIAN LETTER LAMEDH;Lo;0;R;;;;;N;;;;; +10B4C;INSCRIPTIONAL PARTHIAN LETTER MEM;Lo;0;R;;;;;N;;;;; +10B4D;INSCRIPTIONAL PARTHIAN LETTER NUN;Lo;0;R;;;;;N;;;;; +10B4E;INSCRIPTIONAL PARTHIAN LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +10B4F;INSCRIPTIONAL PARTHIAN LETTER AYIN;Lo;0;R;;;;;N;;;;; +10B50;INSCRIPTIONAL PARTHIAN LETTER PE;Lo;0;R;;;;;N;;;;; +10B51;INSCRIPTIONAL PARTHIAN LETTER SADHE;Lo;0;R;;;;;N;;;;; +10B52;INSCRIPTIONAL PARTHIAN LETTER QOPH;Lo;0;R;;;;;N;;;;; +10B53;INSCRIPTIONAL PARTHIAN LETTER RESH;Lo;0;R;;;;;N;;;;; +10B54;INSCRIPTIONAL PARTHIAN LETTER SHIN;Lo;0;R;;;;;N;;;;; +10B55;INSCRIPTIONAL PARTHIAN LETTER TAW;Lo;0;R;;;;;N;;;;; +10B58;INSCRIPTIONAL PARTHIAN NUMBER ONE;No;0;R;;;;1;N;;;;; +10B59;INSCRIPTIONAL PARTHIAN NUMBER TWO;No;0;R;;;;2;N;;;;; +10B5A;INSCRIPTIONAL PARTHIAN NUMBER THREE;No;0;R;;;;3;N;;;;; +10B5B;INSCRIPTIONAL PARTHIAN NUMBER FOUR;No;0;R;;;;4;N;;;;; +10B5C;INSCRIPTIONAL PARTHIAN NUMBER TEN;No;0;R;;;;10;N;;;;; +10B5D;INSCRIPTIONAL PARTHIAN NUMBER TWENTY;No;0;R;;;;20;N;;;;; +10B5E;INSCRIPTIONAL PARTHIAN NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; +10B5F;INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND;No;0;R;;;;1000;N;;;;; +10B60;INSCRIPTIONAL PAHLAVI LETTER ALEPH;Lo;0;R;;;;;N;;;;; +10B61;INSCRIPTIONAL PAHLAVI LETTER BETH;Lo;0;R;;;;;N;;;;; +10B62;INSCRIPTIONAL PAHLAVI LETTER GIMEL;Lo;0;R;;;;;N;;;;; +10B63;INSCRIPTIONAL PAHLAVI LETTER DALETH;Lo;0;R;;;;;N;;;;; +10B64;INSCRIPTIONAL PAHLAVI LETTER HE;Lo;0;R;;;;;N;;;;; +10B65;INSCRIPTIONAL PAHLAVI LETTER WAW-AYIN-RESH;Lo;0;R;;;;;N;;;;; +10B66;INSCRIPTIONAL PAHLAVI LETTER ZAYIN;Lo;0;R;;;;;N;;;;; +10B67;INSCRIPTIONAL PAHLAVI LETTER HETH;Lo;0;R;;;;;N;;;;; +10B68;INSCRIPTIONAL PAHLAVI LETTER TETH;Lo;0;R;;;;;N;;;;; +10B69;INSCRIPTIONAL PAHLAVI LETTER YODH;Lo;0;R;;;;;N;;;;; +10B6A;INSCRIPTIONAL PAHLAVI LETTER KAPH;Lo;0;R;;;;;N;;;;; +10B6B;INSCRIPTIONAL PAHLAVI LETTER LAMEDH;Lo;0;R;;;;;N;;;;; +10B6C;INSCRIPTIONAL PAHLAVI LETTER MEM-QOPH;Lo;0;R;;;;;N;;;;; +10B6D;INSCRIPTIONAL PAHLAVI LETTER NUN;Lo;0;R;;;;;N;;;;; +10B6E;INSCRIPTIONAL PAHLAVI LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +10B6F;INSCRIPTIONAL PAHLAVI LETTER PE;Lo;0;R;;;;;N;;;;; +10B70;INSCRIPTIONAL PAHLAVI LETTER SADHE;Lo;0;R;;;;;N;;;;; +10B71;INSCRIPTIONAL PAHLAVI LETTER SHIN;Lo;0;R;;;;;N;;;;; +10B72;INSCRIPTIONAL PAHLAVI LETTER TAW;Lo;0;R;;;;;N;;;;; +10B78;INSCRIPTIONAL PAHLAVI NUMBER ONE;No;0;R;;;;1;N;;;;; +10B79;INSCRIPTIONAL PAHLAVI NUMBER TWO;No;0;R;;;;2;N;;;;; +10B7A;INSCRIPTIONAL PAHLAVI NUMBER THREE;No;0;R;;;;3;N;;;;; +10B7B;INSCRIPTIONAL PAHLAVI NUMBER FOUR;No;0;R;;;;4;N;;;;; +10B7C;INSCRIPTIONAL PAHLAVI NUMBER TEN;No;0;R;;;;10;N;;;;; +10B7D;INSCRIPTIONAL PAHLAVI NUMBER TWENTY;No;0;R;;;;20;N;;;;; +10B7E;INSCRIPTIONAL PAHLAVI NUMBER ONE HUNDRED;No;0;R;;;;100;N;;;;; +10B7F;INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND;No;0;R;;;;1000;N;;;;; +10C00;OLD TURKIC LETTER ORKHON A;Lo;0;R;;;;;N;;;;; +10C01;OLD TURKIC LETTER YENISEI A;Lo;0;R;;;;;N;;;;; +10C02;OLD TURKIC LETTER YENISEI AE;Lo;0;R;;;;;N;;;;; +10C03;OLD TURKIC LETTER ORKHON I;Lo;0;R;;;;;N;;;;; +10C04;OLD TURKIC LETTER YENISEI I;Lo;0;R;;;;;N;;;;; +10C05;OLD TURKIC LETTER YENISEI E;Lo;0;R;;;;;N;;;;; +10C06;OLD TURKIC LETTER ORKHON O;Lo;0;R;;;;;N;;;;; +10C07;OLD TURKIC LETTER ORKHON OE;Lo;0;R;;;;;N;;;;; +10C08;OLD TURKIC LETTER YENISEI OE;Lo;0;R;;;;;N;;;;; +10C09;OLD TURKIC LETTER ORKHON AB;Lo;0;R;;;;;N;;;;; +10C0A;OLD TURKIC LETTER YENISEI AB;Lo;0;R;;;;;N;;;;; +10C0B;OLD TURKIC LETTER ORKHON AEB;Lo;0;R;;;;;N;;;;; +10C0C;OLD TURKIC LETTER YENISEI AEB;Lo;0;R;;;;;N;;;;; +10C0D;OLD TURKIC LETTER ORKHON AG;Lo;0;R;;;;;N;;;;; +10C0E;OLD TURKIC LETTER YENISEI AG;Lo;0;R;;;;;N;;;;; +10C0F;OLD TURKIC LETTER ORKHON AEG;Lo;0;R;;;;;N;;;;; +10C10;OLD TURKIC LETTER YENISEI AEG;Lo;0;R;;;;;N;;;;; +10C11;OLD TURKIC LETTER ORKHON AD;Lo;0;R;;;;;N;;;;; +10C12;OLD TURKIC LETTER YENISEI AD;Lo;0;R;;;;;N;;;;; +10C13;OLD TURKIC LETTER ORKHON AED;Lo;0;R;;;;;N;;;;; +10C14;OLD TURKIC LETTER ORKHON EZ;Lo;0;R;;;;;N;;;;; +10C15;OLD TURKIC LETTER YENISEI EZ;Lo;0;R;;;;;N;;;;; +10C16;OLD TURKIC LETTER ORKHON AY;Lo;0;R;;;;;N;;;;; +10C17;OLD TURKIC LETTER YENISEI AY;Lo;0;R;;;;;N;;;;; +10C18;OLD TURKIC LETTER ORKHON AEY;Lo;0;R;;;;;N;;;;; +10C19;OLD TURKIC LETTER YENISEI AEY;Lo;0;R;;;;;N;;;;; +10C1A;OLD TURKIC LETTER ORKHON AEK;Lo;0;R;;;;;N;;;;; +10C1B;OLD TURKIC LETTER YENISEI AEK;Lo;0;R;;;;;N;;;;; +10C1C;OLD TURKIC LETTER ORKHON OEK;Lo;0;R;;;;;N;;;;; +10C1D;OLD TURKIC LETTER YENISEI OEK;Lo;0;R;;;;;N;;;;; +10C1E;OLD TURKIC LETTER ORKHON AL;Lo;0;R;;;;;N;;;;; +10C1F;OLD TURKIC LETTER YENISEI AL;Lo;0;R;;;;;N;;;;; +10C20;OLD TURKIC LETTER ORKHON AEL;Lo;0;R;;;;;N;;;;; +10C21;OLD TURKIC LETTER ORKHON ELT;Lo;0;R;;;;;N;;;;; +10C22;OLD TURKIC LETTER ORKHON EM;Lo;0;R;;;;;N;;;;; +10C23;OLD TURKIC LETTER ORKHON AN;Lo;0;R;;;;;N;;;;; +10C24;OLD TURKIC LETTER ORKHON AEN;Lo;0;R;;;;;N;;;;; +10C25;OLD TURKIC LETTER YENISEI AEN;Lo;0;R;;;;;N;;;;; +10C26;OLD TURKIC LETTER ORKHON ENT;Lo;0;R;;;;;N;;;;; +10C27;OLD TURKIC LETTER YENISEI ENT;Lo;0;R;;;;;N;;;;; +10C28;OLD TURKIC LETTER ORKHON ENC;Lo;0;R;;;;;N;;;;; +10C29;OLD TURKIC LETTER YENISEI ENC;Lo;0;R;;;;;N;;;;; +10C2A;OLD TURKIC LETTER ORKHON ENY;Lo;0;R;;;;;N;;;;; +10C2B;OLD TURKIC LETTER YENISEI ENY;Lo;0;R;;;;;N;;;;; +10C2C;OLD TURKIC LETTER YENISEI ANG;Lo;0;R;;;;;N;;;;; +10C2D;OLD TURKIC LETTER ORKHON ENG;Lo;0;R;;;;;N;;;;; +10C2E;OLD TURKIC LETTER YENISEI AENG;Lo;0;R;;;;;N;;;;; +10C2F;OLD TURKIC LETTER ORKHON EP;Lo;0;R;;;;;N;;;;; +10C30;OLD TURKIC LETTER ORKHON OP;Lo;0;R;;;;;N;;;;; +10C31;OLD TURKIC LETTER ORKHON IC;Lo;0;R;;;;;N;;;;; +10C32;OLD TURKIC LETTER ORKHON EC;Lo;0;R;;;;;N;;;;; +10C33;OLD TURKIC LETTER YENISEI EC;Lo;0;R;;;;;N;;;;; +10C34;OLD TURKIC LETTER ORKHON AQ;Lo;0;R;;;;;N;;;;; +10C35;OLD TURKIC LETTER YENISEI AQ;Lo;0;R;;;;;N;;;;; +10C36;OLD TURKIC LETTER ORKHON IQ;Lo;0;R;;;;;N;;;;; +10C37;OLD TURKIC LETTER YENISEI IQ;Lo;0;R;;;;;N;;;;; +10C38;OLD TURKIC LETTER ORKHON OQ;Lo;0;R;;;;;N;;;;; +10C39;OLD TURKIC LETTER YENISEI OQ;Lo;0;R;;;;;N;;;;; +10C3A;OLD TURKIC LETTER ORKHON AR;Lo;0;R;;;;;N;;;;; +10C3B;OLD TURKIC LETTER YENISEI AR;Lo;0;R;;;;;N;;;;; +10C3C;OLD TURKIC LETTER ORKHON AER;Lo;0;R;;;;;N;;;;; +10C3D;OLD TURKIC LETTER ORKHON AS;Lo;0;R;;;;;N;;;;; +10C3E;OLD TURKIC LETTER ORKHON AES;Lo;0;R;;;;;N;;;;; +10C3F;OLD TURKIC LETTER ORKHON ASH;Lo;0;R;;;;;N;;;;; +10C40;OLD TURKIC LETTER YENISEI ASH;Lo;0;R;;;;;N;;;;; +10C41;OLD TURKIC LETTER ORKHON ESH;Lo;0;R;;;;;N;;;;; +10C42;OLD TURKIC LETTER YENISEI ESH;Lo;0;R;;;;;N;;;;; +10C43;OLD TURKIC LETTER ORKHON AT;Lo;0;R;;;;;N;;;;; +10C44;OLD TURKIC LETTER YENISEI AT;Lo;0;R;;;;;N;;;;; +10C45;OLD TURKIC LETTER ORKHON AET;Lo;0;R;;;;;N;;;;; +10C46;OLD TURKIC LETTER YENISEI AET;Lo;0;R;;;;;N;;;;; +10C47;OLD TURKIC LETTER ORKHON OT;Lo;0;R;;;;;N;;;;; +10C48;OLD TURKIC LETTER ORKHON BASH;Lo;0;R;;;;;N;;;;; +10E60;RUMI DIGIT ONE;No;0;AN;;;1;1;N;;;;; +10E61;RUMI DIGIT TWO;No;0;AN;;;2;2;N;;;;; +10E62;RUMI DIGIT THREE;No;0;AN;;;3;3;N;;;;; +10E63;RUMI DIGIT FOUR;No;0;AN;;;4;4;N;;;;; +10E64;RUMI DIGIT FIVE;No;0;AN;;;5;5;N;;;;; +10E65;RUMI DIGIT SIX;No;0;AN;;;6;6;N;;;;; +10E66;RUMI DIGIT SEVEN;No;0;AN;;;7;7;N;;;;; +10E67;RUMI DIGIT EIGHT;No;0;AN;;;8;8;N;;;;; +10E68;RUMI DIGIT NINE;No;0;AN;;;9;9;N;;;;; +10E69;RUMI NUMBER TEN;No;0;AN;;;;10;N;;;;; +10E6A;RUMI NUMBER TWENTY;No;0;AN;;;;20;N;;;;; +10E6B;RUMI NUMBER THIRTY;No;0;AN;;;;30;N;;;;; +10E6C;RUMI NUMBER FORTY;No;0;AN;;;;40;N;;;;; +10E6D;RUMI NUMBER FIFTY;No;0;AN;;;;50;N;;;;; +10E6E;RUMI NUMBER SIXTY;No;0;AN;;;;60;N;;;;; +10E6F;RUMI NUMBER SEVENTY;No;0;AN;;;;70;N;;;;; +10E70;RUMI NUMBER EIGHTY;No;0;AN;;;;80;N;;;;; +10E71;RUMI NUMBER NINETY;No;0;AN;;;;90;N;;;;; +10E72;RUMI NUMBER ONE HUNDRED;No;0;AN;;;;100;N;;;;; +10E73;RUMI NUMBER TWO HUNDRED;No;0;AN;;;;200;N;;;;; +10E74;RUMI NUMBER THREE HUNDRED;No;0;AN;;;;300;N;;;;; +10E75;RUMI NUMBER FOUR HUNDRED;No;0;AN;;;;400;N;;;;; +10E76;RUMI NUMBER FIVE HUNDRED;No;0;AN;;;;500;N;;;;; +10E77;RUMI NUMBER SIX HUNDRED;No;0;AN;;;;600;N;;;;; +10E78;RUMI NUMBER SEVEN HUNDRED;No;0;AN;;;;700;N;;;;; +10E79;RUMI NUMBER EIGHT HUNDRED;No;0;AN;;;;800;N;;;;; +10E7A;RUMI NUMBER NINE HUNDRED;No;0;AN;;;;900;N;;;;; +10E7B;RUMI FRACTION ONE HALF;No;0;AN;;;;1/2;N;;;;; +10E7C;RUMI FRACTION ONE QUARTER;No;0;AN;;;;1/4;N;;;;; +10E7D;RUMI FRACTION ONE THIRD;No;0;AN;;;;1/3;N;;;;; +10E7E;RUMI FRACTION TWO THIRDS;No;0;AN;;;;2/3;N;;;;; +11000;BRAHMI SIGN CANDRABINDU;Mc;0;L;;;;;N;;;;; +11001;BRAHMI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11002;BRAHMI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11003;BRAHMI SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; +11004;BRAHMI SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +11005;BRAHMI LETTER A;Lo;0;L;;;;;N;;;;; +11006;BRAHMI LETTER AA;Lo;0;L;;;;;N;;;;; +11007;BRAHMI LETTER I;Lo;0;L;;;;;N;;;;; +11008;BRAHMI LETTER II;Lo;0;L;;;;;N;;;;; +11009;BRAHMI LETTER U;Lo;0;L;;;;;N;;;;; +1100A;BRAHMI LETTER UU;Lo;0;L;;;;;N;;;;; +1100B;BRAHMI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +1100C;BRAHMI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +1100D;BRAHMI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +1100E;BRAHMI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +1100F;BRAHMI LETTER E;Lo;0;L;;;;;N;;;;; +11010;BRAHMI LETTER AI;Lo;0;L;;;;;N;;;;; +11011;BRAHMI LETTER O;Lo;0;L;;;;;N;;;;; +11012;BRAHMI LETTER AU;Lo;0;L;;;;;N;;;;; +11013;BRAHMI LETTER KA;Lo;0;L;;;;;N;;;;; +11014;BRAHMI LETTER KHA;Lo;0;L;;;;;N;;;;; +11015;BRAHMI LETTER GA;Lo;0;L;;;;;N;;;;; +11016;BRAHMI LETTER GHA;Lo;0;L;;;;;N;;;;; +11017;BRAHMI LETTER NGA;Lo;0;L;;;;;N;;;;; +11018;BRAHMI LETTER CA;Lo;0;L;;;;;N;;;;; +11019;BRAHMI LETTER CHA;Lo;0;L;;;;;N;;;;; +1101A;BRAHMI LETTER JA;Lo;0;L;;;;;N;;;;; +1101B;BRAHMI LETTER JHA;Lo;0;L;;;;;N;;;;; +1101C;BRAHMI LETTER NYA;Lo;0;L;;;;;N;;;;; +1101D;BRAHMI LETTER TTA;Lo;0;L;;;;;N;;;;; +1101E;BRAHMI LETTER TTHA;Lo;0;L;;;;;N;;;;; +1101F;BRAHMI LETTER DDA;Lo;0;L;;;;;N;;;;; +11020;BRAHMI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11021;BRAHMI LETTER NNA;Lo;0;L;;;;;N;;;;; +11022;BRAHMI LETTER TA;Lo;0;L;;;;;N;;;;; +11023;BRAHMI LETTER THA;Lo;0;L;;;;;N;;;;; +11024;BRAHMI LETTER DA;Lo;0;L;;;;;N;;;;; +11025;BRAHMI LETTER DHA;Lo;0;L;;;;;N;;;;; +11026;BRAHMI LETTER NA;Lo;0;L;;;;;N;;;;; +11027;BRAHMI LETTER PA;Lo;0;L;;;;;N;;;;; +11028;BRAHMI LETTER PHA;Lo;0;L;;;;;N;;;;; +11029;BRAHMI LETTER BA;Lo;0;L;;;;;N;;;;; +1102A;BRAHMI LETTER BHA;Lo;0;L;;;;;N;;;;; +1102B;BRAHMI LETTER MA;Lo;0;L;;;;;N;;;;; +1102C;BRAHMI LETTER YA;Lo;0;L;;;;;N;;;;; +1102D;BRAHMI LETTER RA;Lo;0;L;;;;;N;;;;; +1102E;BRAHMI LETTER LA;Lo;0;L;;;;;N;;;;; +1102F;BRAHMI LETTER VA;Lo;0;L;;;;;N;;;;; +11030;BRAHMI LETTER SHA;Lo;0;L;;;;;N;;;;; +11031;BRAHMI LETTER SSA;Lo;0;L;;;;;N;;;;; +11032;BRAHMI LETTER SA;Lo;0;L;;;;;N;;;;; +11033;BRAHMI LETTER HA;Lo;0;L;;;;;N;;;;; +11034;BRAHMI LETTER LLA;Lo;0;L;;;;;N;;;;; +11035;BRAHMI LETTER OLD TAMIL LLLA;Lo;0;L;;;;;N;;;;; +11036;BRAHMI LETTER OLD TAMIL RRA;Lo;0;L;;;;;N;;;;; +11037;BRAHMI LETTER OLD TAMIL NNNA;Lo;0;L;;;;;N;;;;; +11038;BRAHMI VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +11039;BRAHMI VOWEL SIGN BHATTIPROLU AA;Mn;0;NSM;;;;;N;;;;; +1103A;BRAHMI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +1103B;BRAHMI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +1103C;BRAHMI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1103D;BRAHMI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +1103E;BRAHMI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +1103F;BRAHMI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +11040;BRAHMI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +11041;BRAHMI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +11042;BRAHMI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +11043;BRAHMI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +11044;BRAHMI VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +11045;BRAHMI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +11046;BRAHMI VIRAMA;Mn;9;NSM;;;;;N;;;;; +11047;BRAHMI DANDA;Po;0;L;;;;;N;;;;; +11048;BRAHMI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11049;BRAHMI PUNCTUATION DOT;Po;0;L;;;;;N;;;;; +1104A;BRAHMI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;; +1104B;BRAHMI PUNCTUATION LINE;Po;0;L;;;;;N;;;;; +1104C;BRAHMI PUNCTUATION CRESCENT BAR;Po;0;L;;;;;N;;;;; +1104D;BRAHMI PUNCTUATION LOTUS;Po;0;L;;;;;N;;;;; +11052;BRAHMI NUMBER ONE;No;0;ON;;;1;1;N;;;;; +11053;BRAHMI NUMBER TWO;No;0;ON;;;2;2;N;;;;; +11054;BRAHMI NUMBER THREE;No;0;ON;;;3;3;N;;;;; +11055;BRAHMI NUMBER FOUR;No;0;ON;;;4;4;N;;;;; +11056;BRAHMI NUMBER FIVE;No;0;ON;;;5;5;N;;;;; +11057;BRAHMI NUMBER SIX;No;0;ON;;;6;6;N;;;;; +11058;BRAHMI NUMBER SEVEN;No;0;ON;;;7;7;N;;;;; +11059;BRAHMI NUMBER EIGHT;No;0;ON;;;8;8;N;;;;; +1105A;BRAHMI NUMBER NINE;No;0;ON;;;9;9;N;;;;; +1105B;BRAHMI NUMBER TEN;No;0;ON;;;;10;N;;;;; +1105C;BRAHMI NUMBER TWENTY;No;0;ON;;;;20;N;;;;; +1105D;BRAHMI NUMBER THIRTY;No;0;ON;;;;30;N;;;;; +1105E;BRAHMI NUMBER FORTY;No;0;ON;;;;40;N;;;;; +1105F;BRAHMI NUMBER FIFTY;No;0;ON;;;;50;N;;;;; +11060;BRAHMI NUMBER SIXTY;No;0;ON;;;;60;N;;;;; +11061;BRAHMI NUMBER SEVENTY;No;0;ON;;;;70;N;;;;; +11062;BRAHMI NUMBER EIGHTY;No;0;ON;;;;80;N;;;;; +11063;BRAHMI NUMBER NINETY;No;0;ON;;;;90;N;;;;; +11064;BRAHMI NUMBER ONE HUNDRED;No;0;ON;;;;100;N;;;;; +11065;BRAHMI NUMBER ONE THOUSAND;No;0;ON;;;;1000;N;;;;; +11066;BRAHMI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11067;BRAHMI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11068;BRAHMI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11069;BRAHMI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1106A;BRAHMI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1106B;BRAHMI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1106C;BRAHMI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1106D;BRAHMI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1106E;BRAHMI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1106F;BRAHMI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11080;KAITHI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11081;KAITHI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11082;KAITHI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11083;KAITHI LETTER A;Lo;0;L;;;;;N;;;;; +11084;KAITHI LETTER AA;Lo;0;L;;;;;N;;;;; +11085;KAITHI LETTER I;Lo;0;L;;;;;N;;;;; +11086;KAITHI LETTER II;Lo;0;L;;;;;N;;;;; +11087;KAITHI LETTER U;Lo;0;L;;;;;N;;;;; +11088;KAITHI LETTER UU;Lo;0;L;;;;;N;;;;; +11089;KAITHI LETTER E;Lo;0;L;;;;;N;;;;; +1108A;KAITHI LETTER AI;Lo;0;L;;;;;N;;;;; +1108B;KAITHI LETTER O;Lo;0;L;;;;;N;;;;; +1108C;KAITHI LETTER AU;Lo;0;L;;;;;N;;;;; +1108D;KAITHI LETTER KA;Lo;0;L;;;;;N;;;;; +1108E;KAITHI LETTER KHA;Lo;0;L;;;;;N;;;;; +1108F;KAITHI LETTER GA;Lo;0;L;;;;;N;;;;; +11090;KAITHI LETTER GHA;Lo;0;L;;;;;N;;;;; +11091;KAITHI LETTER NGA;Lo;0;L;;;;;N;;;;; +11092;KAITHI LETTER CA;Lo;0;L;;;;;N;;;;; +11093;KAITHI LETTER CHA;Lo;0;L;;;;;N;;;;; +11094;KAITHI LETTER JA;Lo;0;L;;;;;N;;;;; +11095;KAITHI LETTER JHA;Lo;0;L;;;;;N;;;;; +11096;KAITHI LETTER NYA;Lo;0;L;;;;;N;;;;; +11097;KAITHI LETTER TTA;Lo;0;L;;;;;N;;;;; +11098;KAITHI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11099;KAITHI LETTER DDA;Lo;0;L;;;;;N;;;;; +1109A;KAITHI LETTER DDDHA;Lo;0;L;11099 110BA;;;;N;;;;; +1109B;KAITHI LETTER DDHA;Lo;0;L;;;;;N;;;;; +1109C;KAITHI LETTER RHA;Lo;0;L;1109B 110BA;;;;N;;;;; +1109D;KAITHI LETTER NNA;Lo;0;L;;;;;N;;;;; +1109E;KAITHI LETTER TA;Lo;0;L;;;;;N;;;;; +1109F;KAITHI LETTER THA;Lo;0;L;;;;;N;;;;; +110A0;KAITHI LETTER DA;Lo;0;L;;;;;N;;;;; +110A1;KAITHI LETTER DHA;Lo;0;L;;;;;N;;;;; +110A2;KAITHI LETTER NA;Lo;0;L;;;;;N;;;;; +110A3;KAITHI LETTER PA;Lo;0;L;;;;;N;;;;; +110A4;KAITHI LETTER PHA;Lo;0;L;;;;;N;;;;; +110A5;KAITHI LETTER BA;Lo;0;L;;;;;N;;;;; +110A6;KAITHI LETTER BHA;Lo;0;L;;;;;N;;;;; +110A7;KAITHI LETTER MA;Lo;0;L;;;;;N;;;;; +110A8;KAITHI LETTER YA;Lo;0;L;;;;;N;;;;; +110A9;KAITHI LETTER RA;Lo;0;L;;;;;N;;;;; +110AA;KAITHI LETTER LA;Lo;0;L;;;;;N;;;;; +110AB;KAITHI LETTER VA;Lo;0;L;110A5 110BA;;;;N;;;;; +110AC;KAITHI LETTER SHA;Lo;0;L;;;;;N;;;;; +110AD;KAITHI LETTER SSA;Lo;0;L;;;;;N;;;;; +110AE;KAITHI LETTER SA;Lo;0;L;;;;;N;;;;; +110AF;KAITHI LETTER HA;Lo;0;L;;;;;N;;;;; +110B0;KAITHI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +110B1;KAITHI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +110B2;KAITHI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +110B3;KAITHI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +110B4;KAITHI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +110B5;KAITHI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +110B6;KAITHI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +110B7;KAITHI VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +110B8;KAITHI VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +110B9;KAITHI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +110BA;KAITHI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +110BB;KAITHI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +110BC;KAITHI ENUMERATION SIGN;Po;0;L;;;;;N;;;;; +110BD;KAITHI NUMBER SIGN;Cf;0;L;;;;;N;;;;; +110BE;KAITHI SECTION MARK;Po;0;L;;;;;N;;;;; +110BF;KAITHI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; +110C0;KAITHI DANDA;Po;0;L;;;;;N;;;;; +110C1;KAITHI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +110D0;SORA SOMPENG LETTER SAH;Lo;0;L;;;;;N;;;;; +110D1;SORA SOMPENG LETTER TAH;Lo;0;L;;;;;N;;;;; +110D2;SORA SOMPENG LETTER BAH;Lo;0;L;;;;;N;;;;; +110D3;SORA SOMPENG LETTER CAH;Lo;0;L;;;;;N;;;;; +110D4;SORA SOMPENG LETTER DAH;Lo;0;L;;;;;N;;;;; +110D5;SORA SOMPENG LETTER GAH;Lo;0;L;;;;;N;;;;; +110D6;SORA SOMPENG LETTER MAH;Lo;0;L;;;;;N;;;;; +110D7;SORA SOMPENG LETTER NGAH;Lo;0;L;;;;;N;;;;; +110D8;SORA SOMPENG LETTER LAH;Lo;0;L;;;;;N;;;;; +110D9;SORA SOMPENG LETTER NAH;Lo;0;L;;;;;N;;;;; +110DA;SORA SOMPENG LETTER VAH;Lo;0;L;;;;;N;;;;; +110DB;SORA SOMPENG LETTER PAH;Lo;0;L;;;;;N;;;;; +110DC;SORA SOMPENG LETTER YAH;Lo;0;L;;;;;N;;;;; +110DD;SORA SOMPENG LETTER RAH;Lo;0;L;;;;;N;;;;; +110DE;SORA SOMPENG LETTER HAH;Lo;0;L;;;;;N;;;;; +110DF;SORA SOMPENG LETTER KAH;Lo;0;L;;;;;N;;;;; +110E0;SORA SOMPENG LETTER JAH;Lo;0;L;;;;;N;;;;; +110E1;SORA SOMPENG LETTER NYAH;Lo;0;L;;;;;N;;;;; +110E2;SORA SOMPENG LETTER AH;Lo;0;L;;;;;N;;;;; +110E3;SORA SOMPENG LETTER EEH;Lo;0;L;;;;;N;;;;; +110E4;SORA SOMPENG LETTER IH;Lo;0;L;;;;;N;;;;; +110E5;SORA SOMPENG LETTER UH;Lo;0;L;;;;;N;;;;; +110E6;SORA SOMPENG LETTER OH;Lo;0;L;;;;;N;;;;; +110E7;SORA SOMPENG LETTER EH;Lo;0;L;;;;;N;;;;; +110E8;SORA SOMPENG LETTER MAE;Lo;0;L;;;;;N;;;;; +110F0;SORA SOMPENG DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +110F1;SORA SOMPENG DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +110F2;SORA SOMPENG DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +110F3;SORA SOMPENG DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +110F4;SORA SOMPENG DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +110F5;SORA SOMPENG DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +110F6;SORA SOMPENG DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +110F7;SORA SOMPENG DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +110F8;SORA SOMPENG DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +110F9;SORA SOMPENG DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11100;CHAKMA SIGN CANDRABINDU;Mn;230;NSM;;;;;N;;;;; +11101;CHAKMA SIGN ANUSVARA;Mn;230;NSM;;;;;N;;;;; +11102;CHAKMA SIGN VISARGA;Mn;230;NSM;;;;;N;;;;; +11103;CHAKMA LETTER AA;Lo;0;L;;;;;N;;;;; +11104;CHAKMA LETTER I;Lo;0;L;;;;;N;;;;; +11105;CHAKMA LETTER U;Lo;0;L;;;;;N;;;;; +11106;CHAKMA LETTER E;Lo;0;L;;;;;N;;;;; +11107;CHAKMA LETTER KAA;Lo;0;L;;;;;N;;;;; +11108;CHAKMA LETTER KHAA;Lo;0;L;;;;;N;;;;; +11109;CHAKMA LETTER GAA;Lo;0;L;;;;;N;;;;; +1110A;CHAKMA LETTER GHAA;Lo;0;L;;;;;N;;;;; +1110B;CHAKMA LETTER NGAA;Lo;0;L;;;;;N;;;;; +1110C;CHAKMA LETTER CAA;Lo;0;L;;;;;N;;;;; +1110D;CHAKMA LETTER CHAA;Lo;0;L;;;;;N;;;;; +1110E;CHAKMA LETTER JAA;Lo;0;L;;;;;N;;;;; +1110F;CHAKMA LETTER JHAA;Lo;0;L;;;;;N;;;;; +11110;CHAKMA LETTER NYAA;Lo;0;L;;;;;N;;;;; +11111;CHAKMA LETTER TTAA;Lo;0;L;;;;;N;;;;; +11112;CHAKMA LETTER TTHAA;Lo;0;L;;;;;N;;;;; +11113;CHAKMA LETTER DDAA;Lo;0;L;;;;;N;;;;; +11114;CHAKMA LETTER DDHAA;Lo;0;L;;;;;N;;;;; +11115;CHAKMA LETTER NNAA;Lo;0;L;;;;;N;;;;; +11116;CHAKMA LETTER TAA;Lo;0;L;;;;;N;;;;; +11117;CHAKMA LETTER THAA;Lo;0;L;;;;;N;;;;; +11118;CHAKMA LETTER DAA;Lo;0;L;;;;;N;;;;; +11119;CHAKMA LETTER DHAA;Lo;0;L;;;;;N;;;;; +1111A;CHAKMA LETTER NAA;Lo;0;L;;;;;N;;;;; +1111B;CHAKMA LETTER PAA;Lo;0;L;;;;;N;;;;; +1111C;CHAKMA LETTER PHAA;Lo;0;L;;;;;N;;;;; +1111D;CHAKMA LETTER BAA;Lo;0;L;;;;;N;;;;; +1111E;CHAKMA LETTER BHAA;Lo;0;L;;;;;N;;;;; +1111F;CHAKMA LETTER MAA;Lo;0;L;;;;;N;;;;; +11120;CHAKMA LETTER YYAA;Lo;0;L;;;;;N;;;;; +11121;CHAKMA LETTER YAA;Lo;0;L;;;;;N;;;;; +11122;CHAKMA LETTER RAA;Lo;0;L;;;;;N;;;;; +11123;CHAKMA LETTER LAA;Lo;0;L;;;;;N;;;;; +11124;CHAKMA LETTER WAA;Lo;0;L;;;;;N;;;;; +11125;CHAKMA LETTER SAA;Lo;0;L;;;;;N;;;;; +11126;CHAKMA LETTER HAA;Lo;0;L;;;;;N;;;;; +11127;CHAKMA VOWEL SIGN A;Mn;0;NSM;;;;;N;;;;; +11128;CHAKMA VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11129;CHAKMA VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +1112A;CHAKMA VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +1112B;CHAKMA VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +1112C;CHAKMA VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +1112D;CHAKMA VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +1112E;CHAKMA VOWEL SIGN O;Mn;0;NSM;11131 11127;;;;N;;;;; +1112F;CHAKMA VOWEL SIGN AU;Mn;0;NSM;11132 11127;;;;N;;;;; +11130;CHAKMA VOWEL SIGN OI;Mn;0;NSM;;;;;N;;;;; +11131;CHAKMA O MARK;Mn;0;NSM;;;;;N;;;;; +11132;CHAKMA AU MARK;Mn;0;NSM;;;;;N;;;;; +11133;CHAKMA VIRAMA;Mn;9;NSM;;;;;N;;;;; +11134;CHAKMA MAAYYAA;Mn;9;NSM;;;;;N;;;;; +11136;CHAKMA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11137;CHAKMA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11138;CHAKMA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11139;CHAKMA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1113A;CHAKMA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1113B;CHAKMA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1113C;CHAKMA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1113D;CHAKMA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1113E;CHAKMA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1113F;CHAKMA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11140;CHAKMA SECTION MARK;Po;0;L;;;;;N;;;;; +11141;CHAKMA DANDA;Po;0;L;;;;;N;;;;; +11142;CHAKMA DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11143;CHAKMA QUESTION MARK;Po;0;L;;;;;N;;;;; +11180;SHARADA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11181;SHARADA SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11182;SHARADA SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11183;SHARADA LETTER A;Lo;0;L;;;;;N;;;;; +11184;SHARADA LETTER AA;Lo;0;L;;;;;N;;;;; +11185;SHARADA LETTER I;Lo;0;L;;;;;N;;;;; +11186;SHARADA LETTER II;Lo;0;L;;;;;N;;;;; +11187;SHARADA LETTER U;Lo;0;L;;;;;N;;;;; +11188;SHARADA LETTER UU;Lo;0;L;;;;;N;;;;; +11189;SHARADA LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +1118A;SHARADA LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +1118B;SHARADA LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +1118C;SHARADA LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +1118D;SHARADA LETTER E;Lo;0;L;;;;;N;;;;; +1118E;SHARADA LETTER AI;Lo;0;L;;;;;N;;;;; +1118F;SHARADA LETTER O;Lo;0;L;;;;;N;;;;; +11190;SHARADA LETTER AU;Lo;0;L;;;;;N;;;;; +11191;SHARADA LETTER KA;Lo;0;L;;;;;N;;;;; +11192;SHARADA LETTER KHA;Lo;0;L;;;;;N;;;;; +11193;SHARADA LETTER GA;Lo;0;L;;;;;N;;;;; +11194;SHARADA LETTER GHA;Lo;0;L;;;;;N;;;;; +11195;SHARADA LETTER NGA;Lo;0;L;;;;;N;;;;; +11196;SHARADA LETTER CA;Lo;0;L;;;;;N;;;;; +11197;SHARADA LETTER CHA;Lo;0;L;;;;;N;;;;; +11198;SHARADA LETTER JA;Lo;0;L;;;;;N;;;;; +11199;SHARADA LETTER JHA;Lo;0;L;;;;;N;;;;; +1119A;SHARADA LETTER NYA;Lo;0;L;;;;;N;;;;; +1119B;SHARADA LETTER TTA;Lo;0;L;;;;;N;;;;; +1119C;SHARADA LETTER TTHA;Lo;0;L;;;;;N;;;;; +1119D;SHARADA LETTER DDA;Lo;0;L;;;;;N;;;;; +1119E;SHARADA LETTER DDHA;Lo;0;L;;;;;N;;;;; +1119F;SHARADA LETTER NNA;Lo;0;L;;;;;N;;;;; +111A0;SHARADA LETTER TA;Lo;0;L;;;;;N;;;;; +111A1;SHARADA LETTER THA;Lo;0;L;;;;;N;;;;; +111A2;SHARADA LETTER DA;Lo;0;L;;;;;N;;;;; +111A3;SHARADA LETTER DHA;Lo;0;L;;;;;N;;;;; +111A4;SHARADA LETTER NA;Lo;0;L;;;;;N;;;;; +111A5;SHARADA LETTER PA;Lo;0;L;;;;;N;;;;; +111A6;SHARADA LETTER PHA;Lo;0;L;;;;;N;;;;; +111A7;SHARADA LETTER BA;Lo;0;L;;;;;N;;;;; +111A8;SHARADA LETTER BHA;Lo;0;L;;;;;N;;;;; +111A9;SHARADA LETTER MA;Lo;0;L;;;;;N;;;;; +111AA;SHARADA LETTER YA;Lo;0;L;;;;;N;;;;; +111AB;SHARADA LETTER RA;Lo;0;L;;;;;N;;;;; +111AC;SHARADA LETTER LA;Lo;0;L;;;;;N;;;;; +111AD;SHARADA LETTER LLA;Lo;0;L;;;;;N;;;;; +111AE;SHARADA LETTER VA;Lo;0;L;;;;;N;;;;; +111AF;SHARADA LETTER SHA;Lo;0;L;;;;;N;;;;; +111B0;SHARADA LETTER SSA;Lo;0;L;;;;;N;;;;; +111B1;SHARADA LETTER SA;Lo;0;L;;;;;N;;;;; +111B2;SHARADA LETTER HA;Lo;0;L;;;;;N;;;;; +111B3;SHARADA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +111B4;SHARADA VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +111B5;SHARADA VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +111B6;SHARADA VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +111B7;SHARADA VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +111B8;SHARADA VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +111B9;SHARADA VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +111BA;SHARADA VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +111BB;SHARADA VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +111BC;SHARADA VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +111BD;SHARADA VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +111BE;SHARADA VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +111BF;SHARADA VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +111C0;SHARADA SIGN VIRAMA;Mc;9;L;;;;;N;;;;; +111C1;SHARADA SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +111C2;SHARADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; +111C3;SHARADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +111C4;SHARADA OM;Lo;0;L;;;;;N;;;;; +111C5;SHARADA DANDA;Po;0;L;;;;;N;;;;; +111C6;SHARADA DOUBLE DANDA;Po;0;L;;;;;N;;;;; +111C7;SHARADA ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +111C8;SHARADA SEPARATOR;Po;0;L;;;;;N;;;;; +111D0;SHARADA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +111D1;SHARADA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +111D2;SHARADA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +111D3;SHARADA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +111D4;SHARADA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +111D5;SHARADA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +111D6;SHARADA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +111D7;SHARADA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +111D8;SHARADA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +111D9;SHARADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11680;TAKRI LETTER A;Lo;0;L;;;;;N;;;;; +11681;TAKRI LETTER AA;Lo;0;L;;;;;N;;;;; +11682;TAKRI LETTER I;Lo;0;L;;;;;N;;;;; +11683;TAKRI LETTER II;Lo;0;L;;;;;N;;;;; +11684;TAKRI LETTER U;Lo;0;L;;;;;N;;;;; +11685;TAKRI LETTER UU;Lo;0;L;;;;;N;;;;; +11686;TAKRI LETTER E;Lo;0;L;;;;;N;;;;; +11687;TAKRI LETTER AI;Lo;0;L;;;;;N;;;;; +11688;TAKRI LETTER O;Lo;0;L;;;;;N;;;;; +11689;TAKRI LETTER AU;Lo;0;L;;;;;N;;;;; +1168A;TAKRI LETTER KA;Lo;0;L;;;;;N;;;;; +1168B;TAKRI LETTER KHA;Lo;0;L;;;;;N;;;;; +1168C;TAKRI LETTER GA;Lo;0;L;;;;;N;;;;; +1168D;TAKRI LETTER GHA;Lo;0;L;;;;;N;;;;; +1168E;TAKRI LETTER NGA;Lo;0;L;;;;;N;;;;; +1168F;TAKRI LETTER CA;Lo;0;L;;;;;N;;;;; +11690;TAKRI LETTER CHA;Lo;0;L;;;;;N;;;;; +11691;TAKRI LETTER JA;Lo;0;L;;;;;N;;;;; +11692;TAKRI LETTER JHA;Lo;0;L;;;;;N;;;;; +11693;TAKRI LETTER NYA;Lo;0;L;;;;;N;;;;; +11694;TAKRI LETTER TTA;Lo;0;L;;;;;N;;;;; +11695;TAKRI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11696;TAKRI LETTER DDA;Lo;0;L;;;;;N;;;;; +11697;TAKRI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11698;TAKRI LETTER NNA;Lo;0;L;;;;;N;;;;; +11699;TAKRI LETTER TA;Lo;0;L;;;;;N;;;;; +1169A;TAKRI LETTER THA;Lo;0;L;;;;;N;;;;; +1169B;TAKRI LETTER DA;Lo;0;L;;;;;N;;;;; +1169C;TAKRI LETTER DHA;Lo;0;L;;;;;N;;;;; +1169D;TAKRI LETTER NA;Lo;0;L;;;;;N;;;;; +1169E;TAKRI LETTER PA;Lo;0;L;;;;;N;;;;; +1169F;TAKRI LETTER PHA;Lo;0;L;;;;;N;;;;; +116A0;TAKRI LETTER BA;Lo;0;L;;;;;N;;;;; +116A1;TAKRI LETTER BHA;Lo;0;L;;;;;N;;;;; +116A2;TAKRI LETTER MA;Lo;0;L;;;;;N;;;;; +116A3;TAKRI LETTER YA;Lo;0;L;;;;;N;;;;; +116A4;TAKRI LETTER RA;Lo;0;L;;;;;N;;;;; +116A5;TAKRI LETTER LA;Lo;0;L;;;;;N;;;;; +116A6;TAKRI LETTER VA;Lo;0;L;;;;;N;;;;; +116A7;TAKRI LETTER SHA;Lo;0;L;;;;;N;;;;; +116A8;TAKRI LETTER SA;Lo;0;L;;;;;N;;;;; +116A9;TAKRI LETTER HA;Lo;0;L;;;;;N;;;;; +116AA;TAKRI LETTER RRA;Lo;0;L;;;;;N;;;;; +116AB;TAKRI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +116AC;TAKRI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +116AD;TAKRI VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +116AE;TAKRI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +116AF;TAKRI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +116B0;TAKRI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +116B1;TAKRI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +116B2;TAKRI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +116B3;TAKRI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +116B4;TAKRI VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +116B5;TAKRI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +116B6;TAKRI SIGN VIRAMA;Mc;9;L;;;;;N;;;;; +116B7;TAKRI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +116C0;TAKRI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +116C1;TAKRI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +116C2;TAKRI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +116C3;TAKRI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +116C4;TAKRI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +116C5;TAKRI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +116C6;TAKRI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +116C7;TAKRI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +116C8;TAKRI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +116C9;TAKRI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +12000;CUNEIFORM SIGN A;Lo;0;L;;;;;N;;;;; +12001;CUNEIFORM SIGN A TIMES A;Lo;0;L;;;;;N;;;;; +12002;CUNEIFORM SIGN A TIMES BAD;Lo;0;L;;;;;N;;;;; +12003;CUNEIFORM SIGN A TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +12004;CUNEIFORM SIGN A TIMES HA;Lo;0;L;;;;;N;;;;; +12005;CUNEIFORM SIGN A TIMES IGI;Lo;0;L;;;;;N;;;;; +12006;CUNEIFORM SIGN A TIMES LAGAR GUNU;Lo;0;L;;;;;N;;;;; +12007;CUNEIFORM SIGN A TIMES MUSH;Lo;0;L;;;;;N;;;;; +12008;CUNEIFORM SIGN A TIMES SAG;Lo;0;L;;;;;N;;;;; +12009;CUNEIFORM SIGN A2;Lo;0;L;;;;;N;;;;; +1200A;CUNEIFORM SIGN AB;Lo;0;L;;;;;N;;;;; +1200B;CUNEIFORM SIGN AB TIMES ASH2;Lo;0;L;;;;;N;;;;; +1200C;CUNEIFORM SIGN AB TIMES DUN3 GUNU;Lo;0;L;;;;;N;;;;; +1200D;CUNEIFORM SIGN AB TIMES GAL;Lo;0;L;;;;;N;;;;; +1200E;CUNEIFORM SIGN AB TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +1200F;CUNEIFORM SIGN AB TIMES HA;Lo;0;L;;;;;N;;;;; +12010;CUNEIFORM SIGN AB TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +12011;CUNEIFORM SIGN AB TIMES IMIN;Lo;0;L;;;;;N;;;;; +12012;CUNEIFORM SIGN AB TIMES LAGAB;Lo;0;L;;;;;N;;;;; +12013;CUNEIFORM SIGN AB TIMES SHESH;Lo;0;L;;;;;N;;;;; +12014;CUNEIFORM SIGN AB TIMES U PLUS U PLUS U;Lo;0;L;;;;;N;;;;; +12015;CUNEIFORM SIGN AB GUNU;Lo;0;L;;;;;N;;;;; +12016;CUNEIFORM SIGN AB2;Lo;0;L;;;;;N;;;;; +12017;CUNEIFORM SIGN AB2 TIMES BALAG;Lo;0;L;;;;;N;;;;; +12018;CUNEIFORM SIGN AB2 TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +12019;CUNEIFORM SIGN AB2 TIMES ME PLUS EN;Lo;0;L;;;;;N;;;;; +1201A;CUNEIFORM SIGN AB2 TIMES SHA3;Lo;0;L;;;;;N;;;;; +1201B;CUNEIFORM SIGN AB2 TIMES TAK4;Lo;0;L;;;;;N;;;;; +1201C;CUNEIFORM SIGN AD;Lo;0;L;;;;;N;;;;; +1201D;CUNEIFORM SIGN AK;Lo;0;L;;;;;N;;;;; +1201E;CUNEIFORM SIGN AK TIMES ERIN2;Lo;0;L;;;;;N;;;;; +1201F;CUNEIFORM SIGN AK TIMES SHITA PLUS GISH;Lo;0;L;;;;;N;;;;; +12020;CUNEIFORM SIGN AL;Lo;0;L;;;;;N;;;;; +12021;CUNEIFORM SIGN AL TIMES AL;Lo;0;L;;;;;N;;;;; +12022;CUNEIFORM SIGN AL TIMES DIM2;Lo;0;L;;;;;N;;;;; +12023;CUNEIFORM SIGN AL TIMES GISH;Lo;0;L;;;;;N;;;;; +12024;CUNEIFORM SIGN AL TIMES HA;Lo;0;L;;;;;N;;;;; +12025;CUNEIFORM SIGN AL TIMES KAD3;Lo;0;L;;;;;N;;;;; +12026;CUNEIFORM SIGN AL TIMES KI;Lo;0;L;;;;;N;;;;; +12027;CUNEIFORM SIGN AL TIMES SHE;Lo;0;L;;;;;N;;;;; +12028;CUNEIFORM SIGN AL TIMES USH;Lo;0;L;;;;;N;;;;; +12029;CUNEIFORM SIGN ALAN;Lo;0;L;;;;;N;;;;; +1202A;CUNEIFORM SIGN ALEPH;Lo;0;L;;;;;N;;;;; +1202B;CUNEIFORM SIGN AMAR;Lo;0;L;;;;;N;;;;; +1202C;CUNEIFORM SIGN AMAR TIMES SHE;Lo;0;L;;;;;N;;;;; +1202D;CUNEIFORM SIGN AN;Lo;0;L;;;;;N;;;;; +1202E;CUNEIFORM SIGN AN OVER AN;Lo;0;L;;;;;N;;;;; +1202F;CUNEIFORM SIGN AN THREE TIMES;Lo;0;L;;;;;N;;;;; +12030;CUNEIFORM SIGN AN PLUS NAGA OPPOSING AN PLUS NAGA;Lo;0;L;;;;;N;;;;; +12031;CUNEIFORM SIGN AN PLUS NAGA SQUARED;Lo;0;L;;;;;N;;;;; +12032;CUNEIFORM SIGN ANSHE;Lo;0;L;;;;;N;;;;; +12033;CUNEIFORM SIGN APIN;Lo;0;L;;;;;N;;;;; +12034;CUNEIFORM SIGN ARAD;Lo;0;L;;;;;N;;;;; +12035;CUNEIFORM SIGN ARAD TIMES KUR;Lo;0;L;;;;;N;;;;; +12036;CUNEIFORM SIGN ARKAB;Lo;0;L;;;;;N;;;;; +12037;CUNEIFORM SIGN ASAL2;Lo;0;L;;;;;N;;;;; +12038;CUNEIFORM SIGN ASH;Lo;0;L;;;;;N;;;;; +12039;CUNEIFORM SIGN ASH ZIDA TENU;Lo;0;L;;;;;N;;;;; +1203A;CUNEIFORM SIGN ASH KABA TENU;Lo;0;L;;;;;N;;;;; +1203B;CUNEIFORM SIGN ASH OVER ASH TUG2 OVER TUG2 TUG2 OVER TUG2 PAP;Lo;0;L;;;;;N;;;;; +1203C;CUNEIFORM SIGN ASH OVER ASH OVER ASH;Lo;0;L;;;;;N;;;;; +1203D;CUNEIFORM SIGN ASH OVER ASH OVER ASH CROSSING ASH OVER ASH OVER ASH;Lo;0;L;;;;;N;;;;; +1203E;CUNEIFORM SIGN ASH2;Lo;0;L;;;;;N;;;;; +1203F;CUNEIFORM SIGN ASHGAB;Lo;0;L;;;;;N;;;;; +12040;CUNEIFORM SIGN BA;Lo;0;L;;;;;N;;;;; +12041;CUNEIFORM SIGN BAD;Lo;0;L;;;;;N;;;;; +12042;CUNEIFORM SIGN BAG3;Lo;0;L;;;;;N;;;;; +12043;CUNEIFORM SIGN BAHAR2;Lo;0;L;;;;;N;;;;; +12044;CUNEIFORM SIGN BAL;Lo;0;L;;;;;N;;;;; +12045;CUNEIFORM SIGN BAL OVER BAL;Lo;0;L;;;;;N;;;;; +12046;CUNEIFORM SIGN BALAG;Lo;0;L;;;;;N;;;;; +12047;CUNEIFORM SIGN BAR;Lo;0;L;;;;;N;;;;; +12048;CUNEIFORM SIGN BARA2;Lo;0;L;;;;;N;;;;; +12049;CUNEIFORM SIGN BI;Lo;0;L;;;;;N;;;;; +1204A;CUNEIFORM SIGN BI TIMES A;Lo;0;L;;;;;N;;;;; +1204B;CUNEIFORM SIGN BI TIMES GAR;Lo;0;L;;;;;N;;;;; +1204C;CUNEIFORM SIGN BI TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +1204D;CUNEIFORM SIGN BU;Lo;0;L;;;;;N;;;;; +1204E;CUNEIFORM SIGN BU OVER BU AB;Lo;0;L;;;;;N;;;;; +1204F;CUNEIFORM SIGN BU OVER BU UN;Lo;0;L;;;;;N;;;;; +12050;CUNEIFORM SIGN BU CROSSING BU;Lo;0;L;;;;;N;;;;; +12051;CUNEIFORM SIGN BULUG;Lo;0;L;;;;;N;;;;; +12052;CUNEIFORM SIGN BULUG OVER BULUG;Lo;0;L;;;;;N;;;;; +12053;CUNEIFORM SIGN BUR;Lo;0;L;;;;;N;;;;; +12054;CUNEIFORM SIGN BUR2;Lo;0;L;;;;;N;;;;; +12055;CUNEIFORM SIGN DA;Lo;0;L;;;;;N;;;;; +12056;CUNEIFORM SIGN DAG;Lo;0;L;;;;;N;;;;; +12057;CUNEIFORM SIGN DAG KISIM5 TIMES A PLUS MASH;Lo;0;L;;;;;N;;;;; +12058;CUNEIFORM SIGN DAG KISIM5 TIMES AMAR;Lo;0;L;;;;;N;;;;; +12059;CUNEIFORM SIGN DAG KISIM5 TIMES BALAG;Lo;0;L;;;;;N;;;;; +1205A;CUNEIFORM SIGN DAG KISIM5 TIMES BI;Lo;0;L;;;;;N;;;;; +1205B;CUNEIFORM SIGN DAG KISIM5 TIMES GA;Lo;0;L;;;;;N;;;;; +1205C;CUNEIFORM SIGN DAG KISIM5 TIMES GA PLUS MASH;Lo;0;L;;;;;N;;;;; +1205D;CUNEIFORM SIGN DAG KISIM5 TIMES GI;Lo;0;L;;;;;N;;;;; +1205E;CUNEIFORM SIGN DAG KISIM5 TIMES GIR2;Lo;0;L;;;;;N;;;;; +1205F;CUNEIFORM SIGN DAG KISIM5 TIMES GUD;Lo;0;L;;;;;N;;;;; +12060;CUNEIFORM SIGN DAG KISIM5 TIMES HA;Lo;0;L;;;;;N;;;;; +12061;CUNEIFORM SIGN DAG KISIM5 TIMES IR;Lo;0;L;;;;;N;;;;; +12062;CUNEIFORM SIGN DAG KISIM5 TIMES IR PLUS LU;Lo;0;L;;;;;N;;;;; +12063;CUNEIFORM SIGN DAG KISIM5 TIMES KAK;Lo;0;L;;;;;N;;;;; +12064;CUNEIFORM SIGN DAG KISIM5 TIMES LA;Lo;0;L;;;;;N;;;;; +12065;CUNEIFORM SIGN DAG KISIM5 TIMES LU;Lo;0;L;;;;;N;;;;; +12066;CUNEIFORM SIGN DAG KISIM5 TIMES LU PLUS MASH2;Lo;0;L;;;;;N;;;;; +12067;CUNEIFORM SIGN DAG KISIM5 TIMES LUM;Lo;0;L;;;;;N;;;;; +12068;CUNEIFORM SIGN DAG KISIM5 TIMES NE;Lo;0;L;;;;;N;;;;; +12069;CUNEIFORM SIGN DAG KISIM5 TIMES PAP PLUS PAP;Lo;0;L;;;;;N;;;;; +1206A;CUNEIFORM SIGN DAG KISIM5 TIMES SI;Lo;0;L;;;;;N;;;;; +1206B;CUNEIFORM SIGN DAG KISIM5 TIMES TAK4;Lo;0;L;;;;;N;;;;; +1206C;CUNEIFORM SIGN DAG KISIM5 TIMES U2 PLUS GIR2;Lo;0;L;;;;;N;;;;; +1206D;CUNEIFORM SIGN DAG KISIM5 TIMES USH;Lo;0;L;;;;;N;;;;; +1206E;CUNEIFORM SIGN DAM;Lo;0;L;;;;;N;;;;; +1206F;CUNEIFORM SIGN DAR;Lo;0;L;;;;;N;;;;; +12070;CUNEIFORM SIGN DARA3;Lo;0;L;;;;;N;;;;; +12071;CUNEIFORM SIGN DARA4;Lo;0;L;;;;;N;;;;; +12072;CUNEIFORM SIGN DI;Lo;0;L;;;;;N;;;;; +12073;CUNEIFORM SIGN DIB;Lo;0;L;;;;;N;;;;; +12074;CUNEIFORM SIGN DIM;Lo;0;L;;;;;N;;;;; +12075;CUNEIFORM SIGN DIM TIMES SHE;Lo;0;L;;;;;N;;;;; +12076;CUNEIFORM SIGN DIM2;Lo;0;L;;;;;N;;;;; +12077;CUNEIFORM SIGN DIN;Lo;0;L;;;;;N;;;;; +12078;CUNEIFORM SIGN DIN KASKAL U GUNU DISH;Lo;0;L;;;;;N;;;;; +12079;CUNEIFORM SIGN DISH;Lo;0;L;;;;;N;;;;; +1207A;CUNEIFORM SIGN DU;Lo;0;L;;;;;N;;;;; +1207B;CUNEIFORM SIGN DU OVER DU;Lo;0;L;;;;;N;;;;; +1207C;CUNEIFORM SIGN DU GUNU;Lo;0;L;;;;;N;;;;; +1207D;CUNEIFORM SIGN DU SHESHIG;Lo;0;L;;;;;N;;;;; +1207E;CUNEIFORM SIGN DUB;Lo;0;L;;;;;N;;;;; +1207F;CUNEIFORM SIGN DUB TIMES ESH2;Lo;0;L;;;;;N;;;;; +12080;CUNEIFORM SIGN DUB2;Lo;0;L;;;;;N;;;;; +12081;CUNEIFORM SIGN DUG;Lo;0;L;;;;;N;;;;; +12082;CUNEIFORM SIGN DUGUD;Lo;0;L;;;;;N;;;;; +12083;CUNEIFORM SIGN DUH;Lo;0;L;;;;;N;;;;; +12084;CUNEIFORM SIGN DUN;Lo;0;L;;;;;N;;;;; +12085;CUNEIFORM SIGN DUN3;Lo;0;L;;;;;N;;;;; +12086;CUNEIFORM SIGN DUN3 GUNU;Lo;0;L;;;;;N;;;;; +12087;CUNEIFORM SIGN DUN3 GUNU GUNU;Lo;0;L;;;;;N;;;;; +12088;CUNEIFORM SIGN DUN4;Lo;0;L;;;;;N;;;;; +12089;CUNEIFORM SIGN DUR2;Lo;0;L;;;;;N;;;;; +1208A;CUNEIFORM SIGN E;Lo;0;L;;;;;N;;;;; +1208B;CUNEIFORM SIGN E TIMES PAP;Lo;0;L;;;;;N;;;;; +1208C;CUNEIFORM SIGN E OVER E NUN OVER NUN;Lo;0;L;;;;;N;;;;; +1208D;CUNEIFORM SIGN E2;Lo;0;L;;;;;N;;;;; +1208E;CUNEIFORM SIGN E2 TIMES A PLUS HA PLUS DA;Lo;0;L;;;;;N;;;;; +1208F;CUNEIFORM SIGN E2 TIMES GAR;Lo;0;L;;;;;N;;;;; +12090;CUNEIFORM SIGN E2 TIMES MI;Lo;0;L;;;;;N;;;;; +12091;CUNEIFORM SIGN E2 TIMES SAL;Lo;0;L;;;;;N;;;;; +12092;CUNEIFORM SIGN E2 TIMES SHE;Lo;0;L;;;;;N;;;;; +12093;CUNEIFORM SIGN E2 TIMES U;Lo;0;L;;;;;N;;;;; +12094;CUNEIFORM SIGN EDIN;Lo;0;L;;;;;N;;;;; +12095;CUNEIFORM SIGN EGIR;Lo;0;L;;;;;N;;;;; +12096;CUNEIFORM SIGN EL;Lo;0;L;;;;;N;;;;; +12097;CUNEIFORM SIGN EN;Lo;0;L;;;;;N;;;;; +12098;CUNEIFORM SIGN EN TIMES GAN2;Lo;0;L;;;;;N;;;;; +12099;CUNEIFORM SIGN EN TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +1209A;CUNEIFORM SIGN EN TIMES ME;Lo;0;L;;;;;N;;;;; +1209B;CUNEIFORM SIGN EN CROSSING EN;Lo;0;L;;;;;N;;;;; +1209C;CUNEIFORM SIGN EN OPPOSING EN;Lo;0;L;;;;;N;;;;; +1209D;CUNEIFORM SIGN EN SQUARED;Lo;0;L;;;;;N;;;;; +1209E;CUNEIFORM SIGN EREN;Lo;0;L;;;;;N;;;;; +1209F;CUNEIFORM SIGN ERIN2;Lo;0;L;;;;;N;;;;; +120A0;CUNEIFORM SIGN ESH2;Lo;0;L;;;;;N;;;;; +120A1;CUNEIFORM SIGN EZEN;Lo;0;L;;;;;N;;;;; +120A2;CUNEIFORM SIGN EZEN TIMES A;Lo;0;L;;;;;N;;;;; +120A3;CUNEIFORM SIGN EZEN TIMES A PLUS LAL;Lo;0;L;;;;;N;;;;; +120A4;CUNEIFORM SIGN EZEN TIMES A PLUS LAL TIMES LAL;Lo;0;L;;;;;N;;;;; +120A5;CUNEIFORM SIGN EZEN TIMES AN;Lo;0;L;;;;;N;;;;; +120A6;CUNEIFORM SIGN EZEN TIMES BAD;Lo;0;L;;;;;N;;;;; +120A7;CUNEIFORM SIGN EZEN TIMES DUN3 GUNU;Lo;0;L;;;;;N;;;;; +120A8;CUNEIFORM SIGN EZEN TIMES DUN3 GUNU GUNU;Lo;0;L;;;;;N;;;;; +120A9;CUNEIFORM SIGN EZEN TIMES HA;Lo;0;L;;;;;N;;;;; +120AA;CUNEIFORM SIGN EZEN TIMES HA GUNU;Lo;0;L;;;;;N;;;;; +120AB;CUNEIFORM SIGN EZEN TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +120AC;CUNEIFORM SIGN EZEN TIMES KASKAL;Lo;0;L;;;;;N;;;;; +120AD;CUNEIFORM SIGN EZEN TIMES KASKAL SQUARED;Lo;0;L;;;;;N;;;;; +120AE;CUNEIFORM SIGN EZEN TIMES KU3;Lo;0;L;;;;;N;;;;; +120AF;CUNEIFORM SIGN EZEN TIMES LA;Lo;0;L;;;;;N;;;;; +120B0;CUNEIFORM SIGN EZEN TIMES LAL TIMES LAL;Lo;0;L;;;;;N;;;;; +120B1;CUNEIFORM SIGN EZEN TIMES LI;Lo;0;L;;;;;N;;;;; +120B2;CUNEIFORM SIGN EZEN TIMES LU;Lo;0;L;;;;;N;;;;; +120B3;CUNEIFORM SIGN EZEN TIMES U2;Lo;0;L;;;;;N;;;;; +120B4;CUNEIFORM SIGN EZEN TIMES UD;Lo;0;L;;;;;N;;;;; +120B5;CUNEIFORM SIGN GA;Lo;0;L;;;;;N;;;;; +120B6;CUNEIFORM SIGN GA GUNU;Lo;0;L;;;;;N;;;;; +120B7;CUNEIFORM SIGN GA2;Lo;0;L;;;;;N;;;;; +120B8;CUNEIFORM SIGN GA2 TIMES A PLUS DA PLUS HA;Lo;0;L;;;;;N;;;;; +120B9;CUNEIFORM SIGN GA2 TIMES A PLUS HA;Lo;0;L;;;;;N;;;;; +120BA;CUNEIFORM SIGN GA2 TIMES A PLUS IGI;Lo;0;L;;;;;N;;;;; +120BB;CUNEIFORM SIGN GA2 TIMES AB2 TENU PLUS TAB;Lo;0;L;;;;;N;;;;; +120BC;CUNEIFORM SIGN GA2 TIMES AN;Lo;0;L;;;;;N;;;;; +120BD;CUNEIFORM SIGN GA2 TIMES ASH;Lo;0;L;;;;;N;;;;; +120BE;CUNEIFORM SIGN GA2 TIMES ASH2 PLUS GAL;Lo;0;L;;;;;N;;;;; +120BF;CUNEIFORM SIGN GA2 TIMES BAD;Lo;0;L;;;;;N;;;;; +120C0;CUNEIFORM SIGN GA2 TIMES BAR PLUS RA;Lo;0;L;;;;;N;;;;; +120C1;CUNEIFORM SIGN GA2 TIMES BUR;Lo;0;L;;;;;N;;;;; +120C2;CUNEIFORM SIGN GA2 TIMES BUR PLUS RA;Lo;0;L;;;;;N;;;;; +120C3;CUNEIFORM SIGN GA2 TIMES DA;Lo;0;L;;;;;N;;;;; +120C4;CUNEIFORM SIGN GA2 TIMES DI;Lo;0;L;;;;;N;;;;; +120C5;CUNEIFORM SIGN GA2 TIMES DIM TIMES SHE;Lo;0;L;;;;;N;;;;; +120C6;CUNEIFORM SIGN GA2 TIMES DUB;Lo;0;L;;;;;N;;;;; +120C7;CUNEIFORM SIGN GA2 TIMES EL;Lo;0;L;;;;;N;;;;; +120C8;CUNEIFORM SIGN GA2 TIMES EL PLUS LA;Lo;0;L;;;;;N;;;;; +120C9;CUNEIFORM SIGN GA2 TIMES EN;Lo;0;L;;;;;N;;;;; +120CA;CUNEIFORM SIGN GA2 TIMES EN TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +120CB;CUNEIFORM SIGN GA2 TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +120CC;CUNEIFORM SIGN GA2 TIMES GAR;Lo;0;L;;;;;N;;;;; +120CD;CUNEIFORM SIGN GA2 TIMES GI;Lo;0;L;;;;;N;;;;; +120CE;CUNEIFORM SIGN GA2 TIMES GI4;Lo;0;L;;;;;N;;;;; +120CF;CUNEIFORM SIGN GA2 TIMES GI4 PLUS A;Lo;0;L;;;;;N;;;;; +120D0;CUNEIFORM SIGN GA2 TIMES GIR2 PLUS SU;Lo;0;L;;;;;N;;;;; +120D1;CUNEIFORM SIGN GA2 TIMES HA PLUS LU PLUS ESH2;Lo;0;L;;;;;N;;;;; +120D2;CUNEIFORM SIGN GA2 TIMES HAL;Lo;0;L;;;;;N;;;;; +120D3;CUNEIFORM SIGN GA2 TIMES HAL PLUS LA;Lo;0;L;;;;;N;;;;; +120D4;CUNEIFORM SIGN GA2 TIMES HI PLUS LI;Lo;0;L;;;;;N;;;;; +120D5;CUNEIFORM SIGN GA2 TIMES HUB2;Lo;0;L;;;;;N;;;;; +120D6;CUNEIFORM SIGN GA2 TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +120D7;CUNEIFORM SIGN GA2 TIMES ISH PLUS HU PLUS ASH;Lo;0;L;;;;;N;;;;; +120D8;CUNEIFORM SIGN GA2 TIMES KAK;Lo;0;L;;;;;N;;;;; +120D9;CUNEIFORM SIGN GA2 TIMES KASKAL;Lo;0;L;;;;;N;;;;; +120DA;CUNEIFORM SIGN GA2 TIMES KID;Lo;0;L;;;;;N;;;;; +120DB;CUNEIFORM SIGN GA2 TIMES KID PLUS LAL;Lo;0;L;;;;;N;;;;; +120DC;CUNEIFORM SIGN GA2 TIMES KU3 PLUS AN;Lo;0;L;;;;;N;;;;; +120DD;CUNEIFORM SIGN GA2 TIMES LA;Lo;0;L;;;;;N;;;;; +120DE;CUNEIFORM SIGN GA2 TIMES ME PLUS EN;Lo;0;L;;;;;N;;;;; +120DF;CUNEIFORM SIGN GA2 TIMES MI;Lo;0;L;;;;;N;;;;; +120E0;CUNEIFORM SIGN GA2 TIMES NUN;Lo;0;L;;;;;N;;;;; +120E1;CUNEIFORM SIGN GA2 TIMES NUN OVER NUN;Lo;0;L;;;;;N;;;;; +120E2;CUNEIFORM SIGN GA2 TIMES PA;Lo;0;L;;;;;N;;;;; +120E3;CUNEIFORM SIGN GA2 TIMES SAL;Lo;0;L;;;;;N;;;;; +120E4;CUNEIFORM SIGN GA2 TIMES SAR;Lo;0;L;;;;;N;;;;; +120E5;CUNEIFORM SIGN GA2 TIMES SHE;Lo;0;L;;;;;N;;;;; +120E6;CUNEIFORM SIGN GA2 TIMES SHE PLUS TUR;Lo;0;L;;;;;N;;;;; +120E7;CUNEIFORM SIGN GA2 TIMES SHID;Lo;0;L;;;;;N;;;;; +120E8;CUNEIFORM SIGN GA2 TIMES SUM;Lo;0;L;;;;;N;;;;; +120E9;CUNEIFORM SIGN GA2 TIMES TAK4;Lo;0;L;;;;;N;;;;; +120EA;CUNEIFORM SIGN GA2 TIMES U;Lo;0;L;;;;;N;;;;; +120EB;CUNEIFORM SIGN GA2 TIMES UD;Lo;0;L;;;;;N;;;;; +120EC;CUNEIFORM SIGN GA2 TIMES UD PLUS DU;Lo;0;L;;;;;N;;;;; +120ED;CUNEIFORM SIGN GA2 OVER GA2;Lo;0;L;;;;;N;;;;; +120EE;CUNEIFORM SIGN GABA;Lo;0;L;;;;;N;;;;; +120EF;CUNEIFORM SIGN GABA CROSSING GABA;Lo;0;L;;;;;N;;;;; +120F0;CUNEIFORM SIGN GAD;Lo;0;L;;;;;N;;;;; +120F1;CUNEIFORM SIGN GAD OVER GAD GAR OVER GAR;Lo;0;L;;;;;N;;;;; +120F2;CUNEIFORM SIGN GAL;Lo;0;L;;;;;N;;;;; +120F3;CUNEIFORM SIGN GAL GAD OVER GAD GAR OVER GAR;Lo;0;L;;;;;N;;;;; +120F4;CUNEIFORM SIGN GALAM;Lo;0;L;;;;;N;;;;; +120F5;CUNEIFORM SIGN GAM;Lo;0;L;;;;;N;;;;; +120F6;CUNEIFORM SIGN GAN;Lo;0;L;;;;;N;;;;; +120F7;CUNEIFORM SIGN GAN2;Lo;0;L;;;;;N;;;;; +120F8;CUNEIFORM SIGN GAN2 TENU;Lo;0;L;;;;;N;;;;; +120F9;CUNEIFORM SIGN GAN2 OVER GAN2;Lo;0;L;;;;;N;;;;; +120FA;CUNEIFORM SIGN GAN2 CROSSING GAN2;Lo;0;L;;;;;N;;;;; +120FB;CUNEIFORM SIGN GAR;Lo;0;L;;;;;N;;;;; +120FC;CUNEIFORM SIGN GAR3;Lo;0;L;;;;;N;;;;; +120FD;CUNEIFORM SIGN GASHAN;Lo;0;L;;;;;N;;;;; +120FE;CUNEIFORM SIGN GESHTIN;Lo;0;L;;;;;N;;;;; +120FF;CUNEIFORM SIGN GESHTIN TIMES KUR;Lo;0;L;;;;;N;;;;; +12100;CUNEIFORM SIGN GI;Lo;0;L;;;;;N;;;;; +12101;CUNEIFORM SIGN GI TIMES E;Lo;0;L;;;;;N;;;;; +12102;CUNEIFORM SIGN GI TIMES U;Lo;0;L;;;;;N;;;;; +12103;CUNEIFORM SIGN GI CROSSING GI;Lo;0;L;;;;;N;;;;; +12104;CUNEIFORM SIGN GI4;Lo;0;L;;;;;N;;;;; +12105;CUNEIFORM SIGN GI4 OVER GI4;Lo;0;L;;;;;N;;;;; +12106;CUNEIFORM SIGN GI4 CROSSING GI4;Lo;0;L;;;;;N;;;;; +12107;CUNEIFORM SIGN GIDIM;Lo;0;L;;;;;N;;;;; +12108;CUNEIFORM SIGN GIR2;Lo;0;L;;;;;N;;;;; +12109;CUNEIFORM SIGN GIR2 GUNU;Lo;0;L;;;;;N;;;;; +1210A;CUNEIFORM SIGN GIR3;Lo;0;L;;;;;N;;;;; +1210B;CUNEIFORM SIGN GIR3 TIMES A PLUS IGI;Lo;0;L;;;;;N;;;;; +1210C;CUNEIFORM SIGN GIR3 TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +1210D;CUNEIFORM SIGN GIR3 TIMES IGI;Lo;0;L;;;;;N;;;;; +1210E;CUNEIFORM SIGN GIR3 TIMES LU PLUS IGI;Lo;0;L;;;;;N;;;;; +1210F;CUNEIFORM SIGN GIR3 TIMES PA;Lo;0;L;;;;;N;;;;; +12110;CUNEIFORM SIGN GISAL;Lo;0;L;;;;;N;;;;; +12111;CUNEIFORM SIGN GISH;Lo;0;L;;;;;N;;;;; +12112;CUNEIFORM SIGN GISH CROSSING GISH;Lo;0;L;;;;;N;;;;; +12113;CUNEIFORM SIGN GISH TIMES BAD;Lo;0;L;;;;;N;;;;; +12114;CUNEIFORM SIGN GISH TIMES TAK4;Lo;0;L;;;;;N;;;;; +12115;CUNEIFORM SIGN GISH TENU;Lo;0;L;;;;;N;;;;; +12116;CUNEIFORM SIGN GU;Lo;0;L;;;;;N;;;;; +12117;CUNEIFORM SIGN GU CROSSING GU;Lo;0;L;;;;;N;;;;; +12118;CUNEIFORM SIGN GU2;Lo;0;L;;;;;N;;;;; +12119;CUNEIFORM SIGN GU2 TIMES KAK;Lo;0;L;;;;;N;;;;; +1211A;CUNEIFORM SIGN GU2 TIMES KAK TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +1211B;CUNEIFORM SIGN GU2 TIMES NUN;Lo;0;L;;;;;N;;;;; +1211C;CUNEIFORM SIGN GU2 TIMES SAL PLUS TUG2;Lo;0;L;;;;;N;;;;; +1211D;CUNEIFORM SIGN GU2 GUNU;Lo;0;L;;;;;N;;;;; +1211E;CUNEIFORM SIGN GUD;Lo;0;L;;;;;N;;;;; +1211F;CUNEIFORM SIGN GUD TIMES A PLUS KUR;Lo;0;L;;;;;N;;;;; +12120;CUNEIFORM SIGN GUD TIMES KUR;Lo;0;L;;;;;N;;;;; +12121;CUNEIFORM SIGN GUD OVER GUD LUGAL;Lo;0;L;;;;;N;;;;; +12122;CUNEIFORM SIGN GUL;Lo;0;L;;;;;N;;;;; +12123;CUNEIFORM SIGN GUM;Lo;0;L;;;;;N;;;;; +12124;CUNEIFORM SIGN GUM TIMES SHE;Lo;0;L;;;;;N;;;;; +12125;CUNEIFORM SIGN GUR;Lo;0;L;;;;;N;;;;; +12126;CUNEIFORM SIGN GUR7;Lo;0;L;;;;;N;;;;; +12127;CUNEIFORM SIGN GURUN;Lo;0;L;;;;;N;;;;; +12128;CUNEIFORM SIGN GURUSH;Lo;0;L;;;;;N;;;;; +12129;CUNEIFORM SIGN HA;Lo;0;L;;;;;N;;;;; +1212A;CUNEIFORM SIGN HA TENU;Lo;0;L;;;;;N;;;;; +1212B;CUNEIFORM SIGN HA GUNU;Lo;0;L;;;;;N;;;;; +1212C;CUNEIFORM SIGN HAL;Lo;0;L;;;;;N;;;;; +1212D;CUNEIFORM SIGN HI;Lo;0;L;;;;;N;;;;; +1212E;CUNEIFORM SIGN HI TIMES ASH;Lo;0;L;;;;;N;;;;; +1212F;CUNEIFORM SIGN HI TIMES ASH2;Lo;0;L;;;;;N;;;;; +12130;CUNEIFORM SIGN HI TIMES BAD;Lo;0;L;;;;;N;;;;; +12131;CUNEIFORM SIGN HI TIMES DISH;Lo;0;L;;;;;N;;;;; +12132;CUNEIFORM SIGN HI TIMES GAD;Lo;0;L;;;;;N;;;;; +12133;CUNEIFORM SIGN HI TIMES KIN;Lo;0;L;;;;;N;;;;; +12134;CUNEIFORM SIGN HI TIMES NUN;Lo;0;L;;;;;N;;;;; +12135;CUNEIFORM SIGN HI TIMES SHE;Lo;0;L;;;;;N;;;;; +12136;CUNEIFORM SIGN HI TIMES U;Lo;0;L;;;;;N;;;;; +12137;CUNEIFORM SIGN HU;Lo;0;L;;;;;N;;;;; +12138;CUNEIFORM SIGN HUB2;Lo;0;L;;;;;N;;;;; +12139;CUNEIFORM SIGN HUB2 TIMES AN;Lo;0;L;;;;;N;;;;; +1213A;CUNEIFORM SIGN HUB2 TIMES HAL;Lo;0;L;;;;;N;;;;; +1213B;CUNEIFORM SIGN HUB2 TIMES KASKAL;Lo;0;L;;;;;N;;;;; +1213C;CUNEIFORM SIGN HUB2 TIMES LISH;Lo;0;L;;;;;N;;;;; +1213D;CUNEIFORM SIGN HUB2 TIMES UD;Lo;0;L;;;;;N;;;;; +1213E;CUNEIFORM SIGN HUL2;Lo;0;L;;;;;N;;;;; +1213F;CUNEIFORM SIGN I;Lo;0;L;;;;;N;;;;; +12140;CUNEIFORM SIGN I A;Lo;0;L;;;;;N;;;;; +12141;CUNEIFORM SIGN IB;Lo;0;L;;;;;N;;;;; +12142;CUNEIFORM SIGN IDIM;Lo;0;L;;;;;N;;;;; +12143;CUNEIFORM SIGN IDIM OVER IDIM BUR;Lo;0;L;;;;;N;;;;; +12144;CUNEIFORM SIGN IDIM OVER IDIM SQUARED;Lo;0;L;;;;;N;;;;; +12145;CUNEIFORM SIGN IG;Lo;0;L;;;;;N;;;;; +12146;CUNEIFORM SIGN IGI;Lo;0;L;;;;;N;;;;; +12147;CUNEIFORM SIGN IGI DIB;Lo;0;L;;;;;N;;;;; +12148;CUNEIFORM SIGN IGI RI;Lo;0;L;;;;;N;;;;; +12149;CUNEIFORM SIGN IGI OVER IGI SHIR OVER SHIR UD OVER UD;Lo;0;L;;;;;N;;;;; +1214A;CUNEIFORM SIGN IGI GUNU;Lo;0;L;;;;;N;;;;; +1214B;CUNEIFORM SIGN IL;Lo;0;L;;;;;N;;;;; +1214C;CUNEIFORM SIGN IL TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +1214D;CUNEIFORM SIGN IL2;Lo;0;L;;;;;N;;;;; +1214E;CUNEIFORM SIGN IM;Lo;0;L;;;;;N;;;;; +1214F;CUNEIFORM SIGN IM TIMES TAK4;Lo;0;L;;;;;N;;;;; +12150;CUNEIFORM SIGN IM CROSSING IM;Lo;0;L;;;;;N;;;;; +12151;CUNEIFORM SIGN IM OPPOSING IM;Lo;0;L;;;;;N;;;;; +12152;CUNEIFORM SIGN IM SQUARED;Lo;0;L;;;;;N;;;;; +12153;CUNEIFORM SIGN IMIN;Lo;0;L;;;;;N;;;;; +12154;CUNEIFORM SIGN IN;Lo;0;L;;;;;N;;;;; +12155;CUNEIFORM SIGN IR;Lo;0;L;;;;;N;;;;; +12156;CUNEIFORM SIGN ISH;Lo;0;L;;;;;N;;;;; +12157;CUNEIFORM SIGN KA;Lo;0;L;;;;;N;;;;; +12158;CUNEIFORM SIGN KA TIMES A;Lo;0;L;;;;;N;;;;; +12159;CUNEIFORM SIGN KA TIMES AD;Lo;0;L;;;;;N;;;;; +1215A;CUNEIFORM SIGN KA TIMES AD PLUS KU3;Lo;0;L;;;;;N;;;;; +1215B;CUNEIFORM SIGN KA TIMES ASH2;Lo;0;L;;;;;N;;;;; +1215C;CUNEIFORM SIGN KA TIMES BAD;Lo;0;L;;;;;N;;;;; +1215D;CUNEIFORM SIGN KA TIMES BALAG;Lo;0;L;;;;;N;;;;; +1215E;CUNEIFORM SIGN KA TIMES BAR;Lo;0;L;;;;;N;;;;; +1215F;CUNEIFORM SIGN KA TIMES BI;Lo;0;L;;;;;N;;;;; +12160;CUNEIFORM SIGN KA TIMES ERIN2;Lo;0;L;;;;;N;;;;; +12161;CUNEIFORM SIGN KA TIMES ESH2;Lo;0;L;;;;;N;;;;; +12162;CUNEIFORM SIGN KA TIMES GA;Lo;0;L;;;;;N;;;;; +12163;CUNEIFORM SIGN KA TIMES GAL;Lo;0;L;;;;;N;;;;; +12164;CUNEIFORM SIGN KA TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +12165;CUNEIFORM SIGN KA TIMES GAR;Lo;0;L;;;;;N;;;;; +12166;CUNEIFORM SIGN KA TIMES GAR PLUS SHA3 PLUS A;Lo;0;L;;;;;N;;;;; +12167;CUNEIFORM SIGN KA TIMES GI;Lo;0;L;;;;;N;;;;; +12168;CUNEIFORM SIGN KA TIMES GIR2;Lo;0;L;;;;;N;;;;; +12169;CUNEIFORM SIGN KA TIMES GISH PLUS SAR;Lo;0;L;;;;;N;;;;; +1216A;CUNEIFORM SIGN KA TIMES GISH CROSSING GISH;Lo;0;L;;;;;N;;;;; +1216B;CUNEIFORM SIGN KA TIMES GU;Lo;0;L;;;;;N;;;;; +1216C;CUNEIFORM SIGN KA TIMES GUR7;Lo;0;L;;;;;N;;;;; +1216D;CUNEIFORM SIGN KA TIMES IGI;Lo;0;L;;;;;N;;;;; +1216E;CUNEIFORM SIGN KA TIMES IM;Lo;0;L;;;;;N;;;;; +1216F;CUNEIFORM SIGN KA TIMES KAK;Lo;0;L;;;;;N;;;;; +12170;CUNEIFORM SIGN KA TIMES KI;Lo;0;L;;;;;N;;;;; +12171;CUNEIFORM SIGN KA TIMES KID;Lo;0;L;;;;;N;;;;; +12172;CUNEIFORM SIGN KA TIMES LI;Lo;0;L;;;;;N;;;;; +12173;CUNEIFORM SIGN KA TIMES LU;Lo;0;L;;;;;N;;;;; +12174;CUNEIFORM SIGN KA TIMES ME;Lo;0;L;;;;;N;;;;; +12175;CUNEIFORM SIGN KA TIMES ME PLUS DU;Lo;0;L;;;;;N;;;;; +12176;CUNEIFORM SIGN KA TIMES ME PLUS GI;Lo;0;L;;;;;N;;;;; +12177;CUNEIFORM SIGN KA TIMES ME PLUS TE;Lo;0;L;;;;;N;;;;; +12178;CUNEIFORM SIGN KA TIMES MI;Lo;0;L;;;;;N;;;;; +12179;CUNEIFORM SIGN KA TIMES MI PLUS NUNUZ;Lo;0;L;;;;;N;;;;; +1217A;CUNEIFORM SIGN KA TIMES NE;Lo;0;L;;;;;N;;;;; +1217B;CUNEIFORM SIGN KA TIMES NUN;Lo;0;L;;;;;N;;;;; +1217C;CUNEIFORM SIGN KA TIMES PI;Lo;0;L;;;;;N;;;;; +1217D;CUNEIFORM SIGN KA TIMES RU;Lo;0;L;;;;;N;;;;; +1217E;CUNEIFORM SIGN KA TIMES SA;Lo;0;L;;;;;N;;;;; +1217F;CUNEIFORM SIGN KA TIMES SAR;Lo;0;L;;;;;N;;;;; +12180;CUNEIFORM SIGN KA TIMES SHA;Lo;0;L;;;;;N;;;;; +12181;CUNEIFORM SIGN KA TIMES SHE;Lo;0;L;;;;;N;;;;; +12182;CUNEIFORM SIGN KA TIMES SHID;Lo;0;L;;;;;N;;;;; +12183;CUNEIFORM SIGN KA TIMES SHU;Lo;0;L;;;;;N;;;;; +12184;CUNEIFORM SIGN KA TIMES SIG;Lo;0;L;;;;;N;;;;; +12185;CUNEIFORM SIGN KA TIMES SUHUR;Lo;0;L;;;;;N;;;;; +12186;CUNEIFORM SIGN KA TIMES TAR;Lo;0;L;;;;;N;;;;; +12187;CUNEIFORM SIGN KA TIMES U;Lo;0;L;;;;;N;;;;; +12188;CUNEIFORM SIGN KA TIMES U2;Lo;0;L;;;;;N;;;;; +12189;CUNEIFORM SIGN KA TIMES UD;Lo;0;L;;;;;N;;;;; +1218A;CUNEIFORM SIGN KA TIMES UMUM TIMES PA;Lo;0;L;;;;;N;;;;; +1218B;CUNEIFORM SIGN KA TIMES USH;Lo;0;L;;;;;N;;;;; +1218C;CUNEIFORM SIGN KA TIMES ZI;Lo;0;L;;;;;N;;;;; +1218D;CUNEIFORM SIGN KA2;Lo;0;L;;;;;N;;;;; +1218E;CUNEIFORM SIGN KA2 CROSSING KA2;Lo;0;L;;;;;N;;;;; +1218F;CUNEIFORM SIGN KAB;Lo;0;L;;;;;N;;;;; +12190;CUNEIFORM SIGN KAD2;Lo;0;L;;;;;N;;;;; +12191;CUNEIFORM SIGN KAD3;Lo;0;L;;;;;N;;;;; +12192;CUNEIFORM SIGN KAD4;Lo;0;L;;;;;N;;;;; +12193;CUNEIFORM SIGN KAD5;Lo;0;L;;;;;N;;;;; +12194;CUNEIFORM SIGN KAD5 OVER KAD5;Lo;0;L;;;;;N;;;;; +12195;CUNEIFORM SIGN KAK;Lo;0;L;;;;;N;;;;; +12196;CUNEIFORM SIGN KAK TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +12197;CUNEIFORM SIGN KAL;Lo;0;L;;;;;N;;;;; +12198;CUNEIFORM SIGN KAL TIMES BAD;Lo;0;L;;;;;N;;;;; +12199;CUNEIFORM SIGN KAL CROSSING KAL;Lo;0;L;;;;;N;;;;; +1219A;CUNEIFORM SIGN KAM2;Lo;0;L;;;;;N;;;;; +1219B;CUNEIFORM SIGN KAM4;Lo;0;L;;;;;N;;;;; +1219C;CUNEIFORM SIGN KASKAL;Lo;0;L;;;;;N;;;;; +1219D;CUNEIFORM SIGN KASKAL LAGAB TIMES U OVER LAGAB TIMES U;Lo;0;L;;;;;N;;;;; +1219E;CUNEIFORM SIGN KASKAL OVER KASKAL LAGAB TIMES U OVER LAGAB TIMES U;Lo;0;L;;;;;N;;;;; +1219F;CUNEIFORM SIGN KESH2;Lo;0;L;;;;;N;;;;; +121A0;CUNEIFORM SIGN KI;Lo;0;L;;;;;N;;;;; +121A1;CUNEIFORM SIGN KI TIMES BAD;Lo;0;L;;;;;N;;;;; +121A2;CUNEIFORM SIGN KI TIMES U;Lo;0;L;;;;;N;;;;; +121A3;CUNEIFORM SIGN KI TIMES UD;Lo;0;L;;;;;N;;;;; +121A4;CUNEIFORM SIGN KID;Lo;0;L;;;;;N;;;;; +121A5;CUNEIFORM SIGN KIN;Lo;0;L;;;;;N;;;;; +121A6;CUNEIFORM SIGN KISAL;Lo;0;L;;;;;N;;;;; +121A7;CUNEIFORM SIGN KISH;Lo;0;L;;;;;N;;;;; +121A8;CUNEIFORM SIGN KISIM5;Lo;0;L;;;;;N;;;;; +121A9;CUNEIFORM SIGN KISIM5 OVER KISIM5;Lo;0;L;;;;;N;;;;; +121AA;CUNEIFORM SIGN KU;Lo;0;L;;;;;N;;;;; +121AB;CUNEIFORM SIGN KU OVER HI TIMES ASH2 KU OVER HI TIMES ASH2;Lo;0;L;;;;;N;;;;; +121AC;CUNEIFORM SIGN KU3;Lo;0;L;;;;;N;;;;; +121AD;CUNEIFORM SIGN KU4;Lo;0;L;;;;;N;;;;; +121AE;CUNEIFORM SIGN KU4 VARIANT FORM;Lo;0;L;;;;;N;;;;; +121AF;CUNEIFORM SIGN KU7;Lo;0;L;;;;;N;;;;; +121B0;CUNEIFORM SIGN KUL;Lo;0;L;;;;;N;;;;; +121B1;CUNEIFORM SIGN KUL GUNU;Lo;0;L;;;;;N;;;;; +121B2;CUNEIFORM SIGN KUN;Lo;0;L;;;;;N;;;;; +121B3;CUNEIFORM SIGN KUR;Lo;0;L;;;;;N;;;;; +121B4;CUNEIFORM SIGN KUR OPPOSING KUR;Lo;0;L;;;;;N;;;;; +121B5;CUNEIFORM SIGN KUSHU2;Lo;0;L;;;;;N;;;;; +121B6;CUNEIFORM SIGN KWU318;Lo;0;L;;;;;N;;;;; +121B7;CUNEIFORM SIGN LA;Lo;0;L;;;;;N;;;;; +121B8;CUNEIFORM SIGN LAGAB;Lo;0;L;;;;;N;;;;; +121B9;CUNEIFORM SIGN LAGAB TIMES A;Lo;0;L;;;;;N;;;;; +121BA;CUNEIFORM SIGN LAGAB TIMES A PLUS DA PLUS HA;Lo;0;L;;;;;N;;;;; +121BB;CUNEIFORM SIGN LAGAB TIMES A PLUS GAR;Lo;0;L;;;;;N;;;;; +121BC;CUNEIFORM SIGN LAGAB TIMES A PLUS LAL;Lo;0;L;;;;;N;;;;; +121BD;CUNEIFORM SIGN LAGAB TIMES AL;Lo;0;L;;;;;N;;;;; +121BE;CUNEIFORM SIGN LAGAB TIMES AN;Lo;0;L;;;;;N;;;;; +121BF;CUNEIFORM SIGN LAGAB TIMES ASH ZIDA TENU;Lo;0;L;;;;;N;;;;; +121C0;CUNEIFORM SIGN LAGAB TIMES BAD;Lo;0;L;;;;;N;;;;; +121C1;CUNEIFORM SIGN LAGAB TIMES BI;Lo;0;L;;;;;N;;;;; +121C2;CUNEIFORM SIGN LAGAB TIMES DAR;Lo;0;L;;;;;N;;;;; +121C3;CUNEIFORM SIGN LAGAB TIMES EN;Lo;0;L;;;;;N;;;;; +121C4;CUNEIFORM SIGN LAGAB TIMES GA;Lo;0;L;;;;;N;;;;; +121C5;CUNEIFORM SIGN LAGAB TIMES GAR;Lo;0;L;;;;;N;;;;; +121C6;CUNEIFORM SIGN LAGAB TIMES GUD;Lo;0;L;;;;;N;;;;; +121C7;CUNEIFORM SIGN LAGAB TIMES GUD PLUS GUD;Lo;0;L;;;;;N;;;;; +121C8;CUNEIFORM SIGN LAGAB TIMES HA;Lo;0;L;;;;;N;;;;; +121C9;CUNEIFORM SIGN LAGAB TIMES HAL;Lo;0;L;;;;;N;;;;; +121CA;CUNEIFORM SIGN LAGAB TIMES HI TIMES NUN;Lo;0;L;;;;;N;;;;; +121CB;CUNEIFORM SIGN LAGAB TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +121CC;CUNEIFORM SIGN LAGAB TIMES IM;Lo;0;L;;;;;N;;;;; +121CD;CUNEIFORM SIGN LAGAB TIMES IM PLUS HA;Lo;0;L;;;;;N;;;;; +121CE;CUNEIFORM SIGN LAGAB TIMES IM PLUS LU;Lo;0;L;;;;;N;;;;; +121CF;CUNEIFORM SIGN LAGAB TIMES KI;Lo;0;L;;;;;N;;;;; +121D0;CUNEIFORM SIGN LAGAB TIMES KIN;Lo;0;L;;;;;N;;;;; +121D1;CUNEIFORM SIGN LAGAB TIMES KU3;Lo;0;L;;;;;N;;;;; +121D2;CUNEIFORM SIGN LAGAB TIMES KUL;Lo;0;L;;;;;N;;;;; +121D3;CUNEIFORM SIGN LAGAB TIMES KUL PLUS HI PLUS A;Lo;0;L;;;;;N;;;;; +121D4;CUNEIFORM SIGN LAGAB TIMES LAGAB;Lo;0;L;;;;;N;;;;; +121D5;CUNEIFORM SIGN LAGAB TIMES LISH;Lo;0;L;;;;;N;;;;; +121D6;CUNEIFORM SIGN LAGAB TIMES LU;Lo;0;L;;;;;N;;;;; +121D7;CUNEIFORM SIGN LAGAB TIMES LUL;Lo;0;L;;;;;N;;;;; +121D8;CUNEIFORM SIGN LAGAB TIMES ME;Lo;0;L;;;;;N;;;;; +121D9;CUNEIFORM SIGN LAGAB TIMES ME PLUS EN;Lo;0;L;;;;;N;;;;; +121DA;CUNEIFORM SIGN LAGAB TIMES MUSH;Lo;0;L;;;;;N;;;;; +121DB;CUNEIFORM SIGN LAGAB TIMES NE;Lo;0;L;;;;;N;;;;; +121DC;CUNEIFORM SIGN LAGAB TIMES SHE PLUS SUM;Lo;0;L;;;;;N;;;;; +121DD;CUNEIFORM SIGN LAGAB TIMES SHITA PLUS GISH PLUS ERIN2;Lo;0;L;;;;;N;;;;; +121DE;CUNEIFORM SIGN LAGAB TIMES SHITA PLUS GISH TENU;Lo;0;L;;;;;N;;;;; +121DF;CUNEIFORM SIGN LAGAB TIMES SHU2;Lo;0;L;;;;;N;;;;; +121E0;CUNEIFORM SIGN LAGAB TIMES SHU2 PLUS SHU2;Lo;0;L;;;;;N;;;;; +121E1;CUNEIFORM SIGN LAGAB TIMES SUM;Lo;0;L;;;;;N;;;;; +121E2;CUNEIFORM SIGN LAGAB TIMES TAG;Lo;0;L;;;;;N;;;;; +121E3;CUNEIFORM SIGN LAGAB TIMES TAK4;Lo;0;L;;;;;N;;;;; +121E4;CUNEIFORM SIGN LAGAB TIMES TE PLUS A PLUS SU PLUS NA;Lo;0;L;;;;;N;;;;; +121E5;CUNEIFORM SIGN LAGAB TIMES U;Lo;0;L;;;;;N;;;;; +121E6;CUNEIFORM SIGN LAGAB TIMES U PLUS A;Lo;0;L;;;;;N;;;;; +121E7;CUNEIFORM SIGN LAGAB TIMES U PLUS U PLUS U;Lo;0;L;;;;;N;;;;; +121E8;CUNEIFORM SIGN LAGAB TIMES U2 PLUS ASH;Lo;0;L;;;;;N;;;;; +121E9;CUNEIFORM SIGN LAGAB TIMES UD;Lo;0;L;;;;;N;;;;; +121EA;CUNEIFORM SIGN LAGAB TIMES USH;Lo;0;L;;;;;N;;;;; +121EB;CUNEIFORM SIGN LAGAB SQUARED;Lo;0;L;;;;;N;;;;; +121EC;CUNEIFORM SIGN LAGAR;Lo;0;L;;;;;N;;;;; +121ED;CUNEIFORM SIGN LAGAR TIMES SHE;Lo;0;L;;;;;N;;;;; +121EE;CUNEIFORM SIGN LAGAR TIMES SHE PLUS SUM;Lo;0;L;;;;;N;;;;; +121EF;CUNEIFORM SIGN LAGAR GUNU;Lo;0;L;;;;;N;;;;; +121F0;CUNEIFORM SIGN LAGAR GUNU OVER LAGAR GUNU SHE;Lo;0;L;;;;;N;;;;; +121F1;CUNEIFORM SIGN LAHSHU;Lo;0;L;;;;;N;;;;; +121F2;CUNEIFORM SIGN LAL;Lo;0;L;;;;;N;;;;; +121F3;CUNEIFORM SIGN LAL TIMES LAL;Lo;0;L;;;;;N;;;;; +121F4;CUNEIFORM SIGN LAM;Lo;0;L;;;;;N;;;;; +121F5;CUNEIFORM SIGN LAM TIMES KUR;Lo;0;L;;;;;N;;;;; +121F6;CUNEIFORM SIGN LAM TIMES KUR PLUS RU;Lo;0;L;;;;;N;;;;; +121F7;CUNEIFORM SIGN LI;Lo;0;L;;;;;N;;;;; +121F8;CUNEIFORM SIGN LIL;Lo;0;L;;;;;N;;;;; +121F9;CUNEIFORM SIGN LIMMU2;Lo;0;L;;;;;N;;;;; +121FA;CUNEIFORM SIGN LISH;Lo;0;L;;;;;N;;;;; +121FB;CUNEIFORM SIGN LU;Lo;0;L;;;;;N;;;;; +121FC;CUNEIFORM SIGN LU TIMES BAD;Lo;0;L;;;;;N;;;;; +121FD;CUNEIFORM SIGN LU2;Lo;0;L;;;;;N;;;;; +121FE;CUNEIFORM SIGN LU2 TIMES AL;Lo;0;L;;;;;N;;;;; +121FF;CUNEIFORM SIGN LU2 TIMES BAD;Lo;0;L;;;;;N;;;;; +12200;CUNEIFORM SIGN LU2 TIMES ESH2;Lo;0;L;;;;;N;;;;; +12201;CUNEIFORM SIGN LU2 TIMES ESH2 TENU;Lo;0;L;;;;;N;;;;; +12202;CUNEIFORM SIGN LU2 TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +12203;CUNEIFORM SIGN LU2 TIMES HI TIMES BAD;Lo;0;L;;;;;N;;;;; +12204;CUNEIFORM SIGN LU2 TIMES IM;Lo;0;L;;;;;N;;;;; +12205;CUNEIFORM SIGN LU2 TIMES KAD2;Lo;0;L;;;;;N;;;;; +12206;CUNEIFORM SIGN LU2 TIMES KAD3;Lo;0;L;;;;;N;;;;; +12207;CUNEIFORM SIGN LU2 TIMES KAD3 PLUS ASH;Lo;0;L;;;;;N;;;;; +12208;CUNEIFORM SIGN LU2 TIMES KI;Lo;0;L;;;;;N;;;;; +12209;CUNEIFORM SIGN LU2 TIMES LA PLUS ASH;Lo;0;L;;;;;N;;;;; +1220A;CUNEIFORM SIGN LU2 TIMES LAGAB;Lo;0;L;;;;;N;;;;; +1220B;CUNEIFORM SIGN LU2 TIMES ME PLUS EN;Lo;0;L;;;;;N;;;;; +1220C;CUNEIFORM SIGN LU2 TIMES NE;Lo;0;L;;;;;N;;;;; +1220D;CUNEIFORM SIGN LU2 TIMES NU;Lo;0;L;;;;;N;;;;; +1220E;CUNEIFORM SIGN LU2 TIMES SI PLUS ASH;Lo;0;L;;;;;N;;;;; +1220F;CUNEIFORM SIGN LU2 TIMES SIK2 PLUS BU;Lo;0;L;;;;;N;;;;; +12210;CUNEIFORM SIGN LU2 TIMES TUG2;Lo;0;L;;;;;N;;;;; +12211;CUNEIFORM SIGN LU2 TENU;Lo;0;L;;;;;N;;;;; +12212;CUNEIFORM SIGN LU2 CROSSING LU2;Lo;0;L;;;;;N;;;;; +12213;CUNEIFORM SIGN LU2 OPPOSING LU2;Lo;0;L;;;;;N;;;;; +12214;CUNEIFORM SIGN LU2 SQUARED;Lo;0;L;;;;;N;;;;; +12215;CUNEIFORM SIGN LU2 SHESHIG;Lo;0;L;;;;;N;;;;; +12216;CUNEIFORM SIGN LU3;Lo;0;L;;;;;N;;;;; +12217;CUNEIFORM SIGN LUGAL;Lo;0;L;;;;;N;;;;; +12218;CUNEIFORM SIGN LUGAL OVER LUGAL;Lo;0;L;;;;;N;;;;; +12219;CUNEIFORM SIGN LUGAL OPPOSING LUGAL;Lo;0;L;;;;;N;;;;; +1221A;CUNEIFORM SIGN LUGAL SHESHIG;Lo;0;L;;;;;N;;;;; +1221B;CUNEIFORM SIGN LUH;Lo;0;L;;;;;N;;;;; +1221C;CUNEIFORM SIGN LUL;Lo;0;L;;;;;N;;;;; +1221D;CUNEIFORM SIGN LUM;Lo;0;L;;;;;N;;;;; +1221E;CUNEIFORM SIGN LUM OVER LUM;Lo;0;L;;;;;N;;;;; +1221F;CUNEIFORM SIGN LUM OVER LUM GAR OVER GAR;Lo;0;L;;;;;N;;;;; +12220;CUNEIFORM SIGN MA;Lo;0;L;;;;;N;;;;; +12221;CUNEIFORM SIGN MA TIMES TAK4;Lo;0;L;;;;;N;;;;; +12222;CUNEIFORM SIGN MA GUNU;Lo;0;L;;;;;N;;;;; +12223;CUNEIFORM SIGN MA2;Lo;0;L;;;;;N;;;;; +12224;CUNEIFORM SIGN MAH;Lo;0;L;;;;;N;;;;; +12225;CUNEIFORM SIGN MAR;Lo;0;L;;;;;N;;;;; +12226;CUNEIFORM SIGN MASH;Lo;0;L;;;;;N;;;;; +12227;CUNEIFORM SIGN MASH2;Lo;0;L;;;;;N;;;;; +12228;CUNEIFORM SIGN ME;Lo;0;L;;;;;N;;;;; +12229;CUNEIFORM SIGN MES;Lo;0;L;;;;;N;;;;; +1222A;CUNEIFORM SIGN MI;Lo;0;L;;;;;N;;;;; +1222B;CUNEIFORM SIGN MIN;Lo;0;L;;;;;N;;;;; +1222C;CUNEIFORM SIGN MU;Lo;0;L;;;;;N;;;;; +1222D;CUNEIFORM SIGN MU OVER MU;Lo;0;L;;;;;N;;;;; +1222E;CUNEIFORM SIGN MUG;Lo;0;L;;;;;N;;;;; +1222F;CUNEIFORM SIGN MUG GUNU;Lo;0;L;;;;;N;;;;; +12230;CUNEIFORM SIGN MUNSUB;Lo;0;L;;;;;N;;;;; +12231;CUNEIFORM SIGN MURGU2;Lo;0;L;;;;;N;;;;; +12232;CUNEIFORM SIGN MUSH;Lo;0;L;;;;;N;;;;; +12233;CUNEIFORM SIGN MUSH TIMES A;Lo;0;L;;;;;N;;;;; +12234;CUNEIFORM SIGN MUSH TIMES KUR;Lo;0;L;;;;;N;;;;; +12235;CUNEIFORM SIGN MUSH TIMES ZA;Lo;0;L;;;;;N;;;;; +12236;CUNEIFORM SIGN MUSH OVER MUSH;Lo;0;L;;;;;N;;;;; +12237;CUNEIFORM SIGN MUSH OVER MUSH TIMES A PLUS NA;Lo;0;L;;;;;N;;;;; +12238;CUNEIFORM SIGN MUSH CROSSING MUSH;Lo;0;L;;;;;N;;;;; +12239;CUNEIFORM SIGN MUSH3;Lo;0;L;;;;;N;;;;; +1223A;CUNEIFORM SIGN MUSH3 TIMES A;Lo;0;L;;;;;N;;;;; +1223B;CUNEIFORM SIGN MUSH3 TIMES A PLUS DI;Lo;0;L;;;;;N;;;;; +1223C;CUNEIFORM SIGN MUSH3 TIMES DI;Lo;0;L;;;;;N;;;;; +1223D;CUNEIFORM SIGN MUSH3 GUNU;Lo;0;L;;;;;N;;;;; +1223E;CUNEIFORM SIGN NA;Lo;0;L;;;;;N;;;;; +1223F;CUNEIFORM SIGN NA2;Lo;0;L;;;;;N;;;;; +12240;CUNEIFORM SIGN NAGA;Lo;0;L;;;;;N;;;;; +12241;CUNEIFORM SIGN NAGA INVERTED;Lo;0;L;;;;;N;;;;; +12242;CUNEIFORM SIGN NAGA TIMES SHU TENU;Lo;0;L;;;;;N;;;;; +12243;CUNEIFORM SIGN NAGA OPPOSING NAGA;Lo;0;L;;;;;N;;;;; +12244;CUNEIFORM SIGN NAGAR;Lo;0;L;;;;;N;;;;; +12245;CUNEIFORM SIGN NAM NUTILLU;Lo;0;L;;;;;N;;;;; +12246;CUNEIFORM SIGN NAM;Lo;0;L;;;;;N;;;;; +12247;CUNEIFORM SIGN NAM2;Lo;0;L;;;;;N;;;;; +12248;CUNEIFORM SIGN NE;Lo;0;L;;;;;N;;;;; +12249;CUNEIFORM SIGN NE TIMES A;Lo;0;L;;;;;N;;;;; +1224A;CUNEIFORM SIGN NE TIMES UD;Lo;0;L;;;;;N;;;;; +1224B;CUNEIFORM SIGN NE SHESHIG;Lo;0;L;;;;;N;;;;; +1224C;CUNEIFORM SIGN NI;Lo;0;L;;;;;N;;;;; +1224D;CUNEIFORM SIGN NI TIMES E;Lo;0;L;;;;;N;;;;; +1224E;CUNEIFORM SIGN NI2;Lo;0;L;;;;;N;;;;; +1224F;CUNEIFORM SIGN NIM;Lo;0;L;;;;;N;;;;; +12250;CUNEIFORM SIGN NIM TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +12251;CUNEIFORM SIGN NIM TIMES GAR PLUS GAN2 TENU;Lo;0;L;;;;;N;;;;; +12252;CUNEIFORM SIGN NINDA2;Lo;0;L;;;;;N;;;;; +12253;CUNEIFORM SIGN NINDA2 TIMES AN;Lo;0;L;;;;;N;;;;; +12254;CUNEIFORM SIGN NINDA2 TIMES ASH;Lo;0;L;;;;;N;;;;; +12255;CUNEIFORM SIGN NINDA2 TIMES ASH PLUS ASH;Lo;0;L;;;;;N;;;;; +12256;CUNEIFORM SIGN NINDA2 TIMES GUD;Lo;0;L;;;;;N;;;;; +12257;CUNEIFORM SIGN NINDA2 TIMES ME PLUS GAN2 TENU;Lo;0;L;;;;;N;;;;; +12258;CUNEIFORM SIGN NINDA2 TIMES NE;Lo;0;L;;;;;N;;;;; +12259;CUNEIFORM SIGN NINDA2 TIMES NUN;Lo;0;L;;;;;N;;;;; +1225A;CUNEIFORM SIGN NINDA2 TIMES SHE;Lo;0;L;;;;;N;;;;; +1225B;CUNEIFORM SIGN NINDA2 TIMES SHE PLUS A AN;Lo;0;L;;;;;N;;;;; +1225C;CUNEIFORM SIGN NINDA2 TIMES SHE PLUS ASH;Lo;0;L;;;;;N;;;;; +1225D;CUNEIFORM SIGN NINDA2 TIMES SHE PLUS ASH PLUS ASH;Lo;0;L;;;;;N;;;;; +1225E;CUNEIFORM SIGN NINDA2 TIMES U2 PLUS ASH;Lo;0;L;;;;;N;;;;; +1225F;CUNEIFORM SIGN NINDA2 TIMES USH;Lo;0;L;;;;;N;;;;; +12260;CUNEIFORM SIGN NISAG;Lo;0;L;;;;;N;;;;; +12261;CUNEIFORM SIGN NU;Lo;0;L;;;;;N;;;;; +12262;CUNEIFORM SIGN NU11;Lo;0;L;;;;;N;;;;; +12263;CUNEIFORM SIGN NUN;Lo;0;L;;;;;N;;;;; +12264;CUNEIFORM SIGN NUN LAGAR TIMES GAR;Lo;0;L;;;;;N;;;;; +12265;CUNEIFORM SIGN NUN LAGAR TIMES MASH;Lo;0;L;;;;;N;;;;; +12266;CUNEIFORM SIGN NUN LAGAR TIMES SAL;Lo;0;L;;;;;N;;;;; +12267;CUNEIFORM SIGN NUN LAGAR TIMES SAL OVER NUN LAGAR TIMES SAL;Lo;0;L;;;;;N;;;;; +12268;CUNEIFORM SIGN NUN LAGAR TIMES USH;Lo;0;L;;;;;N;;;;; +12269;CUNEIFORM SIGN NUN TENU;Lo;0;L;;;;;N;;;;; +1226A;CUNEIFORM SIGN NUN OVER NUN;Lo;0;L;;;;;N;;;;; +1226B;CUNEIFORM SIGN NUN CROSSING NUN;Lo;0;L;;;;;N;;;;; +1226C;CUNEIFORM SIGN NUN CROSSING NUN LAGAR OVER LAGAR;Lo;0;L;;;;;N;;;;; +1226D;CUNEIFORM SIGN NUNUZ;Lo;0;L;;;;;N;;;;; +1226E;CUNEIFORM SIGN NUNUZ AB2 TIMES ASHGAB;Lo;0;L;;;;;N;;;;; +1226F;CUNEIFORM SIGN NUNUZ AB2 TIMES BI;Lo;0;L;;;;;N;;;;; +12270;CUNEIFORM SIGN NUNUZ AB2 TIMES DUG;Lo;0;L;;;;;N;;;;; +12271;CUNEIFORM SIGN NUNUZ AB2 TIMES GUD;Lo;0;L;;;;;N;;;;; +12272;CUNEIFORM SIGN NUNUZ AB2 TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +12273;CUNEIFORM SIGN NUNUZ AB2 TIMES KAD3;Lo;0;L;;;;;N;;;;; +12274;CUNEIFORM SIGN NUNUZ AB2 TIMES LA;Lo;0;L;;;;;N;;;;; +12275;CUNEIFORM SIGN NUNUZ AB2 TIMES NE;Lo;0;L;;;;;N;;;;; +12276;CUNEIFORM SIGN NUNUZ AB2 TIMES SILA3;Lo;0;L;;;;;N;;;;; +12277;CUNEIFORM SIGN NUNUZ AB2 TIMES U2;Lo;0;L;;;;;N;;;;; +12278;CUNEIFORM SIGN NUNUZ KISIM5 TIMES BI;Lo;0;L;;;;;N;;;;; +12279;CUNEIFORM SIGN NUNUZ KISIM5 TIMES BI U;Lo;0;L;;;;;N;;;;; +1227A;CUNEIFORM SIGN PA;Lo;0;L;;;;;N;;;;; +1227B;CUNEIFORM SIGN PAD;Lo;0;L;;;;;N;;;;; +1227C;CUNEIFORM SIGN PAN;Lo;0;L;;;;;N;;;;; +1227D;CUNEIFORM SIGN PAP;Lo;0;L;;;;;N;;;;; +1227E;CUNEIFORM SIGN PESH2;Lo;0;L;;;;;N;;;;; +1227F;CUNEIFORM SIGN PI;Lo;0;L;;;;;N;;;;; +12280;CUNEIFORM SIGN PI TIMES A;Lo;0;L;;;;;N;;;;; +12281;CUNEIFORM SIGN PI TIMES AB;Lo;0;L;;;;;N;;;;; +12282;CUNEIFORM SIGN PI TIMES BI;Lo;0;L;;;;;N;;;;; +12283;CUNEIFORM SIGN PI TIMES BU;Lo;0;L;;;;;N;;;;; +12284;CUNEIFORM SIGN PI TIMES E;Lo;0;L;;;;;N;;;;; +12285;CUNEIFORM SIGN PI TIMES I;Lo;0;L;;;;;N;;;;; +12286;CUNEIFORM SIGN PI TIMES IB;Lo;0;L;;;;;N;;;;; +12287;CUNEIFORM SIGN PI TIMES U;Lo;0;L;;;;;N;;;;; +12288;CUNEIFORM SIGN PI TIMES U2;Lo;0;L;;;;;N;;;;; +12289;CUNEIFORM SIGN PI CROSSING PI;Lo;0;L;;;;;N;;;;; +1228A;CUNEIFORM SIGN PIRIG;Lo;0;L;;;;;N;;;;; +1228B;CUNEIFORM SIGN PIRIG TIMES KAL;Lo;0;L;;;;;N;;;;; +1228C;CUNEIFORM SIGN PIRIG TIMES UD;Lo;0;L;;;;;N;;;;; +1228D;CUNEIFORM SIGN PIRIG TIMES ZA;Lo;0;L;;;;;N;;;;; +1228E;CUNEIFORM SIGN PIRIG OPPOSING PIRIG;Lo;0;L;;;;;N;;;;; +1228F;CUNEIFORM SIGN RA;Lo;0;L;;;;;N;;;;; +12290;CUNEIFORM SIGN RAB;Lo;0;L;;;;;N;;;;; +12291;CUNEIFORM SIGN RI;Lo;0;L;;;;;N;;;;; +12292;CUNEIFORM SIGN RU;Lo;0;L;;;;;N;;;;; +12293;CUNEIFORM SIGN SA;Lo;0;L;;;;;N;;;;; +12294;CUNEIFORM SIGN SAG NUTILLU;Lo;0;L;;;;;N;;;;; +12295;CUNEIFORM SIGN SAG;Lo;0;L;;;;;N;;;;; +12296;CUNEIFORM SIGN SAG TIMES A;Lo;0;L;;;;;N;;;;; +12297;CUNEIFORM SIGN SAG TIMES DU;Lo;0;L;;;;;N;;;;; +12298;CUNEIFORM SIGN SAG TIMES DUB;Lo;0;L;;;;;N;;;;; +12299;CUNEIFORM SIGN SAG TIMES HA;Lo;0;L;;;;;N;;;;; +1229A;CUNEIFORM SIGN SAG TIMES KAK;Lo;0;L;;;;;N;;;;; +1229B;CUNEIFORM SIGN SAG TIMES KUR;Lo;0;L;;;;;N;;;;; +1229C;CUNEIFORM SIGN SAG TIMES LUM;Lo;0;L;;;;;N;;;;; +1229D;CUNEIFORM SIGN SAG TIMES MI;Lo;0;L;;;;;N;;;;; +1229E;CUNEIFORM SIGN SAG TIMES NUN;Lo;0;L;;;;;N;;;;; +1229F;CUNEIFORM SIGN SAG TIMES SAL;Lo;0;L;;;;;N;;;;; +122A0;CUNEIFORM SIGN SAG TIMES SHID;Lo;0;L;;;;;N;;;;; +122A1;CUNEIFORM SIGN SAG TIMES TAB;Lo;0;L;;;;;N;;;;; +122A2;CUNEIFORM SIGN SAG TIMES U2;Lo;0;L;;;;;N;;;;; +122A3;CUNEIFORM SIGN SAG TIMES UB;Lo;0;L;;;;;N;;;;; +122A4;CUNEIFORM SIGN SAG TIMES UM;Lo;0;L;;;;;N;;;;; +122A5;CUNEIFORM SIGN SAG TIMES UR;Lo;0;L;;;;;N;;;;; +122A6;CUNEIFORM SIGN SAG TIMES USH;Lo;0;L;;;;;N;;;;; +122A7;CUNEIFORM SIGN SAG OVER SAG;Lo;0;L;;;;;N;;;;; +122A8;CUNEIFORM SIGN SAG GUNU;Lo;0;L;;;;;N;;;;; +122A9;CUNEIFORM SIGN SAL;Lo;0;L;;;;;N;;;;; +122AA;CUNEIFORM SIGN SAL LAGAB TIMES ASH2;Lo;0;L;;;;;N;;;;; +122AB;CUNEIFORM SIGN SANGA2;Lo;0;L;;;;;N;;;;; +122AC;CUNEIFORM SIGN SAR;Lo;0;L;;;;;N;;;;; +122AD;CUNEIFORM SIGN SHA;Lo;0;L;;;;;N;;;;; +122AE;CUNEIFORM SIGN SHA3;Lo;0;L;;;;;N;;;;; +122AF;CUNEIFORM SIGN SHA3 TIMES A;Lo;0;L;;;;;N;;;;; +122B0;CUNEIFORM SIGN SHA3 TIMES BAD;Lo;0;L;;;;;N;;;;; +122B1;CUNEIFORM SIGN SHA3 TIMES GISH;Lo;0;L;;;;;N;;;;; +122B2;CUNEIFORM SIGN SHA3 TIMES NE;Lo;0;L;;;;;N;;;;; +122B3;CUNEIFORM SIGN SHA3 TIMES SHU2;Lo;0;L;;;;;N;;;;; +122B4;CUNEIFORM SIGN SHA3 TIMES TUR;Lo;0;L;;;;;N;;;;; +122B5;CUNEIFORM SIGN SHA3 TIMES U;Lo;0;L;;;;;N;;;;; +122B6;CUNEIFORM SIGN SHA3 TIMES U PLUS A;Lo;0;L;;;;;N;;;;; +122B7;CUNEIFORM SIGN SHA6;Lo;0;L;;;;;N;;;;; +122B8;CUNEIFORM SIGN SHAB6;Lo;0;L;;;;;N;;;;; +122B9;CUNEIFORM SIGN SHAR2;Lo;0;L;;;;;N;;;;; +122BA;CUNEIFORM SIGN SHE;Lo;0;L;;;;;N;;;;; +122BB;CUNEIFORM SIGN SHE HU;Lo;0;L;;;;;N;;;;; +122BC;CUNEIFORM SIGN SHE OVER SHE GAD OVER GAD GAR OVER GAR;Lo;0;L;;;;;N;;;;; +122BD;CUNEIFORM SIGN SHE OVER SHE TAB OVER TAB GAR OVER GAR;Lo;0;L;;;;;N;;;;; +122BE;CUNEIFORM SIGN SHEG9;Lo;0;L;;;;;N;;;;; +122BF;CUNEIFORM SIGN SHEN;Lo;0;L;;;;;N;;;;; +122C0;CUNEIFORM SIGN SHESH;Lo;0;L;;;;;N;;;;; +122C1;CUNEIFORM SIGN SHESH2;Lo;0;L;;;;;N;;;;; +122C2;CUNEIFORM SIGN SHESHLAM;Lo;0;L;;;;;N;;;;; +122C3;CUNEIFORM SIGN SHID;Lo;0;L;;;;;N;;;;; +122C4;CUNEIFORM SIGN SHID TIMES A;Lo;0;L;;;;;N;;;;; +122C5;CUNEIFORM SIGN SHID TIMES IM;Lo;0;L;;;;;N;;;;; +122C6;CUNEIFORM SIGN SHIM;Lo;0;L;;;;;N;;;;; +122C7;CUNEIFORM SIGN SHIM TIMES A;Lo;0;L;;;;;N;;;;; +122C8;CUNEIFORM SIGN SHIM TIMES BAL;Lo;0;L;;;;;N;;;;; +122C9;CUNEIFORM SIGN SHIM TIMES BULUG;Lo;0;L;;;;;N;;;;; +122CA;CUNEIFORM SIGN SHIM TIMES DIN;Lo;0;L;;;;;N;;;;; +122CB;CUNEIFORM SIGN SHIM TIMES GAR;Lo;0;L;;;;;N;;;;; +122CC;CUNEIFORM SIGN SHIM TIMES IGI;Lo;0;L;;;;;N;;;;; +122CD;CUNEIFORM SIGN SHIM TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; +122CE;CUNEIFORM SIGN SHIM TIMES KUSHU2;Lo;0;L;;;;;N;;;;; +122CF;CUNEIFORM SIGN SHIM TIMES LUL;Lo;0;L;;;;;N;;;;; +122D0;CUNEIFORM SIGN SHIM TIMES MUG;Lo;0;L;;;;;N;;;;; +122D1;CUNEIFORM SIGN SHIM TIMES SAL;Lo;0;L;;;;;N;;;;; +122D2;CUNEIFORM SIGN SHINIG;Lo;0;L;;;;;N;;;;; +122D3;CUNEIFORM SIGN SHIR;Lo;0;L;;;;;N;;;;; +122D4;CUNEIFORM SIGN SHIR TENU;Lo;0;L;;;;;N;;;;; +122D5;CUNEIFORM SIGN SHIR OVER SHIR BUR OVER BUR;Lo;0;L;;;;;N;;;;; +122D6;CUNEIFORM SIGN SHITA;Lo;0;L;;;;;N;;;;; +122D7;CUNEIFORM SIGN SHU;Lo;0;L;;;;;N;;;;; +122D8;CUNEIFORM SIGN SHU OVER INVERTED SHU;Lo;0;L;;;;;N;;;;; +122D9;CUNEIFORM SIGN SHU2;Lo;0;L;;;;;N;;;;; +122DA;CUNEIFORM SIGN SHUBUR;Lo;0;L;;;;;N;;;;; +122DB;CUNEIFORM SIGN SI;Lo;0;L;;;;;N;;;;; +122DC;CUNEIFORM SIGN SI GUNU;Lo;0;L;;;;;N;;;;; +122DD;CUNEIFORM SIGN SIG;Lo;0;L;;;;;N;;;;; +122DE;CUNEIFORM SIGN SIG4;Lo;0;L;;;;;N;;;;; +122DF;CUNEIFORM SIGN SIG4 OVER SIG4 SHU2;Lo;0;L;;;;;N;;;;; +122E0;CUNEIFORM SIGN SIK2;Lo;0;L;;;;;N;;;;; +122E1;CUNEIFORM SIGN SILA3;Lo;0;L;;;;;N;;;;; +122E2;CUNEIFORM SIGN SU;Lo;0;L;;;;;N;;;;; +122E3;CUNEIFORM SIGN SU OVER SU;Lo;0;L;;;;;N;;;;; +122E4;CUNEIFORM SIGN SUD;Lo;0;L;;;;;N;;;;; +122E5;CUNEIFORM SIGN SUD2;Lo;0;L;;;;;N;;;;; +122E6;CUNEIFORM SIGN SUHUR;Lo;0;L;;;;;N;;;;; +122E7;CUNEIFORM SIGN SUM;Lo;0;L;;;;;N;;;;; +122E8;CUNEIFORM SIGN SUMASH;Lo;0;L;;;;;N;;;;; +122E9;CUNEIFORM SIGN SUR;Lo;0;L;;;;;N;;;;; +122EA;CUNEIFORM SIGN SUR9;Lo;0;L;;;;;N;;;;; +122EB;CUNEIFORM SIGN TA;Lo;0;L;;;;;N;;;;; +122EC;CUNEIFORM SIGN TA ASTERISK;Lo;0;L;;;;;N;;;;; +122ED;CUNEIFORM SIGN TA TIMES HI;Lo;0;L;;;;;N;;;;; +122EE;CUNEIFORM SIGN TA TIMES MI;Lo;0;L;;;;;N;;;;; +122EF;CUNEIFORM SIGN TA GUNU;Lo;0;L;;;;;N;;;;; +122F0;CUNEIFORM SIGN TAB;Lo;0;L;;;;;N;;;;; +122F1;CUNEIFORM SIGN TAB OVER TAB NI OVER NI DISH OVER DISH;Lo;0;L;;;;;N;;;;; +122F2;CUNEIFORM SIGN TAB SQUARED;Lo;0;L;;;;;N;;;;; +122F3;CUNEIFORM SIGN TAG;Lo;0;L;;;;;N;;;;; +122F4;CUNEIFORM SIGN TAG TIMES BI;Lo;0;L;;;;;N;;;;; +122F5;CUNEIFORM SIGN TAG TIMES GUD;Lo;0;L;;;;;N;;;;; +122F6;CUNEIFORM SIGN TAG TIMES SHE;Lo;0;L;;;;;N;;;;; +122F7;CUNEIFORM SIGN TAG TIMES SHU;Lo;0;L;;;;;N;;;;; +122F8;CUNEIFORM SIGN TAG TIMES TUG2;Lo;0;L;;;;;N;;;;; +122F9;CUNEIFORM SIGN TAG TIMES UD;Lo;0;L;;;;;N;;;;; +122FA;CUNEIFORM SIGN TAK4;Lo;0;L;;;;;N;;;;; +122FB;CUNEIFORM SIGN TAR;Lo;0;L;;;;;N;;;;; +122FC;CUNEIFORM SIGN TE;Lo;0;L;;;;;N;;;;; +122FD;CUNEIFORM SIGN TE GUNU;Lo;0;L;;;;;N;;;;; +122FE;CUNEIFORM SIGN TI;Lo;0;L;;;;;N;;;;; +122FF;CUNEIFORM SIGN TI TENU;Lo;0;L;;;;;N;;;;; +12300;CUNEIFORM SIGN TIL;Lo;0;L;;;;;N;;;;; +12301;CUNEIFORM SIGN TIR;Lo;0;L;;;;;N;;;;; +12302;CUNEIFORM SIGN TIR TIMES TAK4;Lo;0;L;;;;;N;;;;; +12303;CUNEIFORM SIGN TIR OVER TIR;Lo;0;L;;;;;N;;;;; +12304;CUNEIFORM SIGN TIR OVER TIR GAD OVER GAD GAR OVER GAR;Lo;0;L;;;;;N;;;;; +12305;CUNEIFORM SIGN TU;Lo;0;L;;;;;N;;;;; +12306;CUNEIFORM SIGN TUG2;Lo;0;L;;;;;N;;;;; +12307;CUNEIFORM SIGN TUK;Lo;0;L;;;;;N;;;;; +12308;CUNEIFORM SIGN TUM;Lo;0;L;;;;;N;;;;; +12309;CUNEIFORM SIGN TUR;Lo;0;L;;;;;N;;;;; +1230A;CUNEIFORM SIGN TUR OVER TUR ZA OVER ZA;Lo;0;L;;;;;N;;;;; +1230B;CUNEIFORM SIGN U;Lo;0;L;;;;;N;;;;; +1230C;CUNEIFORM SIGN U GUD;Lo;0;L;;;;;N;;;;; +1230D;CUNEIFORM SIGN U U U;Lo;0;L;;;;;N;;;;; +1230E;CUNEIFORM SIGN U OVER U PA OVER PA GAR OVER GAR;Lo;0;L;;;;;N;;;;; +1230F;CUNEIFORM SIGN U OVER U SUR OVER SUR;Lo;0;L;;;;;N;;;;; +12310;CUNEIFORM SIGN U OVER U U REVERSED OVER U REVERSED;Lo;0;L;;;;;N;;;;; +12311;CUNEIFORM SIGN U2;Lo;0;L;;;;;N;;;;; +12312;CUNEIFORM SIGN UB;Lo;0;L;;;;;N;;;;; +12313;CUNEIFORM SIGN UD;Lo;0;L;;;;;N;;;;; +12314;CUNEIFORM SIGN UD KUSHU2;Lo;0;L;;;;;N;;;;; +12315;CUNEIFORM SIGN UD TIMES BAD;Lo;0;L;;;;;N;;;;; +12316;CUNEIFORM SIGN UD TIMES MI;Lo;0;L;;;;;N;;;;; +12317;CUNEIFORM SIGN UD TIMES U PLUS U PLUS U;Lo;0;L;;;;;N;;;;; +12318;CUNEIFORM SIGN UD TIMES U PLUS U PLUS U GUNU;Lo;0;L;;;;;N;;;;; +12319;CUNEIFORM SIGN UD GUNU;Lo;0;L;;;;;N;;;;; +1231A;CUNEIFORM SIGN UD SHESHIG;Lo;0;L;;;;;N;;;;; +1231B;CUNEIFORM SIGN UD SHESHIG TIMES BAD;Lo;0;L;;;;;N;;;;; +1231C;CUNEIFORM SIGN UDUG;Lo;0;L;;;;;N;;;;; +1231D;CUNEIFORM SIGN UM;Lo;0;L;;;;;N;;;;; +1231E;CUNEIFORM SIGN UM TIMES LAGAB;Lo;0;L;;;;;N;;;;; +1231F;CUNEIFORM SIGN UM TIMES ME PLUS DA;Lo;0;L;;;;;N;;;;; +12320;CUNEIFORM SIGN UM TIMES SHA3;Lo;0;L;;;;;N;;;;; +12321;CUNEIFORM SIGN UM TIMES U;Lo;0;L;;;;;N;;;;; +12322;CUNEIFORM SIGN UMBIN;Lo;0;L;;;;;N;;;;; +12323;CUNEIFORM SIGN UMUM;Lo;0;L;;;;;N;;;;; +12324;CUNEIFORM SIGN UMUM TIMES KASKAL;Lo;0;L;;;;;N;;;;; +12325;CUNEIFORM SIGN UMUM TIMES PA;Lo;0;L;;;;;N;;;;; +12326;CUNEIFORM SIGN UN;Lo;0;L;;;;;N;;;;; +12327;CUNEIFORM SIGN UN GUNU;Lo;0;L;;;;;N;;;;; +12328;CUNEIFORM SIGN UR;Lo;0;L;;;;;N;;;;; +12329;CUNEIFORM SIGN UR CROSSING UR;Lo;0;L;;;;;N;;;;; +1232A;CUNEIFORM SIGN UR SHESHIG;Lo;0;L;;;;;N;;;;; +1232B;CUNEIFORM SIGN UR2;Lo;0;L;;;;;N;;;;; +1232C;CUNEIFORM SIGN UR2 TIMES A PLUS HA;Lo;0;L;;;;;N;;;;; +1232D;CUNEIFORM SIGN UR2 TIMES A PLUS NA;Lo;0;L;;;;;N;;;;; +1232E;CUNEIFORM SIGN UR2 TIMES AL;Lo;0;L;;;;;N;;;;; +1232F;CUNEIFORM SIGN UR2 TIMES HA;Lo;0;L;;;;;N;;;;; +12330;CUNEIFORM SIGN UR2 TIMES NUN;Lo;0;L;;;;;N;;;;; +12331;CUNEIFORM SIGN UR2 TIMES U2;Lo;0;L;;;;;N;;;;; +12332;CUNEIFORM SIGN UR2 TIMES U2 PLUS ASH;Lo;0;L;;;;;N;;;;; +12333;CUNEIFORM SIGN UR2 TIMES U2 PLUS BI;Lo;0;L;;;;;N;;;;; +12334;CUNEIFORM SIGN UR4;Lo;0;L;;;;;N;;;;; +12335;CUNEIFORM SIGN URI;Lo;0;L;;;;;N;;;;; +12336;CUNEIFORM SIGN URI3;Lo;0;L;;;;;N;;;;; +12337;CUNEIFORM SIGN URU;Lo;0;L;;;;;N;;;;; +12338;CUNEIFORM SIGN URU TIMES A;Lo;0;L;;;;;N;;;;; +12339;CUNEIFORM SIGN URU TIMES ASHGAB;Lo;0;L;;;;;N;;;;; +1233A;CUNEIFORM SIGN URU TIMES BAR;Lo;0;L;;;;;N;;;;; +1233B;CUNEIFORM SIGN URU TIMES DUN;Lo;0;L;;;;;N;;;;; +1233C;CUNEIFORM SIGN URU TIMES GA;Lo;0;L;;;;;N;;;;; +1233D;CUNEIFORM SIGN URU TIMES GAL;Lo;0;L;;;;;N;;;;; +1233E;CUNEIFORM SIGN URU TIMES GAN2 TENU;Lo;0;L;;;;;N;;;;; +1233F;CUNEIFORM SIGN URU TIMES GAR;Lo;0;L;;;;;N;;;;; +12340;CUNEIFORM SIGN URU TIMES GU;Lo;0;L;;;;;N;;;;; +12341;CUNEIFORM SIGN URU TIMES HA;Lo;0;L;;;;;N;;;;; +12342;CUNEIFORM SIGN URU TIMES IGI;Lo;0;L;;;;;N;;;;; +12343;CUNEIFORM SIGN URU TIMES IM;Lo;0;L;;;;;N;;;;; +12344;CUNEIFORM SIGN URU TIMES ISH;Lo;0;L;;;;;N;;;;; +12345;CUNEIFORM SIGN URU TIMES KI;Lo;0;L;;;;;N;;;;; +12346;CUNEIFORM SIGN URU TIMES LUM;Lo;0;L;;;;;N;;;;; +12347;CUNEIFORM SIGN URU TIMES MIN;Lo;0;L;;;;;N;;;;; +12348;CUNEIFORM SIGN URU TIMES PA;Lo;0;L;;;;;N;;;;; +12349;CUNEIFORM SIGN URU TIMES SHE;Lo;0;L;;;;;N;;;;; +1234A;CUNEIFORM SIGN URU TIMES SIG4;Lo;0;L;;;;;N;;;;; +1234B;CUNEIFORM SIGN URU TIMES TU;Lo;0;L;;;;;N;;;;; +1234C;CUNEIFORM SIGN URU TIMES U PLUS GUD;Lo;0;L;;;;;N;;;;; +1234D;CUNEIFORM SIGN URU TIMES UD;Lo;0;L;;;;;N;;;;; +1234E;CUNEIFORM SIGN URU TIMES URUDA;Lo;0;L;;;;;N;;;;; +1234F;CUNEIFORM SIGN URUDA;Lo;0;L;;;;;N;;;;; +12350;CUNEIFORM SIGN URUDA TIMES U;Lo;0;L;;;;;N;;;;; +12351;CUNEIFORM SIGN USH;Lo;0;L;;;;;N;;;;; +12352;CUNEIFORM SIGN USH TIMES A;Lo;0;L;;;;;N;;;;; +12353;CUNEIFORM SIGN USH TIMES KU;Lo;0;L;;;;;N;;;;; +12354;CUNEIFORM SIGN USH TIMES KUR;Lo;0;L;;;;;N;;;;; +12355;CUNEIFORM SIGN USH TIMES TAK4;Lo;0;L;;;;;N;;;;; +12356;CUNEIFORM SIGN USHX;Lo;0;L;;;;;N;;;;; +12357;CUNEIFORM SIGN USH2;Lo;0;L;;;;;N;;;;; +12358;CUNEIFORM SIGN USHUMX;Lo;0;L;;;;;N;;;;; +12359;CUNEIFORM SIGN UTUKI;Lo;0;L;;;;;N;;;;; +1235A;CUNEIFORM SIGN UZ3;Lo;0;L;;;;;N;;;;; +1235B;CUNEIFORM SIGN UZ3 TIMES KASKAL;Lo;0;L;;;;;N;;;;; +1235C;CUNEIFORM SIGN UZU;Lo;0;L;;;;;N;;;;; +1235D;CUNEIFORM SIGN ZA;Lo;0;L;;;;;N;;;;; +1235E;CUNEIFORM SIGN ZA TENU;Lo;0;L;;;;;N;;;;; +1235F;CUNEIFORM SIGN ZA SQUARED TIMES KUR;Lo;0;L;;;;;N;;;;; +12360;CUNEIFORM SIGN ZAG;Lo;0;L;;;;;N;;;;; +12361;CUNEIFORM SIGN ZAMX;Lo;0;L;;;;;N;;;;; +12362;CUNEIFORM SIGN ZE2;Lo;0;L;;;;;N;;;;; +12363;CUNEIFORM SIGN ZI;Lo;0;L;;;;;N;;;;; +12364;CUNEIFORM SIGN ZI OVER ZI;Lo;0;L;;;;;N;;;;; +12365;CUNEIFORM SIGN ZI3;Lo;0;L;;;;;N;;;;; +12366;CUNEIFORM SIGN ZIB;Lo;0;L;;;;;N;;;;; +12367;CUNEIFORM SIGN ZIB KABA TENU;Lo;0;L;;;;;N;;;;; +12368;CUNEIFORM SIGN ZIG;Lo;0;L;;;;;N;;;;; +12369;CUNEIFORM SIGN ZIZ2;Lo;0;L;;;;;N;;;;; +1236A;CUNEIFORM SIGN ZU;Lo;0;L;;;;;N;;;;; +1236B;CUNEIFORM SIGN ZU5;Lo;0;L;;;;;N;;;;; +1236C;CUNEIFORM SIGN ZU5 TIMES A;Lo;0;L;;;;;N;;;;; +1236D;CUNEIFORM SIGN ZUBUR;Lo;0;L;;;;;N;;;;; +1236E;CUNEIFORM SIGN ZUM;Lo;0;L;;;;;N;;;;; +12400;CUNEIFORM NUMERIC SIGN TWO ASH;Nl;0;L;;;;2;N;;;;; +12401;CUNEIFORM NUMERIC SIGN THREE ASH;Nl;0;L;;;;3;N;;;;; +12402;CUNEIFORM NUMERIC SIGN FOUR ASH;Nl;0;L;;;;4;N;;;;; +12403;CUNEIFORM NUMERIC SIGN FIVE ASH;Nl;0;L;;;;5;N;;;;; +12404;CUNEIFORM NUMERIC SIGN SIX ASH;Nl;0;L;;;;6;N;;;;; +12405;CUNEIFORM NUMERIC SIGN SEVEN ASH;Nl;0;L;;;;7;N;;;;; +12406;CUNEIFORM NUMERIC SIGN EIGHT ASH;Nl;0;L;;;;8;N;;;;; +12407;CUNEIFORM NUMERIC SIGN NINE ASH;Nl;0;L;;;;9;N;;;;; +12408;CUNEIFORM NUMERIC SIGN THREE DISH;Nl;0;L;;;;3;N;;;;; +12409;CUNEIFORM NUMERIC SIGN FOUR DISH;Nl;0;L;;;;4;N;;;;; +1240A;CUNEIFORM NUMERIC SIGN FIVE DISH;Nl;0;L;;;;5;N;;;;; +1240B;CUNEIFORM NUMERIC SIGN SIX DISH;Nl;0;L;;;;6;N;;;;; +1240C;CUNEIFORM NUMERIC SIGN SEVEN DISH;Nl;0;L;;;;7;N;;;;; +1240D;CUNEIFORM NUMERIC SIGN EIGHT DISH;Nl;0;L;;;;8;N;;;;; +1240E;CUNEIFORM NUMERIC SIGN NINE DISH;Nl;0;L;;;;9;N;;;;; +1240F;CUNEIFORM NUMERIC SIGN FOUR U;Nl;0;L;;;;4;N;;;;; +12410;CUNEIFORM NUMERIC SIGN FIVE U;Nl;0;L;;;;5;N;;;;; +12411;CUNEIFORM NUMERIC SIGN SIX U;Nl;0;L;;;;6;N;;;;; +12412;CUNEIFORM NUMERIC SIGN SEVEN U;Nl;0;L;;;;7;N;;;;; +12413;CUNEIFORM NUMERIC SIGN EIGHT U;Nl;0;L;;;;8;N;;;;; +12414;CUNEIFORM NUMERIC SIGN NINE U;Nl;0;L;;;;9;N;;;;; +12415;CUNEIFORM NUMERIC SIGN ONE GESH2;Nl;0;L;;;;1;N;;;;; +12416;CUNEIFORM NUMERIC SIGN TWO GESH2;Nl;0;L;;;;2;N;;;;; +12417;CUNEIFORM NUMERIC SIGN THREE GESH2;Nl;0;L;;;;3;N;;;;; +12418;CUNEIFORM NUMERIC SIGN FOUR GESH2;Nl;0;L;;;;4;N;;;;; +12419;CUNEIFORM NUMERIC SIGN FIVE GESH2;Nl;0;L;;;;5;N;;;;; +1241A;CUNEIFORM NUMERIC SIGN SIX GESH2;Nl;0;L;;;;6;N;;;;; +1241B;CUNEIFORM NUMERIC SIGN SEVEN GESH2;Nl;0;L;;;;7;N;;;;; +1241C;CUNEIFORM NUMERIC SIGN EIGHT GESH2;Nl;0;L;;;;8;N;;;;; +1241D;CUNEIFORM NUMERIC SIGN NINE GESH2;Nl;0;L;;;;9;N;;;;; +1241E;CUNEIFORM NUMERIC SIGN ONE GESHU;Nl;0;L;;;;1;N;;;;; +1241F;CUNEIFORM NUMERIC SIGN TWO GESHU;Nl;0;L;;;;2;N;;;;; +12420;CUNEIFORM NUMERIC SIGN THREE GESHU;Nl;0;L;;;;3;N;;;;; +12421;CUNEIFORM NUMERIC SIGN FOUR GESHU;Nl;0;L;;;;4;N;;;;; +12422;CUNEIFORM NUMERIC SIGN FIVE GESHU;Nl;0;L;;;;5;N;;;;; +12423;CUNEIFORM NUMERIC SIGN TWO SHAR2;Nl;0;L;;;;2;N;;;;; +12424;CUNEIFORM NUMERIC SIGN THREE SHAR2;Nl;0;L;;;;3;N;;;;; +12425;CUNEIFORM NUMERIC SIGN THREE SHAR2 VARIANT FORM;Nl;0;L;;;;3;N;;;;; +12426;CUNEIFORM NUMERIC SIGN FOUR SHAR2;Nl;0;L;;;;4;N;;;;; +12427;CUNEIFORM NUMERIC SIGN FIVE SHAR2;Nl;0;L;;;;5;N;;;;; +12428;CUNEIFORM NUMERIC SIGN SIX SHAR2;Nl;0;L;;;;6;N;;;;; +12429;CUNEIFORM NUMERIC SIGN SEVEN SHAR2;Nl;0;L;;;;7;N;;;;; +1242A;CUNEIFORM NUMERIC SIGN EIGHT SHAR2;Nl;0;L;;;;8;N;;;;; +1242B;CUNEIFORM NUMERIC SIGN NINE SHAR2;Nl;0;L;;;;9;N;;;;; +1242C;CUNEIFORM NUMERIC SIGN ONE SHARU;Nl;0;L;;;;1;N;;;;; +1242D;CUNEIFORM NUMERIC SIGN TWO SHARU;Nl;0;L;;;;2;N;;;;; +1242E;CUNEIFORM NUMERIC SIGN THREE SHARU;Nl;0;L;;;;3;N;;;;; +1242F;CUNEIFORM NUMERIC SIGN THREE SHARU VARIANT FORM;Nl;0;L;;;;3;N;;;;; +12430;CUNEIFORM NUMERIC SIGN FOUR SHARU;Nl;0;L;;;;4;N;;;;; +12431;CUNEIFORM NUMERIC SIGN FIVE SHARU;Nl;0;L;;;;5;N;;;;; +12432;CUNEIFORM NUMERIC SIGN SHAR2 TIMES GAL PLUS DISH;Nl;0;L;;;;;N;;;;; +12433;CUNEIFORM NUMERIC SIGN SHAR2 TIMES GAL PLUS MIN;Nl;0;L;;;;;N;;;;; +12434;CUNEIFORM NUMERIC SIGN ONE BURU;Nl;0;L;;;;1;N;;;;; +12435;CUNEIFORM NUMERIC SIGN TWO BURU;Nl;0;L;;;;2;N;;;;; +12436;CUNEIFORM NUMERIC SIGN THREE BURU;Nl;0;L;;;;3;N;;;;; +12437;CUNEIFORM NUMERIC SIGN THREE BURU VARIANT FORM;Nl;0;L;;;;3;N;;;;; +12438;CUNEIFORM NUMERIC SIGN FOUR BURU;Nl;0;L;;;;4;N;;;;; +12439;CUNEIFORM NUMERIC SIGN FIVE BURU;Nl;0;L;;;;5;N;;;;; +1243A;CUNEIFORM NUMERIC SIGN THREE VARIANT FORM ESH16;Nl;0;L;;;;3;N;;;;; +1243B;CUNEIFORM NUMERIC SIGN THREE VARIANT FORM ESH21;Nl;0;L;;;;3;N;;;;; +1243C;CUNEIFORM NUMERIC SIGN FOUR VARIANT FORM LIMMU;Nl;0;L;;;;4;N;;;;; +1243D;CUNEIFORM NUMERIC SIGN FOUR VARIANT FORM LIMMU4;Nl;0;L;;;;4;N;;;;; +1243E;CUNEIFORM NUMERIC SIGN FOUR VARIANT FORM LIMMU A;Nl;0;L;;;;4;N;;;;; +1243F;CUNEIFORM NUMERIC SIGN FOUR VARIANT FORM LIMMU B;Nl;0;L;;;;4;N;;;;; +12440;CUNEIFORM NUMERIC SIGN SIX VARIANT FORM ASH9;Nl;0;L;;;;6;N;;;;; +12441;CUNEIFORM NUMERIC SIGN SEVEN VARIANT FORM IMIN3;Nl;0;L;;;;7;N;;;;; +12442;CUNEIFORM NUMERIC SIGN SEVEN VARIANT FORM IMIN A;Nl;0;L;;;;7;N;;;;; +12443;CUNEIFORM NUMERIC SIGN SEVEN VARIANT FORM IMIN B;Nl;0;L;;;;7;N;;;;; +12444;CUNEIFORM NUMERIC SIGN EIGHT VARIANT FORM USSU;Nl;0;L;;;;8;N;;;;; +12445;CUNEIFORM NUMERIC SIGN EIGHT VARIANT FORM USSU3;Nl;0;L;;;;8;N;;;;; +12446;CUNEIFORM NUMERIC SIGN NINE VARIANT FORM ILIMMU;Nl;0;L;;;;9;N;;;;; +12447;CUNEIFORM NUMERIC SIGN NINE VARIANT FORM ILIMMU3;Nl;0;L;;;;9;N;;;;; +12448;CUNEIFORM NUMERIC SIGN NINE VARIANT FORM ILIMMU4;Nl;0;L;;;;9;N;;;;; +12449;CUNEIFORM NUMERIC SIGN NINE VARIANT FORM ILIMMU A;Nl;0;L;;;;9;N;;;;; +1244A;CUNEIFORM NUMERIC SIGN TWO ASH TENU;Nl;0;L;;;;2;N;;;;; +1244B;CUNEIFORM NUMERIC SIGN THREE ASH TENU;Nl;0;L;;;;3;N;;;;; +1244C;CUNEIFORM NUMERIC SIGN FOUR ASH TENU;Nl;0;L;;;;4;N;;;;; +1244D;CUNEIFORM NUMERIC SIGN FIVE ASH TENU;Nl;0;L;;;;5;N;;;;; +1244E;CUNEIFORM NUMERIC SIGN SIX ASH TENU;Nl;0;L;;;;6;N;;;;; +1244F;CUNEIFORM NUMERIC SIGN ONE BAN2;Nl;0;L;;;;1;N;;;;; +12450;CUNEIFORM NUMERIC SIGN TWO BAN2;Nl;0;L;;;;2;N;;;;; +12451;CUNEIFORM NUMERIC SIGN THREE BAN2;Nl;0;L;;;;3;N;;;;; +12452;CUNEIFORM NUMERIC SIGN FOUR BAN2;Nl;0;L;;;;4;N;;;;; +12453;CUNEIFORM NUMERIC SIGN FOUR BAN2 VARIANT FORM;Nl;0;L;;;;4;N;;;;; +12454;CUNEIFORM NUMERIC SIGN FIVE BAN2;Nl;0;L;;;;5;N;;;;; +12455;CUNEIFORM NUMERIC SIGN FIVE BAN2 VARIANT FORM;Nl;0;L;;;;5;N;;;;; +12456;CUNEIFORM NUMERIC SIGN NIGIDAMIN;Nl;0;L;;;;;N;;;;; +12457;CUNEIFORM NUMERIC SIGN NIGIDAESH;Nl;0;L;;;;;N;;;;; +12458;CUNEIFORM NUMERIC SIGN ONE ESHE3;Nl;0;L;;;;1;N;;;;; +12459;CUNEIFORM NUMERIC SIGN TWO ESHE3;Nl;0;L;;;;2;N;;;;; +1245A;CUNEIFORM NUMERIC SIGN ONE THIRD DISH;Nl;0;L;;;;1/3;N;;;;; +1245B;CUNEIFORM NUMERIC SIGN TWO THIRDS DISH;Nl;0;L;;;;2/3;N;;;;; +1245C;CUNEIFORM NUMERIC SIGN FIVE SIXTHS DISH;Nl;0;L;;;;5/6;N;;;;; +1245D;CUNEIFORM NUMERIC SIGN ONE THIRD VARIANT FORM A;Nl;0;L;;;;1/3;N;;;;; +1245E;CUNEIFORM NUMERIC SIGN TWO THIRDS VARIANT FORM A;Nl;0;L;;;;2/3;N;;;;; +1245F;CUNEIFORM NUMERIC SIGN ONE EIGHTH ASH;Nl;0;L;;;;1/8;N;;;;; +12460;CUNEIFORM NUMERIC SIGN ONE QUARTER ASH;Nl;0;L;;;;1/4;N;;;;; +12461;CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE SIXTH;Nl;0;L;;;;1/6;N;;;;; +12462;CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER;Nl;0;L;;;;1/4;N;;;;; +12470;CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER;Po;0;L;;;;;N;;;;; +12471;CUNEIFORM PUNCTUATION SIGN VERTICAL COLON;Po;0;L;;;;;N;;;;; +12472;CUNEIFORM PUNCTUATION SIGN DIAGONAL COLON;Po;0;L;;;;;N;;;;; +12473;CUNEIFORM PUNCTUATION SIGN DIAGONAL TRICOLON;Po;0;L;;;;;N;;;;; +13000;EGYPTIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; +13001;EGYPTIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; +13002;EGYPTIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; +13003;EGYPTIAN HIEROGLYPH A004;Lo;0;L;;;;;N;;;;; +13004;EGYPTIAN HIEROGLYPH A005;Lo;0;L;;;;;N;;;;; +13005;EGYPTIAN HIEROGLYPH A005A;Lo;0;L;;;;;N;;;;; +13006;EGYPTIAN HIEROGLYPH A006;Lo;0;L;;;;;N;;;;; +13007;EGYPTIAN HIEROGLYPH A006A;Lo;0;L;;;;;N;;;;; +13008;EGYPTIAN HIEROGLYPH A006B;Lo;0;L;;;;;N;;;;; +13009;EGYPTIAN HIEROGLYPH A007;Lo;0;L;;;;;N;;;;; +1300A;EGYPTIAN HIEROGLYPH A008;Lo;0;L;;;;;N;;;;; +1300B;EGYPTIAN HIEROGLYPH A009;Lo;0;L;;;;;N;;;;; +1300C;EGYPTIAN HIEROGLYPH A010;Lo;0;L;;;;;N;;;;; +1300D;EGYPTIAN HIEROGLYPH A011;Lo;0;L;;;;;N;;;;; +1300E;EGYPTIAN HIEROGLYPH A012;Lo;0;L;;;;;N;;;;; +1300F;EGYPTIAN HIEROGLYPH A013;Lo;0;L;;;;;N;;;;; +13010;EGYPTIAN HIEROGLYPH A014;Lo;0;L;;;;;N;;;;; +13011;EGYPTIAN HIEROGLYPH A014A;Lo;0;L;;;;;N;;;;; +13012;EGYPTIAN HIEROGLYPH A015;Lo;0;L;;;;;N;;;;; +13013;EGYPTIAN HIEROGLYPH A016;Lo;0;L;;;;;N;;;;; +13014;EGYPTIAN HIEROGLYPH A017;Lo;0;L;;;;;N;;;;; +13015;EGYPTIAN HIEROGLYPH A017A;Lo;0;L;;;;;N;;;;; +13016;EGYPTIAN HIEROGLYPH A018;Lo;0;L;;;;;N;;;;; +13017;EGYPTIAN HIEROGLYPH A019;Lo;0;L;;;;;N;;;;; +13018;EGYPTIAN HIEROGLYPH A020;Lo;0;L;;;;;N;;;;; +13019;EGYPTIAN HIEROGLYPH A021;Lo;0;L;;;;;N;;;;; +1301A;EGYPTIAN HIEROGLYPH A022;Lo;0;L;;;;;N;;;;; +1301B;EGYPTIAN HIEROGLYPH A023;Lo;0;L;;;;;N;;;;; +1301C;EGYPTIAN HIEROGLYPH A024;Lo;0;L;;;;;N;;;;; +1301D;EGYPTIAN HIEROGLYPH A025;Lo;0;L;;;;;N;;;;; +1301E;EGYPTIAN HIEROGLYPH A026;Lo;0;L;;;;;N;;;;; +1301F;EGYPTIAN HIEROGLYPH A027;Lo;0;L;;;;;N;;;;; +13020;EGYPTIAN HIEROGLYPH A028;Lo;0;L;;;;;N;;;;; +13021;EGYPTIAN HIEROGLYPH A029;Lo;0;L;;;;;N;;;;; +13022;EGYPTIAN HIEROGLYPH A030;Lo;0;L;;;;;N;;;;; +13023;EGYPTIAN HIEROGLYPH A031;Lo;0;L;;;;;N;;;;; +13024;EGYPTIAN HIEROGLYPH A032;Lo;0;L;;;;;N;;;;; +13025;EGYPTIAN HIEROGLYPH A032A;Lo;0;L;;;;;N;;;;; +13026;EGYPTIAN HIEROGLYPH A033;Lo;0;L;;;;;N;;;;; +13027;EGYPTIAN HIEROGLYPH A034;Lo;0;L;;;;;N;;;;; +13028;EGYPTIAN HIEROGLYPH A035;Lo;0;L;;;;;N;;;;; +13029;EGYPTIAN HIEROGLYPH A036;Lo;0;L;;;;;N;;;;; +1302A;EGYPTIAN HIEROGLYPH A037;Lo;0;L;;;;;N;;;;; +1302B;EGYPTIAN HIEROGLYPH A038;Lo;0;L;;;;;N;;;;; +1302C;EGYPTIAN HIEROGLYPH A039;Lo;0;L;;;;;N;;;;; +1302D;EGYPTIAN HIEROGLYPH A040;Lo;0;L;;;;;N;;;;; +1302E;EGYPTIAN HIEROGLYPH A040A;Lo;0;L;;;;;N;;;;; +1302F;EGYPTIAN HIEROGLYPH A041;Lo;0;L;;;;;N;;;;; +13030;EGYPTIAN HIEROGLYPH A042;Lo;0;L;;;;;N;;;;; +13031;EGYPTIAN HIEROGLYPH A042A;Lo;0;L;;;;;N;;;;; +13032;EGYPTIAN HIEROGLYPH A043;Lo;0;L;;;;;N;;;;; +13033;EGYPTIAN HIEROGLYPH A043A;Lo;0;L;;;;;N;;;;; +13034;EGYPTIAN HIEROGLYPH A044;Lo;0;L;;;;;N;;;;; +13035;EGYPTIAN HIEROGLYPH A045;Lo;0;L;;;;;N;;;;; +13036;EGYPTIAN HIEROGLYPH A045A;Lo;0;L;;;;;N;;;;; +13037;EGYPTIAN HIEROGLYPH A046;Lo;0;L;;;;;N;;;;; +13038;EGYPTIAN HIEROGLYPH A047;Lo;0;L;;;;;N;;;;; +13039;EGYPTIAN HIEROGLYPH A048;Lo;0;L;;;;;N;;;;; +1303A;EGYPTIAN HIEROGLYPH A049;Lo;0;L;;;;;N;;;;; +1303B;EGYPTIAN HIEROGLYPH A050;Lo;0;L;;;;;N;;;;; +1303C;EGYPTIAN HIEROGLYPH A051;Lo;0;L;;;;;N;;;;; +1303D;EGYPTIAN HIEROGLYPH A052;Lo;0;L;;;;;N;;;;; +1303E;EGYPTIAN HIEROGLYPH A053;Lo;0;L;;;;;N;;;;; +1303F;EGYPTIAN HIEROGLYPH A054;Lo;0;L;;;;;N;;;;; +13040;EGYPTIAN HIEROGLYPH A055;Lo;0;L;;;;;N;;;;; +13041;EGYPTIAN HIEROGLYPH A056;Lo;0;L;;;;;N;;;;; +13042;EGYPTIAN HIEROGLYPH A057;Lo;0;L;;;;;N;;;;; +13043;EGYPTIAN HIEROGLYPH A058;Lo;0;L;;;;;N;;;;; +13044;EGYPTIAN HIEROGLYPH A059;Lo;0;L;;;;;N;;;;; +13045;EGYPTIAN HIEROGLYPH A060;Lo;0;L;;;;;N;;;;; +13046;EGYPTIAN HIEROGLYPH A061;Lo;0;L;;;;;N;;;;; +13047;EGYPTIAN HIEROGLYPH A062;Lo;0;L;;;;;N;;;;; +13048;EGYPTIAN HIEROGLYPH A063;Lo;0;L;;;;;N;;;;; +13049;EGYPTIAN HIEROGLYPH A064;Lo;0;L;;;;;N;;;;; +1304A;EGYPTIAN HIEROGLYPH A065;Lo;0;L;;;;;N;;;;; +1304B;EGYPTIAN HIEROGLYPH A066;Lo;0;L;;;;;N;;;;; +1304C;EGYPTIAN HIEROGLYPH A067;Lo;0;L;;;;;N;;;;; +1304D;EGYPTIAN HIEROGLYPH A068;Lo;0;L;;;;;N;;;;; +1304E;EGYPTIAN HIEROGLYPH A069;Lo;0;L;;;;;N;;;;; +1304F;EGYPTIAN HIEROGLYPH A070;Lo;0;L;;;;;N;;;;; +13050;EGYPTIAN HIEROGLYPH B001;Lo;0;L;;;;;N;;;;; +13051;EGYPTIAN HIEROGLYPH B002;Lo;0;L;;;;;N;;;;; +13052;EGYPTIAN HIEROGLYPH B003;Lo;0;L;;;;;N;;;;; +13053;EGYPTIAN HIEROGLYPH B004;Lo;0;L;;;;;N;;;;; +13054;EGYPTIAN HIEROGLYPH B005;Lo;0;L;;;;;N;;;;; +13055;EGYPTIAN HIEROGLYPH B005A;Lo;0;L;;;;;N;;;;; +13056;EGYPTIAN HIEROGLYPH B006;Lo;0;L;;;;;N;;;;; +13057;EGYPTIAN HIEROGLYPH B007;Lo;0;L;;;;;N;;;;; +13058;EGYPTIAN HIEROGLYPH B008;Lo;0;L;;;;;N;;;;; +13059;EGYPTIAN HIEROGLYPH B009;Lo;0;L;;;;;N;;;;; +1305A;EGYPTIAN HIEROGLYPH C001;Lo;0;L;;;;;N;;;;; +1305B;EGYPTIAN HIEROGLYPH C002;Lo;0;L;;;;;N;;;;; +1305C;EGYPTIAN HIEROGLYPH C002A;Lo;0;L;;;;;N;;;;; +1305D;EGYPTIAN HIEROGLYPH C002B;Lo;0;L;;;;;N;;;;; +1305E;EGYPTIAN HIEROGLYPH C002C;Lo;0;L;;;;;N;;;;; +1305F;EGYPTIAN HIEROGLYPH C003;Lo;0;L;;;;;N;;;;; +13060;EGYPTIAN HIEROGLYPH C004;Lo;0;L;;;;;N;;;;; +13061;EGYPTIAN HIEROGLYPH C005;Lo;0;L;;;;;N;;;;; +13062;EGYPTIAN HIEROGLYPH C006;Lo;0;L;;;;;N;;;;; +13063;EGYPTIAN HIEROGLYPH C007;Lo;0;L;;;;;N;;;;; +13064;EGYPTIAN HIEROGLYPH C008;Lo;0;L;;;;;N;;;;; +13065;EGYPTIAN HIEROGLYPH C009;Lo;0;L;;;;;N;;;;; +13066;EGYPTIAN HIEROGLYPH C010;Lo;0;L;;;;;N;;;;; +13067;EGYPTIAN HIEROGLYPH C010A;Lo;0;L;;;;;N;;;;; +13068;EGYPTIAN HIEROGLYPH C011;Lo;0;L;;;;;N;;;;; +13069;EGYPTIAN HIEROGLYPH C012;Lo;0;L;;;;;N;;;;; +1306A;EGYPTIAN HIEROGLYPH C013;Lo;0;L;;;;;N;;;;; +1306B;EGYPTIAN HIEROGLYPH C014;Lo;0;L;;;;;N;;;;; +1306C;EGYPTIAN HIEROGLYPH C015;Lo;0;L;;;;;N;;;;; +1306D;EGYPTIAN HIEROGLYPH C016;Lo;0;L;;;;;N;;;;; +1306E;EGYPTIAN HIEROGLYPH C017;Lo;0;L;;;;;N;;;;; +1306F;EGYPTIAN HIEROGLYPH C018;Lo;0;L;;;;;N;;;;; +13070;EGYPTIAN HIEROGLYPH C019;Lo;0;L;;;;;N;;;;; +13071;EGYPTIAN HIEROGLYPH C020;Lo;0;L;;;;;N;;;;; +13072;EGYPTIAN HIEROGLYPH C021;Lo;0;L;;;;;N;;;;; +13073;EGYPTIAN HIEROGLYPH C022;Lo;0;L;;;;;N;;;;; +13074;EGYPTIAN HIEROGLYPH C023;Lo;0;L;;;;;N;;;;; +13075;EGYPTIAN HIEROGLYPH C024;Lo;0;L;;;;;N;;;;; +13076;EGYPTIAN HIEROGLYPH D001;Lo;0;L;;;;;N;;;;; +13077;EGYPTIAN HIEROGLYPH D002;Lo;0;L;;;;;N;;;;; +13078;EGYPTIAN HIEROGLYPH D003;Lo;0;L;;;;;N;;;;; +13079;EGYPTIAN HIEROGLYPH D004;Lo;0;L;;;;;N;;;;; +1307A;EGYPTIAN HIEROGLYPH D005;Lo;0;L;;;;;N;;;;; +1307B;EGYPTIAN HIEROGLYPH D006;Lo;0;L;;;;;N;;;;; +1307C;EGYPTIAN HIEROGLYPH D007;Lo;0;L;;;;;N;;;;; +1307D;EGYPTIAN HIEROGLYPH D008;Lo;0;L;;;;;N;;;;; +1307E;EGYPTIAN HIEROGLYPH D008A;Lo;0;L;;;;;N;;;;; +1307F;EGYPTIAN HIEROGLYPH D009;Lo;0;L;;;;;N;;;;; +13080;EGYPTIAN HIEROGLYPH D010;Lo;0;L;;;;;N;;;;; +13081;EGYPTIAN HIEROGLYPH D011;Lo;0;L;;;;;N;;;;; +13082;EGYPTIAN HIEROGLYPH D012;Lo;0;L;;;;;N;;;;; +13083;EGYPTIAN HIEROGLYPH D013;Lo;0;L;;;;;N;;;;; +13084;EGYPTIAN HIEROGLYPH D014;Lo;0;L;;;;;N;;;;; +13085;EGYPTIAN HIEROGLYPH D015;Lo;0;L;;;;;N;;;;; +13086;EGYPTIAN HIEROGLYPH D016;Lo;0;L;;;;;N;;;;; +13087;EGYPTIAN HIEROGLYPH D017;Lo;0;L;;;;;N;;;;; +13088;EGYPTIAN HIEROGLYPH D018;Lo;0;L;;;;;N;;;;; +13089;EGYPTIAN HIEROGLYPH D019;Lo;0;L;;;;;N;;;;; +1308A;EGYPTIAN HIEROGLYPH D020;Lo;0;L;;;;;N;;;;; +1308B;EGYPTIAN HIEROGLYPH D021;Lo;0;L;;;;;N;;;;; +1308C;EGYPTIAN HIEROGLYPH D022;Lo;0;L;;;;;N;;;;; +1308D;EGYPTIAN HIEROGLYPH D023;Lo;0;L;;;;;N;;;;; +1308E;EGYPTIAN HIEROGLYPH D024;Lo;0;L;;;;;N;;;;; +1308F;EGYPTIAN HIEROGLYPH D025;Lo;0;L;;;;;N;;;;; +13090;EGYPTIAN HIEROGLYPH D026;Lo;0;L;;;;;N;;;;; +13091;EGYPTIAN HIEROGLYPH D027;Lo;0;L;;;;;N;;;;; +13092;EGYPTIAN HIEROGLYPH D027A;Lo;0;L;;;;;N;;;;; +13093;EGYPTIAN HIEROGLYPH D028;Lo;0;L;;;;;N;;;;; +13094;EGYPTIAN HIEROGLYPH D029;Lo;0;L;;;;;N;;;;; +13095;EGYPTIAN HIEROGLYPH D030;Lo;0;L;;;;;N;;;;; +13096;EGYPTIAN HIEROGLYPH D031;Lo;0;L;;;;;N;;;;; +13097;EGYPTIAN HIEROGLYPH D031A;Lo;0;L;;;;;N;;;;; +13098;EGYPTIAN HIEROGLYPH D032;Lo;0;L;;;;;N;;;;; +13099;EGYPTIAN HIEROGLYPH D033;Lo;0;L;;;;;N;;;;; +1309A;EGYPTIAN HIEROGLYPH D034;Lo;0;L;;;;;N;;;;; +1309B;EGYPTIAN HIEROGLYPH D034A;Lo;0;L;;;;;N;;;;; +1309C;EGYPTIAN HIEROGLYPH D035;Lo;0;L;;;;;N;;;;; +1309D;EGYPTIAN HIEROGLYPH D036;Lo;0;L;;;;;N;;;;; +1309E;EGYPTIAN HIEROGLYPH D037;Lo;0;L;;;;;N;;;;; +1309F;EGYPTIAN HIEROGLYPH D038;Lo;0;L;;;;;N;;;;; +130A0;EGYPTIAN HIEROGLYPH D039;Lo;0;L;;;;;N;;;;; +130A1;EGYPTIAN HIEROGLYPH D040;Lo;0;L;;;;;N;;;;; +130A2;EGYPTIAN HIEROGLYPH D041;Lo;0;L;;;;;N;;;;; +130A3;EGYPTIAN HIEROGLYPH D042;Lo;0;L;;;;;N;;;;; +130A4;EGYPTIAN HIEROGLYPH D043;Lo;0;L;;;;;N;;;;; +130A5;EGYPTIAN HIEROGLYPH D044;Lo;0;L;;;;;N;;;;; +130A6;EGYPTIAN HIEROGLYPH D045;Lo;0;L;;;;;N;;;;; +130A7;EGYPTIAN HIEROGLYPH D046;Lo;0;L;;;;;N;;;;; +130A8;EGYPTIAN HIEROGLYPH D046A;Lo;0;L;;;;;N;;;;; +130A9;EGYPTIAN HIEROGLYPH D047;Lo;0;L;;;;;N;;;;; +130AA;EGYPTIAN HIEROGLYPH D048;Lo;0;L;;;;;N;;;;; +130AB;EGYPTIAN HIEROGLYPH D048A;Lo;0;L;;;;;N;;;;; +130AC;EGYPTIAN HIEROGLYPH D049;Lo;0;L;;;;;N;;;;; +130AD;EGYPTIAN HIEROGLYPH D050;Lo;0;L;;;;;N;;;;; +130AE;EGYPTIAN HIEROGLYPH D050A;Lo;0;L;;;;;N;;;;; +130AF;EGYPTIAN HIEROGLYPH D050B;Lo;0;L;;;;;N;;;;; +130B0;EGYPTIAN HIEROGLYPH D050C;Lo;0;L;;;;;N;;;;; +130B1;EGYPTIAN HIEROGLYPH D050D;Lo;0;L;;;;;N;;;;; +130B2;EGYPTIAN HIEROGLYPH D050E;Lo;0;L;;;;;N;;;;; +130B3;EGYPTIAN HIEROGLYPH D050F;Lo;0;L;;;;;N;;;;; +130B4;EGYPTIAN HIEROGLYPH D050G;Lo;0;L;;;;;N;;;;; +130B5;EGYPTIAN HIEROGLYPH D050H;Lo;0;L;;;;;N;;;;; +130B6;EGYPTIAN HIEROGLYPH D050I;Lo;0;L;;;;;N;;;;; +130B7;EGYPTIAN HIEROGLYPH D051;Lo;0;L;;;;;N;;;;; +130B8;EGYPTIAN HIEROGLYPH D052;Lo;0;L;;;;;N;;;;; +130B9;EGYPTIAN HIEROGLYPH D052A;Lo;0;L;;;;;N;;;;; +130BA;EGYPTIAN HIEROGLYPH D053;Lo;0;L;;;;;N;;;;; +130BB;EGYPTIAN HIEROGLYPH D054;Lo;0;L;;;;;N;;;;; +130BC;EGYPTIAN HIEROGLYPH D054A;Lo;0;L;;;;;N;;;;; +130BD;EGYPTIAN HIEROGLYPH D055;Lo;0;L;;;;;N;;;;; +130BE;EGYPTIAN HIEROGLYPH D056;Lo;0;L;;;;;N;;;;; +130BF;EGYPTIAN HIEROGLYPH D057;Lo;0;L;;;;;N;;;;; +130C0;EGYPTIAN HIEROGLYPH D058;Lo;0;L;;;;;N;;;;; +130C1;EGYPTIAN HIEROGLYPH D059;Lo;0;L;;;;;N;;;;; +130C2;EGYPTIAN HIEROGLYPH D060;Lo;0;L;;;;;N;;;;; +130C3;EGYPTIAN HIEROGLYPH D061;Lo;0;L;;;;;N;;;;; +130C4;EGYPTIAN HIEROGLYPH D062;Lo;0;L;;;;;N;;;;; +130C5;EGYPTIAN HIEROGLYPH D063;Lo;0;L;;;;;N;;;;; +130C6;EGYPTIAN HIEROGLYPH D064;Lo;0;L;;;;;N;;;;; +130C7;EGYPTIAN HIEROGLYPH D065;Lo;0;L;;;;;N;;;;; +130C8;EGYPTIAN HIEROGLYPH D066;Lo;0;L;;;;;N;;;;; +130C9;EGYPTIAN HIEROGLYPH D067;Lo;0;L;;;;;N;;;;; +130CA;EGYPTIAN HIEROGLYPH D067A;Lo;0;L;;;;;N;;;;; +130CB;EGYPTIAN HIEROGLYPH D067B;Lo;0;L;;;;;N;;;;; +130CC;EGYPTIAN HIEROGLYPH D067C;Lo;0;L;;;;;N;;;;; +130CD;EGYPTIAN HIEROGLYPH D067D;Lo;0;L;;;;;N;;;;; +130CE;EGYPTIAN HIEROGLYPH D067E;Lo;0;L;;;;;N;;;;; +130CF;EGYPTIAN HIEROGLYPH D067F;Lo;0;L;;;;;N;;;;; +130D0;EGYPTIAN HIEROGLYPH D067G;Lo;0;L;;;;;N;;;;; +130D1;EGYPTIAN HIEROGLYPH D067H;Lo;0;L;;;;;N;;;;; +130D2;EGYPTIAN HIEROGLYPH E001;Lo;0;L;;;;;N;;;;; +130D3;EGYPTIAN HIEROGLYPH E002;Lo;0;L;;;;;N;;;;; +130D4;EGYPTIAN HIEROGLYPH E003;Lo;0;L;;;;;N;;;;; +130D5;EGYPTIAN HIEROGLYPH E004;Lo;0;L;;;;;N;;;;; +130D6;EGYPTIAN HIEROGLYPH E005;Lo;0;L;;;;;N;;;;; +130D7;EGYPTIAN HIEROGLYPH E006;Lo;0;L;;;;;N;;;;; +130D8;EGYPTIAN HIEROGLYPH E007;Lo;0;L;;;;;N;;;;; +130D9;EGYPTIAN HIEROGLYPH E008;Lo;0;L;;;;;N;;;;; +130DA;EGYPTIAN HIEROGLYPH E008A;Lo;0;L;;;;;N;;;;; +130DB;EGYPTIAN HIEROGLYPH E009;Lo;0;L;;;;;N;;;;; +130DC;EGYPTIAN HIEROGLYPH E009A;Lo;0;L;;;;;N;;;;; +130DD;EGYPTIAN HIEROGLYPH E010;Lo;0;L;;;;;N;;;;; +130DE;EGYPTIAN HIEROGLYPH E011;Lo;0;L;;;;;N;;;;; +130DF;EGYPTIAN HIEROGLYPH E012;Lo;0;L;;;;;N;;;;; +130E0;EGYPTIAN HIEROGLYPH E013;Lo;0;L;;;;;N;;;;; +130E1;EGYPTIAN HIEROGLYPH E014;Lo;0;L;;;;;N;;;;; +130E2;EGYPTIAN HIEROGLYPH E015;Lo;0;L;;;;;N;;;;; +130E3;EGYPTIAN HIEROGLYPH E016;Lo;0;L;;;;;N;;;;; +130E4;EGYPTIAN HIEROGLYPH E016A;Lo;0;L;;;;;N;;;;; +130E5;EGYPTIAN HIEROGLYPH E017;Lo;0;L;;;;;N;;;;; +130E6;EGYPTIAN HIEROGLYPH E017A;Lo;0;L;;;;;N;;;;; +130E7;EGYPTIAN HIEROGLYPH E018;Lo;0;L;;;;;N;;;;; +130E8;EGYPTIAN HIEROGLYPH E019;Lo;0;L;;;;;N;;;;; +130E9;EGYPTIAN HIEROGLYPH E020;Lo;0;L;;;;;N;;;;; +130EA;EGYPTIAN HIEROGLYPH E020A;Lo;0;L;;;;;N;;;;; +130EB;EGYPTIAN HIEROGLYPH E021;Lo;0;L;;;;;N;;;;; +130EC;EGYPTIAN HIEROGLYPH E022;Lo;0;L;;;;;N;;;;; +130ED;EGYPTIAN HIEROGLYPH E023;Lo;0;L;;;;;N;;;;; +130EE;EGYPTIAN HIEROGLYPH E024;Lo;0;L;;;;;N;;;;; +130EF;EGYPTIAN HIEROGLYPH E025;Lo;0;L;;;;;N;;;;; +130F0;EGYPTIAN HIEROGLYPH E026;Lo;0;L;;;;;N;;;;; +130F1;EGYPTIAN HIEROGLYPH E027;Lo;0;L;;;;;N;;;;; +130F2;EGYPTIAN HIEROGLYPH E028;Lo;0;L;;;;;N;;;;; +130F3;EGYPTIAN HIEROGLYPH E028A;Lo;0;L;;;;;N;;;;; +130F4;EGYPTIAN HIEROGLYPH E029;Lo;0;L;;;;;N;;;;; +130F5;EGYPTIAN HIEROGLYPH E030;Lo;0;L;;;;;N;;;;; +130F6;EGYPTIAN HIEROGLYPH E031;Lo;0;L;;;;;N;;;;; +130F7;EGYPTIAN HIEROGLYPH E032;Lo;0;L;;;;;N;;;;; +130F8;EGYPTIAN HIEROGLYPH E033;Lo;0;L;;;;;N;;;;; +130F9;EGYPTIAN HIEROGLYPH E034;Lo;0;L;;;;;N;;;;; +130FA;EGYPTIAN HIEROGLYPH E034A;Lo;0;L;;;;;N;;;;; +130FB;EGYPTIAN HIEROGLYPH E036;Lo;0;L;;;;;N;;;;; +130FC;EGYPTIAN HIEROGLYPH E037;Lo;0;L;;;;;N;;;;; +130FD;EGYPTIAN HIEROGLYPH E038;Lo;0;L;;;;;N;;;;; +130FE;EGYPTIAN HIEROGLYPH F001;Lo;0;L;;;;;N;;;;; +130FF;EGYPTIAN HIEROGLYPH F001A;Lo;0;L;;;;;N;;;;; +13100;EGYPTIAN HIEROGLYPH F002;Lo;0;L;;;;;N;;;;; +13101;EGYPTIAN HIEROGLYPH F003;Lo;0;L;;;;;N;;;;; +13102;EGYPTIAN HIEROGLYPH F004;Lo;0;L;;;;;N;;;;; +13103;EGYPTIAN HIEROGLYPH F005;Lo;0;L;;;;;N;;;;; +13104;EGYPTIAN HIEROGLYPH F006;Lo;0;L;;;;;N;;;;; +13105;EGYPTIAN HIEROGLYPH F007;Lo;0;L;;;;;N;;;;; +13106;EGYPTIAN HIEROGLYPH F008;Lo;0;L;;;;;N;;;;; +13107;EGYPTIAN HIEROGLYPH F009;Lo;0;L;;;;;N;;;;; +13108;EGYPTIAN HIEROGLYPH F010;Lo;0;L;;;;;N;;;;; +13109;EGYPTIAN HIEROGLYPH F011;Lo;0;L;;;;;N;;;;; +1310A;EGYPTIAN HIEROGLYPH F012;Lo;0;L;;;;;N;;;;; +1310B;EGYPTIAN HIEROGLYPH F013;Lo;0;L;;;;;N;;;;; +1310C;EGYPTIAN HIEROGLYPH F013A;Lo;0;L;;;;;N;;;;; +1310D;EGYPTIAN HIEROGLYPH F014;Lo;0;L;;;;;N;;;;; +1310E;EGYPTIAN HIEROGLYPH F015;Lo;0;L;;;;;N;;;;; +1310F;EGYPTIAN HIEROGLYPH F016;Lo;0;L;;;;;N;;;;; +13110;EGYPTIAN HIEROGLYPH F017;Lo;0;L;;;;;N;;;;; +13111;EGYPTIAN HIEROGLYPH F018;Lo;0;L;;;;;N;;;;; +13112;EGYPTIAN HIEROGLYPH F019;Lo;0;L;;;;;N;;;;; +13113;EGYPTIAN HIEROGLYPH F020;Lo;0;L;;;;;N;;;;; +13114;EGYPTIAN HIEROGLYPH F021;Lo;0;L;;;;;N;;;;; +13115;EGYPTIAN HIEROGLYPH F021A;Lo;0;L;;;;;N;;;;; +13116;EGYPTIAN HIEROGLYPH F022;Lo;0;L;;;;;N;;;;; +13117;EGYPTIAN HIEROGLYPH F023;Lo;0;L;;;;;N;;;;; +13118;EGYPTIAN HIEROGLYPH F024;Lo;0;L;;;;;N;;;;; +13119;EGYPTIAN HIEROGLYPH F025;Lo;0;L;;;;;N;;;;; +1311A;EGYPTIAN HIEROGLYPH F026;Lo;0;L;;;;;N;;;;; +1311B;EGYPTIAN HIEROGLYPH F027;Lo;0;L;;;;;N;;;;; +1311C;EGYPTIAN HIEROGLYPH F028;Lo;0;L;;;;;N;;;;; +1311D;EGYPTIAN HIEROGLYPH F029;Lo;0;L;;;;;N;;;;; +1311E;EGYPTIAN HIEROGLYPH F030;Lo;0;L;;;;;N;;;;; +1311F;EGYPTIAN HIEROGLYPH F031;Lo;0;L;;;;;N;;;;; +13120;EGYPTIAN HIEROGLYPH F031A;Lo;0;L;;;;;N;;;;; +13121;EGYPTIAN HIEROGLYPH F032;Lo;0;L;;;;;N;;;;; +13122;EGYPTIAN HIEROGLYPH F033;Lo;0;L;;;;;N;;;;; +13123;EGYPTIAN HIEROGLYPH F034;Lo;0;L;;;;;N;;;;; +13124;EGYPTIAN HIEROGLYPH F035;Lo;0;L;;;;;N;;;;; +13125;EGYPTIAN HIEROGLYPH F036;Lo;0;L;;;;;N;;;;; +13126;EGYPTIAN HIEROGLYPH F037;Lo;0;L;;;;;N;;;;; +13127;EGYPTIAN HIEROGLYPH F037A;Lo;0;L;;;;;N;;;;; +13128;EGYPTIAN HIEROGLYPH F038;Lo;0;L;;;;;N;;;;; +13129;EGYPTIAN HIEROGLYPH F038A;Lo;0;L;;;;;N;;;;; +1312A;EGYPTIAN HIEROGLYPH F039;Lo;0;L;;;;;N;;;;; +1312B;EGYPTIAN HIEROGLYPH F040;Lo;0;L;;;;;N;;;;; +1312C;EGYPTIAN HIEROGLYPH F041;Lo;0;L;;;;;N;;;;; +1312D;EGYPTIAN HIEROGLYPH F042;Lo;0;L;;;;;N;;;;; +1312E;EGYPTIAN HIEROGLYPH F043;Lo;0;L;;;;;N;;;;; +1312F;EGYPTIAN HIEROGLYPH F044;Lo;0;L;;;;;N;;;;; +13130;EGYPTIAN HIEROGLYPH F045;Lo;0;L;;;;;N;;;;; +13131;EGYPTIAN HIEROGLYPH F045A;Lo;0;L;;;;;N;;;;; +13132;EGYPTIAN HIEROGLYPH F046;Lo;0;L;;;;;N;;;;; +13133;EGYPTIAN HIEROGLYPH F046A;Lo;0;L;;;;;N;;;;; +13134;EGYPTIAN HIEROGLYPH F047;Lo;0;L;;;;;N;;;;; +13135;EGYPTIAN HIEROGLYPH F047A;Lo;0;L;;;;;N;;;;; +13136;EGYPTIAN HIEROGLYPH F048;Lo;0;L;;;;;N;;;;; +13137;EGYPTIAN HIEROGLYPH F049;Lo;0;L;;;;;N;;;;; +13138;EGYPTIAN HIEROGLYPH F050;Lo;0;L;;;;;N;;;;; +13139;EGYPTIAN HIEROGLYPH F051;Lo;0;L;;;;;N;;;;; +1313A;EGYPTIAN HIEROGLYPH F051A;Lo;0;L;;;;;N;;;;; +1313B;EGYPTIAN HIEROGLYPH F051B;Lo;0;L;;;;;N;;;;; +1313C;EGYPTIAN HIEROGLYPH F051C;Lo;0;L;;;;;N;;;;; +1313D;EGYPTIAN HIEROGLYPH F052;Lo;0;L;;;;;N;;;;; +1313E;EGYPTIAN HIEROGLYPH F053;Lo;0;L;;;;;N;;;;; +1313F;EGYPTIAN HIEROGLYPH G001;Lo;0;L;;;;;N;;;;; +13140;EGYPTIAN HIEROGLYPH G002;Lo;0;L;;;;;N;;;;; +13141;EGYPTIAN HIEROGLYPH G003;Lo;0;L;;;;;N;;;;; +13142;EGYPTIAN HIEROGLYPH G004;Lo;0;L;;;;;N;;;;; +13143;EGYPTIAN HIEROGLYPH G005;Lo;0;L;;;;;N;;;;; +13144;EGYPTIAN HIEROGLYPH G006;Lo;0;L;;;;;N;;;;; +13145;EGYPTIAN HIEROGLYPH G006A;Lo;0;L;;;;;N;;;;; +13146;EGYPTIAN HIEROGLYPH G007;Lo;0;L;;;;;N;;;;; +13147;EGYPTIAN HIEROGLYPH G007A;Lo;0;L;;;;;N;;;;; +13148;EGYPTIAN HIEROGLYPH G007B;Lo;0;L;;;;;N;;;;; +13149;EGYPTIAN HIEROGLYPH G008;Lo;0;L;;;;;N;;;;; +1314A;EGYPTIAN HIEROGLYPH G009;Lo;0;L;;;;;N;;;;; +1314B;EGYPTIAN HIEROGLYPH G010;Lo;0;L;;;;;N;;;;; +1314C;EGYPTIAN HIEROGLYPH G011;Lo;0;L;;;;;N;;;;; +1314D;EGYPTIAN HIEROGLYPH G011A;Lo;0;L;;;;;N;;;;; +1314E;EGYPTIAN HIEROGLYPH G012;Lo;0;L;;;;;N;;;;; +1314F;EGYPTIAN HIEROGLYPH G013;Lo;0;L;;;;;N;;;;; +13150;EGYPTIAN HIEROGLYPH G014;Lo;0;L;;;;;N;;;;; +13151;EGYPTIAN HIEROGLYPH G015;Lo;0;L;;;;;N;;;;; +13152;EGYPTIAN HIEROGLYPH G016;Lo;0;L;;;;;N;;;;; +13153;EGYPTIAN HIEROGLYPH G017;Lo;0;L;;;;;N;;;;; +13154;EGYPTIAN HIEROGLYPH G018;Lo;0;L;;;;;N;;;;; +13155;EGYPTIAN HIEROGLYPH G019;Lo;0;L;;;;;N;;;;; +13156;EGYPTIAN HIEROGLYPH G020;Lo;0;L;;;;;N;;;;; +13157;EGYPTIAN HIEROGLYPH G020A;Lo;0;L;;;;;N;;;;; +13158;EGYPTIAN HIEROGLYPH G021;Lo;0;L;;;;;N;;;;; +13159;EGYPTIAN HIEROGLYPH G022;Lo;0;L;;;;;N;;;;; +1315A;EGYPTIAN HIEROGLYPH G023;Lo;0;L;;;;;N;;;;; +1315B;EGYPTIAN HIEROGLYPH G024;Lo;0;L;;;;;N;;;;; +1315C;EGYPTIAN HIEROGLYPH G025;Lo;0;L;;;;;N;;;;; +1315D;EGYPTIAN HIEROGLYPH G026;Lo;0;L;;;;;N;;;;; +1315E;EGYPTIAN HIEROGLYPH G026A;Lo;0;L;;;;;N;;;;; +1315F;EGYPTIAN HIEROGLYPH G027;Lo;0;L;;;;;N;;;;; +13160;EGYPTIAN HIEROGLYPH G028;Lo;0;L;;;;;N;;;;; +13161;EGYPTIAN HIEROGLYPH G029;Lo;0;L;;;;;N;;;;; +13162;EGYPTIAN HIEROGLYPH G030;Lo;0;L;;;;;N;;;;; +13163;EGYPTIAN HIEROGLYPH G031;Lo;0;L;;;;;N;;;;; +13164;EGYPTIAN HIEROGLYPH G032;Lo;0;L;;;;;N;;;;; +13165;EGYPTIAN HIEROGLYPH G033;Lo;0;L;;;;;N;;;;; +13166;EGYPTIAN HIEROGLYPH G034;Lo;0;L;;;;;N;;;;; +13167;EGYPTIAN HIEROGLYPH G035;Lo;0;L;;;;;N;;;;; +13168;EGYPTIAN HIEROGLYPH G036;Lo;0;L;;;;;N;;;;; +13169;EGYPTIAN HIEROGLYPH G036A;Lo;0;L;;;;;N;;;;; +1316A;EGYPTIAN HIEROGLYPH G037;Lo;0;L;;;;;N;;;;; +1316B;EGYPTIAN HIEROGLYPH G037A;Lo;0;L;;;;;N;;;;; +1316C;EGYPTIAN HIEROGLYPH G038;Lo;0;L;;;;;N;;;;; +1316D;EGYPTIAN HIEROGLYPH G039;Lo;0;L;;;;;N;;;;; +1316E;EGYPTIAN HIEROGLYPH G040;Lo;0;L;;;;;N;;;;; +1316F;EGYPTIAN HIEROGLYPH G041;Lo;0;L;;;;;N;;;;; +13170;EGYPTIAN HIEROGLYPH G042;Lo;0;L;;;;;N;;;;; +13171;EGYPTIAN HIEROGLYPH G043;Lo;0;L;;;;;N;;;;; +13172;EGYPTIAN HIEROGLYPH G043A;Lo;0;L;;;;;N;;;;; +13173;EGYPTIAN HIEROGLYPH G044;Lo;0;L;;;;;N;;;;; +13174;EGYPTIAN HIEROGLYPH G045;Lo;0;L;;;;;N;;;;; +13175;EGYPTIAN HIEROGLYPH G045A;Lo;0;L;;;;;N;;;;; +13176;EGYPTIAN HIEROGLYPH G046;Lo;0;L;;;;;N;;;;; +13177;EGYPTIAN HIEROGLYPH G047;Lo;0;L;;;;;N;;;;; +13178;EGYPTIAN HIEROGLYPH G048;Lo;0;L;;;;;N;;;;; +13179;EGYPTIAN HIEROGLYPH G049;Lo;0;L;;;;;N;;;;; +1317A;EGYPTIAN HIEROGLYPH G050;Lo;0;L;;;;;N;;;;; +1317B;EGYPTIAN HIEROGLYPH G051;Lo;0;L;;;;;N;;;;; +1317C;EGYPTIAN HIEROGLYPH G052;Lo;0;L;;;;;N;;;;; +1317D;EGYPTIAN HIEROGLYPH G053;Lo;0;L;;;;;N;;;;; +1317E;EGYPTIAN HIEROGLYPH G054;Lo;0;L;;;;;N;;;;; +1317F;EGYPTIAN HIEROGLYPH H001;Lo;0;L;;;;;N;;;;; +13180;EGYPTIAN HIEROGLYPH H002;Lo;0;L;;;;;N;;;;; +13181;EGYPTIAN HIEROGLYPH H003;Lo;0;L;;;;;N;;;;; +13182;EGYPTIAN HIEROGLYPH H004;Lo;0;L;;;;;N;;;;; +13183;EGYPTIAN HIEROGLYPH H005;Lo;0;L;;;;;N;;;;; +13184;EGYPTIAN HIEROGLYPH H006;Lo;0;L;;;;;N;;;;; +13185;EGYPTIAN HIEROGLYPH H006A;Lo;0;L;;;;;N;;;;; +13186;EGYPTIAN HIEROGLYPH H007;Lo;0;L;;;;;N;;;;; +13187;EGYPTIAN HIEROGLYPH H008;Lo;0;L;;;;;N;;;;; +13188;EGYPTIAN HIEROGLYPH I001;Lo;0;L;;;;;N;;;;; +13189;EGYPTIAN HIEROGLYPH I002;Lo;0;L;;;;;N;;;;; +1318A;EGYPTIAN HIEROGLYPH I003;Lo;0;L;;;;;N;;;;; +1318B;EGYPTIAN HIEROGLYPH I004;Lo;0;L;;;;;N;;;;; +1318C;EGYPTIAN HIEROGLYPH I005;Lo;0;L;;;;;N;;;;; +1318D;EGYPTIAN HIEROGLYPH I005A;Lo;0;L;;;;;N;;;;; +1318E;EGYPTIAN HIEROGLYPH I006;Lo;0;L;;;;;N;;;;; +1318F;EGYPTIAN HIEROGLYPH I007;Lo;0;L;;;;;N;;;;; +13190;EGYPTIAN HIEROGLYPH I008;Lo;0;L;;;;;N;;;;; +13191;EGYPTIAN HIEROGLYPH I009;Lo;0;L;;;;;N;;;;; +13192;EGYPTIAN HIEROGLYPH I009A;Lo;0;L;;;;;N;;;;; +13193;EGYPTIAN HIEROGLYPH I010;Lo;0;L;;;;;N;;;;; +13194;EGYPTIAN HIEROGLYPH I010A;Lo;0;L;;;;;N;;;;; +13195;EGYPTIAN HIEROGLYPH I011;Lo;0;L;;;;;N;;;;; +13196;EGYPTIAN HIEROGLYPH I011A;Lo;0;L;;;;;N;;;;; +13197;EGYPTIAN HIEROGLYPH I012;Lo;0;L;;;;;N;;;;; +13198;EGYPTIAN HIEROGLYPH I013;Lo;0;L;;;;;N;;;;; +13199;EGYPTIAN HIEROGLYPH I014;Lo;0;L;;;;;N;;;;; +1319A;EGYPTIAN HIEROGLYPH I015;Lo;0;L;;;;;N;;;;; +1319B;EGYPTIAN HIEROGLYPH K001;Lo;0;L;;;;;N;;;;; +1319C;EGYPTIAN HIEROGLYPH K002;Lo;0;L;;;;;N;;;;; +1319D;EGYPTIAN HIEROGLYPH K003;Lo;0;L;;;;;N;;;;; +1319E;EGYPTIAN HIEROGLYPH K004;Lo;0;L;;;;;N;;;;; +1319F;EGYPTIAN HIEROGLYPH K005;Lo;0;L;;;;;N;;;;; +131A0;EGYPTIAN HIEROGLYPH K006;Lo;0;L;;;;;N;;;;; +131A1;EGYPTIAN HIEROGLYPH K007;Lo;0;L;;;;;N;;;;; +131A2;EGYPTIAN HIEROGLYPH K008;Lo;0;L;;;;;N;;;;; +131A3;EGYPTIAN HIEROGLYPH L001;Lo;0;L;;;;;N;;;;; +131A4;EGYPTIAN HIEROGLYPH L002;Lo;0;L;;;;;N;;;;; +131A5;EGYPTIAN HIEROGLYPH L002A;Lo;0;L;;;;;N;;;;; +131A6;EGYPTIAN HIEROGLYPH L003;Lo;0;L;;;;;N;;;;; +131A7;EGYPTIAN HIEROGLYPH L004;Lo;0;L;;;;;N;;;;; +131A8;EGYPTIAN HIEROGLYPH L005;Lo;0;L;;;;;N;;;;; +131A9;EGYPTIAN HIEROGLYPH L006;Lo;0;L;;;;;N;;;;; +131AA;EGYPTIAN HIEROGLYPH L006A;Lo;0;L;;;;;N;;;;; +131AB;EGYPTIAN HIEROGLYPH L007;Lo;0;L;;;;;N;;;;; +131AC;EGYPTIAN HIEROGLYPH L008;Lo;0;L;;;;;N;;;;; +131AD;EGYPTIAN HIEROGLYPH M001;Lo;0;L;;;;;N;;;;; +131AE;EGYPTIAN HIEROGLYPH M001A;Lo;0;L;;;;;N;;;;; +131AF;EGYPTIAN HIEROGLYPH M001B;Lo;0;L;;;;;N;;;;; +131B0;EGYPTIAN HIEROGLYPH M002;Lo;0;L;;;;;N;;;;; +131B1;EGYPTIAN HIEROGLYPH M003;Lo;0;L;;;;;N;;;;; +131B2;EGYPTIAN HIEROGLYPH M003A;Lo;0;L;;;;;N;;;;; +131B3;EGYPTIAN HIEROGLYPH M004;Lo;0;L;;;;;N;;;;; +131B4;EGYPTIAN HIEROGLYPH M005;Lo;0;L;;;;;N;;;;; +131B5;EGYPTIAN HIEROGLYPH M006;Lo;0;L;;;;;N;;;;; +131B6;EGYPTIAN HIEROGLYPH M007;Lo;0;L;;;;;N;;;;; +131B7;EGYPTIAN HIEROGLYPH M008;Lo;0;L;;;;;N;;;;; +131B8;EGYPTIAN HIEROGLYPH M009;Lo;0;L;;;;;N;;;;; +131B9;EGYPTIAN HIEROGLYPH M010;Lo;0;L;;;;;N;;;;; +131BA;EGYPTIAN HIEROGLYPH M010A;Lo;0;L;;;;;N;;;;; +131BB;EGYPTIAN HIEROGLYPH M011;Lo;0;L;;;;;N;;;;; +131BC;EGYPTIAN HIEROGLYPH M012;Lo;0;L;;;;;N;;;;; +131BD;EGYPTIAN HIEROGLYPH M012A;Lo;0;L;;;;;N;;;;; +131BE;EGYPTIAN HIEROGLYPH M012B;Lo;0;L;;;;;N;;;;; +131BF;EGYPTIAN HIEROGLYPH M012C;Lo;0;L;;;;;N;;;;; +131C0;EGYPTIAN HIEROGLYPH M012D;Lo;0;L;;;;;N;;;;; +131C1;EGYPTIAN HIEROGLYPH M012E;Lo;0;L;;;;;N;;;;; +131C2;EGYPTIAN HIEROGLYPH M012F;Lo;0;L;;;;;N;;;;; +131C3;EGYPTIAN HIEROGLYPH M012G;Lo;0;L;;;;;N;;;;; +131C4;EGYPTIAN HIEROGLYPH M012H;Lo;0;L;;;;;N;;;;; +131C5;EGYPTIAN HIEROGLYPH M013;Lo;0;L;;;;;N;;;;; +131C6;EGYPTIAN HIEROGLYPH M014;Lo;0;L;;;;;N;;;;; +131C7;EGYPTIAN HIEROGLYPH M015;Lo;0;L;;;;;N;;;;; +131C8;EGYPTIAN HIEROGLYPH M015A;Lo;0;L;;;;;N;;;;; +131C9;EGYPTIAN HIEROGLYPH M016;Lo;0;L;;;;;N;;;;; +131CA;EGYPTIAN HIEROGLYPH M016A;Lo;0;L;;;;;N;;;;; +131CB;EGYPTIAN HIEROGLYPH M017;Lo;0;L;;;;;N;;;;; +131CC;EGYPTIAN HIEROGLYPH M017A;Lo;0;L;;;;;N;;;;; +131CD;EGYPTIAN HIEROGLYPH M018;Lo;0;L;;;;;N;;;;; +131CE;EGYPTIAN HIEROGLYPH M019;Lo;0;L;;;;;N;;;;; +131CF;EGYPTIAN HIEROGLYPH M020;Lo;0;L;;;;;N;;;;; +131D0;EGYPTIAN HIEROGLYPH M021;Lo;0;L;;;;;N;;;;; +131D1;EGYPTIAN HIEROGLYPH M022;Lo;0;L;;;;;N;;;;; +131D2;EGYPTIAN HIEROGLYPH M022A;Lo;0;L;;;;;N;;;;; +131D3;EGYPTIAN HIEROGLYPH M023;Lo;0;L;;;;;N;;;;; +131D4;EGYPTIAN HIEROGLYPH M024;Lo;0;L;;;;;N;;;;; +131D5;EGYPTIAN HIEROGLYPH M024A;Lo;0;L;;;;;N;;;;; +131D6;EGYPTIAN HIEROGLYPH M025;Lo;0;L;;;;;N;;;;; +131D7;EGYPTIAN HIEROGLYPH M026;Lo;0;L;;;;;N;;;;; +131D8;EGYPTIAN HIEROGLYPH M027;Lo;0;L;;;;;N;;;;; +131D9;EGYPTIAN HIEROGLYPH M028;Lo;0;L;;;;;N;;;;; +131DA;EGYPTIAN HIEROGLYPH M028A;Lo;0;L;;;;;N;;;;; +131DB;EGYPTIAN HIEROGLYPH M029;Lo;0;L;;;;;N;;;;; +131DC;EGYPTIAN HIEROGLYPH M030;Lo;0;L;;;;;N;;;;; +131DD;EGYPTIAN HIEROGLYPH M031;Lo;0;L;;;;;N;;;;; +131DE;EGYPTIAN HIEROGLYPH M031A;Lo;0;L;;;;;N;;;;; +131DF;EGYPTIAN HIEROGLYPH M032;Lo;0;L;;;;;N;;;;; +131E0;EGYPTIAN HIEROGLYPH M033;Lo;0;L;;;;;N;;;;; +131E1;EGYPTIAN HIEROGLYPH M033A;Lo;0;L;;;;;N;;;;; +131E2;EGYPTIAN HIEROGLYPH M033B;Lo;0;L;;;;;N;;;;; +131E3;EGYPTIAN HIEROGLYPH M034;Lo;0;L;;;;;N;;;;; +131E4;EGYPTIAN HIEROGLYPH M035;Lo;0;L;;;;;N;;;;; +131E5;EGYPTIAN HIEROGLYPH M036;Lo;0;L;;;;;N;;;;; +131E6;EGYPTIAN HIEROGLYPH M037;Lo;0;L;;;;;N;;;;; +131E7;EGYPTIAN HIEROGLYPH M038;Lo;0;L;;;;;N;;;;; +131E8;EGYPTIAN HIEROGLYPH M039;Lo;0;L;;;;;N;;;;; +131E9;EGYPTIAN HIEROGLYPH M040;Lo;0;L;;;;;N;;;;; +131EA;EGYPTIAN HIEROGLYPH M040A;Lo;0;L;;;;;N;;;;; +131EB;EGYPTIAN HIEROGLYPH M041;Lo;0;L;;;;;N;;;;; +131EC;EGYPTIAN HIEROGLYPH M042;Lo;0;L;;;;;N;;;;; +131ED;EGYPTIAN HIEROGLYPH M043;Lo;0;L;;;;;N;;;;; +131EE;EGYPTIAN HIEROGLYPH M044;Lo;0;L;;;;;N;;;;; +131EF;EGYPTIAN HIEROGLYPH N001;Lo;0;L;;;;;N;;;;; +131F0;EGYPTIAN HIEROGLYPH N002;Lo;0;L;;;;;N;;;;; +131F1;EGYPTIAN HIEROGLYPH N003;Lo;0;L;;;;;N;;;;; +131F2;EGYPTIAN HIEROGLYPH N004;Lo;0;L;;;;;N;;;;; +131F3;EGYPTIAN HIEROGLYPH N005;Lo;0;L;;;;;N;;;;; +131F4;EGYPTIAN HIEROGLYPH N006;Lo;0;L;;;;;N;;;;; +131F5;EGYPTIAN HIEROGLYPH N007;Lo;0;L;;;;;N;;;;; +131F6;EGYPTIAN HIEROGLYPH N008;Lo;0;L;;;;;N;;;;; +131F7;EGYPTIAN HIEROGLYPH N009;Lo;0;L;;;;;N;;;;; +131F8;EGYPTIAN HIEROGLYPH N010;Lo;0;L;;;;;N;;;;; +131F9;EGYPTIAN HIEROGLYPH N011;Lo;0;L;;;;;N;;;;; +131FA;EGYPTIAN HIEROGLYPH N012;Lo;0;L;;;;;N;;;;; +131FB;EGYPTIAN HIEROGLYPH N013;Lo;0;L;;;;;N;;;;; +131FC;EGYPTIAN HIEROGLYPH N014;Lo;0;L;;;;;N;;;;; +131FD;EGYPTIAN HIEROGLYPH N015;Lo;0;L;;;;;N;;;;; +131FE;EGYPTIAN HIEROGLYPH N016;Lo;0;L;;;;;N;;;;; +131FF;EGYPTIAN HIEROGLYPH N017;Lo;0;L;;;;;N;;;;; +13200;EGYPTIAN HIEROGLYPH N018;Lo;0;L;;;;;N;;;;; +13201;EGYPTIAN HIEROGLYPH N018A;Lo;0;L;;;;;N;;;;; +13202;EGYPTIAN HIEROGLYPH N018B;Lo;0;L;;;;;N;;;;; +13203;EGYPTIAN HIEROGLYPH N019;Lo;0;L;;;;;N;;;;; +13204;EGYPTIAN HIEROGLYPH N020;Lo;0;L;;;;;N;;;;; +13205;EGYPTIAN HIEROGLYPH N021;Lo;0;L;;;;;N;;;;; +13206;EGYPTIAN HIEROGLYPH N022;Lo;0;L;;;;;N;;;;; +13207;EGYPTIAN HIEROGLYPH N023;Lo;0;L;;;;;N;;;;; +13208;EGYPTIAN HIEROGLYPH N024;Lo;0;L;;;;;N;;;;; +13209;EGYPTIAN HIEROGLYPH N025;Lo;0;L;;;;;N;;;;; +1320A;EGYPTIAN HIEROGLYPH N025A;Lo;0;L;;;;;N;;;;; +1320B;EGYPTIAN HIEROGLYPH N026;Lo;0;L;;;;;N;;;;; +1320C;EGYPTIAN HIEROGLYPH N027;Lo;0;L;;;;;N;;;;; +1320D;EGYPTIAN HIEROGLYPH N028;Lo;0;L;;;;;N;;;;; +1320E;EGYPTIAN HIEROGLYPH N029;Lo;0;L;;;;;N;;;;; +1320F;EGYPTIAN HIEROGLYPH N030;Lo;0;L;;;;;N;;;;; +13210;EGYPTIAN HIEROGLYPH N031;Lo;0;L;;;;;N;;;;; +13211;EGYPTIAN HIEROGLYPH N032;Lo;0;L;;;;;N;;;;; +13212;EGYPTIAN HIEROGLYPH N033;Lo;0;L;;;;;N;;;;; +13213;EGYPTIAN HIEROGLYPH N033A;Lo;0;L;;;;;N;;;;; +13214;EGYPTIAN HIEROGLYPH N034;Lo;0;L;;;;;N;;;;; +13215;EGYPTIAN HIEROGLYPH N034A;Lo;0;L;;;;;N;;;;; +13216;EGYPTIAN HIEROGLYPH N035;Lo;0;L;;;;;N;;;;; +13217;EGYPTIAN HIEROGLYPH N035A;Lo;0;L;;;;;N;;;;; +13218;EGYPTIAN HIEROGLYPH N036;Lo;0;L;;;;;N;;;;; +13219;EGYPTIAN HIEROGLYPH N037;Lo;0;L;;;;;N;;;;; +1321A;EGYPTIAN HIEROGLYPH N037A;Lo;0;L;;;;;N;;;;; +1321B;EGYPTIAN HIEROGLYPH N038;Lo;0;L;;;;;N;;;;; +1321C;EGYPTIAN HIEROGLYPH N039;Lo;0;L;;;;;N;;;;; +1321D;EGYPTIAN HIEROGLYPH N040;Lo;0;L;;;;;N;;;;; +1321E;EGYPTIAN HIEROGLYPH N041;Lo;0;L;;;;;N;;;;; +1321F;EGYPTIAN HIEROGLYPH N042;Lo;0;L;;;;;N;;;;; +13220;EGYPTIAN HIEROGLYPH NL001;Lo;0;L;;;;;N;;;;; +13221;EGYPTIAN HIEROGLYPH NL002;Lo;0;L;;;;;N;;;;; +13222;EGYPTIAN HIEROGLYPH NL003;Lo;0;L;;;;;N;;;;; +13223;EGYPTIAN HIEROGLYPH NL004;Lo;0;L;;;;;N;;;;; +13224;EGYPTIAN HIEROGLYPH NL005;Lo;0;L;;;;;N;;;;; +13225;EGYPTIAN HIEROGLYPH NL005A;Lo;0;L;;;;;N;;;;; +13226;EGYPTIAN HIEROGLYPH NL006;Lo;0;L;;;;;N;;;;; +13227;EGYPTIAN HIEROGLYPH NL007;Lo;0;L;;;;;N;;;;; +13228;EGYPTIAN HIEROGLYPH NL008;Lo;0;L;;;;;N;;;;; +13229;EGYPTIAN HIEROGLYPH NL009;Lo;0;L;;;;;N;;;;; +1322A;EGYPTIAN HIEROGLYPH NL010;Lo;0;L;;;;;N;;;;; +1322B;EGYPTIAN HIEROGLYPH NL011;Lo;0;L;;;;;N;;;;; +1322C;EGYPTIAN HIEROGLYPH NL012;Lo;0;L;;;;;N;;;;; +1322D;EGYPTIAN HIEROGLYPH NL013;Lo;0;L;;;;;N;;;;; +1322E;EGYPTIAN HIEROGLYPH NL014;Lo;0;L;;;;;N;;;;; +1322F;EGYPTIAN HIEROGLYPH NL015;Lo;0;L;;;;;N;;;;; +13230;EGYPTIAN HIEROGLYPH NL016;Lo;0;L;;;;;N;;;;; +13231;EGYPTIAN HIEROGLYPH NL017;Lo;0;L;;;;;N;;;;; +13232;EGYPTIAN HIEROGLYPH NL017A;Lo;0;L;;;;;N;;;;; +13233;EGYPTIAN HIEROGLYPH NL018;Lo;0;L;;;;;N;;;;; +13234;EGYPTIAN HIEROGLYPH NL019;Lo;0;L;;;;;N;;;;; +13235;EGYPTIAN HIEROGLYPH NL020;Lo;0;L;;;;;N;;;;; +13236;EGYPTIAN HIEROGLYPH NU001;Lo;0;L;;;;;N;;;;; +13237;EGYPTIAN HIEROGLYPH NU002;Lo;0;L;;;;;N;;;;; +13238;EGYPTIAN HIEROGLYPH NU003;Lo;0;L;;;;;N;;;;; +13239;EGYPTIAN HIEROGLYPH NU004;Lo;0;L;;;;;N;;;;; +1323A;EGYPTIAN HIEROGLYPH NU005;Lo;0;L;;;;;N;;;;; +1323B;EGYPTIAN HIEROGLYPH NU006;Lo;0;L;;;;;N;;;;; +1323C;EGYPTIAN HIEROGLYPH NU007;Lo;0;L;;;;;N;;;;; +1323D;EGYPTIAN HIEROGLYPH NU008;Lo;0;L;;;;;N;;;;; +1323E;EGYPTIAN HIEROGLYPH NU009;Lo;0;L;;;;;N;;;;; +1323F;EGYPTIAN HIEROGLYPH NU010;Lo;0;L;;;;;N;;;;; +13240;EGYPTIAN HIEROGLYPH NU010A;Lo;0;L;;;;;N;;;;; +13241;EGYPTIAN HIEROGLYPH NU011;Lo;0;L;;;;;N;;;;; +13242;EGYPTIAN HIEROGLYPH NU011A;Lo;0;L;;;;;N;;;;; +13243;EGYPTIAN HIEROGLYPH NU012;Lo;0;L;;;;;N;;;;; +13244;EGYPTIAN HIEROGLYPH NU013;Lo;0;L;;;;;N;;;;; +13245;EGYPTIAN HIEROGLYPH NU014;Lo;0;L;;;;;N;;;;; +13246;EGYPTIAN HIEROGLYPH NU015;Lo;0;L;;;;;N;;;;; +13247;EGYPTIAN HIEROGLYPH NU016;Lo;0;L;;;;;N;;;;; +13248;EGYPTIAN HIEROGLYPH NU017;Lo;0;L;;;;;N;;;;; +13249;EGYPTIAN HIEROGLYPH NU018;Lo;0;L;;;;;N;;;;; +1324A;EGYPTIAN HIEROGLYPH NU018A;Lo;0;L;;;;;N;;;;; +1324B;EGYPTIAN HIEROGLYPH NU019;Lo;0;L;;;;;N;;;;; +1324C;EGYPTIAN HIEROGLYPH NU020;Lo;0;L;;;;;N;;;;; +1324D;EGYPTIAN HIEROGLYPH NU021;Lo;0;L;;;;;N;;;;; +1324E;EGYPTIAN HIEROGLYPH NU022;Lo;0;L;;;;;N;;;;; +1324F;EGYPTIAN HIEROGLYPH NU022A;Lo;0;L;;;;;N;;;;; +13250;EGYPTIAN HIEROGLYPH O001;Lo;0;L;;;;;N;;;;; +13251;EGYPTIAN HIEROGLYPH O001A;Lo;0;L;;;;;N;;;;; +13252;EGYPTIAN HIEROGLYPH O002;Lo;0;L;;;;;N;;;;; +13253;EGYPTIAN HIEROGLYPH O003;Lo;0;L;;;;;N;;;;; +13254;EGYPTIAN HIEROGLYPH O004;Lo;0;L;;;;;N;;;;; +13255;EGYPTIAN HIEROGLYPH O005;Lo;0;L;;;;;N;;;;; +13256;EGYPTIAN HIEROGLYPH O005A;Lo;0;L;;;;;N;;;;; +13257;EGYPTIAN HIEROGLYPH O006;Lo;0;L;;;;;N;;;;; +13258;EGYPTIAN HIEROGLYPH O006A;Lo;0;L;;;;;N;;;;; +13259;EGYPTIAN HIEROGLYPH O006B;Lo;0;L;;;;;N;;;;; +1325A;EGYPTIAN HIEROGLYPH O006C;Lo;0;L;;;;;N;;;;; +1325B;EGYPTIAN HIEROGLYPH O006D;Lo;0;L;;;;;N;;;;; +1325C;EGYPTIAN HIEROGLYPH O006E;Lo;0;L;;;;;N;;;;; +1325D;EGYPTIAN HIEROGLYPH O006F;Lo;0;L;;;;;N;;;;; +1325E;EGYPTIAN HIEROGLYPH O007;Lo;0;L;;;;;N;;;;; +1325F;EGYPTIAN HIEROGLYPH O008;Lo;0;L;;;;;N;;;;; +13260;EGYPTIAN HIEROGLYPH O009;Lo;0;L;;;;;N;;;;; +13261;EGYPTIAN HIEROGLYPH O010;Lo;0;L;;;;;N;;;;; +13262;EGYPTIAN HIEROGLYPH O010A;Lo;0;L;;;;;N;;;;; +13263;EGYPTIAN HIEROGLYPH O010B;Lo;0;L;;;;;N;;;;; +13264;EGYPTIAN HIEROGLYPH O010C;Lo;0;L;;;;;N;;;;; +13265;EGYPTIAN HIEROGLYPH O011;Lo;0;L;;;;;N;;;;; +13266;EGYPTIAN HIEROGLYPH O012;Lo;0;L;;;;;N;;;;; +13267;EGYPTIAN HIEROGLYPH O013;Lo;0;L;;;;;N;;;;; +13268;EGYPTIAN HIEROGLYPH O014;Lo;0;L;;;;;N;;;;; +13269;EGYPTIAN HIEROGLYPH O015;Lo;0;L;;;;;N;;;;; +1326A;EGYPTIAN HIEROGLYPH O016;Lo;0;L;;;;;N;;;;; +1326B;EGYPTIAN HIEROGLYPH O017;Lo;0;L;;;;;N;;;;; +1326C;EGYPTIAN HIEROGLYPH O018;Lo;0;L;;;;;N;;;;; +1326D;EGYPTIAN HIEROGLYPH O019;Lo;0;L;;;;;N;;;;; +1326E;EGYPTIAN HIEROGLYPH O019A;Lo;0;L;;;;;N;;;;; +1326F;EGYPTIAN HIEROGLYPH O020;Lo;0;L;;;;;N;;;;; +13270;EGYPTIAN HIEROGLYPH O020A;Lo;0;L;;;;;N;;;;; +13271;EGYPTIAN HIEROGLYPH O021;Lo;0;L;;;;;N;;;;; +13272;EGYPTIAN HIEROGLYPH O022;Lo;0;L;;;;;N;;;;; +13273;EGYPTIAN HIEROGLYPH O023;Lo;0;L;;;;;N;;;;; +13274;EGYPTIAN HIEROGLYPH O024;Lo;0;L;;;;;N;;;;; +13275;EGYPTIAN HIEROGLYPH O024A;Lo;0;L;;;;;N;;;;; +13276;EGYPTIAN HIEROGLYPH O025;Lo;0;L;;;;;N;;;;; +13277;EGYPTIAN HIEROGLYPH O025A;Lo;0;L;;;;;N;;;;; +13278;EGYPTIAN HIEROGLYPH O026;Lo;0;L;;;;;N;;;;; +13279;EGYPTIAN HIEROGLYPH O027;Lo;0;L;;;;;N;;;;; +1327A;EGYPTIAN HIEROGLYPH O028;Lo;0;L;;;;;N;;;;; +1327B;EGYPTIAN HIEROGLYPH O029;Lo;0;L;;;;;N;;;;; +1327C;EGYPTIAN HIEROGLYPH O029A;Lo;0;L;;;;;N;;;;; +1327D;EGYPTIAN HIEROGLYPH O030;Lo;0;L;;;;;N;;;;; +1327E;EGYPTIAN HIEROGLYPH O030A;Lo;0;L;;;;;N;;;;; +1327F;EGYPTIAN HIEROGLYPH O031;Lo;0;L;;;;;N;;;;; +13280;EGYPTIAN HIEROGLYPH O032;Lo;0;L;;;;;N;;;;; +13281;EGYPTIAN HIEROGLYPH O033;Lo;0;L;;;;;N;;;;; +13282;EGYPTIAN HIEROGLYPH O033A;Lo;0;L;;;;;N;;;;; +13283;EGYPTIAN HIEROGLYPH O034;Lo;0;L;;;;;N;;;;; +13284;EGYPTIAN HIEROGLYPH O035;Lo;0;L;;;;;N;;;;; +13285;EGYPTIAN HIEROGLYPH O036;Lo;0;L;;;;;N;;;;; +13286;EGYPTIAN HIEROGLYPH O036A;Lo;0;L;;;;;N;;;;; +13287;EGYPTIAN HIEROGLYPH O036B;Lo;0;L;;;;;N;;;;; +13288;EGYPTIAN HIEROGLYPH O036C;Lo;0;L;;;;;N;;;;; +13289;EGYPTIAN HIEROGLYPH O036D;Lo;0;L;;;;;N;;;;; +1328A;EGYPTIAN HIEROGLYPH O037;Lo;0;L;;;;;N;;;;; +1328B;EGYPTIAN HIEROGLYPH O038;Lo;0;L;;;;;N;;;;; +1328C;EGYPTIAN HIEROGLYPH O039;Lo;0;L;;;;;N;;;;; +1328D;EGYPTIAN HIEROGLYPH O040;Lo;0;L;;;;;N;;;;; +1328E;EGYPTIAN HIEROGLYPH O041;Lo;0;L;;;;;N;;;;; +1328F;EGYPTIAN HIEROGLYPH O042;Lo;0;L;;;;;N;;;;; +13290;EGYPTIAN HIEROGLYPH O043;Lo;0;L;;;;;N;;;;; +13291;EGYPTIAN HIEROGLYPH O044;Lo;0;L;;;;;N;;;;; +13292;EGYPTIAN HIEROGLYPH O045;Lo;0;L;;;;;N;;;;; +13293;EGYPTIAN HIEROGLYPH O046;Lo;0;L;;;;;N;;;;; +13294;EGYPTIAN HIEROGLYPH O047;Lo;0;L;;;;;N;;;;; +13295;EGYPTIAN HIEROGLYPH O048;Lo;0;L;;;;;N;;;;; +13296;EGYPTIAN HIEROGLYPH O049;Lo;0;L;;;;;N;;;;; +13297;EGYPTIAN HIEROGLYPH O050;Lo;0;L;;;;;N;;;;; +13298;EGYPTIAN HIEROGLYPH O050A;Lo;0;L;;;;;N;;;;; +13299;EGYPTIAN HIEROGLYPH O050B;Lo;0;L;;;;;N;;;;; +1329A;EGYPTIAN HIEROGLYPH O051;Lo;0;L;;;;;N;;;;; +1329B;EGYPTIAN HIEROGLYPH P001;Lo;0;L;;;;;N;;;;; +1329C;EGYPTIAN HIEROGLYPH P001A;Lo;0;L;;;;;N;;;;; +1329D;EGYPTIAN HIEROGLYPH P002;Lo;0;L;;;;;N;;;;; +1329E;EGYPTIAN HIEROGLYPH P003;Lo;0;L;;;;;N;;;;; +1329F;EGYPTIAN HIEROGLYPH P003A;Lo;0;L;;;;;N;;;;; +132A0;EGYPTIAN HIEROGLYPH P004;Lo;0;L;;;;;N;;;;; +132A1;EGYPTIAN HIEROGLYPH P005;Lo;0;L;;;;;N;;;;; +132A2;EGYPTIAN HIEROGLYPH P006;Lo;0;L;;;;;N;;;;; +132A3;EGYPTIAN HIEROGLYPH P007;Lo;0;L;;;;;N;;;;; +132A4;EGYPTIAN HIEROGLYPH P008;Lo;0;L;;;;;N;;;;; +132A5;EGYPTIAN HIEROGLYPH P009;Lo;0;L;;;;;N;;;;; +132A6;EGYPTIAN HIEROGLYPH P010;Lo;0;L;;;;;N;;;;; +132A7;EGYPTIAN HIEROGLYPH P011;Lo;0;L;;;;;N;;;;; +132A8;EGYPTIAN HIEROGLYPH Q001;Lo;0;L;;;;;N;;;;; +132A9;EGYPTIAN HIEROGLYPH Q002;Lo;0;L;;;;;N;;;;; +132AA;EGYPTIAN HIEROGLYPH Q003;Lo;0;L;;;;;N;;;;; +132AB;EGYPTIAN HIEROGLYPH Q004;Lo;0;L;;;;;N;;;;; +132AC;EGYPTIAN HIEROGLYPH Q005;Lo;0;L;;;;;N;;;;; +132AD;EGYPTIAN HIEROGLYPH Q006;Lo;0;L;;;;;N;;;;; +132AE;EGYPTIAN HIEROGLYPH Q007;Lo;0;L;;;;;N;;;;; +132AF;EGYPTIAN HIEROGLYPH R001;Lo;0;L;;;;;N;;;;; +132B0;EGYPTIAN HIEROGLYPH R002;Lo;0;L;;;;;N;;;;; +132B1;EGYPTIAN HIEROGLYPH R002A;Lo;0;L;;;;;N;;;;; +132B2;EGYPTIAN HIEROGLYPH R003;Lo;0;L;;;;;N;;;;; +132B3;EGYPTIAN HIEROGLYPH R003A;Lo;0;L;;;;;N;;;;; +132B4;EGYPTIAN HIEROGLYPH R003B;Lo;0;L;;;;;N;;;;; +132B5;EGYPTIAN HIEROGLYPH R004;Lo;0;L;;;;;N;;;;; +132B6;EGYPTIAN HIEROGLYPH R005;Lo;0;L;;;;;N;;;;; +132B7;EGYPTIAN HIEROGLYPH R006;Lo;0;L;;;;;N;;;;; +132B8;EGYPTIAN HIEROGLYPH R007;Lo;0;L;;;;;N;;;;; +132B9;EGYPTIAN HIEROGLYPH R008;Lo;0;L;;;;;N;;;;; +132BA;EGYPTIAN HIEROGLYPH R009;Lo;0;L;;;;;N;;;;; +132BB;EGYPTIAN HIEROGLYPH R010;Lo;0;L;;;;;N;;;;; +132BC;EGYPTIAN HIEROGLYPH R010A;Lo;0;L;;;;;N;;;;; +132BD;EGYPTIAN HIEROGLYPH R011;Lo;0;L;;;;;N;;;;; +132BE;EGYPTIAN HIEROGLYPH R012;Lo;0;L;;;;;N;;;;; +132BF;EGYPTIAN HIEROGLYPH R013;Lo;0;L;;;;;N;;;;; +132C0;EGYPTIAN HIEROGLYPH R014;Lo;0;L;;;;;N;;;;; +132C1;EGYPTIAN HIEROGLYPH R015;Lo;0;L;;;;;N;;;;; +132C2;EGYPTIAN HIEROGLYPH R016;Lo;0;L;;;;;N;;;;; +132C3;EGYPTIAN HIEROGLYPH R016A;Lo;0;L;;;;;N;;;;; +132C4;EGYPTIAN HIEROGLYPH R017;Lo;0;L;;;;;N;;;;; +132C5;EGYPTIAN HIEROGLYPH R018;Lo;0;L;;;;;N;;;;; +132C6;EGYPTIAN HIEROGLYPH R019;Lo;0;L;;;;;N;;;;; +132C7;EGYPTIAN HIEROGLYPH R020;Lo;0;L;;;;;N;;;;; +132C8;EGYPTIAN HIEROGLYPH R021;Lo;0;L;;;;;N;;;;; +132C9;EGYPTIAN HIEROGLYPH R022;Lo;0;L;;;;;N;;;;; +132CA;EGYPTIAN HIEROGLYPH R023;Lo;0;L;;;;;N;;;;; +132CB;EGYPTIAN HIEROGLYPH R024;Lo;0;L;;;;;N;;;;; +132CC;EGYPTIAN HIEROGLYPH R025;Lo;0;L;;;;;N;;;;; +132CD;EGYPTIAN HIEROGLYPH R026;Lo;0;L;;;;;N;;;;; +132CE;EGYPTIAN HIEROGLYPH R027;Lo;0;L;;;;;N;;;;; +132CF;EGYPTIAN HIEROGLYPH R028;Lo;0;L;;;;;N;;;;; +132D0;EGYPTIAN HIEROGLYPH R029;Lo;0;L;;;;;N;;;;; +132D1;EGYPTIAN HIEROGLYPH S001;Lo;0;L;;;;;N;;;;; +132D2;EGYPTIAN HIEROGLYPH S002;Lo;0;L;;;;;N;;;;; +132D3;EGYPTIAN HIEROGLYPH S002A;Lo;0;L;;;;;N;;;;; +132D4;EGYPTIAN HIEROGLYPH S003;Lo;0;L;;;;;N;;;;; +132D5;EGYPTIAN HIEROGLYPH S004;Lo;0;L;;;;;N;;;;; +132D6;EGYPTIAN HIEROGLYPH S005;Lo;0;L;;;;;N;;;;; +132D7;EGYPTIAN HIEROGLYPH S006;Lo;0;L;;;;;N;;;;; +132D8;EGYPTIAN HIEROGLYPH S006A;Lo;0;L;;;;;N;;;;; +132D9;EGYPTIAN HIEROGLYPH S007;Lo;0;L;;;;;N;;;;; +132DA;EGYPTIAN HIEROGLYPH S008;Lo;0;L;;;;;N;;;;; +132DB;EGYPTIAN HIEROGLYPH S009;Lo;0;L;;;;;N;;;;; +132DC;EGYPTIAN HIEROGLYPH S010;Lo;0;L;;;;;N;;;;; +132DD;EGYPTIAN HIEROGLYPH S011;Lo;0;L;;;;;N;;;;; +132DE;EGYPTIAN HIEROGLYPH S012;Lo;0;L;;;;;N;;;;; +132DF;EGYPTIAN HIEROGLYPH S013;Lo;0;L;;;;;N;;;;; +132E0;EGYPTIAN HIEROGLYPH S014;Lo;0;L;;;;;N;;;;; +132E1;EGYPTIAN HIEROGLYPH S014A;Lo;0;L;;;;;N;;;;; +132E2;EGYPTIAN HIEROGLYPH S014B;Lo;0;L;;;;;N;;;;; +132E3;EGYPTIAN HIEROGLYPH S015;Lo;0;L;;;;;N;;;;; +132E4;EGYPTIAN HIEROGLYPH S016;Lo;0;L;;;;;N;;;;; +132E5;EGYPTIAN HIEROGLYPH S017;Lo;0;L;;;;;N;;;;; +132E6;EGYPTIAN HIEROGLYPH S017A;Lo;0;L;;;;;N;;;;; +132E7;EGYPTIAN HIEROGLYPH S018;Lo;0;L;;;;;N;;;;; +132E8;EGYPTIAN HIEROGLYPH S019;Lo;0;L;;;;;N;;;;; +132E9;EGYPTIAN HIEROGLYPH S020;Lo;0;L;;;;;N;;;;; +132EA;EGYPTIAN HIEROGLYPH S021;Lo;0;L;;;;;N;;;;; +132EB;EGYPTIAN HIEROGLYPH S022;Lo;0;L;;;;;N;;;;; +132EC;EGYPTIAN HIEROGLYPH S023;Lo;0;L;;;;;N;;;;; +132ED;EGYPTIAN HIEROGLYPH S024;Lo;0;L;;;;;N;;;;; +132EE;EGYPTIAN HIEROGLYPH S025;Lo;0;L;;;;;N;;;;; +132EF;EGYPTIAN HIEROGLYPH S026;Lo;0;L;;;;;N;;;;; +132F0;EGYPTIAN HIEROGLYPH S026A;Lo;0;L;;;;;N;;;;; +132F1;EGYPTIAN HIEROGLYPH S026B;Lo;0;L;;;;;N;;;;; +132F2;EGYPTIAN HIEROGLYPH S027;Lo;0;L;;;;;N;;;;; +132F3;EGYPTIAN HIEROGLYPH S028;Lo;0;L;;;;;N;;;;; +132F4;EGYPTIAN HIEROGLYPH S029;Lo;0;L;;;;;N;;;;; +132F5;EGYPTIAN HIEROGLYPH S030;Lo;0;L;;;;;N;;;;; +132F6;EGYPTIAN HIEROGLYPH S031;Lo;0;L;;;;;N;;;;; +132F7;EGYPTIAN HIEROGLYPH S032;Lo;0;L;;;;;N;;;;; +132F8;EGYPTIAN HIEROGLYPH S033;Lo;0;L;;;;;N;;;;; +132F9;EGYPTIAN HIEROGLYPH S034;Lo;0;L;;;;;N;;;;; +132FA;EGYPTIAN HIEROGLYPH S035;Lo;0;L;;;;;N;;;;; +132FB;EGYPTIAN HIEROGLYPH S035A;Lo;0;L;;;;;N;;;;; +132FC;EGYPTIAN HIEROGLYPH S036;Lo;0;L;;;;;N;;;;; +132FD;EGYPTIAN HIEROGLYPH S037;Lo;0;L;;;;;N;;;;; +132FE;EGYPTIAN HIEROGLYPH S038;Lo;0;L;;;;;N;;;;; +132FF;EGYPTIAN HIEROGLYPH S039;Lo;0;L;;;;;N;;;;; +13300;EGYPTIAN HIEROGLYPH S040;Lo;0;L;;;;;N;;;;; +13301;EGYPTIAN HIEROGLYPH S041;Lo;0;L;;;;;N;;;;; +13302;EGYPTIAN HIEROGLYPH S042;Lo;0;L;;;;;N;;;;; +13303;EGYPTIAN HIEROGLYPH S043;Lo;0;L;;;;;N;;;;; +13304;EGYPTIAN HIEROGLYPH S044;Lo;0;L;;;;;N;;;;; +13305;EGYPTIAN HIEROGLYPH S045;Lo;0;L;;;;;N;;;;; +13306;EGYPTIAN HIEROGLYPH S046;Lo;0;L;;;;;N;;;;; +13307;EGYPTIAN HIEROGLYPH T001;Lo;0;L;;;;;N;;;;; +13308;EGYPTIAN HIEROGLYPH T002;Lo;0;L;;;;;N;;;;; +13309;EGYPTIAN HIEROGLYPH T003;Lo;0;L;;;;;N;;;;; +1330A;EGYPTIAN HIEROGLYPH T003A;Lo;0;L;;;;;N;;;;; +1330B;EGYPTIAN HIEROGLYPH T004;Lo;0;L;;;;;N;;;;; +1330C;EGYPTIAN HIEROGLYPH T005;Lo;0;L;;;;;N;;;;; +1330D;EGYPTIAN HIEROGLYPH T006;Lo;0;L;;;;;N;;;;; +1330E;EGYPTIAN HIEROGLYPH T007;Lo;0;L;;;;;N;;;;; +1330F;EGYPTIAN HIEROGLYPH T007A;Lo;0;L;;;;;N;;;;; +13310;EGYPTIAN HIEROGLYPH T008;Lo;0;L;;;;;N;;;;; +13311;EGYPTIAN HIEROGLYPH T008A;Lo;0;L;;;;;N;;;;; +13312;EGYPTIAN HIEROGLYPH T009;Lo;0;L;;;;;N;;;;; +13313;EGYPTIAN HIEROGLYPH T009A;Lo;0;L;;;;;N;;;;; +13314;EGYPTIAN HIEROGLYPH T010;Lo;0;L;;;;;N;;;;; +13315;EGYPTIAN HIEROGLYPH T011;Lo;0;L;;;;;N;;;;; +13316;EGYPTIAN HIEROGLYPH T011A;Lo;0;L;;;;;N;;;;; +13317;EGYPTIAN HIEROGLYPH T012;Lo;0;L;;;;;N;;;;; +13318;EGYPTIAN HIEROGLYPH T013;Lo;0;L;;;;;N;;;;; +13319;EGYPTIAN HIEROGLYPH T014;Lo;0;L;;;;;N;;;;; +1331A;EGYPTIAN HIEROGLYPH T015;Lo;0;L;;;;;N;;;;; +1331B;EGYPTIAN HIEROGLYPH T016;Lo;0;L;;;;;N;;;;; +1331C;EGYPTIAN HIEROGLYPH T016A;Lo;0;L;;;;;N;;;;; +1331D;EGYPTIAN HIEROGLYPH T017;Lo;0;L;;;;;N;;;;; +1331E;EGYPTIAN HIEROGLYPH T018;Lo;0;L;;;;;N;;;;; +1331F;EGYPTIAN HIEROGLYPH T019;Lo;0;L;;;;;N;;;;; +13320;EGYPTIAN HIEROGLYPH T020;Lo;0;L;;;;;N;;;;; +13321;EGYPTIAN HIEROGLYPH T021;Lo;0;L;;;;;N;;;;; +13322;EGYPTIAN HIEROGLYPH T022;Lo;0;L;;;;;N;;;;; +13323;EGYPTIAN HIEROGLYPH T023;Lo;0;L;;;;;N;;;;; +13324;EGYPTIAN HIEROGLYPH T024;Lo;0;L;;;;;N;;;;; +13325;EGYPTIAN HIEROGLYPH T025;Lo;0;L;;;;;N;;;;; +13326;EGYPTIAN HIEROGLYPH T026;Lo;0;L;;;;;N;;;;; +13327;EGYPTIAN HIEROGLYPH T027;Lo;0;L;;;;;N;;;;; +13328;EGYPTIAN HIEROGLYPH T028;Lo;0;L;;;;;N;;;;; +13329;EGYPTIAN HIEROGLYPH T029;Lo;0;L;;;;;N;;;;; +1332A;EGYPTIAN HIEROGLYPH T030;Lo;0;L;;;;;N;;;;; +1332B;EGYPTIAN HIEROGLYPH T031;Lo;0;L;;;;;N;;;;; +1332C;EGYPTIAN HIEROGLYPH T032;Lo;0;L;;;;;N;;;;; +1332D;EGYPTIAN HIEROGLYPH T032A;Lo;0;L;;;;;N;;;;; +1332E;EGYPTIAN HIEROGLYPH T033;Lo;0;L;;;;;N;;;;; +1332F;EGYPTIAN HIEROGLYPH T033A;Lo;0;L;;;;;N;;;;; +13330;EGYPTIAN HIEROGLYPH T034;Lo;0;L;;;;;N;;;;; +13331;EGYPTIAN HIEROGLYPH T035;Lo;0;L;;;;;N;;;;; +13332;EGYPTIAN HIEROGLYPH T036;Lo;0;L;;;;;N;;;;; +13333;EGYPTIAN HIEROGLYPH U001;Lo;0;L;;;;;N;;;;; +13334;EGYPTIAN HIEROGLYPH U002;Lo;0;L;;;;;N;;;;; +13335;EGYPTIAN HIEROGLYPH U003;Lo;0;L;;;;;N;;;;; +13336;EGYPTIAN HIEROGLYPH U004;Lo;0;L;;;;;N;;;;; +13337;EGYPTIAN HIEROGLYPH U005;Lo;0;L;;;;;N;;;;; +13338;EGYPTIAN HIEROGLYPH U006;Lo;0;L;;;;;N;;;;; +13339;EGYPTIAN HIEROGLYPH U006A;Lo;0;L;;;;;N;;;;; +1333A;EGYPTIAN HIEROGLYPH U006B;Lo;0;L;;;;;N;;;;; +1333B;EGYPTIAN HIEROGLYPH U007;Lo;0;L;;;;;N;;;;; +1333C;EGYPTIAN HIEROGLYPH U008;Lo;0;L;;;;;N;;;;; +1333D;EGYPTIAN HIEROGLYPH U009;Lo;0;L;;;;;N;;;;; +1333E;EGYPTIAN HIEROGLYPH U010;Lo;0;L;;;;;N;;;;; +1333F;EGYPTIAN HIEROGLYPH U011;Lo;0;L;;;;;N;;;;; +13340;EGYPTIAN HIEROGLYPH U012;Lo;0;L;;;;;N;;;;; +13341;EGYPTIAN HIEROGLYPH U013;Lo;0;L;;;;;N;;;;; +13342;EGYPTIAN HIEROGLYPH U014;Lo;0;L;;;;;N;;;;; +13343;EGYPTIAN HIEROGLYPH U015;Lo;0;L;;;;;N;;;;; +13344;EGYPTIAN HIEROGLYPH U016;Lo;0;L;;;;;N;;;;; +13345;EGYPTIAN HIEROGLYPH U017;Lo;0;L;;;;;N;;;;; +13346;EGYPTIAN HIEROGLYPH U018;Lo;0;L;;;;;N;;;;; +13347;EGYPTIAN HIEROGLYPH U019;Lo;0;L;;;;;N;;;;; +13348;EGYPTIAN HIEROGLYPH U020;Lo;0;L;;;;;N;;;;; +13349;EGYPTIAN HIEROGLYPH U021;Lo;0;L;;;;;N;;;;; +1334A;EGYPTIAN HIEROGLYPH U022;Lo;0;L;;;;;N;;;;; +1334B;EGYPTIAN HIEROGLYPH U023;Lo;0;L;;;;;N;;;;; +1334C;EGYPTIAN HIEROGLYPH U023A;Lo;0;L;;;;;N;;;;; +1334D;EGYPTIAN HIEROGLYPH U024;Lo;0;L;;;;;N;;;;; +1334E;EGYPTIAN HIEROGLYPH U025;Lo;0;L;;;;;N;;;;; +1334F;EGYPTIAN HIEROGLYPH U026;Lo;0;L;;;;;N;;;;; +13350;EGYPTIAN HIEROGLYPH U027;Lo;0;L;;;;;N;;;;; +13351;EGYPTIAN HIEROGLYPH U028;Lo;0;L;;;;;N;;;;; +13352;EGYPTIAN HIEROGLYPH U029;Lo;0;L;;;;;N;;;;; +13353;EGYPTIAN HIEROGLYPH U029A;Lo;0;L;;;;;N;;;;; +13354;EGYPTIAN HIEROGLYPH U030;Lo;0;L;;;;;N;;;;; +13355;EGYPTIAN HIEROGLYPH U031;Lo;0;L;;;;;N;;;;; +13356;EGYPTIAN HIEROGLYPH U032;Lo;0;L;;;;;N;;;;; +13357;EGYPTIAN HIEROGLYPH U032A;Lo;0;L;;;;;N;;;;; +13358;EGYPTIAN HIEROGLYPH U033;Lo;0;L;;;;;N;;;;; +13359;EGYPTIAN HIEROGLYPH U034;Lo;0;L;;;;;N;;;;; +1335A;EGYPTIAN HIEROGLYPH U035;Lo;0;L;;;;;N;;;;; +1335B;EGYPTIAN HIEROGLYPH U036;Lo;0;L;;;;;N;;;;; +1335C;EGYPTIAN HIEROGLYPH U037;Lo;0;L;;;;;N;;;;; +1335D;EGYPTIAN HIEROGLYPH U038;Lo;0;L;;;;;N;;;;; +1335E;EGYPTIAN HIEROGLYPH U039;Lo;0;L;;;;;N;;;;; +1335F;EGYPTIAN HIEROGLYPH U040;Lo;0;L;;;;;N;;;;; +13360;EGYPTIAN HIEROGLYPH U041;Lo;0;L;;;;;N;;;;; +13361;EGYPTIAN HIEROGLYPH U042;Lo;0;L;;;;;N;;;;; +13362;EGYPTIAN HIEROGLYPH V001;Lo;0;L;;;;;N;;;;; +13363;EGYPTIAN HIEROGLYPH V001A;Lo;0;L;;;;;N;;;;; +13364;EGYPTIAN HIEROGLYPH V001B;Lo;0;L;;;;;N;;;;; +13365;EGYPTIAN HIEROGLYPH V001C;Lo;0;L;;;;;N;;;;; +13366;EGYPTIAN HIEROGLYPH V001D;Lo;0;L;;;;;N;;;;; +13367;EGYPTIAN HIEROGLYPH V001E;Lo;0;L;;;;;N;;;;; +13368;EGYPTIAN HIEROGLYPH V001F;Lo;0;L;;;;;N;;;;; +13369;EGYPTIAN HIEROGLYPH V001G;Lo;0;L;;;;;N;;;;; +1336A;EGYPTIAN HIEROGLYPH V001H;Lo;0;L;;;;;N;;;;; +1336B;EGYPTIAN HIEROGLYPH V001I;Lo;0;L;;;;;N;;;;; +1336C;EGYPTIAN HIEROGLYPH V002;Lo;0;L;;;;;N;;;;; +1336D;EGYPTIAN HIEROGLYPH V002A;Lo;0;L;;;;;N;;;;; +1336E;EGYPTIAN HIEROGLYPH V003;Lo;0;L;;;;;N;;;;; +1336F;EGYPTIAN HIEROGLYPH V004;Lo;0;L;;;;;N;;;;; +13370;EGYPTIAN HIEROGLYPH V005;Lo;0;L;;;;;N;;;;; +13371;EGYPTIAN HIEROGLYPH V006;Lo;0;L;;;;;N;;;;; +13372;EGYPTIAN HIEROGLYPH V007;Lo;0;L;;;;;N;;;;; +13373;EGYPTIAN HIEROGLYPH V007A;Lo;0;L;;;;;N;;;;; +13374;EGYPTIAN HIEROGLYPH V007B;Lo;0;L;;;;;N;;;;; +13375;EGYPTIAN HIEROGLYPH V008;Lo;0;L;;;;;N;;;;; +13376;EGYPTIAN HIEROGLYPH V009;Lo;0;L;;;;;N;;;;; +13377;EGYPTIAN HIEROGLYPH V010;Lo;0;L;;;;;N;;;;; +13378;EGYPTIAN HIEROGLYPH V011;Lo;0;L;;;;;N;;;;; +13379;EGYPTIAN HIEROGLYPH V011A;Lo;0;L;;;;;N;;;;; +1337A;EGYPTIAN HIEROGLYPH V011B;Lo;0;L;;;;;N;;;;; +1337B;EGYPTIAN HIEROGLYPH V011C;Lo;0;L;;;;;N;;;;; +1337C;EGYPTIAN HIEROGLYPH V012;Lo;0;L;;;;;N;;;;; +1337D;EGYPTIAN HIEROGLYPH V012A;Lo;0;L;;;;;N;;;;; +1337E;EGYPTIAN HIEROGLYPH V012B;Lo;0;L;;;;;N;;;;; +1337F;EGYPTIAN HIEROGLYPH V013;Lo;0;L;;;;;N;;;;; +13380;EGYPTIAN HIEROGLYPH V014;Lo;0;L;;;;;N;;;;; +13381;EGYPTIAN HIEROGLYPH V015;Lo;0;L;;;;;N;;;;; +13382;EGYPTIAN HIEROGLYPH V016;Lo;0;L;;;;;N;;;;; +13383;EGYPTIAN HIEROGLYPH V017;Lo;0;L;;;;;N;;;;; +13384;EGYPTIAN HIEROGLYPH V018;Lo;0;L;;;;;N;;;;; +13385;EGYPTIAN HIEROGLYPH V019;Lo;0;L;;;;;N;;;;; +13386;EGYPTIAN HIEROGLYPH V020;Lo;0;L;;;;;N;;;;; +13387;EGYPTIAN HIEROGLYPH V020A;Lo;0;L;;;;;N;;;;; +13388;EGYPTIAN HIEROGLYPH V020B;Lo;0;L;;;;;N;;;;; +13389;EGYPTIAN HIEROGLYPH V020C;Lo;0;L;;;;;N;;;;; +1338A;EGYPTIAN HIEROGLYPH V020D;Lo;0;L;;;;;N;;;;; +1338B;EGYPTIAN HIEROGLYPH V020E;Lo;0;L;;;;;N;;;;; +1338C;EGYPTIAN HIEROGLYPH V020F;Lo;0;L;;;;;N;;;;; +1338D;EGYPTIAN HIEROGLYPH V020G;Lo;0;L;;;;;N;;;;; +1338E;EGYPTIAN HIEROGLYPH V020H;Lo;0;L;;;;;N;;;;; +1338F;EGYPTIAN HIEROGLYPH V020I;Lo;0;L;;;;;N;;;;; +13390;EGYPTIAN HIEROGLYPH V020J;Lo;0;L;;;;;N;;;;; +13391;EGYPTIAN HIEROGLYPH V020K;Lo;0;L;;;;;N;;;;; +13392;EGYPTIAN HIEROGLYPH V020L;Lo;0;L;;;;;N;;;;; +13393;EGYPTIAN HIEROGLYPH V021;Lo;0;L;;;;;N;;;;; +13394;EGYPTIAN HIEROGLYPH V022;Lo;0;L;;;;;N;;;;; +13395;EGYPTIAN HIEROGLYPH V023;Lo;0;L;;;;;N;;;;; +13396;EGYPTIAN HIEROGLYPH V023A;Lo;0;L;;;;;N;;;;; +13397;EGYPTIAN HIEROGLYPH V024;Lo;0;L;;;;;N;;;;; +13398;EGYPTIAN HIEROGLYPH V025;Lo;0;L;;;;;N;;;;; +13399;EGYPTIAN HIEROGLYPH V026;Lo;0;L;;;;;N;;;;; +1339A;EGYPTIAN HIEROGLYPH V027;Lo;0;L;;;;;N;;;;; +1339B;EGYPTIAN HIEROGLYPH V028;Lo;0;L;;;;;N;;;;; +1339C;EGYPTIAN HIEROGLYPH V028A;Lo;0;L;;;;;N;;;;; +1339D;EGYPTIAN HIEROGLYPH V029;Lo;0;L;;;;;N;;;;; +1339E;EGYPTIAN HIEROGLYPH V029A;Lo;0;L;;;;;N;;;;; +1339F;EGYPTIAN HIEROGLYPH V030;Lo;0;L;;;;;N;;;;; +133A0;EGYPTIAN HIEROGLYPH V030A;Lo;0;L;;;;;N;;;;; +133A1;EGYPTIAN HIEROGLYPH V031;Lo;0;L;;;;;N;;;;; +133A2;EGYPTIAN HIEROGLYPH V031A;Lo;0;L;;;;;N;;;;; +133A3;EGYPTIAN HIEROGLYPH V032;Lo;0;L;;;;;N;;;;; +133A4;EGYPTIAN HIEROGLYPH V033;Lo;0;L;;;;;N;;;;; +133A5;EGYPTIAN HIEROGLYPH V033A;Lo;0;L;;;;;N;;;;; +133A6;EGYPTIAN HIEROGLYPH V034;Lo;0;L;;;;;N;;;;; +133A7;EGYPTIAN HIEROGLYPH V035;Lo;0;L;;;;;N;;;;; +133A8;EGYPTIAN HIEROGLYPH V036;Lo;0;L;;;;;N;;;;; +133A9;EGYPTIAN HIEROGLYPH V037;Lo;0;L;;;;;N;;;;; +133AA;EGYPTIAN HIEROGLYPH V037A;Lo;0;L;;;;;N;;;;; +133AB;EGYPTIAN HIEROGLYPH V038;Lo;0;L;;;;;N;;;;; +133AC;EGYPTIAN HIEROGLYPH V039;Lo;0;L;;;;;N;;;;; +133AD;EGYPTIAN HIEROGLYPH V040;Lo;0;L;;;;;N;;;;; +133AE;EGYPTIAN HIEROGLYPH V040A;Lo;0;L;;;;;N;;;;; +133AF;EGYPTIAN HIEROGLYPH W001;Lo;0;L;;;;;N;;;;; +133B0;EGYPTIAN HIEROGLYPH W002;Lo;0;L;;;;;N;;;;; +133B1;EGYPTIAN HIEROGLYPH W003;Lo;0;L;;;;;N;;;;; +133B2;EGYPTIAN HIEROGLYPH W003A;Lo;0;L;;;;;N;;;;; +133B3;EGYPTIAN HIEROGLYPH W004;Lo;0;L;;;;;N;;;;; +133B4;EGYPTIAN HIEROGLYPH W005;Lo;0;L;;;;;N;;;;; +133B5;EGYPTIAN HIEROGLYPH W006;Lo;0;L;;;;;N;;;;; +133B6;EGYPTIAN HIEROGLYPH W007;Lo;0;L;;;;;N;;;;; +133B7;EGYPTIAN HIEROGLYPH W008;Lo;0;L;;;;;N;;;;; +133B8;EGYPTIAN HIEROGLYPH W009;Lo;0;L;;;;;N;;;;; +133B9;EGYPTIAN HIEROGLYPH W009A;Lo;0;L;;;;;N;;;;; +133BA;EGYPTIAN HIEROGLYPH W010;Lo;0;L;;;;;N;;;;; +133BB;EGYPTIAN HIEROGLYPH W010A;Lo;0;L;;;;;N;;;;; +133BC;EGYPTIAN HIEROGLYPH W011;Lo;0;L;;;;;N;;;;; +133BD;EGYPTIAN HIEROGLYPH W012;Lo;0;L;;;;;N;;;;; +133BE;EGYPTIAN HIEROGLYPH W013;Lo;0;L;;;;;N;;;;; +133BF;EGYPTIAN HIEROGLYPH W014;Lo;0;L;;;;;N;;;;; +133C0;EGYPTIAN HIEROGLYPH W014A;Lo;0;L;;;;;N;;;;; +133C1;EGYPTIAN HIEROGLYPH W015;Lo;0;L;;;;;N;;;;; +133C2;EGYPTIAN HIEROGLYPH W016;Lo;0;L;;;;;N;;;;; +133C3;EGYPTIAN HIEROGLYPH W017;Lo;0;L;;;;;N;;;;; +133C4;EGYPTIAN HIEROGLYPH W017A;Lo;0;L;;;;;N;;;;; +133C5;EGYPTIAN HIEROGLYPH W018;Lo;0;L;;;;;N;;;;; +133C6;EGYPTIAN HIEROGLYPH W018A;Lo;0;L;;;;;N;;;;; +133C7;EGYPTIAN HIEROGLYPH W019;Lo;0;L;;;;;N;;;;; +133C8;EGYPTIAN HIEROGLYPH W020;Lo;0;L;;;;;N;;;;; +133C9;EGYPTIAN HIEROGLYPH W021;Lo;0;L;;;;;N;;;;; +133CA;EGYPTIAN HIEROGLYPH W022;Lo;0;L;;;;;N;;;;; +133CB;EGYPTIAN HIEROGLYPH W023;Lo;0;L;;;;;N;;;;; +133CC;EGYPTIAN HIEROGLYPH W024;Lo;0;L;;;;;N;;;;; +133CD;EGYPTIAN HIEROGLYPH W024A;Lo;0;L;;;;;N;;;;; +133CE;EGYPTIAN HIEROGLYPH W025;Lo;0;L;;;;;N;;;;; +133CF;EGYPTIAN HIEROGLYPH X001;Lo;0;L;;;;;N;;;;; +133D0;EGYPTIAN HIEROGLYPH X002;Lo;0;L;;;;;N;;;;; +133D1;EGYPTIAN HIEROGLYPH X003;Lo;0;L;;;;;N;;;;; +133D2;EGYPTIAN HIEROGLYPH X004;Lo;0;L;;;;;N;;;;; +133D3;EGYPTIAN HIEROGLYPH X004A;Lo;0;L;;;;;N;;;;; +133D4;EGYPTIAN HIEROGLYPH X004B;Lo;0;L;;;;;N;;;;; +133D5;EGYPTIAN HIEROGLYPH X005;Lo;0;L;;;;;N;;;;; +133D6;EGYPTIAN HIEROGLYPH X006;Lo;0;L;;;;;N;;;;; +133D7;EGYPTIAN HIEROGLYPH X006A;Lo;0;L;;;;;N;;;;; +133D8;EGYPTIAN HIEROGLYPH X007;Lo;0;L;;;;;N;;;;; +133D9;EGYPTIAN HIEROGLYPH X008;Lo;0;L;;;;;N;;;;; +133DA;EGYPTIAN HIEROGLYPH X008A;Lo;0;L;;;;;N;;;;; +133DB;EGYPTIAN HIEROGLYPH Y001;Lo;0;L;;;;;N;;;;; +133DC;EGYPTIAN HIEROGLYPH Y001A;Lo;0;L;;;;;N;;;;; +133DD;EGYPTIAN HIEROGLYPH Y002;Lo;0;L;;;;;N;;;;; +133DE;EGYPTIAN HIEROGLYPH Y003;Lo;0;L;;;;;N;;;;; +133DF;EGYPTIAN HIEROGLYPH Y004;Lo;0;L;;;;;N;;;;; +133E0;EGYPTIAN HIEROGLYPH Y005;Lo;0;L;;;;;N;;;;; +133E1;EGYPTIAN HIEROGLYPH Y006;Lo;0;L;;;;;N;;;;; +133E2;EGYPTIAN HIEROGLYPH Y007;Lo;0;L;;;;;N;;;;; +133E3;EGYPTIAN HIEROGLYPH Y008;Lo;0;L;;;;;N;;;;; +133E4;EGYPTIAN HIEROGLYPH Z001;Lo;0;L;;;;;N;;;;; +133E5;EGYPTIAN HIEROGLYPH Z002;Lo;0;L;;;;;N;;;;; +133E6;EGYPTIAN HIEROGLYPH Z002A;Lo;0;L;;;;;N;;;;; +133E7;EGYPTIAN HIEROGLYPH Z002B;Lo;0;L;;;;;N;;;;; +133E8;EGYPTIAN HIEROGLYPH Z002C;Lo;0;L;;;;;N;;;;; +133E9;EGYPTIAN HIEROGLYPH Z002D;Lo;0;L;;;;;N;;;;; +133EA;EGYPTIAN HIEROGLYPH Z003;Lo;0;L;;;;;N;;;;; +133EB;EGYPTIAN HIEROGLYPH Z003A;Lo;0;L;;;;;N;;;;; +133EC;EGYPTIAN HIEROGLYPH Z003B;Lo;0;L;;;;;N;;;;; +133ED;EGYPTIAN HIEROGLYPH Z004;Lo;0;L;;;;;N;;;;; +133EE;EGYPTIAN HIEROGLYPH Z004A;Lo;0;L;;;;;N;;;;; +133EF;EGYPTIAN HIEROGLYPH Z005;Lo;0;L;;;;;N;;;;; +133F0;EGYPTIAN HIEROGLYPH Z005A;Lo;0;L;;;;;N;;;;; +133F1;EGYPTIAN HIEROGLYPH Z006;Lo;0;L;;;;;N;;;;; +133F2;EGYPTIAN HIEROGLYPH Z007;Lo;0;L;;;;;N;;;;; +133F3;EGYPTIAN HIEROGLYPH Z008;Lo;0;L;;;;;N;;;;; +133F4;EGYPTIAN HIEROGLYPH Z009;Lo;0;L;;;;;N;;;;; +133F5;EGYPTIAN HIEROGLYPH Z010;Lo;0;L;;;;;N;;;;; +133F6;EGYPTIAN HIEROGLYPH Z011;Lo;0;L;;;;;N;;;;; +133F7;EGYPTIAN HIEROGLYPH Z012;Lo;0;L;;;;;N;;;;; +133F8;EGYPTIAN HIEROGLYPH Z013;Lo;0;L;;;;;N;;;;; +133F9;EGYPTIAN HIEROGLYPH Z014;Lo;0;L;;;;;N;;;;; +133FA;EGYPTIAN HIEROGLYPH Z015;Lo;0;L;;;;;N;;;;; +133FB;EGYPTIAN HIEROGLYPH Z015A;Lo;0;L;;;;;N;;;;; +133FC;EGYPTIAN HIEROGLYPH Z015B;Lo;0;L;;;;;N;;;;; +133FD;EGYPTIAN HIEROGLYPH Z015C;Lo;0;L;;;;;N;;;;; +133FE;EGYPTIAN HIEROGLYPH Z015D;Lo;0;L;;;;;N;;;;; +133FF;EGYPTIAN HIEROGLYPH Z015E;Lo;0;L;;;;;N;;;;; +13400;EGYPTIAN HIEROGLYPH Z015F;Lo;0;L;;;;;N;;;;; +13401;EGYPTIAN HIEROGLYPH Z015G;Lo;0;L;;;;;N;;;;; +13402;EGYPTIAN HIEROGLYPH Z015H;Lo;0;L;;;;;N;;;;; +13403;EGYPTIAN HIEROGLYPH Z015I;Lo;0;L;;;;;N;;;;; +13404;EGYPTIAN HIEROGLYPH Z016;Lo;0;L;;;;;N;;;;; +13405;EGYPTIAN HIEROGLYPH Z016A;Lo;0;L;;;;;N;;;;; +13406;EGYPTIAN HIEROGLYPH Z016B;Lo;0;L;;;;;N;;;;; +13407;EGYPTIAN HIEROGLYPH Z016C;Lo;0;L;;;;;N;;;;; +13408;EGYPTIAN HIEROGLYPH Z016D;Lo;0;L;;;;;N;;;;; +13409;EGYPTIAN HIEROGLYPH Z016E;Lo;0;L;;;;;N;;;;; +1340A;EGYPTIAN HIEROGLYPH Z016F;Lo;0;L;;;;;N;;;;; +1340B;EGYPTIAN HIEROGLYPH Z016G;Lo;0;L;;;;;N;;;;; +1340C;EGYPTIAN HIEROGLYPH Z016H;Lo;0;L;;;;;N;;;;; +1340D;EGYPTIAN HIEROGLYPH AA001;Lo;0;L;;;;;N;;;;; +1340E;EGYPTIAN HIEROGLYPH AA002;Lo;0;L;;;;;N;;;;; +1340F;EGYPTIAN HIEROGLYPH AA003;Lo;0;L;;;;;N;;;;; +13410;EGYPTIAN HIEROGLYPH AA004;Lo;0;L;;;;;N;;;;; +13411;EGYPTIAN HIEROGLYPH AA005;Lo;0;L;;;;;N;;;;; +13412;EGYPTIAN HIEROGLYPH AA006;Lo;0;L;;;;;N;;;;; +13413;EGYPTIAN HIEROGLYPH AA007;Lo;0;L;;;;;N;;;;; +13414;EGYPTIAN HIEROGLYPH AA007A;Lo;0;L;;;;;N;;;;; +13415;EGYPTIAN HIEROGLYPH AA007B;Lo;0;L;;;;;N;;;;; +13416;EGYPTIAN HIEROGLYPH AA008;Lo;0;L;;;;;N;;;;; +13417;EGYPTIAN HIEROGLYPH AA009;Lo;0;L;;;;;N;;;;; +13418;EGYPTIAN HIEROGLYPH AA010;Lo;0;L;;;;;N;;;;; +13419;EGYPTIAN HIEROGLYPH AA011;Lo;0;L;;;;;N;;;;; +1341A;EGYPTIAN HIEROGLYPH AA012;Lo;0;L;;;;;N;;;;; +1341B;EGYPTIAN HIEROGLYPH AA013;Lo;0;L;;;;;N;;;;; +1341C;EGYPTIAN HIEROGLYPH AA014;Lo;0;L;;;;;N;;;;; +1341D;EGYPTIAN HIEROGLYPH AA015;Lo;0;L;;;;;N;;;;; +1341E;EGYPTIAN HIEROGLYPH AA016;Lo;0;L;;;;;N;;;;; +1341F;EGYPTIAN HIEROGLYPH AA017;Lo;0;L;;;;;N;;;;; +13420;EGYPTIAN HIEROGLYPH AA018;Lo;0;L;;;;;N;;;;; +13421;EGYPTIAN HIEROGLYPH AA019;Lo;0;L;;;;;N;;;;; +13422;EGYPTIAN HIEROGLYPH AA020;Lo;0;L;;;;;N;;;;; +13423;EGYPTIAN HIEROGLYPH AA021;Lo;0;L;;;;;N;;;;; +13424;EGYPTIAN HIEROGLYPH AA022;Lo;0;L;;;;;N;;;;; +13425;EGYPTIAN HIEROGLYPH AA023;Lo;0;L;;;;;N;;;;; +13426;EGYPTIAN HIEROGLYPH AA024;Lo;0;L;;;;;N;;;;; +13427;EGYPTIAN HIEROGLYPH AA025;Lo;0;L;;;;;N;;;;; +13428;EGYPTIAN HIEROGLYPH AA026;Lo;0;L;;;;;N;;;;; +13429;EGYPTIAN HIEROGLYPH AA027;Lo;0;L;;;;;N;;;;; +1342A;EGYPTIAN HIEROGLYPH AA028;Lo;0;L;;;;;N;;;;; +1342B;EGYPTIAN HIEROGLYPH AA029;Lo;0;L;;;;;N;;;;; +1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; +1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; +1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +16800;BAMUM LETTER PHASE-A NGKUE MFON;Lo;0;L;;;;;N;;;;; +16801;BAMUM LETTER PHASE-A GBIEE FON;Lo;0;L;;;;;N;;;;; +16802;BAMUM LETTER PHASE-A PON MFON PIPAEMGBIEE;Lo;0;L;;;;;N;;;;; +16803;BAMUM LETTER PHASE-A PON MFON PIPAEMBA;Lo;0;L;;;;;N;;;;; +16804;BAMUM LETTER PHASE-A NAA MFON;Lo;0;L;;;;;N;;;;; +16805;BAMUM LETTER PHASE-A SHUENSHUET;Lo;0;L;;;;;N;;;;; +16806;BAMUM LETTER PHASE-A TITA MFON;Lo;0;L;;;;;N;;;;; +16807;BAMUM LETTER PHASE-A NZA MFON;Lo;0;L;;;;;N;;;;; +16808;BAMUM LETTER PHASE-A SHINDA PA NJI;Lo;0;L;;;;;N;;;;; +16809;BAMUM LETTER PHASE-A PON PA NJI PIPAEMGBIEE;Lo;0;L;;;;;N;;;;; +1680A;BAMUM LETTER PHASE-A PON PA NJI PIPAEMBA;Lo;0;L;;;;;N;;;;; +1680B;BAMUM LETTER PHASE-A MAEMBGBIEE;Lo;0;L;;;;;N;;;;; +1680C;BAMUM LETTER PHASE-A TU MAEMBA;Lo;0;L;;;;;N;;;;; +1680D;BAMUM LETTER PHASE-A NGANGU;Lo;0;L;;;;;N;;;;; +1680E;BAMUM LETTER PHASE-A MAEMVEUX;Lo;0;L;;;;;N;;;;; +1680F;BAMUM LETTER PHASE-A MANSUAE;Lo;0;L;;;;;N;;;;; +16810;BAMUM LETTER PHASE-A MVEUAENGAM;Lo;0;L;;;;;N;;;;; +16811;BAMUM LETTER PHASE-A SEUNYAM;Lo;0;L;;;;;N;;;;; +16812;BAMUM LETTER PHASE-A NTOQPEN;Lo;0;L;;;;;N;;;;; +16813;BAMUM LETTER PHASE-A KEUKEUTNDA;Lo;0;L;;;;;N;;;;; +16814;BAMUM LETTER PHASE-A NKINDI;Lo;0;L;;;;;N;;;;; +16815;BAMUM LETTER PHASE-A SUU;Lo;0;L;;;;;N;;;;; +16816;BAMUM LETTER PHASE-A NGKUENZEUM;Lo;0;L;;;;;N;;;;; +16817;BAMUM LETTER PHASE-A LAPAQ;Lo;0;L;;;;;N;;;;; +16818;BAMUM LETTER PHASE-A LET KUT;Lo;0;L;;;;;N;;;;; +16819;BAMUM LETTER PHASE-A NTAP MFAA;Lo;0;L;;;;;N;;;;; +1681A;BAMUM LETTER PHASE-A MAEKEUP;Lo;0;L;;;;;N;;;;; +1681B;BAMUM LETTER PHASE-A PASHAE;Lo;0;L;;;;;N;;;;; +1681C;BAMUM LETTER PHASE-A GHEUAERAE;Lo;0;L;;;;;N;;;;; +1681D;BAMUM LETTER PHASE-A PAMSHAE;Lo;0;L;;;;;N;;;;; +1681E;BAMUM LETTER PHASE-A MON NGGEUAET;Lo;0;L;;;;;N;;;;; +1681F;BAMUM LETTER PHASE-A NZUN MEUT;Lo;0;L;;;;;N;;;;; +16820;BAMUM LETTER PHASE-A U YUQ NAE;Lo;0;L;;;;;N;;;;; +16821;BAMUM LETTER PHASE-A GHEUAEGHEUAE;Lo;0;L;;;;;N;;;;; +16822;BAMUM LETTER PHASE-A NTAP NTAA;Lo;0;L;;;;;N;;;;; +16823;BAMUM LETTER PHASE-A SISA;Lo;0;L;;;;;N;;;;; +16824;BAMUM LETTER PHASE-A MGBASA;Lo;0;L;;;;;N;;;;; +16825;BAMUM LETTER PHASE-A MEUNJOMNDEUQ;Lo;0;L;;;;;N;;;;; +16826;BAMUM LETTER PHASE-A MOOMPUQ;Lo;0;L;;;;;N;;;;; +16827;BAMUM LETTER PHASE-A KAFA;Lo;0;L;;;;;N;;;;; +16828;BAMUM LETTER PHASE-A PA LEERAEWA;Lo;0;L;;;;;N;;;;; +16829;BAMUM LETTER PHASE-A NDA LEERAEWA;Lo;0;L;;;;;N;;;;; +1682A;BAMUM LETTER PHASE-A PET;Lo;0;L;;;;;N;;;;; +1682B;BAMUM LETTER PHASE-A MAEMKPEN;Lo;0;L;;;;;N;;;;; +1682C;BAMUM LETTER PHASE-A NIKA;Lo;0;L;;;;;N;;;;; +1682D;BAMUM LETTER PHASE-A PUP;Lo;0;L;;;;;N;;;;; +1682E;BAMUM LETTER PHASE-A TUAEP;Lo;0;L;;;;;N;;;;; +1682F;BAMUM LETTER PHASE-A LUAEP;Lo;0;L;;;;;N;;;;; +16830;BAMUM LETTER PHASE-A SONJAM;Lo;0;L;;;;;N;;;;; +16831;BAMUM LETTER PHASE-A TEUTEUWEN;Lo;0;L;;;;;N;;;;; +16832;BAMUM LETTER PHASE-A MAENYI;Lo;0;L;;;;;N;;;;; +16833;BAMUM LETTER PHASE-A KET;Lo;0;L;;;;;N;;;;; +16834;BAMUM LETTER PHASE-A NDAANGGEUAET;Lo;0;L;;;;;N;;;;; +16835;BAMUM LETTER PHASE-A KUOQ;Lo;0;L;;;;;N;;;;; +16836;BAMUM LETTER PHASE-A MOOMEUT;Lo;0;L;;;;;N;;;;; +16837;BAMUM LETTER PHASE-A SHUM;Lo;0;L;;;;;N;;;;; +16838;BAMUM LETTER PHASE-A LOMMAE;Lo;0;L;;;;;N;;;;; +16839;BAMUM LETTER PHASE-A FIRI;Lo;0;L;;;;;N;;;;; +1683A;BAMUM LETTER PHASE-A ROM;Lo;0;L;;;;;N;;;;; +1683B;BAMUM LETTER PHASE-A KPOQ;Lo;0;L;;;;;N;;;;; +1683C;BAMUM LETTER PHASE-A SOQ;Lo;0;L;;;;;N;;;;; +1683D;BAMUM LETTER PHASE-A MAP PIEET;Lo;0;L;;;;;N;;;;; +1683E;BAMUM LETTER PHASE-A SHIRAE;Lo;0;L;;;;;N;;;;; +1683F;BAMUM LETTER PHASE-A NTAP;Lo;0;L;;;;;N;;;;; +16840;BAMUM LETTER PHASE-A SHOQ NSHUT YUM;Lo;0;L;;;;;N;;;;; +16841;BAMUM LETTER PHASE-A NYIT MONGKEUAEQ;Lo;0;L;;;;;N;;;;; +16842;BAMUM LETTER PHASE-A PAARAE;Lo;0;L;;;;;N;;;;; +16843;BAMUM LETTER PHASE-A NKAARAE;Lo;0;L;;;;;N;;;;; +16844;BAMUM LETTER PHASE-A UNKNOWN;Lo;0;L;;;;;N;;;;; +16845;BAMUM LETTER PHASE-A NGGEN;Lo;0;L;;;;;N;;;;; +16846;BAMUM LETTER PHASE-A MAESI;Lo;0;L;;;;;N;;;;; +16847;BAMUM LETTER PHASE-A NJAM;Lo;0;L;;;;;N;;;;; +16848;BAMUM LETTER PHASE-A MBANYI;Lo;0;L;;;;;N;;;;; +16849;BAMUM LETTER PHASE-A NYET;Lo;0;L;;;;;N;;;;; +1684A;BAMUM LETTER PHASE-A TEUAEN;Lo;0;L;;;;;N;;;;; +1684B;BAMUM LETTER PHASE-A SOT;Lo;0;L;;;;;N;;;;; +1684C;BAMUM LETTER PHASE-A PAAM;Lo;0;L;;;;;N;;;;; +1684D;BAMUM LETTER PHASE-A NSHIEE;Lo;0;L;;;;;N;;;;; +1684E;BAMUM LETTER PHASE-A MAEM;Lo;0;L;;;;;N;;;;; +1684F;BAMUM LETTER PHASE-A NYI;Lo;0;L;;;;;N;;;;; +16850;BAMUM LETTER PHASE-A KAQ;Lo;0;L;;;;;N;;;;; +16851;BAMUM LETTER PHASE-A NSHA;Lo;0;L;;;;;N;;;;; +16852;BAMUM LETTER PHASE-A VEE;Lo;0;L;;;;;N;;;;; +16853;BAMUM LETTER PHASE-A LU;Lo;0;L;;;;;N;;;;; +16854;BAMUM LETTER PHASE-A NEN;Lo;0;L;;;;;N;;;;; +16855;BAMUM LETTER PHASE-A NAQ;Lo;0;L;;;;;N;;;;; +16856;BAMUM LETTER PHASE-A MBAQ;Lo;0;L;;;;;N;;;;; +16857;BAMUM LETTER PHASE-B NSHUET;Lo;0;L;;;;;N;;;;; +16858;BAMUM LETTER PHASE-B TU MAEMGBIEE;Lo;0;L;;;;;N;;;;; +16859;BAMUM LETTER PHASE-B SIEE;Lo;0;L;;;;;N;;;;; +1685A;BAMUM LETTER PHASE-B SET TU;Lo;0;L;;;;;N;;;;; +1685B;BAMUM LETTER PHASE-B LOM NTEUM;Lo;0;L;;;;;N;;;;; +1685C;BAMUM LETTER PHASE-B MBA MAELEE;Lo;0;L;;;;;N;;;;; +1685D;BAMUM LETTER PHASE-B KIEEM;Lo;0;L;;;;;N;;;;; +1685E;BAMUM LETTER PHASE-B YEURAE;Lo;0;L;;;;;N;;;;; +1685F;BAMUM LETTER PHASE-B MBAARAE;Lo;0;L;;;;;N;;;;; +16860;BAMUM LETTER PHASE-B KAM;Lo;0;L;;;;;N;;;;; +16861;BAMUM LETTER PHASE-B PEESHI;Lo;0;L;;;;;N;;;;; +16862;BAMUM LETTER PHASE-B YAFU LEERAEWA;Lo;0;L;;;;;N;;;;; +16863;BAMUM LETTER PHASE-B LAM NSHUT NYAM;Lo;0;L;;;;;N;;;;; +16864;BAMUM LETTER PHASE-B NTIEE SHEUOQ;Lo;0;L;;;;;N;;;;; +16865;BAMUM LETTER PHASE-B NDU NJAA;Lo;0;L;;;;;N;;;;; +16866;BAMUM LETTER PHASE-B GHEUGHEUAEM;Lo;0;L;;;;;N;;;;; +16867;BAMUM LETTER PHASE-B PIT;Lo;0;L;;;;;N;;;;; +16868;BAMUM LETTER PHASE-B TU NSIEE;Lo;0;L;;;;;N;;;;; +16869;BAMUM LETTER PHASE-B SHET NJAQ;Lo;0;L;;;;;N;;;;; +1686A;BAMUM LETTER PHASE-B SHEUAEQTU;Lo;0;L;;;;;N;;;;; +1686B;BAMUM LETTER PHASE-B MFON TEUAEQ;Lo;0;L;;;;;N;;;;; +1686C;BAMUM LETTER PHASE-B MBIT MBAAKET;Lo;0;L;;;;;N;;;;; +1686D;BAMUM LETTER PHASE-B NYI NTEUM;Lo;0;L;;;;;N;;;;; +1686E;BAMUM LETTER PHASE-B KEUPUQ;Lo;0;L;;;;;N;;;;; +1686F;BAMUM LETTER PHASE-B GHEUGHEN;Lo;0;L;;;;;N;;;;; +16870;BAMUM LETTER PHASE-B KEUYEUX;Lo;0;L;;;;;N;;;;; +16871;BAMUM LETTER PHASE-B LAANAE;Lo;0;L;;;;;N;;;;; +16872;BAMUM LETTER PHASE-B PARUM;Lo;0;L;;;;;N;;;;; +16873;BAMUM LETTER PHASE-B VEUM;Lo;0;L;;;;;N;;;;; +16874;BAMUM LETTER PHASE-B NGKINDI MVOP;Lo;0;L;;;;;N;;;;; +16875;BAMUM LETTER PHASE-B NGGEU MBU;Lo;0;L;;;;;N;;;;; +16876;BAMUM LETTER PHASE-B WUAET;Lo;0;L;;;;;N;;;;; +16877;BAMUM LETTER PHASE-B SAKEUAE;Lo;0;L;;;;;N;;;;; +16878;BAMUM LETTER PHASE-B TAAM;Lo;0;L;;;;;N;;;;; +16879;BAMUM LETTER PHASE-B MEUQ;Lo;0;L;;;;;N;;;;; +1687A;BAMUM LETTER PHASE-B NGGUOQ;Lo;0;L;;;;;N;;;;; +1687B;BAMUM LETTER PHASE-B NGGUOQ LARGE;Lo;0;L;;;;;N;;;;; +1687C;BAMUM LETTER PHASE-B MFIYAQ;Lo;0;L;;;;;N;;;;; +1687D;BAMUM LETTER PHASE-B SUE;Lo;0;L;;;;;N;;;;; +1687E;BAMUM LETTER PHASE-B MBEURI;Lo;0;L;;;;;N;;;;; +1687F;BAMUM LETTER PHASE-B MONTIEEN;Lo;0;L;;;;;N;;;;; +16880;BAMUM LETTER PHASE-B NYAEMAE;Lo;0;L;;;;;N;;;;; +16881;BAMUM LETTER PHASE-B PUNGAAM;Lo;0;L;;;;;N;;;;; +16882;BAMUM LETTER PHASE-B MEUT NGGEET;Lo;0;L;;;;;N;;;;; +16883;BAMUM LETTER PHASE-B FEUX;Lo;0;L;;;;;N;;;;; +16884;BAMUM LETTER PHASE-B MBUOQ;Lo;0;L;;;;;N;;;;; +16885;BAMUM LETTER PHASE-B FEE;Lo;0;L;;;;;N;;;;; +16886;BAMUM LETTER PHASE-B KEUAEM;Lo;0;L;;;;;N;;;;; +16887;BAMUM LETTER PHASE-B MA NJEUAENA;Lo;0;L;;;;;N;;;;; +16888;BAMUM LETTER PHASE-B MA NJUQA;Lo;0;L;;;;;N;;;;; +16889;BAMUM LETTER PHASE-B LET;Lo;0;L;;;;;N;;;;; +1688A;BAMUM LETTER PHASE-B NGGAAM;Lo;0;L;;;;;N;;;;; +1688B;BAMUM LETTER PHASE-B NSEN;Lo;0;L;;;;;N;;;;; +1688C;BAMUM LETTER PHASE-B MA;Lo;0;L;;;;;N;;;;; +1688D;BAMUM LETTER PHASE-B KIQ;Lo;0;L;;;;;N;;;;; +1688E;BAMUM LETTER PHASE-B NGOM;Lo;0;L;;;;;N;;;;; +1688F;BAMUM LETTER PHASE-C NGKUE MAEMBA;Lo;0;L;;;;;N;;;;; +16890;BAMUM LETTER PHASE-C NZA;Lo;0;L;;;;;N;;;;; +16891;BAMUM LETTER PHASE-C YUM;Lo;0;L;;;;;N;;;;; +16892;BAMUM LETTER PHASE-C WANGKUOQ;Lo;0;L;;;;;N;;;;; +16893;BAMUM LETTER PHASE-C NGGEN;Lo;0;L;;;;;N;;;;; +16894;BAMUM LETTER PHASE-C NDEUAEREE;Lo;0;L;;;;;N;;;;; +16895;BAMUM LETTER PHASE-C NGKAQ;Lo;0;L;;;;;N;;;;; +16896;BAMUM LETTER PHASE-C GHARAE;Lo;0;L;;;;;N;;;;; +16897;BAMUM LETTER PHASE-C MBEEKEET;Lo;0;L;;;;;N;;;;; +16898;BAMUM LETTER PHASE-C GBAYI;Lo;0;L;;;;;N;;;;; +16899;BAMUM LETTER PHASE-C NYIR MKPARAQ MEUN;Lo;0;L;;;;;N;;;;; +1689A;BAMUM LETTER PHASE-C NTU MBIT;Lo;0;L;;;;;N;;;;; +1689B;BAMUM LETTER PHASE-C MBEUM;Lo;0;L;;;;;N;;;;; +1689C;BAMUM LETTER PHASE-C PIRIEEN;Lo;0;L;;;;;N;;;;; +1689D;BAMUM LETTER PHASE-C NDOMBU;Lo;0;L;;;;;N;;;;; +1689E;BAMUM LETTER PHASE-C MBAA CABBAGE-TREE;Lo;0;L;;;;;N;;;;; +1689F;BAMUM LETTER PHASE-C KEUSHEUAEP;Lo;0;L;;;;;N;;;;; +168A0;BAMUM LETTER PHASE-C GHAP;Lo;0;L;;;;;N;;;;; +168A1;BAMUM LETTER PHASE-C KEUKAQ;Lo;0;L;;;;;N;;;;; +168A2;BAMUM LETTER PHASE-C YU MUOMAE;Lo;0;L;;;;;N;;;;; +168A3;BAMUM LETTER PHASE-C NZEUM;Lo;0;L;;;;;N;;;;; +168A4;BAMUM LETTER PHASE-C MBUE;Lo;0;L;;;;;N;;;;; +168A5;BAMUM LETTER PHASE-C NSEUAEN;Lo;0;L;;;;;N;;;;; +168A6;BAMUM LETTER PHASE-C MBIT;Lo;0;L;;;;;N;;;;; +168A7;BAMUM LETTER PHASE-C YEUQ;Lo;0;L;;;;;N;;;;; +168A8;BAMUM LETTER PHASE-C KPARAQ;Lo;0;L;;;;;N;;;;; +168A9;BAMUM LETTER PHASE-C KAA;Lo;0;L;;;;;N;;;;; +168AA;BAMUM LETTER PHASE-C SEUX;Lo;0;L;;;;;N;;;;; +168AB;BAMUM LETTER PHASE-C NDIDA;Lo;0;L;;;;;N;;;;; +168AC;BAMUM LETTER PHASE-C TAASHAE;Lo;0;L;;;;;N;;;;; +168AD;BAMUM LETTER PHASE-C NJUEQ;Lo;0;L;;;;;N;;;;; +168AE;BAMUM LETTER PHASE-C TITA YUE;Lo;0;L;;;;;N;;;;; +168AF;BAMUM LETTER PHASE-C SUAET;Lo;0;L;;;;;N;;;;; +168B0;BAMUM LETTER PHASE-C NGGUAEN NYAM;Lo;0;L;;;;;N;;;;; +168B1;BAMUM LETTER PHASE-C VEUX;Lo;0;L;;;;;N;;;;; +168B2;BAMUM LETTER PHASE-C NANSANAQ;Lo;0;L;;;;;N;;;;; +168B3;BAMUM LETTER PHASE-C MA KEUAERI;Lo;0;L;;;;;N;;;;; +168B4;BAMUM LETTER PHASE-C NTAA;Lo;0;L;;;;;N;;;;; +168B5;BAMUM LETTER PHASE-C NGGUON;Lo;0;L;;;;;N;;;;; +168B6;BAMUM LETTER PHASE-C LAP;Lo;0;L;;;;;N;;;;; +168B7;BAMUM LETTER PHASE-C MBIRIEEN;Lo;0;L;;;;;N;;;;; +168B8;BAMUM LETTER PHASE-C MGBASAQ;Lo;0;L;;;;;N;;;;; +168B9;BAMUM LETTER PHASE-C NTEUNGBA;Lo;0;L;;;;;N;;;;; +168BA;BAMUM LETTER PHASE-C TEUTEUX;Lo;0;L;;;;;N;;;;; +168BB;BAMUM LETTER PHASE-C NGGUM;Lo;0;L;;;;;N;;;;; +168BC;BAMUM LETTER PHASE-C FUE;Lo;0;L;;;;;N;;;;; +168BD;BAMUM LETTER PHASE-C NDEUT;Lo;0;L;;;;;N;;;;; +168BE;BAMUM LETTER PHASE-C NSA;Lo;0;L;;;;;N;;;;; +168BF;BAMUM LETTER PHASE-C NSHAQ;Lo;0;L;;;;;N;;;;; +168C0;BAMUM LETTER PHASE-C BUNG;Lo;0;L;;;;;N;;;;; +168C1;BAMUM LETTER PHASE-C VEUAEPEN;Lo;0;L;;;;;N;;;;; +168C2;BAMUM LETTER PHASE-C MBERAE;Lo;0;L;;;;;N;;;;; +168C3;BAMUM LETTER PHASE-C RU;Lo;0;L;;;;;N;;;;; +168C4;BAMUM LETTER PHASE-C NJAEM;Lo;0;L;;;;;N;;;;; +168C5;BAMUM LETTER PHASE-C LAM;Lo;0;L;;;;;N;;;;; +168C6;BAMUM LETTER PHASE-C TITUAEP;Lo;0;L;;;;;N;;;;; +168C7;BAMUM LETTER PHASE-C NSUOT NGOM;Lo;0;L;;;;;N;;;;; +168C8;BAMUM LETTER PHASE-C NJEEEE;Lo;0;L;;;;;N;;;;; +168C9;BAMUM LETTER PHASE-C KET;Lo;0;L;;;;;N;;;;; +168CA;BAMUM LETTER PHASE-C NGGU;Lo;0;L;;;;;N;;;;; +168CB;BAMUM LETTER PHASE-C MAESI;Lo;0;L;;;;;N;;;;; +168CC;BAMUM LETTER PHASE-C MBUAEM;Lo;0;L;;;;;N;;;;; +168CD;BAMUM LETTER PHASE-C LU;Lo;0;L;;;;;N;;;;; +168CE;BAMUM LETTER PHASE-C KUT;Lo;0;L;;;;;N;;;;; +168CF;BAMUM LETTER PHASE-C NJAM;Lo;0;L;;;;;N;;;;; +168D0;BAMUM LETTER PHASE-C NGOM;Lo;0;L;;;;;N;;;;; +168D1;BAMUM LETTER PHASE-C WUP;Lo;0;L;;;;;N;;;;; +168D2;BAMUM LETTER PHASE-C NGGUEET;Lo;0;L;;;;;N;;;;; +168D3;BAMUM LETTER PHASE-C NSOM;Lo;0;L;;;;;N;;;;; +168D4;BAMUM LETTER PHASE-C NTEN;Lo;0;L;;;;;N;;;;; +168D5;BAMUM LETTER PHASE-C KUOP NKAARAE;Lo;0;L;;;;;N;;;;; +168D6;BAMUM LETTER PHASE-C NSUN;Lo;0;L;;;;;N;;;;; +168D7;BAMUM LETTER PHASE-C NDAM;Lo;0;L;;;;;N;;;;; +168D8;BAMUM LETTER PHASE-C MA NSIEE;Lo;0;L;;;;;N;;;;; +168D9;BAMUM LETTER PHASE-C YAA;Lo;0;L;;;;;N;;;;; +168DA;BAMUM LETTER PHASE-C NDAP;Lo;0;L;;;;;N;;;;; +168DB;BAMUM LETTER PHASE-C SHUEQ;Lo;0;L;;;;;N;;;;; +168DC;BAMUM LETTER PHASE-C SETFON;Lo;0;L;;;;;N;;;;; +168DD;BAMUM LETTER PHASE-C MBI;Lo;0;L;;;;;N;;;;; +168DE;BAMUM LETTER PHASE-C MAEMBA;Lo;0;L;;;;;N;;;;; +168DF;BAMUM LETTER PHASE-C MBANYI;Lo;0;L;;;;;N;;;;; +168E0;BAMUM LETTER PHASE-C KEUSEUX;Lo;0;L;;;;;N;;;;; +168E1;BAMUM LETTER PHASE-C MBEUX;Lo;0;L;;;;;N;;;;; +168E2;BAMUM LETTER PHASE-C KEUM;Lo;0;L;;;;;N;;;;; +168E3;BAMUM LETTER PHASE-C MBAA PICKET;Lo;0;L;;;;;N;;;;; +168E4;BAMUM LETTER PHASE-C YUWOQ;Lo;0;L;;;;;N;;;;; +168E5;BAMUM LETTER PHASE-C NJEUX;Lo;0;L;;;;;N;;;;; +168E6;BAMUM LETTER PHASE-C MIEE;Lo;0;L;;;;;N;;;;; +168E7;BAMUM LETTER PHASE-C MUAE;Lo;0;L;;;;;N;;;;; +168E8;BAMUM LETTER PHASE-C SHIQ;Lo;0;L;;;;;N;;;;; +168E9;BAMUM LETTER PHASE-C KEN LAW;Lo;0;L;;;;;N;;;;; +168EA;BAMUM LETTER PHASE-C KEN FATIGUE;Lo;0;L;;;;;N;;;;; +168EB;BAMUM LETTER PHASE-C NGAQ;Lo;0;L;;;;;N;;;;; +168EC;BAMUM LETTER PHASE-C NAQ;Lo;0;L;;;;;N;;;;; +168ED;BAMUM LETTER PHASE-C LIQ;Lo;0;L;;;;;N;;;;; +168EE;BAMUM LETTER PHASE-C PIN;Lo;0;L;;;;;N;;;;; +168EF;BAMUM LETTER PHASE-C PEN;Lo;0;L;;;;;N;;;;; +168F0;BAMUM LETTER PHASE-C TET;Lo;0;L;;;;;N;;;;; +168F1;BAMUM LETTER PHASE-D MBUO;Lo;0;L;;;;;N;;;;; +168F2;BAMUM LETTER PHASE-D WAP;Lo;0;L;;;;;N;;;;; +168F3;BAMUM LETTER PHASE-D NJI;Lo;0;L;;;;;N;;;;; +168F4;BAMUM LETTER PHASE-D MFON;Lo;0;L;;;;;N;;;;; +168F5;BAMUM LETTER PHASE-D NJIEE;Lo;0;L;;;;;N;;;;; +168F6;BAMUM LETTER PHASE-D LIEE;Lo;0;L;;;;;N;;;;; +168F7;BAMUM LETTER PHASE-D NJEUT;Lo;0;L;;;;;N;;;;; +168F8;BAMUM LETTER PHASE-D NSHEE;Lo;0;L;;;;;N;;;;; +168F9;BAMUM LETTER PHASE-D NGGAAMAE;Lo;0;L;;;;;N;;;;; +168FA;BAMUM LETTER PHASE-D NYAM;Lo;0;L;;;;;N;;;;; +168FB;BAMUM LETTER PHASE-D WUAEN;Lo;0;L;;;;;N;;;;; +168FC;BAMUM LETTER PHASE-D NGKUN;Lo;0;L;;;;;N;;;;; +168FD;BAMUM LETTER PHASE-D SHEE;Lo;0;L;;;;;N;;;;; +168FE;BAMUM LETTER PHASE-D NGKAP;Lo;0;L;;;;;N;;;;; +168FF;BAMUM LETTER PHASE-D KEUAETMEUN;Lo;0;L;;;;;N;;;;; +16900;BAMUM LETTER PHASE-D TEUT;Lo;0;L;;;;;N;;;;; +16901;BAMUM LETTER PHASE-D SHEUAE;Lo;0;L;;;;;N;;;;; +16902;BAMUM LETTER PHASE-D NJAP;Lo;0;L;;;;;N;;;;; +16903;BAMUM LETTER PHASE-D SUE;Lo;0;L;;;;;N;;;;; +16904;BAMUM LETTER PHASE-D KET;Lo;0;L;;;;;N;;;;; +16905;BAMUM LETTER PHASE-D YAEMMAE;Lo;0;L;;;;;N;;;;; +16906;BAMUM LETTER PHASE-D KUOM;Lo;0;L;;;;;N;;;;; +16907;BAMUM LETTER PHASE-D SAP;Lo;0;L;;;;;N;;;;; +16908;BAMUM LETTER PHASE-D MFEUT;Lo;0;L;;;;;N;;;;; +16909;BAMUM LETTER PHASE-D NDEUX;Lo;0;L;;;;;N;;;;; +1690A;BAMUM LETTER PHASE-D MALEERI;Lo;0;L;;;;;N;;;;; +1690B;BAMUM LETTER PHASE-D MEUT;Lo;0;L;;;;;N;;;;; +1690C;BAMUM LETTER PHASE-D SEUAEQ;Lo;0;L;;;;;N;;;;; +1690D;BAMUM LETTER PHASE-D YEN;Lo;0;L;;;;;N;;;;; +1690E;BAMUM LETTER PHASE-D NJEUAEM;Lo;0;L;;;;;N;;;;; +1690F;BAMUM LETTER PHASE-D KEUOT MBUAE;Lo;0;L;;;;;N;;;;; +16910;BAMUM LETTER PHASE-D NGKEURI;Lo;0;L;;;;;N;;;;; +16911;BAMUM LETTER PHASE-D TU;Lo;0;L;;;;;N;;;;; +16912;BAMUM LETTER PHASE-D GHAA;Lo;0;L;;;;;N;;;;; +16913;BAMUM LETTER PHASE-D NGKYEE;Lo;0;L;;;;;N;;;;; +16914;BAMUM LETTER PHASE-D FEUFEUAET;Lo;0;L;;;;;N;;;;; +16915;BAMUM LETTER PHASE-D NDEE;Lo;0;L;;;;;N;;;;; +16916;BAMUM LETTER PHASE-D MGBOFUM;Lo;0;L;;;;;N;;;;; +16917;BAMUM LETTER PHASE-D LEUAEP;Lo;0;L;;;;;N;;;;; +16918;BAMUM LETTER PHASE-D NDON;Lo;0;L;;;;;N;;;;; +16919;BAMUM LETTER PHASE-D MONI;Lo;0;L;;;;;N;;;;; +1691A;BAMUM LETTER PHASE-D MGBEUN;Lo;0;L;;;;;N;;;;; +1691B;BAMUM LETTER PHASE-D PUUT;Lo;0;L;;;;;N;;;;; +1691C;BAMUM LETTER PHASE-D MGBIEE;Lo;0;L;;;;;N;;;;; +1691D;BAMUM LETTER PHASE-D MFO;Lo;0;L;;;;;N;;;;; +1691E;BAMUM LETTER PHASE-D LUM;Lo;0;L;;;;;N;;;;; +1691F;BAMUM LETTER PHASE-D NSIEEP;Lo;0;L;;;;;N;;;;; +16920;BAMUM LETTER PHASE-D MBAA;Lo;0;L;;;;;N;;;;; +16921;BAMUM LETTER PHASE-D KWAET;Lo;0;L;;;;;N;;;;; +16922;BAMUM LETTER PHASE-D NYET;Lo;0;L;;;;;N;;;;; +16923;BAMUM LETTER PHASE-D TEUAEN;Lo;0;L;;;;;N;;;;; +16924;BAMUM LETTER PHASE-D SOT;Lo;0;L;;;;;N;;;;; +16925;BAMUM LETTER PHASE-D YUWOQ;Lo;0;L;;;;;N;;;;; +16926;BAMUM LETTER PHASE-D KEUM;Lo;0;L;;;;;N;;;;; +16927;BAMUM LETTER PHASE-D RAEM;Lo;0;L;;;;;N;;;;; +16928;BAMUM LETTER PHASE-D TEEEE;Lo;0;L;;;;;N;;;;; +16929;BAMUM LETTER PHASE-D NGKEUAEQ;Lo;0;L;;;;;N;;;;; +1692A;BAMUM LETTER PHASE-D MFEUAE;Lo;0;L;;;;;N;;;;; +1692B;BAMUM LETTER PHASE-D NSIEET;Lo;0;L;;;;;N;;;;; +1692C;BAMUM LETTER PHASE-D KEUP;Lo;0;L;;;;;N;;;;; +1692D;BAMUM LETTER PHASE-D PIP;Lo;0;L;;;;;N;;;;; +1692E;BAMUM LETTER PHASE-D PEUTAE;Lo;0;L;;;;;N;;;;; +1692F;BAMUM LETTER PHASE-D NYUE;Lo;0;L;;;;;N;;;;; +16930;BAMUM LETTER PHASE-D LET;Lo;0;L;;;;;N;;;;; +16931;BAMUM LETTER PHASE-D NGGAAM;Lo;0;L;;;;;N;;;;; +16932;BAMUM LETTER PHASE-D MFIEE;Lo;0;L;;;;;N;;;;; +16933;BAMUM LETTER PHASE-D NGGWAEN;Lo;0;L;;;;;N;;;;; +16934;BAMUM LETTER PHASE-D YUOM;Lo;0;L;;;;;N;;;;; +16935;BAMUM LETTER PHASE-D PAP;Lo;0;L;;;;;N;;;;; +16936;BAMUM LETTER PHASE-D YUOP;Lo;0;L;;;;;N;;;;; +16937;BAMUM LETTER PHASE-D NDAM;Lo;0;L;;;;;N;;;;; +16938;BAMUM LETTER PHASE-D NTEUM;Lo;0;L;;;;;N;;;;; +16939;BAMUM LETTER PHASE-D SUAE;Lo;0;L;;;;;N;;;;; +1693A;BAMUM LETTER PHASE-D KUN;Lo;0;L;;;;;N;;;;; +1693B;BAMUM LETTER PHASE-D NGGEUX;Lo;0;L;;;;;N;;;;; +1693C;BAMUM LETTER PHASE-D NGKIEE;Lo;0;L;;;;;N;;;;; +1693D;BAMUM LETTER PHASE-D TUOT;Lo;0;L;;;;;N;;;;; +1693E;BAMUM LETTER PHASE-D MEUN;Lo;0;L;;;;;N;;;;; +1693F;BAMUM LETTER PHASE-D KUQ;Lo;0;L;;;;;N;;;;; +16940;BAMUM LETTER PHASE-D NSUM;Lo;0;L;;;;;N;;;;; +16941;BAMUM LETTER PHASE-D TEUN;Lo;0;L;;;;;N;;;;; +16942;BAMUM LETTER PHASE-D MAENJET;Lo;0;L;;;;;N;;;;; +16943;BAMUM LETTER PHASE-D NGGAP;Lo;0;L;;;;;N;;;;; +16944;BAMUM LETTER PHASE-D LEUM;Lo;0;L;;;;;N;;;;; +16945;BAMUM LETTER PHASE-D NGGUOM;Lo;0;L;;;;;N;;;;; +16946;BAMUM LETTER PHASE-D NSHUT;Lo;0;L;;;;;N;;;;; +16947;BAMUM LETTER PHASE-D NJUEQ;Lo;0;L;;;;;N;;;;; +16948;BAMUM LETTER PHASE-D GHEUAE;Lo;0;L;;;;;N;;;;; +16949;BAMUM LETTER PHASE-D KU;Lo;0;L;;;;;N;;;;; +1694A;BAMUM LETTER PHASE-D REN OLD;Lo;0;L;;;;;N;;;;; +1694B;BAMUM LETTER PHASE-D TAE;Lo;0;L;;;;;N;;;;; +1694C;BAMUM LETTER PHASE-D TOQ;Lo;0;L;;;;;N;;;;; +1694D;BAMUM LETTER PHASE-D NYI;Lo;0;L;;;;;N;;;;; +1694E;BAMUM LETTER PHASE-D RII;Lo;0;L;;;;;N;;;;; +1694F;BAMUM LETTER PHASE-D LEEEE;Lo;0;L;;;;;N;;;;; +16950;BAMUM LETTER PHASE-D MEEEE;Lo;0;L;;;;;N;;;;; +16951;BAMUM LETTER PHASE-D M;Lo;0;L;;;;;N;;;;; +16952;BAMUM LETTER PHASE-D SUU;Lo;0;L;;;;;N;;;;; +16953;BAMUM LETTER PHASE-D MU;Lo;0;L;;;;;N;;;;; +16954;BAMUM LETTER PHASE-D SHII;Lo;0;L;;;;;N;;;;; +16955;BAMUM LETTER PHASE-D SHEUX;Lo;0;L;;;;;N;;;;; +16956;BAMUM LETTER PHASE-D KYEE;Lo;0;L;;;;;N;;;;; +16957;BAMUM LETTER PHASE-D NU;Lo;0;L;;;;;N;;;;; +16958;BAMUM LETTER PHASE-D SHU;Lo;0;L;;;;;N;;;;; +16959;BAMUM LETTER PHASE-D NTEE;Lo;0;L;;;;;N;;;;; +1695A;BAMUM LETTER PHASE-D PEE;Lo;0;L;;;;;N;;;;; +1695B;BAMUM LETTER PHASE-D NI;Lo;0;L;;;;;N;;;;; +1695C;BAMUM LETTER PHASE-D SHOQ;Lo;0;L;;;;;N;;;;; +1695D;BAMUM LETTER PHASE-D PUQ;Lo;0;L;;;;;N;;;;; +1695E;BAMUM LETTER PHASE-D MVOP;Lo;0;L;;;;;N;;;;; +1695F;BAMUM LETTER PHASE-D LOQ;Lo;0;L;;;;;N;;;;; +16960;BAMUM LETTER PHASE-D REN MUCH;Lo;0;L;;;;;N;;;;; +16961;BAMUM LETTER PHASE-D TI;Lo;0;L;;;;;N;;;;; +16962;BAMUM LETTER PHASE-D NTUU;Lo;0;L;;;;;N;;;;; +16963;BAMUM LETTER PHASE-D MBAA SEVEN;Lo;0;L;;;;;N;;;;; +16964;BAMUM LETTER PHASE-D SAQ;Lo;0;L;;;;;N;;;;; +16965;BAMUM LETTER PHASE-D FAA;Lo;0;L;;;;;N;;;;; +16966;BAMUM LETTER PHASE-E NDAP;Lo;0;L;;;;;N;;;;; +16967;BAMUM LETTER PHASE-E TOON;Lo;0;L;;;;;N;;;;; +16968;BAMUM LETTER PHASE-E MBEUM;Lo;0;L;;;;;N;;;;; +16969;BAMUM LETTER PHASE-E LAP;Lo;0;L;;;;;N;;;;; +1696A;BAMUM LETTER PHASE-E VOM;Lo;0;L;;;;;N;;;;; +1696B;BAMUM LETTER PHASE-E LOON;Lo;0;L;;;;;N;;;;; +1696C;BAMUM LETTER PHASE-E PAA;Lo;0;L;;;;;N;;;;; +1696D;BAMUM LETTER PHASE-E SOM;Lo;0;L;;;;;N;;;;; +1696E;BAMUM LETTER PHASE-E RAQ;Lo;0;L;;;;;N;;;;; +1696F;BAMUM LETTER PHASE-E NSHUOP;Lo;0;L;;;;;N;;;;; +16970;BAMUM LETTER PHASE-E NDUN;Lo;0;L;;;;;N;;;;; +16971;BAMUM LETTER PHASE-E PUAE;Lo;0;L;;;;;N;;;;; +16972;BAMUM LETTER PHASE-E TAM;Lo;0;L;;;;;N;;;;; +16973;BAMUM LETTER PHASE-E NGKA;Lo;0;L;;;;;N;;;;; +16974;BAMUM LETTER PHASE-E KPEUX;Lo;0;L;;;;;N;;;;; +16975;BAMUM LETTER PHASE-E WUO;Lo;0;L;;;;;N;;;;; +16976;BAMUM LETTER PHASE-E SEE;Lo;0;L;;;;;N;;;;; +16977;BAMUM LETTER PHASE-E NGGEUAET;Lo;0;L;;;;;N;;;;; +16978;BAMUM LETTER PHASE-E PAAM;Lo;0;L;;;;;N;;;;; +16979;BAMUM LETTER PHASE-E TOO;Lo;0;L;;;;;N;;;;; +1697A;BAMUM LETTER PHASE-E KUOP;Lo;0;L;;;;;N;;;;; +1697B;BAMUM LETTER PHASE-E LOM;Lo;0;L;;;;;N;;;;; +1697C;BAMUM LETTER PHASE-E NSHIEE;Lo;0;L;;;;;N;;;;; +1697D;BAMUM LETTER PHASE-E NGOP;Lo;0;L;;;;;N;;;;; +1697E;BAMUM LETTER PHASE-E MAEM;Lo;0;L;;;;;N;;;;; +1697F;BAMUM LETTER PHASE-E NGKEUX;Lo;0;L;;;;;N;;;;; +16980;BAMUM LETTER PHASE-E NGOQ;Lo;0;L;;;;;N;;;;; +16981;BAMUM LETTER PHASE-E NSHUE;Lo;0;L;;;;;N;;;;; +16982;BAMUM LETTER PHASE-E RIMGBA;Lo;0;L;;;;;N;;;;; +16983;BAMUM LETTER PHASE-E NJEUX;Lo;0;L;;;;;N;;;;; +16984;BAMUM LETTER PHASE-E PEEM;Lo;0;L;;;;;N;;;;; +16985;BAMUM LETTER PHASE-E SAA;Lo;0;L;;;;;N;;;;; +16986;BAMUM LETTER PHASE-E NGGURAE;Lo;0;L;;;;;N;;;;; +16987;BAMUM LETTER PHASE-E MGBA;Lo;0;L;;;;;N;;;;; +16988;BAMUM LETTER PHASE-E GHEUX;Lo;0;L;;;;;N;;;;; +16989;BAMUM LETTER PHASE-E NGKEUAEM;Lo;0;L;;;;;N;;;;; +1698A;BAMUM LETTER PHASE-E NJAEMLI;Lo;0;L;;;;;N;;;;; +1698B;BAMUM LETTER PHASE-E MAP;Lo;0;L;;;;;N;;;;; +1698C;BAMUM LETTER PHASE-E LOOT;Lo;0;L;;;;;N;;;;; +1698D;BAMUM LETTER PHASE-E NGGEEEE;Lo;0;L;;;;;N;;;;; +1698E;BAMUM LETTER PHASE-E NDIQ;Lo;0;L;;;;;N;;;;; +1698F;BAMUM LETTER PHASE-E TAEN NTEUM;Lo;0;L;;;;;N;;;;; +16990;BAMUM LETTER PHASE-E SET;Lo;0;L;;;;;N;;;;; +16991;BAMUM LETTER PHASE-E PUM;Lo;0;L;;;;;N;;;;; +16992;BAMUM LETTER PHASE-E NDAA SOFTNESS;Lo;0;L;;;;;N;;;;; +16993;BAMUM LETTER PHASE-E NGGUAESHAE NYAM;Lo;0;L;;;;;N;;;;; +16994;BAMUM LETTER PHASE-E YIEE;Lo;0;L;;;;;N;;;;; +16995;BAMUM LETTER PHASE-E GHEUN;Lo;0;L;;;;;N;;;;; +16996;BAMUM LETTER PHASE-E TUAE;Lo;0;L;;;;;N;;;;; +16997;BAMUM LETTER PHASE-E YEUAE;Lo;0;L;;;;;N;;;;; +16998;BAMUM LETTER PHASE-E PO;Lo;0;L;;;;;N;;;;; +16999;BAMUM LETTER PHASE-E TUMAE;Lo;0;L;;;;;N;;;;; +1699A;BAMUM LETTER PHASE-E KEUAE;Lo;0;L;;;;;N;;;;; +1699B;BAMUM LETTER PHASE-E SUAEN;Lo;0;L;;;;;N;;;;; +1699C;BAMUM LETTER PHASE-E TEUAEQ;Lo;0;L;;;;;N;;;;; +1699D;BAMUM LETTER PHASE-E VEUAE;Lo;0;L;;;;;N;;;;; +1699E;BAMUM LETTER PHASE-E WEUX;Lo;0;L;;;;;N;;;;; +1699F;BAMUM LETTER PHASE-E LAAM;Lo;0;L;;;;;N;;;;; +169A0;BAMUM LETTER PHASE-E PU;Lo;0;L;;;;;N;;;;; +169A1;BAMUM LETTER PHASE-E TAAQ;Lo;0;L;;;;;N;;;;; +169A2;BAMUM LETTER PHASE-E GHAAMAE;Lo;0;L;;;;;N;;;;; +169A3;BAMUM LETTER PHASE-E NGEUREUT;Lo;0;L;;;;;N;;;;; +169A4;BAMUM LETTER PHASE-E SHEUAEQ;Lo;0;L;;;;;N;;;;; +169A5;BAMUM LETTER PHASE-E MGBEN;Lo;0;L;;;;;N;;;;; +169A6;BAMUM LETTER PHASE-E MBEE;Lo;0;L;;;;;N;;;;; +169A7;BAMUM LETTER PHASE-E NZAQ;Lo;0;L;;;;;N;;;;; +169A8;BAMUM LETTER PHASE-E NKOM;Lo;0;L;;;;;N;;;;; +169A9;BAMUM LETTER PHASE-E GBET;Lo;0;L;;;;;N;;;;; +169AA;BAMUM LETTER PHASE-E TUM;Lo;0;L;;;;;N;;;;; +169AB;BAMUM LETTER PHASE-E KUET;Lo;0;L;;;;;N;;;;; +169AC;BAMUM LETTER PHASE-E YAP;Lo;0;L;;;;;N;;;;; +169AD;BAMUM LETTER PHASE-E NYI CLEAVER;Lo;0;L;;;;;N;;;;; +169AE;BAMUM LETTER PHASE-E YIT;Lo;0;L;;;;;N;;;;; +169AF;BAMUM LETTER PHASE-E MFEUQ;Lo;0;L;;;;;N;;;;; +169B0;BAMUM LETTER PHASE-E NDIAQ;Lo;0;L;;;;;N;;;;; +169B1;BAMUM LETTER PHASE-E PIEEQ;Lo;0;L;;;;;N;;;;; +169B2;BAMUM LETTER PHASE-E YUEQ;Lo;0;L;;;;;N;;;;; +169B3;BAMUM LETTER PHASE-E LEUAEM;Lo;0;L;;;;;N;;;;; +169B4;BAMUM LETTER PHASE-E FUE;Lo;0;L;;;;;N;;;;; +169B5;BAMUM LETTER PHASE-E GBEUX;Lo;0;L;;;;;N;;;;; +169B6;BAMUM LETTER PHASE-E NGKUP;Lo;0;L;;;;;N;;;;; +169B7;BAMUM LETTER PHASE-E KET;Lo;0;L;;;;;N;;;;; +169B8;BAMUM LETTER PHASE-E MAE;Lo;0;L;;;;;N;;;;; +169B9;BAMUM LETTER PHASE-E NGKAAMI;Lo;0;L;;;;;N;;;;; +169BA;BAMUM LETTER PHASE-E GHET;Lo;0;L;;;;;N;;;;; +169BB;BAMUM LETTER PHASE-E FA;Lo;0;L;;;;;N;;;;; +169BC;BAMUM LETTER PHASE-E NTUM;Lo;0;L;;;;;N;;;;; +169BD;BAMUM LETTER PHASE-E PEUT;Lo;0;L;;;;;N;;;;; +169BE;BAMUM LETTER PHASE-E YEUM;Lo;0;L;;;;;N;;;;; +169BF;BAMUM LETTER PHASE-E NGGEUAE;Lo;0;L;;;;;N;;;;; +169C0;BAMUM LETTER PHASE-E NYI BETWEEN;Lo;0;L;;;;;N;;;;; +169C1;BAMUM LETTER PHASE-E NZUQ;Lo;0;L;;;;;N;;;;; +169C2;BAMUM LETTER PHASE-E POON;Lo;0;L;;;;;N;;;;; +169C3;BAMUM LETTER PHASE-E MIEE;Lo;0;L;;;;;N;;;;; +169C4;BAMUM LETTER PHASE-E FUET;Lo;0;L;;;;;N;;;;; +169C5;BAMUM LETTER PHASE-E NAE;Lo;0;L;;;;;N;;;;; +169C6;BAMUM LETTER PHASE-E MUAE;Lo;0;L;;;;;N;;;;; +169C7;BAMUM LETTER PHASE-E GHEUAE;Lo;0;L;;;;;N;;;;; +169C8;BAMUM LETTER PHASE-E FU I;Lo;0;L;;;;;N;;;;; +169C9;BAMUM LETTER PHASE-E MVI;Lo;0;L;;;;;N;;;;; +169CA;BAMUM LETTER PHASE-E PUAQ;Lo;0;L;;;;;N;;;;; +169CB;BAMUM LETTER PHASE-E NGKUM;Lo;0;L;;;;;N;;;;; +169CC;BAMUM LETTER PHASE-E KUT;Lo;0;L;;;;;N;;;;; +169CD;BAMUM LETTER PHASE-E PIET;Lo;0;L;;;;;N;;;;; +169CE;BAMUM LETTER PHASE-E NTAP;Lo;0;L;;;;;N;;;;; +169CF;BAMUM LETTER PHASE-E YEUAET;Lo;0;L;;;;;N;;;;; +169D0;BAMUM LETTER PHASE-E NGGUP;Lo;0;L;;;;;N;;;;; +169D1;BAMUM LETTER PHASE-E PA PEOPLE;Lo;0;L;;;;;N;;;;; +169D2;BAMUM LETTER PHASE-E FU CALL;Lo;0;L;;;;;N;;;;; +169D3;BAMUM LETTER PHASE-E FOM;Lo;0;L;;;;;N;;;;; +169D4;BAMUM LETTER PHASE-E NJEE;Lo;0;L;;;;;N;;;;; +169D5;BAMUM LETTER PHASE-E A;Lo;0;L;;;;;N;;;;; +169D6;BAMUM LETTER PHASE-E TOQ;Lo;0;L;;;;;N;;;;; +169D7;BAMUM LETTER PHASE-E O;Lo;0;L;;;;;N;;;;; +169D8;BAMUM LETTER PHASE-E I;Lo;0;L;;;;;N;;;;; +169D9;BAMUM LETTER PHASE-E LAQ;Lo;0;L;;;;;N;;;;; +169DA;BAMUM LETTER PHASE-E PA PLURAL;Lo;0;L;;;;;N;;;;; +169DB;BAMUM LETTER PHASE-E TAA;Lo;0;L;;;;;N;;;;; +169DC;BAMUM LETTER PHASE-E TAQ;Lo;0;L;;;;;N;;;;; +169DD;BAMUM LETTER PHASE-E NDAA MY HOUSE;Lo;0;L;;;;;N;;;;; +169DE;BAMUM LETTER PHASE-E SHIQ;Lo;0;L;;;;;N;;;;; +169DF;BAMUM LETTER PHASE-E YEUX;Lo;0;L;;;;;N;;;;; +169E0;BAMUM LETTER PHASE-E NGUAE;Lo;0;L;;;;;N;;;;; +169E1;BAMUM LETTER PHASE-E YUAEN;Lo;0;L;;;;;N;;;;; +169E2;BAMUM LETTER PHASE-E YOQ SWIMMING;Lo;0;L;;;;;N;;;;; +169E3;BAMUM LETTER PHASE-E YOQ COVER;Lo;0;L;;;;;N;;;;; +169E4;BAMUM LETTER PHASE-E YUQ;Lo;0;L;;;;;N;;;;; +169E5;BAMUM LETTER PHASE-E YUN;Lo;0;L;;;;;N;;;;; +169E6;BAMUM LETTER PHASE-E KEUX;Lo;0;L;;;;;N;;;;; +169E7;BAMUM LETTER PHASE-E PEUX;Lo;0;L;;;;;N;;;;; +169E8;BAMUM LETTER PHASE-E NJEE EPOCH;Lo;0;L;;;;;N;;;;; +169E9;BAMUM LETTER PHASE-E PUE;Lo;0;L;;;;;N;;;;; +169EA;BAMUM LETTER PHASE-E WUE;Lo;0;L;;;;;N;;;;; +169EB;BAMUM LETTER PHASE-E FEE;Lo;0;L;;;;;N;;;;; +169EC;BAMUM LETTER PHASE-E VEE;Lo;0;L;;;;;N;;;;; +169ED;BAMUM LETTER PHASE-E LU;Lo;0;L;;;;;N;;;;; +169EE;BAMUM LETTER PHASE-E MI;Lo;0;L;;;;;N;;;;; +169EF;BAMUM LETTER PHASE-E REUX;Lo;0;L;;;;;N;;;;; +169F0;BAMUM LETTER PHASE-E RAE;Lo;0;L;;;;;N;;;;; +169F1;BAMUM LETTER PHASE-E NGUAET;Lo;0;L;;;;;N;;;;; +169F2;BAMUM LETTER PHASE-E NGA;Lo;0;L;;;;;N;;;;; +169F3;BAMUM LETTER PHASE-E SHO;Lo;0;L;;;;;N;;;;; +169F4;BAMUM LETTER PHASE-E SHOQ;Lo;0;L;;;;;N;;;;; +169F5;BAMUM LETTER PHASE-E FU REMEDY;Lo;0;L;;;;;N;;;;; +169F6;BAMUM LETTER PHASE-E NA;Lo;0;L;;;;;N;;;;; +169F7;BAMUM LETTER PHASE-E PI;Lo;0;L;;;;;N;;;;; +169F8;BAMUM LETTER PHASE-E LOQ;Lo;0;L;;;;;N;;;;; +169F9;BAMUM LETTER PHASE-E KO;Lo;0;L;;;;;N;;;;; +169FA;BAMUM LETTER PHASE-E MEN;Lo;0;L;;;;;N;;;;; +169FB;BAMUM LETTER PHASE-E MA;Lo;0;L;;;;;N;;;;; +169FC;BAMUM LETTER PHASE-E MAQ;Lo;0;L;;;;;N;;;;; +169FD;BAMUM LETTER PHASE-E TEU;Lo;0;L;;;;;N;;;;; +169FE;BAMUM LETTER PHASE-E KI;Lo;0;L;;;;;N;;;;; +169FF;BAMUM LETTER PHASE-E MON;Lo;0;L;;;;;N;;;;; +16A00;BAMUM LETTER PHASE-E TEN;Lo;0;L;;;;;N;;;;; +16A01;BAMUM LETTER PHASE-E FAQ;Lo;0;L;;;;;N;;;;; +16A02;BAMUM LETTER PHASE-E GHOM;Lo;0;L;;;;;N;;;;; +16A03;BAMUM LETTER PHASE-F KA;Lo;0;L;;;;;N;;;;; +16A04;BAMUM LETTER PHASE-F U;Lo;0;L;;;;;N;;;;; +16A05;BAMUM LETTER PHASE-F KU;Lo;0;L;;;;;N;;;;; +16A06;BAMUM LETTER PHASE-F EE;Lo;0;L;;;;;N;;;;; +16A07;BAMUM LETTER PHASE-F REE;Lo;0;L;;;;;N;;;;; +16A08;BAMUM LETTER PHASE-F TAE;Lo;0;L;;;;;N;;;;; +16A09;BAMUM LETTER PHASE-F NYI;Lo;0;L;;;;;N;;;;; +16A0A;BAMUM LETTER PHASE-F LA;Lo;0;L;;;;;N;;;;; +16A0B;BAMUM LETTER PHASE-F RII;Lo;0;L;;;;;N;;;;; +16A0C;BAMUM LETTER PHASE-F RIEE;Lo;0;L;;;;;N;;;;; +16A0D;BAMUM LETTER PHASE-F MEEEE;Lo;0;L;;;;;N;;;;; +16A0E;BAMUM LETTER PHASE-F TAA;Lo;0;L;;;;;N;;;;; +16A0F;BAMUM LETTER PHASE-F NDAA;Lo;0;L;;;;;N;;;;; +16A10;BAMUM LETTER PHASE-F NJAEM;Lo;0;L;;;;;N;;;;; +16A11;BAMUM LETTER PHASE-F M;Lo;0;L;;;;;N;;;;; +16A12;BAMUM LETTER PHASE-F SUU;Lo;0;L;;;;;N;;;;; +16A13;BAMUM LETTER PHASE-F SHII;Lo;0;L;;;;;N;;;;; +16A14;BAMUM LETTER PHASE-F SI;Lo;0;L;;;;;N;;;;; +16A15;BAMUM LETTER PHASE-F SEUX;Lo;0;L;;;;;N;;;;; +16A16;BAMUM LETTER PHASE-F KYEE;Lo;0;L;;;;;N;;;;; +16A17;BAMUM LETTER PHASE-F KET;Lo;0;L;;;;;N;;;;; +16A18;BAMUM LETTER PHASE-F NUAE;Lo;0;L;;;;;N;;;;; +16A19;BAMUM LETTER PHASE-F NU;Lo;0;L;;;;;N;;;;; +16A1A;BAMUM LETTER PHASE-F NJUAE;Lo;0;L;;;;;N;;;;; +16A1B;BAMUM LETTER PHASE-F YOQ;Lo;0;L;;;;;N;;;;; +16A1C;BAMUM LETTER PHASE-F SHU;Lo;0;L;;;;;N;;;;; +16A1D;BAMUM LETTER PHASE-F YA;Lo;0;L;;;;;N;;;;; +16A1E;BAMUM LETTER PHASE-F NSHA;Lo;0;L;;;;;N;;;;; +16A1F;BAMUM LETTER PHASE-F PEUX;Lo;0;L;;;;;N;;;;; +16A20;BAMUM LETTER PHASE-F NTEE;Lo;0;L;;;;;N;;;;; +16A21;BAMUM LETTER PHASE-F WUE;Lo;0;L;;;;;N;;;;; +16A22;BAMUM LETTER PHASE-F PEE;Lo;0;L;;;;;N;;;;; +16A23;BAMUM LETTER PHASE-F RU;Lo;0;L;;;;;N;;;;; +16A24;BAMUM LETTER PHASE-F NI;Lo;0;L;;;;;N;;;;; +16A25;BAMUM LETTER PHASE-F REUX;Lo;0;L;;;;;N;;;;; +16A26;BAMUM LETTER PHASE-F KEN;Lo;0;L;;;;;N;;;;; +16A27;BAMUM LETTER PHASE-F NGKWAEN;Lo;0;L;;;;;N;;;;; +16A28;BAMUM LETTER PHASE-F NGGA;Lo;0;L;;;;;N;;;;; +16A29;BAMUM LETTER PHASE-F SHO;Lo;0;L;;;;;N;;;;; +16A2A;BAMUM LETTER PHASE-F PUAE;Lo;0;L;;;;;N;;;;; +16A2B;BAMUM LETTER PHASE-F FOM;Lo;0;L;;;;;N;;;;; +16A2C;BAMUM LETTER PHASE-F WA;Lo;0;L;;;;;N;;;;; +16A2D;BAMUM LETTER PHASE-F LI;Lo;0;L;;;;;N;;;;; +16A2E;BAMUM LETTER PHASE-F LOQ;Lo;0;L;;;;;N;;;;; +16A2F;BAMUM LETTER PHASE-F KO;Lo;0;L;;;;;N;;;;; +16A30;BAMUM LETTER PHASE-F MBEN;Lo;0;L;;;;;N;;;;; +16A31;BAMUM LETTER PHASE-F REN;Lo;0;L;;;;;N;;;;; +16A32;BAMUM LETTER PHASE-F MA;Lo;0;L;;;;;N;;;;; +16A33;BAMUM LETTER PHASE-F MO;Lo;0;L;;;;;N;;;;; +16A34;BAMUM LETTER PHASE-F MBAA;Lo;0;L;;;;;N;;;;; +16A35;BAMUM LETTER PHASE-F TET;Lo;0;L;;;;;N;;;;; +16A36;BAMUM LETTER PHASE-F KPA;Lo;0;L;;;;;N;;;;; +16A37;BAMUM LETTER PHASE-F SAMBA;Lo;0;L;;;;;N;;;;; +16A38;BAMUM LETTER PHASE-F VUEQ;Lo;0;L;;;;;N;;;;; +16F00;MIAO LETTER PA;Lo;0;L;;;;;N;;;;; +16F01;MIAO LETTER BA;Lo;0;L;;;;;N;;;;; +16F02;MIAO LETTER YI PA;Lo;0;L;;;;;N;;;;; +16F03;MIAO LETTER PLA;Lo;0;L;;;;;N;;;;; +16F04;MIAO LETTER MA;Lo;0;L;;;;;N;;;;; +16F05;MIAO LETTER MHA;Lo;0;L;;;;;N;;;;; +16F06;MIAO LETTER ARCHAIC MA;Lo;0;L;;;;;N;;;;; +16F07;MIAO LETTER FA;Lo;0;L;;;;;N;;;;; +16F08;MIAO LETTER VA;Lo;0;L;;;;;N;;;;; +16F09;MIAO LETTER VFA;Lo;0;L;;;;;N;;;;; +16F0A;MIAO LETTER TA;Lo;0;L;;;;;N;;;;; +16F0B;MIAO LETTER DA;Lo;0;L;;;;;N;;;;; +16F0C;MIAO LETTER YI TTA;Lo;0;L;;;;;N;;;;; +16F0D;MIAO LETTER YI TA;Lo;0;L;;;;;N;;;;; +16F0E;MIAO LETTER TTA;Lo;0;L;;;;;N;;;;; +16F0F;MIAO LETTER DDA;Lo;0;L;;;;;N;;;;; +16F10;MIAO LETTER NA;Lo;0;L;;;;;N;;;;; +16F11;MIAO LETTER NHA;Lo;0;L;;;;;N;;;;; +16F12;MIAO LETTER YI NNA;Lo;0;L;;;;;N;;;;; +16F13;MIAO LETTER ARCHAIC NA;Lo;0;L;;;;;N;;;;; +16F14;MIAO LETTER NNA;Lo;0;L;;;;;N;;;;; +16F15;MIAO LETTER NNHA;Lo;0;L;;;;;N;;;;; +16F16;MIAO LETTER LA;Lo;0;L;;;;;N;;;;; +16F17;MIAO LETTER LYA;Lo;0;L;;;;;N;;;;; +16F18;MIAO LETTER LHA;Lo;0;L;;;;;N;;;;; +16F19;MIAO LETTER LHYA;Lo;0;L;;;;;N;;;;; +16F1A;MIAO LETTER TLHA;Lo;0;L;;;;;N;;;;; +16F1B;MIAO LETTER DLHA;Lo;0;L;;;;;N;;;;; +16F1C;MIAO LETTER TLHYA;Lo;0;L;;;;;N;;;;; +16F1D;MIAO LETTER DLHYA;Lo;0;L;;;;;N;;;;; +16F1E;MIAO LETTER KA;Lo;0;L;;;;;N;;;;; +16F1F;MIAO LETTER GA;Lo;0;L;;;;;N;;;;; +16F20;MIAO LETTER YI KA;Lo;0;L;;;;;N;;;;; +16F21;MIAO LETTER QA;Lo;0;L;;;;;N;;;;; +16F22;MIAO LETTER QGA;Lo;0;L;;;;;N;;;;; +16F23;MIAO LETTER NGA;Lo;0;L;;;;;N;;;;; +16F24;MIAO LETTER NGHA;Lo;0;L;;;;;N;;;;; +16F25;MIAO LETTER ARCHAIC NGA;Lo;0;L;;;;;N;;;;; +16F26;MIAO LETTER HA;Lo;0;L;;;;;N;;;;; +16F27;MIAO LETTER XA;Lo;0;L;;;;;N;;;;; +16F28;MIAO LETTER GHA;Lo;0;L;;;;;N;;;;; +16F29;MIAO LETTER GHHA;Lo;0;L;;;;;N;;;;; +16F2A;MIAO LETTER TSSA;Lo;0;L;;;;;N;;;;; +16F2B;MIAO LETTER DZZA;Lo;0;L;;;;;N;;;;; +16F2C;MIAO LETTER NYA;Lo;0;L;;;;;N;;;;; +16F2D;MIAO LETTER NYHA;Lo;0;L;;;;;N;;;;; +16F2E;MIAO LETTER TSHA;Lo;0;L;;;;;N;;;;; +16F2F;MIAO LETTER DZHA;Lo;0;L;;;;;N;;;;; +16F30;MIAO LETTER YI TSHA;Lo;0;L;;;;;N;;;;; +16F31;MIAO LETTER YI DZHA;Lo;0;L;;;;;N;;;;; +16F32;MIAO LETTER REFORMED TSHA;Lo;0;L;;;;;N;;;;; +16F33;MIAO LETTER SHA;Lo;0;L;;;;;N;;;;; +16F34;MIAO LETTER SSA;Lo;0;L;;;;;N;;;;; +16F35;MIAO LETTER ZHA;Lo;0;L;;;;;N;;;;; +16F36;MIAO LETTER ZSHA;Lo;0;L;;;;;N;;;;; +16F37;MIAO LETTER TSA;Lo;0;L;;;;;N;;;;; +16F38;MIAO LETTER DZA;Lo;0;L;;;;;N;;;;; +16F39;MIAO LETTER YI TSA;Lo;0;L;;;;;N;;;;; +16F3A;MIAO LETTER SA;Lo;0;L;;;;;N;;;;; +16F3B;MIAO LETTER ZA;Lo;0;L;;;;;N;;;;; +16F3C;MIAO LETTER ZSA;Lo;0;L;;;;;N;;;;; +16F3D;MIAO LETTER ZZA;Lo;0;L;;;;;N;;;;; +16F3E;MIAO LETTER ZZSA;Lo;0;L;;;;;N;;;;; +16F3F;MIAO LETTER ARCHAIC ZZA;Lo;0;L;;;;;N;;;;; +16F40;MIAO LETTER ZZYA;Lo;0;L;;;;;N;;;;; +16F41;MIAO LETTER ZZSYA;Lo;0;L;;;;;N;;;;; +16F42;MIAO LETTER WA;Lo;0;L;;;;;N;;;;; +16F43;MIAO LETTER AH;Lo;0;L;;;;;N;;;;; +16F44;MIAO LETTER HHA;Lo;0;L;;;;;N;;;;; +16F50;MIAO LETTER NASALIZATION;Lo;0;L;;;;;N;;;;; +16F51;MIAO SIGN ASPIRATION;Mc;0;L;;;;;N;;;;; +16F52;MIAO SIGN REFORMED VOICING;Mc;0;L;;;;;N;;;;; +16F53;MIAO SIGN REFORMED ASPIRATION;Mc;0;L;;;;;N;;;;; +16F54;MIAO VOWEL SIGN A;Mc;0;L;;;;;N;;;;; +16F55;MIAO VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +16F56;MIAO VOWEL SIGN AHH;Mc;0;L;;;;;N;;;;; +16F57;MIAO VOWEL SIGN AN;Mc;0;L;;;;;N;;;;; +16F58;MIAO VOWEL SIGN ANG;Mc;0;L;;;;;N;;;;; +16F59;MIAO VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +16F5A;MIAO VOWEL SIGN OO;Mc;0;L;;;;;N;;;;; +16F5B;MIAO VOWEL SIGN WO;Mc;0;L;;;;;N;;;;; +16F5C;MIAO VOWEL SIGN W;Mc;0;L;;;;;N;;;;; +16F5D;MIAO VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +16F5E;MIAO VOWEL SIGN EN;Mc;0;L;;;;;N;;;;; +16F5F;MIAO VOWEL SIGN ENG;Mc;0;L;;;;;N;;;;; +16F60;MIAO VOWEL SIGN OEY;Mc;0;L;;;;;N;;;;; +16F61;MIAO VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +16F62;MIAO VOWEL SIGN IA;Mc;0;L;;;;;N;;;;; +16F63;MIAO VOWEL SIGN IAN;Mc;0;L;;;;;N;;;;; +16F64;MIAO VOWEL SIGN IANG;Mc;0;L;;;;;N;;;;; +16F65;MIAO VOWEL SIGN IO;Mc;0;L;;;;;N;;;;; +16F66;MIAO VOWEL SIGN IE;Mc;0;L;;;;;N;;;;; +16F67;MIAO VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +16F68;MIAO VOWEL SIGN IU;Mc;0;L;;;;;N;;;;; +16F69;MIAO VOWEL SIGN ING;Mc;0;L;;;;;N;;;;; +16F6A;MIAO VOWEL SIGN U;Mc;0;L;;;;;N;;;;; +16F6B;MIAO VOWEL SIGN UA;Mc;0;L;;;;;N;;;;; +16F6C;MIAO VOWEL SIGN UAN;Mc;0;L;;;;;N;;;;; +16F6D;MIAO VOWEL SIGN UANG;Mc;0;L;;;;;N;;;;; +16F6E;MIAO VOWEL SIGN UU;Mc;0;L;;;;;N;;;;; +16F6F;MIAO VOWEL SIGN UEI;Mc;0;L;;;;;N;;;;; +16F70;MIAO VOWEL SIGN UNG;Mc;0;L;;;;;N;;;;; +16F71;MIAO VOWEL SIGN Y;Mc;0;L;;;;;N;;;;; +16F72;MIAO VOWEL SIGN YI;Mc;0;L;;;;;N;;;;; +16F73;MIAO VOWEL SIGN AE;Mc;0;L;;;;;N;;;;; +16F74;MIAO VOWEL SIGN AEE;Mc;0;L;;;;;N;;;;; +16F75;MIAO VOWEL SIGN ERR;Mc;0;L;;;;;N;;;;; +16F76;MIAO VOWEL SIGN ROUNDED ERR;Mc;0;L;;;;;N;;;;; +16F77;MIAO VOWEL SIGN ER;Mc;0;L;;;;;N;;;;; +16F78;MIAO VOWEL SIGN ROUNDED ER;Mc;0;L;;;;;N;;;;; +16F79;MIAO VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +16F7A;MIAO VOWEL SIGN EI;Mc;0;L;;;;;N;;;;; +16F7B;MIAO VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +16F7C;MIAO VOWEL SIGN OU;Mc;0;L;;;;;N;;;;; +16F7D;MIAO VOWEL SIGN N;Mc;0;L;;;;;N;;;;; +16F7E;MIAO VOWEL SIGN NG;Mc;0;L;;;;;N;;;;; +16F8F;MIAO TONE RIGHT;Mn;0;NSM;;;;;N;;;;; +16F90;MIAO TONE TOP RIGHT;Mn;0;NSM;;;;;N;;;;; +16F91;MIAO TONE ABOVE;Mn;0;NSM;;;;;N;;;;; +16F92;MIAO TONE BELOW;Mn;0;NSM;;;;;N;;;;; +16F93;MIAO LETTER TONE-2;Lm;0;L;;;;;N;;;;; +16F94;MIAO LETTER TONE-3;Lm;0;L;;;;;N;;;;; +16F95;MIAO LETTER TONE-4;Lm;0;L;;;;;N;;;;; +16F96;MIAO LETTER TONE-5;Lm;0;L;;;;;N;;;;; +16F97;MIAO LETTER TONE-6;Lm;0;L;;;;;N;;;;; +16F98;MIAO LETTER TONE-7;Lm;0;L;;;;;N;;;;; +16F99;MIAO LETTER TONE-8;Lm;0;L;;;;;N;;;;; +16F9A;MIAO LETTER REFORMED TONE-1;Lm;0;L;;;;;N;;;;; +16F9B;MIAO LETTER REFORMED TONE-2;Lm;0;L;;;;;N;;;;; +16F9C;MIAO LETTER REFORMED TONE-4;Lm;0;L;;;;;N;;;;; +16F9D;MIAO LETTER REFORMED TONE-5;Lm;0;L;;;;;N;;;;; +16F9E;MIAO LETTER REFORMED TONE-6;Lm;0;L;;;;;N;;;;; +16F9F;MIAO LETTER REFORMED TONE-8;Lm;0;L;;;;;N;;;;; +1B000;KATAKANA LETTER ARCHAIC E;Lo;0;L;;;;;N;;;;; +1B001;HIRAGANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; +1D000;BYZANTINE MUSICAL SYMBOL PSILI;So;0;L;;;;;N;;;;; +1D001;BYZANTINE MUSICAL SYMBOL DASEIA;So;0;L;;;;;N;;;;; +1D002;BYZANTINE MUSICAL SYMBOL PERISPOMENI;So;0;L;;;;;N;;;;; +1D003;BYZANTINE MUSICAL SYMBOL OXEIA EKFONITIKON;So;0;L;;;;;N;;;;; +1D004;BYZANTINE MUSICAL SYMBOL OXEIA DIPLI;So;0;L;;;;;N;;;;; +1D005;BYZANTINE MUSICAL SYMBOL VAREIA EKFONITIKON;So;0;L;;;;;N;;;;; +1D006;BYZANTINE MUSICAL SYMBOL VAREIA DIPLI;So;0;L;;;;;N;;;;; +1D007;BYZANTINE MUSICAL SYMBOL KATHISTI;So;0;L;;;;;N;;;;; +1D008;BYZANTINE MUSICAL SYMBOL SYRMATIKI;So;0;L;;;;;N;;;;; +1D009;BYZANTINE MUSICAL SYMBOL PARAKLITIKI;So;0;L;;;;;N;;;;; +1D00A;BYZANTINE MUSICAL SYMBOL YPOKRISIS;So;0;L;;;;;N;;;;; +1D00B;BYZANTINE MUSICAL SYMBOL YPOKRISIS DIPLI;So;0;L;;;;;N;;;;; +1D00C;BYZANTINE MUSICAL SYMBOL KREMASTI;So;0;L;;;;;N;;;;; +1D00D;BYZANTINE MUSICAL SYMBOL APESO EKFONITIKON;So;0;L;;;;;N;;;;; +1D00E;BYZANTINE MUSICAL SYMBOL EXO EKFONITIKON;So;0;L;;;;;N;;;;; +1D00F;BYZANTINE MUSICAL SYMBOL TELEIA;So;0;L;;;;;N;;;;; +1D010;BYZANTINE MUSICAL SYMBOL KENTIMATA;So;0;L;;;;;N;;;;; +1D011;BYZANTINE MUSICAL SYMBOL APOSTROFOS;So;0;L;;;;;N;;;;; +1D012;BYZANTINE MUSICAL SYMBOL APOSTROFOS DIPLI;So;0;L;;;;;N;;;;; +1D013;BYZANTINE MUSICAL SYMBOL SYNEVMA;So;0;L;;;;;N;;;;; +1D014;BYZANTINE MUSICAL SYMBOL THITA;So;0;L;;;;;N;;;;; +1D015;BYZANTINE MUSICAL SYMBOL OLIGON ARCHAION;So;0;L;;;;;N;;;;; +1D016;BYZANTINE MUSICAL SYMBOL GORGON ARCHAION;So;0;L;;;;;N;;;;; +1D017;BYZANTINE MUSICAL SYMBOL PSILON;So;0;L;;;;;N;;;;; +1D018;BYZANTINE MUSICAL SYMBOL CHAMILON;So;0;L;;;;;N;;;;; +1D019;BYZANTINE MUSICAL SYMBOL VATHY;So;0;L;;;;;N;;;;; +1D01A;BYZANTINE MUSICAL SYMBOL ISON ARCHAION;So;0;L;;;;;N;;;;; +1D01B;BYZANTINE MUSICAL SYMBOL KENTIMA ARCHAION;So;0;L;;;;;N;;;;; +1D01C;BYZANTINE MUSICAL SYMBOL KENTIMATA ARCHAION;So;0;L;;;;;N;;;;; +1D01D;BYZANTINE MUSICAL SYMBOL SAXIMATA;So;0;L;;;;;N;;;;; +1D01E;BYZANTINE MUSICAL SYMBOL PARICHON;So;0;L;;;;;N;;;;; +1D01F;BYZANTINE MUSICAL SYMBOL STAVROS APODEXIA;So;0;L;;;;;N;;;;; +1D020;BYZANTINE MUSICAL SYMBOL OXEIAI ARCHAION;So;0;L;;;;;N;;;;; +1D021;BYZANTINE MUSICAL SYMBOL VAREIAI ARCHAION;So;0;L;;;;;N;;;;; +1D022;BYZANTINE MUSICAL SYMBOL APODERMA ARCHAION;So;0;L;;;;;N;;;;; +1D023;BYZANTINE MUSICAL SYMBOL APOTHEMA;So;0;L;;;;;N;;;;; +1D024;BYZANTINE MUSICAL SYMBOL KLASMA;So;0;L;;;;;N;;;;; +1D025;BYZANTINE MUSICAL SYMBOL REVMA;So;0;L;;;;;N;;;;; +1D026;BYZANTINE MUSICAL SYMBOL PIASMA ARCHAION;So;0;L;;;;;N;;;;; +1D027;BYZANTINE MUSICAL SYMBOL TINAGMA;So;0;L;;;;;N;;;;; +1D028;BYZANTINE MUSICAL SYMBOL ANATRICHISMA;So;0;L;;;;;N;;;;; +1D029;BYZANTINE MUSICAL SYMBOL SEISMA;So;0;L;;;;;N;;;;; +1D02A;BYZANTINE MUSICAL SYMBOL SYNAGMA ARCHAION;So;0;L;;;;;N;;;;; +1D02B;BYZANTINE MUSICAL SYMBOL SYNAGMA META STAVROU;So;0;L;;;;;N;;;;; +1D02C;BYZANTINE MUSICAL SYMBOL OYRANISMA ARCHAION;So;0;L;;;;;N;;;;; +1D02D;BYZANTINE MUSICAL SYMBOL THEMA;So;0;L;;;;;N;;;;; +1D02E;BYZANTINE MUSICAL SYMBOL LEMOI;So;0;L;;;;;N;;;;; +1D02F;BYZANTINE MUSICAL SYMBOL DYO;So;0;L;;;;;N;;;;; +1D030;BYZANTINE MUSICAL SYMBOL TRIA;So;0;L;;;;;N;;;;; +1D031;BYZANTINE MUSICAL SYMBOL TESSERA;So;0;L;;;;;N;;;;; +1D032;BYZANTINE MUSICAL SYMBOL KRATIMATA;So;0;L;;;;;N;;;;; +1D033;BYZANTINE MUSICAL SYMBOL APESO EXO NEO;So;0;L;;;;;N;;;;; +1D034;BYZANTINE MUSICAL SYMBOL FTHORA ARCHAION;So;0;L;;;;;N;;;;; +1D035;BYZANTINE MUSICAL SYMBOL IMIFTHORA;So;0;L;;;;;N;;;;; +1D036;BYZANTINE MUSICAL SYMBOL TROMIKON ARCHAION;So;0;L;;;;;N;;;;; +1D037;BYZANTINE MUSICAL SYMBOL KATAVA TROMIKON;So;0;L;;;;;N;;;;; +1D038;BYZANTINE MUSICAL SYMBOL PELASTON;So;0;L;;;;;N;;;;; +1D039;BYZANTINE MUSICAL SYMBOL PSIFISTON;So;0;L;;;;;N;;;;; +1D03A;BYZANTINE MUSICAL SYMBOL KONTEVMA;So;0;L;;;;;N;;;;; +1D03B;BYZANTINE MUSICAL SYMBOL CHOREVMA ARCHAION;So;0;L;;;;;N;;;;; +1D03C;BYZANTINE MUSICAL SYMBOL RAPISMA;So;0;L;;;;;N;;;;; +1D03D;BYZANTINE MUSICAL SYMBOL PARAKALESMA ARCHAION;So;0;L;;;;;N;;;;; +1D03E;BYZANTINE MUSICAL SYMBOL PARAKLITIKI ARCHAION;So;0;L;;;;;N;;;;; +1D03F;BYZANTINE MUSICAL SYMBOL ICHADIN;So;0;L;;;;;N;;;;; +1D040;BYZANTINE MUSICAL SYMBOL NANA;So;0;L;;;;;N;;;;; +1D041;BYZANTINE MUSICAL SYMBOL PETASMA;So;0;L;;;;;N;;;;; +1D042;BYZANTINE MUSICAL SYMBOL KONTEVMA ALLO;So;0;L;;;;;N;;;;; +1D043;BYZANTINE MUSICAL SYMBOL TROMIKON ALLO;So;0;L;;;;;N;;;;; +1D044;BYZANTINE MUSICAL SYMBOL STRAGGISMATA;So;0;L;;;;;N;;;;; +1D045;BYZANTINE MUSICAL SYMBOL GRONTHISMATA;So;0;L;;;;;N;;;;; +1D046;BYZANTINE MUSICAL SYMBOL ISON NEO;So;0;L;;;;;N;;;;; +1D047;BYZANTINE MUSICAL SYMBOL OLIGON NEO;So;0;L;;;;;N;;;;; +1D048;BYZANTINE MUSICAL SYMBOL OXEIA NEO;So;0;L;;;;;N;;;;; +1D049;BYZANTINE MUSICAL SYMBOL PETASTI;So;0;L;;;;;N;;;;; +1D04A;BYZANTINE MUSICAL SYMBOL KOUFISMA;So;0;L;;;;;N;;;;; +1D04B;BYZANTINE MUSICAL SYMBOL PETASTOKOUFISMA;So;0;L;;;;;N;;;;; +1D04C;BYZANTINE MUSICAL SYMBOL KRATIMOKOUFISMA;So;0;L;;;;;N;;;;; +1D04D;BYZANTINE MUSICAL SYMBOL PELASTON NEO;So;0;L;;;;;N;;;;; +1D04E;BYZANTINE MUSICAL SYMBOL KENTIMATA NEO ANO;So;0;L;;;;;N;;;;; +1D04F;BYZANTINE MUSICAL SYMBOL KENTIMA NEO ANO;So;0;L;;;;;N;;;;; +1D050;BYZANTINE MUSICAL SYMBOL YPSILI;So;0;L;;;;;N;;;;; +1D051;BYZANTINE MUSICAL SYMBOL APOSTROFOS NEO;So;0;L;;;;;N;;;;; +1D052;BYZANTINE MUSICAL SYMBOL APOSTROFOI SYNDESMOS NEO;So;0;L;;;;;N;;;;; +1D053;BYZANTINE MUSICAL SYMBOL YPORROI;So;0;L;;;;;N;;;;; +1D054;BYZANTINE MUSICAL SYMBOL KRATIMOYPORROON;So;0;L;;;;;N;;;;; +1D055;BYZANTINE MUSICAL SYMBOL ELAFRON;So;0;L;;;;;N;;;;; +1D056;BYZANTINE MUSICAL SYMBOL CHAMILI;So;0;L;;;;;N;;;;; +1D057;BYZANTINE MUSICAL SYMBOL MIKRON ISON;So;0;L;;;;;N;;;;; +1D058;BYZANTINE MUSICAL SYMBOL VAREIA NEO;So;0;L;;;;;N;;;;; +1D059;BYZANTINE MUSICAL SYMBOL PIASMA NEO;So;0;L;;;;;N;;;;; +1D05A;BYZANTINE MUSICAL SYMBOL PSIFISTON NEO;So;0;L;;;;;N;;;;; +1D05B;BYZANTINE MUSICAL SYMBOL OMALON;So;0;L;;;;;N;;;;; +1D05C;BYZANTINE MUSICAL SYMBOL ANTIKENOMA;So;0;L;;;;;N;;;;; +1D05D;BYZANTINE MUSICAL SYMBOL LYGISMA;So;0;L;;;;;N;;;;; +1D05E;BYZANTINE MUSICAL SYMBOL PARAKLITIKI NEO;So;0;L;;;;;N;;;;; +1D05F;BYZANTINE MUSICAL SYMBOL PARAKALESMA NEO;So;0;L;;;;;N;;;;; +1D060;BYZANTINE MUSICAL SYMBOL ETERON PARAKALESMA;So;0;L;;;;;N;;;;; +1D061;BYZANTINE MUSICAL SYMBOL KYLISMA;So;0;L;;;;;N;;;;; +1D062;BYZANTINE MUSICAL SYMBOL ANTIKENOKYLISMA;So;0;L;;;;;N;;;;; +1D063;BYZANTINE MUSICAL SYMBOL TROMIKON NEO;So;0;L;;;;;N;;;;; +1D064;BYZANTINE MUSICAL SYMBOL EKSTREPTON;So;0;L;;;;;N;;;;; +1D065;BYZANTINE MUSICAL SYMBOL SYNAGMA NEO;So;0;L;;;;;N;;;;; +1D066;BYZANTINE MUSICAL SYMBOL SYRMA;So;0;L;;;;;N;;;;; +1D067;BYZANTINE MUSICAL SYMBOL CHOREVMA NEO;So;0;L;;;;;N;;;;; +1D068;BYZANTINE MUSICAL SYMBOL EPEGERMA;So;0;L;;;;;N;;;;; +1D069;BYZANTINE MUSICAL SYMBOL SEISMA NEO;So;0;L;;;;;N;;;;; +1D06A;BYZANTINE MUSICAL SYMBOL XIRON KLASMA;So;0;L;;;;;N;;;;; +1D06B;BYZANTINE MUSICAL SYMBOL TROMIKOPSIFISTON;So;0;L;;;;;N;;;;; +1D06C;BYZANTINE MUSICAL SYMBOL PSIFISTOLYGISMA;So;0;L;;;;;N;;;;; +1D06D;BYZANTINE MUSICAL SYMBOL TROMIKOLYGISMA;So;0;L;;;;;N;;;;; +1D06E;BYZANTINE MUSICAL SYMBOL TROMIKOPARAKALESMA;So;0;L;;;;;N;;;;; +1D06F;BYZANTINE MUSICAL SYMBOL PSIFISTOPARAKALESMA;So;0;L;;;;;N;;;;; +1D070;BYZANTINE MUSICAL SYMBOL TROMIKOSYNAGMA;So;0;L;;;;;N;;;;; +1D071;BYZANTINE MUSICAL SYMBOL PSIFISTOSYNAGMA;So;0;L;;;;;N;;;;; +1D072;BYZANTINE MUSICAL SYMBOL GORGOSYNTHETON;So;0;L;;;;;N;;;;; +1D073;BYZANTINE MUSICAL SYMBOL ARGOSYNTHETON;So;0;L;;;;;N;;;;; +1D074;BYZANTINE MUSICAL SYMBOL ETERON ARGOSYNTHETON;So;0;L;;;;;N;;;;; +1D075;BYZANTINE MUSICAL SYMBOL OYRANISMA NEO;So;0;L;;;;;N;;;;; +1D076;BYZANTINE MUSICAL SYMBOL THEMATISMOS ESO;So;0;L;;;;;N;;;;; +1D077;BYZANTINE MUSICAL SYMBOL THEMATISMOS EXO;So;0;L;;;;;N;;;;; +1D078;BYZANTINE MUSICAL SYMBOL THEMA APLOUN;So;0;L;;;;;N;;;;; +1D079;BYZANTINE MUSICAL SYMBOL THES KAI APOTHES;So;0;L;;;;;N;;;;; +1D07A;BYZANTINE MUSICAL SYMBOL KATAVASMA;So;0;L;;;;;N;;;;; +1D07B;BYZANTINE MUSICAL SYMBOL ENDOFONON;So;0;L;;;;;N;;;;; +1D07C;BYZANTINE MUSICAL SYMBOL YFEN KATO;So;0;L;;;;;N;;;;; +1D07D;BYZANTINE MUSICAL SYMBOL YFEN ANO;So;0;L;;;;;N;;;;; +1D07E;BYZANTINE MUSICAL SYMBOL STAVROS;So;0;L;;;;;N;;;;; +1D07F;BYZANTINE MUSICAL SYMBOL KLASMA ANO;So;0;L;;;;;N;;;;; +1D080;BYZANTINE MUSICAL SYMBOL DIPLI ARCHAION;So;0;L;;;;;N;;;;; +1D081;BYZANTINE MUSICAL SYMBOL KRATIMA ARCHAION;So;0;L;;;;;N;;;;; +1D082;BYZANTINE MUSICAL SYMBOL KRATIMA ALLO;So;0;L;;;;;N;;;;; +1D083;BYZANTINE MUSICAL SYMBOL KRATIMA NEO;So;0;L;;;;;N;;;;; +1D084;BYZANTINE MUSICAL SYMBOL APODERMA NEO;So;0;L;;;;;N;;;;; +1D085;BYZANTINE MUSICAL SYMBOL APLI;So;0;L;;;;;N;;;;; +1D086;BYZANTINE MUSICAL SYMBOL DIPLI;So;0;L;;;;;N;;;;; +1D087;BYZANTINE MUSICAL SYMBOL TRIPLI;So;0;L;;;;;N;;;;; +1D088;BYZANTINE MUSICAL SYMBOL TETRAPLI;So;0;L;;;;;N;;;;; +1D089;BYZANTINE MUSICAL SYMBOL KORONIS;So;0;L;;;;;N;;;;; +1D08A;BYZANTINE MUSICAL SYMBOL LEIMMA ENOS CHRONOU;So;0;L;;;;;N;;;;; +1D08B;BYZANTINE MUSICAL SYMBOL LEIMMA DYO CHRONON;So;0;L;;;;;N;;;;; +1D08C;BYZANTINE MUSICAL SYMBOL LEIMMA TRION CHRONON;So;0;L;;;;;N;;;;; +1D08D;BYZANTINE MUSICAL SYMBOL LEIMMA TESSARON CHRONON;So;0;L;;;;;N;;;;; +1D08E;BYZANTINE MUSICAL SYMBOL LEIMMA IMISEOS CHRONOU;So;0;L;;;;;N;;;;; +1D08F;BYZANTINE MUSICAL SYMBOL GORGON NEO ANO;So;0;L;;;;;N;;;;; +1D090;BYZANTINE MUSICAL SYMBOL GORGON PARESTIGMENON ARISTERA;So;0;L;;;;;N;;;;; +1D091;BYZANTINE MUSICAL SYMBOL GORGON PARESTIGMENON DEXIA;So;0;L;;;;;N;;;;; +1D092;BYZANTINE MUSICAL SYMBOL DIGORGON;So;0;L;;;;;N;;;;; +1D093;BYZANTINE MUSICAL SYMBOL DIGORGON PARESTIGMENON ARISTERA KATO;So;0;L;;;;;N;;;;; +1D094;BYZANTINE MUSICAL SYMBOL DIGORGON PARESTIGMENON ARISTERA ANO;So;0;L;;;;;N;;;;; +1D095;BYZANTINE MUSICAL SYMBOL DIGORGON PARESTIGMENON DEXIA;So;0;L;;;;;N;;;;; +1D096;BYZANTINE MUSICAL SYMBOL TRIGORGON;So;0;L;;;;;N;;;;; +1D097;BYZANTINE MUSICAL SYMBOL ARGON;So;0;L;;;;;N;;;;; +1D098;BYZANTINE MUSICAL SYMBOL IMIDIARGON;So;0;L;;;;;N;;;;; +1D099;BYZANTINE MUSICAL SYMBOL DIARGON;So;0;L;;;;;N;;;;; +1D09A;BYZANTINE MUSICAL SYMBOL AGOGI POLI ARGI;So;0;L;;;;;N;;;;; +1D09B;BYZANTINE MUSICAL SYMBOL AGOGI ARGOTERI;So;0;L;;;;;N;;;;; +1D09C;BYZANTINE MUSICAL SYMBOL AGOGI ARGI;So;0;L;;;;;N;;;;; +1D09D;BYZANTINE MUSICAL SYMBOL AGOGI METRIA;So;0;L;;;;;N;;;;; +1D09E;BYZANTINE MUSICAL SYMBOL AGOGI MESI;So;0;L;;;;;N;;;;; +1D09F;BYZANTINE MUSICAL SYMBOL AGOGI GORGI;So;0;L;;;;;N;;;;; +1D0A0;BYZANTINE MUSICAL SYMBOL AGOGI GORGOTERI;So;0;L;;;;;N;;;;; +1D0A1;BYZANTINE MUSICAL SYMBOL AGOGI POLI GORGI;So;0;L;;;;;N;;;;; +1D0A2;BYZANTINE MUSICAL SYMBOL MARTYRIA PROTOS ICHOS;So;0;L;;;;;N;;;;; +1D0A3;BYZANTINE MUSICAL SYMBOL MARTYRIA ALLI PROTOS ICHOS;So;0;L;;;;;N;;;;; +1D0A4;BYZANTINE MUSICAL SYMBOL MARTYRIA DEYTEROS ICHOS;So;0;L;;;;;N;;;;; +1D0A5;BYZANTINE MUSICAL SYMBOL MARTYRIA ALLI DEYTEROS ICHOS;So;0;L;;;;;N;;;;; +1D0A6;BYZANTINE MUSICAL SYMBOL MARTYRIA TRITOS ICHOS;So;0;L;;;;;N;;;;; +1D0A7;BYZANTINE MUSICAL SYMBOL MARTYRIA TRIFONIAS;So;0;L;;;;;N;;;;; +1D0A8;BYZANTINE MUSICAL SYMBOL MARTYRIA TETARTOS ICHOS;So;0;L;;;;;N;;;;; +1D0A9;BYZANTINE MUSICAL SYMBOL MARTYRIA TETARTOS LEGETOS ICHOS;So;0;L;;;;;N;;;;; +1D0AA;BYZANTINE MUSICAL SYMBOL MARTYRIA LEGETOS ICHOS;So;0;L;;;;;N;;;;; +1D0AB;BYZANTINE MUSICAL SYMBOL MARTYRIA PLAGIOS ICHOS;So;0;L;;;;;N;;;;; +1D0AC;BYZANTINE MUSICAL SYMBOL ISAKIA TELOUS ICHIMATOS;So;0;L;;;;;N;;;;; +1D0AD;BYZANTINE MUSICAL SYMBOL APOSTROFOI TELOUS ICHIMATOS;So;0;L;;;;;N;;;;; +1D0AE;BYZANTINE MUSICAL SYMBOL FANEROSIS TETRAFONIAS;So;0;L;;;;;N;;;;; +1D0AF;BYZANTINE MUSICAL SYMBOL FANEROSIS MONOFONIAS;So;0;L;;;;;N;;;;; +1D0B0;BYZANTINE MUSICAL SYMBOL FANEROSIS DIFONIAS;So;0;L;;;;;N;;;;; +1D0B1;BYZANTINE MUSICAL SYMBOL MARTYRIA VARYS ICHOS;So;0;L;;;;;N;;;;; +1D0B2;BYZANTINE MUSICAL SYMBOL MARTYRIA PROTOVARYS ICHOS;So;0;L;;;;;N;;;;; +1D0B3;BYZANTINE MUSICAL SYMBOL MARTYRIA PLAGIOS TETARTOS ICHOS;So;0;L;;;;;N;;;;; +1D0B4;BYZANTINE MUSICAL SYMBOL GORTHMIKON N APLOUN;So;0;L;;;;;N;;;;; +1D0B5;BYZANTINE MUSICAL SYMBOL GORTHMIKON N DIPLOUN;So;0;L;;;;;N;;;;; +1D0B6;BYZANTINE MUSICAL SYMBOL ENARXIS KAI FTHORA VOU;So;0;L;;;;;N;;;;; +1D0B7;BYZANTINE MUSICAL SYMBOL IMIFONON;So;0;L;;;;;N;;;;; +1D0B8;BYZANTINE MUSICAL SYMBOL IMIFTHORON;So;0;L;;;;;N;;;;; +1D0B9;BYZANTINE MUSICAL SYMBOL FTHORA ARCHAION DEYTEROU ICHOU;So;0;L;;;;;N;;;;; +1D0BA;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI PA;So;0;L;;;;;N;;;;; +1D0BB;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI NANA;So;0;L;;;;;N;;;;; +1D0BC;BYZANTINE MUSICAL SYMBOL FTHORA NAOS ICHOS;So;0;L;;;;;N;;;;; +1D0BD;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI DI;So;0;L;;;;;N;;;;; +1D0BE;BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON DIATONON DI;So;0;L;;;;;N;;;;; +1D0BF;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI KE;So;0;L;;;;;N;;;;; +1D0C0;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI ZO;So;0;L;;;;;N;;;;; +1D0C1;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI NI KATO;So;0;L;;;;;N;;;;; +1D0C2;BYZANTINE MUSICAL SYMBOL FTHORA DIATONIKI NI ANO;So;0;L;;;;;N;;;;; +1D0C3;BYZANTINE MUSICAL SYMBOL FTHORA MALAKON CHROMA DIFONIAS;So;0;L;;;;;N;;;;; +1D0C4;BYZANTINE MUSICAL SYMBOL FTHORA MALAKON CHROMA MONOFONIAS;So;0;L;;;;;N;;;;; +1D0C5;BYZANTINE MUSICAL SYMBOL FHTORA SKLIRON CHROMA VASIS;So;0;L;;;;;N;;;;; +1D0C6;BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA SYNAFI;So;0;L;;;;;N;;;;; +1D0C7;BYZANTINE MUSICAL SYMBOL FTHORA NENANO;So;0;L;;;;;N;;;;; +1D0C8;BYZANTINE MUSICAL SYMBOL CHROA ZYGOS;So;0;L;;;;;N;;;;; +1D0C9;BYZANTINE MUSICAL SYMBOL CHROA KLITON;So;0;L;;;;;N;;;;; +1D0CA;BYZANTINE MUSICAL SYMBOL CHROA SPATHI;So;0;L;;;;;N;;;;; +1D0CB;BYZANTINE MUSICAL SYMBOL FTHORA I YFESIS TETARTIMORION;So;0;L;;;;;N;;;;; +1D0CC;BYZANTINE MUSICAL SYMBOL FTHORA ENARMONIOS ANTIFONIA;So;0;L;;;;;N;;;;; +1D0CD;BYZANTINE MUSICAL SYMBOL YFESIS TRITIMORION;So;0;L;;;;;N;;;;; +1D0CE;BYZANTINE MUSICAL SYMBOL DIESIS TRITIMORION;So;0;L;;;;;N;;;;; +1D0CF;BYZANTINE MUSICAL SYMBOL DIESIS TETARTIMORION;So;0;L;;;;;N;;;;; +1D0D0;BYZANTINE MUSICAL SYMBOL DIESIS APLI DYO DODEKATA;So;0;L;;;;;N;;;;; +1D0D1;BYZANTINE MUSICAL SYMBOL DIESIS MONOGRAMMOS TESSERA DODEKATA;So;0;L;;;;;N;;;;; +1D0D2;BYZANTINE MUSICAL SYMBOL DIESIS DIGRAMMOS EX DODEKATA;So;0;L;;;;;N;;;;; +1D0D3;BYZANTINE MUSICAL SYMBOL DIESIS TRIGRAMMOS OKTO DODEKATA;So;0;L;;;;;N;;;;; +1D0D4;BYZANTINE MUSICAL SYMBOL YFESIS APLI DYO DODEKATA;So;0;L;;;;;N;;;;; +1D0D5;BYZANTINE MUSICAL SYMBOL YFESIS MONOGRAMMOS TESSERA DODEKATA;So;0;L;;;;;N;;;;; +1D0D6;BYZANTINE MUSICAL SYMBOL YFESIS DIGRAMMOS EX DODEKATA;So;0;L;;;;;N;;;;; +1D0D7;BYZANTINE MUSICAL SYMBOL YFESIS TRIGRAMMOS OKTO DODEKATA;So;0;L;;;;;N;;;;; +1D0D8;BYZANTINE MUSICAL SYMBOL GENIKI DIESIS;So;0;L;;;;;N;;;;; +1D0D9;BYZANTINE MUSICAL SYMBOL GENIKI YFESIS;So;0;L;;;;;N;;;;; +1D0DA;BYZANTINE MUSICAL SYMBOL DIASTOLI APLI MIKRI;So;0;L;;;;;N;;;;; +1D0DB;BYZANTINE MUSICAL SYMBOL DIASTOLI APLI MEGALI;So;0;L;;;;;N;;;;; +1D0DC;BYZANTINE MUSICAL SYMBOL DIASTOLI DIPLI;So;0;L;;;;;N;;;;; +1D0DD;BYZANTINE MUSICAL SYMBOL DIASTOLI THESEOS;So;0;L;;;;;N;;;;; +1D0DE;BYZANTINE MUSICAL SYMBOL SIMANSIS THESEOS;So;0;L;;;;;N;;;;; +1D0DF;BYZANTINE MUSICAL SYMBOL SIMANSIS THESEOS DISIMOU;So;0;L;;;;;N;;;;; +1D0E0;BYZANTINE MUSICAL SYMBOL SIMANSIS THESEOS TRISIMOU;So;0;L;;;;;N;;;;; +1D0E1;BYZANTINE MUSICAL SYMBOL SIMANSIS THESEOS TETRASIMOU;So;0;L;;;;;N;;;;; +1D0E2;BYZANTINE MUSICAL SYMBOL SIMANSIS ARSEOS;So;0;L;;;;;N;;;;; +1D0E3;BYZANTINE MUSICAL SYMBOL SIMANSIS ARSEOS DISIMOU;So;0;L;;;;;N;;;;; +1D0E4;BYZANTINE MUSICAL SYMBOL SIMANSIS ARSEOS TRISIMOU;So;0;L;;;;;N;;;;; +1D0E5;BYZANTINE MUSICAL SYMBOL SIMANSIS ARSEOS TETRASIMOU;So;0;L;;;;;N;;;;; +1D0E6;BYZANTINE MUSICAL SYMBOL DIGRAMMA GG;So;0;L;;;;;N;;;;; +1D0E7;BYZANTINE MUSICAL SYMBOL DIFTOGGOS OU;So;0;L;;;;;N;;;;; +1D0E8;BYZANTINE MUSICAL SYMBOL STIGMA;So;0;L;;;;;N;;;;; +1D0E9;BYZANTINE MUSICAL SYMBOL ARKTIKO PA;So;0;L;;;;;N;;;;; +1D0EA;BYZANTINE MUSICAL SYMBOL ARKTIKO VOU;So;0;L;;;;;N;;;;; +1D0EB;BYZANTINE MUSICAL SYMBOL ARKTIKO GA;So;0;L;;;;;N;;;;; +1D0EC;BYZANTINE MUSICAL SYMBOL ARKTIKO DI;So;0;L;;;;;N;;;;; +1D0ED;BYZANTINE MUSICAL SYMBOL ARKTIKO KE;So;0;L;;;;;N;;;;; +1D0EE;BYZANTINE MUSICAL SYMBOL ARKTIKO ZO;So;0;L;;;;;N;;;;; +1D0EF;BYZANTINE MUSICAL SYMBOL ARKTIKO NI;So;0;L;;;;;N;;;;; +1D0F0;BYZANTINE MUSICAL SYMBOL KENTIMATA NEO MESO;So;0;L;;;;;N;;;;; +1D0F1;BYZANTINE MUSICAL SYMBOL KENTIMA NEO MESO;So;0;L;;;;;N;;;;; +1D0F2;BYZANTINE MUSICAL SYMBOL KENTIMATA NEO KATO;So;0;L;;;;;N;;;;; +1D0F3;BYZANTINE MUSICAL SYMBOL KENTIMA NEO KATO;So;0;L;;;;;N;;;;; +1D0F4;BYZANTINE MUSICAL SYMBOL KLASMA KATO;So;0;L;;;;;N;;;;; +1D0F5;BYZANTINE MUSICAL SYMBOL GORGON NEO KATO;So;0;L;;;;;N;;;;; +1D100;MUSICAL SYMBOL SINGLE BARLINE;So;0;L;;;;;N;;;;; +1D101;MUSICAL SYMBOL DOUBLE BARLINE;So;0;L;;;;;N;;;;; +1D102;MUSICAL SYMBOL FINAL BARLINE;So;0;L;;;;;N;;;;; +1D103;MUSICAL SYMBOL REVERSE FINAL BARLINE;So;0;L;;;;;N;;;;; +1D104;MUSICAL SYMBOL DASHED BARLINE;So;0;L;;;;;N;;;;; +1D105;MUSICAL SYMBOL SHORT BARLINE;So;0;L;;;;;N;;;;; +1D106;MUSICAL SYMBOL LEFT REPEAT SIGN;So;0;L;;;;;N;;;;; +1D107;MUSICAL SYMBOL RIGHT REPEAT SIGN;So;0;L;;;;;N;;;;; +1D108;MUSICAL SYMBOL REPEAT DOTS;So;0;L;;;;;N;;;;; +1D109;MUSICAL SYMBOL DAL SEGNO;So;0;L;;;;;N;;;;; +1D10A;MUSICAL SYMBOL DA CAPO;So;0;L;;;;;N;;;;; +1D10B;MUSICAL SYMBOL SEGNO;So;0;L;;;;;N;;;;; +1D10C;MUSICAL SYMBOL CODA;So;0;L;;;;;N;;;;; +1D10D;MUSICAL SYMBOL REPEATED FIGURE-1;So;0;L;;;;;N;;;;; +1D10E;MUSICAL SYMBOL REPEATED FIGURE-2;So;0;L;;;;;N;;;;; +1D10F;MUSICAL SYMBOL REPEATED FIGURE-3;So;0;L;;;;;N;;;;; +1D110;MUSICAL SYMBOL FERMATA;So;0;L;;;;;N;;;;; +1D111;MUSICAL SYMBOL FERMATA BELOW;So;0;L;;;;;N;;;;; +1D112;MUSICAL SYMBOL BREATH MARK;So;0;L;;;;;N;;;;; +1D113;MUSICAL SYMBOL CAESURA;So;0;L;;;;;N;;;;; +1D114;MUSICAL SYMBOL BRACE;So;0;L;;;;;N;;;;; +1D115;MUSICAL SYMBOL BRACKET;So;0;L;;;;;N;;;;; +1D116;MUSICAL SYMBOL ONE-LINE STAFF;So;0;L;;;;;N;;;;; +1D117;MUSICAL SYMBOL TWO-LINE STAFF;So;0;L;;;;;N;;;;; +1D118;MUSICAL SYMBOL THREE-LINE STAFF;So;0;L;;;;;N;;;;; +1D119;MUSICAL SYMBOL FOUR-LINE STAFF;So;0;L;;;;;N;;;;; +1D11A;MUSICAL SYMBOL FIVE-LINE STAFF;So;0;L;;;;;N;;;;; +1D11B;MUSICAL SYMBOL SIX-LINE STAFF;So;0;L;;;;;N;;;;; +1D11C;MUSICAL SYMBOL SIX-STRING FRETBOARD;So;0;L;;;;;N;;;;; +1D11D;MUSICAL SYMBOL FOUR-STRING FRETBOARD;So;0;L;;;;;N;;;;; +1D11E;MUSICAL SYMBOL G CLEF;So;0;L;;;;;N;;;;; +1D11F;MUSICAL SYMBOL G CLEF OTTAVA ALTA;So;0;L;;;;;N;;;;; +1D120;MUSICAL SYMBOL G CLEF OTTAVA BASSA;So;0;L;;;;;N;;;;; +1D121;MUSICAL SYMBOL C CLEF;So;0;L;;;;;N;;;;; +1D122;MUSICAL SYMBOL F CLEF;So;0;L;;;;;N;;;;; +1D123;MUSICAL SYMBOL F CLEF OTTAVA ALTA;So;0;L;;;;;N;;;;; +1D124;MUSICAL SYMBOL F CLEF OTTAVA BASSA;So;0;L;;;;;N;;;;; +1D125;MUSICAL SYMBOL DRUM CLEF-1;So;0;L;;;;;N;;;;; +1D126;MUSICAL SYMBOL DRUM CLEF-2;So;0;L;;;;;N;;;;; +1D129;MUSICAL SYMBOL MULTIPLE MEASURE REST;So;0;L;;;;;N;;;;; +1D12A;MUSICAL SYMBOL DOUBLE SHARP;So;0;L;;;;;N;;;;; +1D12B;MUSICAL SYMBOL DOUBLE FLAT;So;0;L;;;;;N;;;;; +1D12C;MUSICAL SYMBOL FLAT UP;So;0;L;;;;;N;;;;; +1D12D;MUSICAL SYMBOL FLAT DOWN;So;0;L;;;;;N;;;;; +1D12E;MUSICAL SYMBOL NATURAL UP;So;0;L;;;;;N;;;;; +1D12F;MUSICAL SYMBOL NATURAL DOWN;So;0;L;;;;;N;;;;; +1D130;MUSICAL SYMBOL SHARP UP;So;0;L;;;;;N;;;;; +1D131;MUSICAL SYMBOL SHARP DOWN;So;0;L;;;;;N;;;;; +1D132;MUSICAL SYMBOL QUARTER TONE SHARP;So;0;L;;;;;N;;;;; +1D133;MUSICAL SYMBOL QUARTER TONE FLAT;So;0;L;;;;;N;;;;; +1D134;MUSICAL SYMBOL COMMON TIME;So;0;L;;;;;N;;;;; +1D135;MUSICAL SYMBOL CUT TIME;So;0;L;;;;;N;;;;; +1D136;MUSICAL SYMBOL OTTAVA ALTA;So;0;L;;;;;N;;;;; +1D137;MUSICAL SYMBOL OTTAVA BASSA;So;0;L;;;;;N;;;;; +1D138;MUSICAL SYMBOL QUINDICESIMA ALTA;So;0;L;;;;;N;;;;; +1D139;MUSICAL SYMBOL QUINDICESIMA BASSA;So;0;L;;;;;N;;;;; +1D13A;MUSICAL SYMBOL MULTI REST;So;0;L;;;;;N;;;;; +1D13B;MUSICAL SYMBOL WHOLE REST;So;0;L;;;;;N;;;;; +1D13C;MUSICAL SYMBOL HALF REST;So;0;L;;;;;N;;;;; +1D13D;MUSICAL SYMBOL QUARTER REST;So;0;L;;;;;N;;;;; +1D13E;MUSICAL SYMBOL EIGHTH REST;So;0;L;;;;;N;;;;; +1D13F;MUSICAL SYMBOL SIXTEENTH REST;So;0;L;;;;;N;;;;; +1D140;MUSICAL SYMBOL THIRTY-SECOND REST;So;0;L;;;;;N;;;;; +1D141;MUSICAL SYMBOL SIXTY-FOURTH REST;So;0;L;;;;;N;;;;; +1D142;MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH REST;So;0;L;;;;;N;;;;; +1D143;MUSICAL SYMBOL X NOTEHEAD;So;0;L;;;;;N;;;;; +1D144;MUSICAL SYMBOL PLUS NOTEHEAD;So;0;L;;;;;N;;;;; +1D145;MUSICAL SYMBOL CIRCLE X NOTEHEAD;So;0;L;;;;;N;;;;; +1D146;MUSICAL SYMBOL SQUARE NOTEHEAD WHITE;So;0;L;;;;;N;;;;; +1D147;MUSICAL SYMBOL SQUARE NOTEHEAD BLACK;So;0;L;;;;;N;;;;; +1D148;MUSICAL SYMBOL TRIANGLE NOTEHEAD UP WHITE;So;0;L;;;;;N;;;;; +1D149;MUSICAL SYMBOL TRIANGLE NOTEHEAD UP BLACK;So;0;L;;;;;N;;;;; +1D14A;MUSICAL SYMBOL TRIANGLE NOTEHEAD LEFT WHITE;So;0;L;;;;;N;;;;; +1D14B;MUSICAL SYMBOL TRIANGLE NOTEHEAD LEFT BLACK;So;0;L;;;;;N;;;;; +1D14C;MUSICAL SYMBOL TRIANGLE NOTEHEAD RIGHT WHITE;So;0;L;;;;;N;;;;; +1D14D;MUSICAL SYMBOL TRIANGLE NOTEHEAD RIGHT BLACK;So;0;L;;;;;N;;;;; +1D14E;MUSICAL SYMBOL TRIANGLE NOTEHEAD DOWN WHITE;So;0;L;;;;;N;;;;; +1D14F;MUSICAL SYMBOL TRIANGLE NOTEHEAD DOWN BLACK;So;0;L;;;;;N;;;;; +1D150;MUSICAL SYMBOL TRIANGLE NOTEHEAD UP RIGHT WHITE;So;0;L;;;;;N;;;;; +1D151;MUSICAL SYMBOL TRIANGLE NOTEHEAD UP RIGHT BLACK;So;0;L;;;;;N;;;;; +1D152;MUSICAL SYMBOL MOON NOTEHEAD WHITE;So;0;L;;;;;N;;;;; +1D153;MUSICAL SYMBOL MOON NOTEHEAD BLACK;So;0;L;;;;;N;;;;; +1D154;MUSICAL SYMBOL TRIANGLE-ROUND NOTEHEAD DOWN WHITE;So;0;L;;;;;N;;;;; +1D155;MUSICAL SYMBOL TRIANGLE-ROUND NOTEHEAD DOWN BLACK;So;0;L;;;;;N;;;;; +1D156;MUSICAL SYMBOL PARENTHESIS NOTEHEAD;So;0;L;;;;;N;;;;; +1D157;MUSICAL SYMBOL VOID NOTEHEAD;So;0;L;;;;;N;;;;; +1D158;MUSICAL SYMBOL NOTEHEAD BLACK;So;0;L;;;;;N;;;;; +1D159;MUSICAL SYMBOL NULL NOTEHEAD;So;0;L;;;;;N;;;;; +1D15A;MUSICAL SYMBOL CLUSTER NOTEHEAD WHITE;So;0;L;;;;;N;;;;; +1D15B;MUSICAL SYMBOL CLUSTER NOTEHEAD BLACK;So;0;L;;;;;N;;;;; +1D15C;MUSICAL SYMBOL BREVE;So;0;L;;;;;N;;;;; +1D15D;MUSICAL SYMBOL WHOLE NOTE;So;0;L;;;;;N;;;;; +1D15E;MUSICAL SYMBOL HALF NOTE;So;0;L;1D157 1D165;;;;N;;;;; +1D15F;MUSICAL SYMBOL QUARTER NOTE;So;0;L;1D158 1D165;;;;N;;;;; +1D160;MUSICAL SYMBOL EIGHTH NOTE;So;0;L;1D15F 1D16E;;;;N;;;;; +1D161;MUSICAL SYMBOL SIXTEENTH NOTE;So;0;L;1D15F 1D16F;;;;N;;;;; +1D162;MUSICAL SYMBOL THIRTY-SECOND NOTE;So;0;L;1D15F 1D170;;;;N;;;;; +1D163;MUSICAL SYMBOL SIXTY-FOURTH NOTE;So;0;L;1D15F 1D171;;;;N;;;;; +1D164;MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE;So;0;L;1D15F 1D172;;;;N;;;;; +1D165;MUSICAL SYMBOL COMBINING STEM;Mc;216;L;;;;;N;;;;; +1D166;MUSICAL SYMBOL COMBINING SPRECHGESANG STEM;Mc;216;L;;;;;N;;;;; +1D167;MUSICAL SYMBOL COMBINING TREMOLO-1;Mn;1;NSM;;;;;N;;;;; +1D168;MUSICAL SYMBOL COMBINING TREMOLO-2;Mn;1;NSM;;;;;N;;;;; +1D169;MUSICAL SYMBOL COMBINING TREMOLO-3;Mn;1;NSM;;;;;N;;;;; +1D16A;MUSICAL SYMBOL FINGERED TREMOLO-1;So;0;L;;;;;N;;;;; +1D16B;MUSICAL SYMBOL FINGERED TREMOLO-2;So;0;L;;;;;N;;;;; +1D16C;MUSICAL SYMBOL FINGERED TREMOLO-3;So;0;L;;;;;N;;;;; +1D16D;MUSICAL SYMBOL COMBINING AUGMENTATION DOT;Mc;226;L;;;;;N;;;;; +1D16E;MUSICAL SYMBOL COMBINING FLAG-1;Mc;216;L;;;;;N;;;;; +1D16F;MUSICAL SYMBOL COMBINING FLAG-2;Mc;216;L;;;;;N;;;;; +1D170;MUSICAL SYMBOL COMBINING FLAG-3;Mc;216;L;;;;;N;;;;; +1D171;MUSICAL SYMBOL COMBINING FLAG-4;Mc;216;L;;;;;N;;;;; +1D172;MUSICAL SYMBOL COMBINING FLAG-5;Mc;216;L;;;;;N;;;;; +1D173;MUSICAL SYMBOL BEGIN BEAM;Cf;0;BN;;;;;N;;;;; +1D174;MUSICAL SYMBOL END BEAM;Cf;0;BN;;;;;N;;;;; +1D175;MUSICAL SYMBOL BEGIN TIE;Cf;0;BN;;;;;N;;;;; +1D176;MUSICAL SYMBOL END TIE;Cf;0;BN;;;;;N;;;;; +1D177;MUSICAL SYMBOL BEGIN SLUR;Cf;0;BN;;;;;N;;;;; +1D178;MUSICAL SYMBOL END SLUR;Cf;0;BN;;;;;N;;;;; +1D179;MUSICAL SYMBOL BEGIN PHRASE;Cf;0;BN;;;;;N;;;;; +1D17A;MUSICAL SYMBOL END PHRASE;Cf;0;BN;;;;;N;;;;; +1D17B;MUSICAL SYMBOL COMBINING ACCENT;Mn;220;NSM;;;;;N;;;;; +1D17C;MUSICAL SYMBOL COMBINING STACCATO;Mn;220;NSM;;;;;N;;;;; +1D17D;MUSICAL SYMBOL COMBINING TENUTO;Mn;220;NSM;;;;;N;;;;; +1D17E;MUSICAL SYMBOL COMBINING STACCATISSIMO;Mn;220;NSM;;;;;N;;;;; +1D17F;MUSICAL SYMBOL COMBINING MARCATO;Mn;220;NSM;;;;;N;;;;; +1D180;MUSICAL SYMBOL COMBINING MARCATO-STACCATO;Mn;220;NSM;;;;;N;;;;; +1D181;MUSICAL SYMBOL COMBINING ACCENT-STACCATO;Mn;220;NSM;;;;;N;;;;; +1D182;MUSICAL SYMBOL COMBINING LOURE;Mn;220;NSM;;;;;N;;;;; +1D183;MUSICAL SYMBOL ARPEGGIATO UP;So;0;L;;;;;N;;;;; +1D184;MUSICAL SYMBOL ARPEGGIATO DOWN;So;0;L;;;;;N;;;;; +1D185;MUSICAL SYMBOL COMBINING DOIT;Mn;230;NSM;;;;;N;;;;; +1D186;MUSICAL SYMBOL COMBINING RIP;Mn;230;NSM;;;;;N;;;;; +1D187;MUSICAL SYMBOL COMBINING FLIP;Mn;230;NSM;;;;;N;;;;; +1D188;MUSICAL SYMBOL COMBINING SMEAR;Mn;230;NSM;;;;;N;;;;; +1D189;MUSICAL SYMBOL COMBINING BEND;Mn;230;NSM;;;;;N;;;;; +1D18A;MUSICAL SYMBOL COMBINING DOUBLE TONGUE;Mn;220;NSM;;;;;N;;;;; +1D18B;MUSICAL SYMBOL COMBINING TRIPLE TONGUE;Mn;220;NSM;;;;;N;;;;; +1D18C;MUSICAL SYMBOL RINFORZANDO;So;0;L;;;;;N;;;;; +1D18D;MUSICAL SYMBOL SUBITO;So;0;L;;;;;N;;;;; +1D18E;MUSICAL SYMBOL Z;So;0;L;;;;;N;;;;; +1D18F;MUSICAL SYMBOL PIANO;So;0;L;;;;;N;;;;; +1D190;MUSICAL SYMBOL MEZZO;So;0;L;;;;;N;;;;; +1D191;MUSICAL SYMBOL FORTE;So;0;L;;;;;N;;;;; +1D192;MUSICAL SYMBOL CRESCENDO;So;0;L;;;;;N;;;;; +1D193;MUSICAL SYMBOL DECRESCENDO;So;0;L;;;;;N;;;;; +1D194;MUSICAL SYMBOL GRACE NOTE SLASH;So;0;L;;;;;N;;;;; +1D195;MUSICAL SYMBOL GRACE NOTE NO SLASH;So;0;L;;;;;N;;;;; +1D196;MUSICAL SYMBOL TR;So;0;L;;;;;N;;;;; +1D197;MUSICAL SYMBOL TURN;So;0;L;;;;;N;;;;; +1D198;MUSICAL SYMBOL INVERTED TURN;So;0;L;;;;;N;;;;; +1D199;MUSICAL SYMBOL TURN SLASH;So;0;L;;;;;N;;;;; +1D19A;MUSICAL SYMBOL TURN UP;So;0;L;;;;;N;;;;; +1D19B;MUSICAL SYMBOL ORNAMENT STROKE-1;So;0;L;;;;;N;;;;; +1D19C;MUSICAL SYMBOL ORNAMENT STROKE-2;So;0;L;;;;;N;;;;; +1D19D;MUSICAL SYMBOL ORNAMENT STROKE-3;So;0;L;;;;;N;;;;; +1D19E;MUSICAL SYMBOL ORNAMENT STROKE-4;So;0;L;;;;;N;;;;; +1D19F;MUSICAL SYMBOL ORNAMENT STROKE-5;So;0;L;;;;;N;;;;; +1D1A0;MUSICAL SYMBOL ORNAMENT STROKE-6;So;0;L;;;;;N;;;;; +1D1A1;MUSICAL SYMBOL ORNAMENT STROKE-7;So;0;L;;;;;N;;;;; +1D1A2;MUSICAL SYMBOL ORNAMENT STROKE-8;So;0;L;;;;;N;;;;; +1D1A3;MUSICAL SYMBOL ORNAMENT STROKE-9;So;0;L;;;;;N;;;;; +1D1A4;MUSICAL SYMBOL ORNAMENT STROKE-10;So;0;L;;;;;N;;;;; +1D1A5;MUSICAL SYMBOL ORNAMENT STROKE-11;So;0;L;;;;;N;;;;; +1D1A6;MUSICAL SYMBOL HAUPTSTIMME;So;0;L;;;;;N;;;;; +1D1A7;MUSICAL SYMBOL NEBENSTIMME;So;0;L;;;;;N;;;;; +1D1A8;MUSICAL SYMBOL END OF STIMME;So;0;L;;;;;N;;;;; +1D1A9;MUSICAL SYMBOL DEGREE SLASH;So;0;L;;;;;N;;;;; +1D1AA;MUSICAL SYMBOL COMBINING DOWN BOW;Mn;230;NSM;;;;;N;;;;; +1D1AB;MUSICAL SYMBOL COMBINING UP BOW;Mn;230;NSM;;;;;N;;;;; +1D1AC;MUSICAL SYMBOL COMBINING HARMONIC;Mn;230;NSM;;;;;N;;;;; +1D1AD;MUSICAL SYMBOL COMBINING SNAP PIZZICATO;Mn;230;NSM;;;;;N;;;;; +1D1AE;MUSICAL SYMBOL PEDAL MARK;So;0;L;;;;;N;;;;; +1D1AF;MUSICAL SYMBOL PEDAL UP MARK;So;0;L;;;;;N;;;;; +1D1B0;MUSICAL SYMBOL HALF PEDAL MARK;So;0;L;;;;;N;;;;; +1D1B1;MUSICAL SYMBOL GLISSANDO UP;So;0;L;;;;;N;;;;; +1D1B2;MUSICAL SYMBOL GLISSANDO DOWN;So;0;L;;;;;N;;;;; +1D1B3;MUSICAL SYMBOL WITH FINGERNAILS;So;0;L;;;;;N;;;;; +1D1B4;MUSICAL SYMBOL DAMP;So;0;L;;;;;N;;;;; +1D1B5;MUSICAL SYMBOL DAMP ALL;So;0;L;;;;;N;;;;; +1D1B6;MUSICAL SYMBOL MAXIMA;So;0;L;;;;;N;;;;; +1D1B7;MUSICAL SYMBOL LONGA;So;0;L;;;;;N;;;;; +1D1B8;MUSICAL SYMBOL BREVIS;So;0;L;;;;;N;;;;; +1D1B9;MUSICAL SYMBOL SEMIBREVIS WHITE;So;0;L;;;;;N;;;;; +1D1BA;MUSICAL SYMBOL SEMIBREVIS BLACK;So;0;L;;;;;N;;;;; +1D1BB;MUSICAL SYMBOL MINIMA;So;0;L;1D1B9 1D165;;;;N;;;;; +1D1BC;MUSICAL SYMBOL MINIMA BLACK;So;0;L;1D1BA 1D165;;;;N;;;;; +1D1BD;MUSICAL SYMBOL SEMIMINIMA WHITE;So;0;L;1D1BB 1D16E;;;;N;;;;; +1D1BE;MUSICAL SYMBOL SEMIMINIMA BLACK;So;0;L;1D1BC 1D16E;;;;N;;;;; +1D1BF;MUSICAL SYMBOL FUSA WHITE;So;0;L;1D1BB 1D16F;;;;N;;;;; +1D1C0;MUSICAL SYMBOL FUSA BLACK;So;0;L;1D1BC 1D16F;;;;N;;;;; +1D1C1;MUSICAL SYMBOL LONGA PERFECTA REST;So;0;L;;;;;N;;;;; +1D1C2;MUSICAL SYMBOL LONGA IMPERFECTA REST;So;0;L;;;;;N;;;;; +1D1C3;MUSICAL SYMBOL BREVIS REST;So;0;L;;;;;N;;;;; +1D1C4;MUSICAL SYMBOL SEMIBREVIS REST;So;0;L;;;;;N;;;;; +1D1C5;MUSICAL SYMBOL MINIMA REST;So;0;L;;;;;N;;;;; +1D1C6;MUSICAL SYMBOL SEMIMINIMA REST;So;0;L;;;;;N;;;;; +1D1C7;MUSICAL SYMBOL TEMPUS PERFECTUM CUM PROLATIONE PERFECTA;So;0;L;;;;;N;;;;; +1D1C8;MUSICAL SYMBOL TEMPUS PERFECTUM CUM PROLATIONE IMPERFECTA;So;0;L;;;;;N;;;;; +1D1C9;MUSICAL SYMBOL TEMPUS PERFECTUM CUM PROLATIONE PERFECTA DIMINUTION-1;So;0;L;;;;;N;;;;; +1D1CA;MUSICAL SYMBOL TEMPUS IMPERFECTUM CUM PROLATIONE PERFECTA;So;0;L;;;;;N;;;;; +1D1CB;MUSICAL SYMBOL TEMPUS IMPERFECTUM CUM PROLATIONE IMPERFECTA;So;0;L;;;;;N;;;;; +1D1CC;MUSICAL SYMBOL TEMPUS IMPERFECTUM CUM PROLATIONE IMPERFECTA DIMINUTION-1;So;0;L;;;;;N;;;;; +1D1CD;MUSICAL SYMBOL TEMPUS IMPERFECTUM CUM PROLATIONE IMPERFECTA DIMINUTION-2;So;0;L;;;;;N;;;;; +1D1CE;MUSICAL SYMBOL TEMPUS IMPERFECTUM CUM PROLATIONE IMPERFECTA DIMINUTION-3;So;0;L;;;;;N;;;;; +1D1CF;MUSICAL SYMBOL CROIX;So;0;L;;;;;N;;;;; +1D1D0;MUSICAL SYMBOL GREGORIAN C CLEF;So;0;L;;;;;N;;;;; +1D1D1;MUSICAL SYMBOL GREGORIAN F CLEF;So;0;L;;;;;N;;;;; +1D1D2;MUSICAL SYMBOL SQUARE B;So;0;L;;;;;N;;;;; +1D1D3;MUSICAL SYMBOL VIRGA;So;0;L;;;;;N;;;;; +1D1D4;MUSICAL SYMBOL PODATUS;So;0;L;;;;;N;;;;; +1D1D5;MUSICAL SYMBOL CLIVIS;So;0;L;;;;;N;;;;; +1D1D6;MUSICAL SYMBOL SCANDICUS;So;0;L;;;;;N;;;;; +1D1D7;MUSICAL SYMBOL CLIMACUS;So;0;L;;;;;N;;;;; +1D1D8;MUSICAL SYMBOL TORCULUS;So;0;L;;;;;N;;;;; +1D1D9;MUSICAL SYMBOL PORRECTUS;So;0;L;;;;;N;;;;; +1D1DA;MUSICAL SYMBOL PORRECTUS FLEXUS;So;0;L;;;;;N;;;;; +1D1DB;MUSICAL SYMBOL SCANDICUS FLEXUS;So;0;L;;;;;N;;;;; +1D1DC;MUSICAL SYMBOL TORCULUS RESUPINUS;So;0;L;;;;;N;;;;; +1D1DD;MUSICAL SYMBOL PES SUBPUNCTIS;So;0;L;;;;;N;;;;; +1D200;GREEK VOCAL NOTATION SYMBOL-1;So;0;ON;;;;;N;;;;; +1D201;GREEK VOCAL NOTATION SYMBOL-2;So;0;ON;;;;;N;;;;; +1D202;GREEK VOCAL NOTATION SYMBOL-3;So;0;ON;;;;;N;;;;; +1D203;GREEK VOCAL NOTATION SYMBOL-4;So;0;ON;;;;;N;;;;; +1D204;GREEK VOCAL NOTATION SYMBOL-5;So;0;ON;;;;;N;;;;; +1D205;GREEK VOCAL NOTATION SYMBOL-6;So;0;ON;;;;;N;;;;; +1D206;GREEK VOCAL NOTATION SYMBOL-7;So;0;ON;;;;;N;;;;; +1D207;GREEK VOCAL NOTATION SYMBOL-8;So;0;ON;;;;;N;;;;; +1D208;GREEK VOCAL NOTATION SYMBOL-9;So;0;ON;;;;;N;;;;; +1D209;GREEK VOCAL NOTATION SYMBOL-10;So;0;ON;;;;;N;;;;; +1D20A;GREEK VOCAL NOTATION SYMBOL-11;So;0;ON;;;;;N;;;;; +1D20B;GREEK VOCAL NOTATION SYMBOL-12;So;0;ON;;;;;N;;;;; +1D20C;GREEK VOCAL NOTATION SYMBOL-13;So;0;ON;;;;;N;;;;; +1D20D;GREEK VOCAL NOTATION SYMBOL-14;So;0;ON;;;;;N;;;;; +1D20E;GREEK VOCAL NOTATION SYMBOL-15;So;0;ON;;;;;N;;;;; +1D20F;GREEK VOCAL NOTATION SYMBOL-16;So;0;ON;;;;;N;;;;; +1D210;GREEK VOCAL NOTATION SYMBOL-17;So;0;ON;;;;;N;;;;; +1D211;GREEK VOCAL NOTATION SYMBOL-18;So;0;ON;;;;;N;;;;; +1D212;GREEK VOCAL NOTATION SYMBOL-19;So;0;ON;;;;;N;;;;; +1D213;GREEK VOCAL NOTATION SYMBOL-20;So;0;ON;;;;;N;;;;; +1D214;GREEK VOCAL NOTATION SYMBOL-21;So;0;ON;;;;;N;;;;; +1D215;GREEK VOCAL NOTATION SYMBOL-22;So;0;ON;;;;;N;;;;; +1D216;GREEK VOCAL NOTATION SYMBOL-23;So;0;ON;;;;;N;;;;; +1D217;GREEK VOCAL NOTATION SYMBOL-24;So;0;ON;;;;;N;;;;; +1D218;GREEK VOCAL NOTATION SYMBOL-50;So;0;ON;;;;;N;;;;; +1D219;GREEK VOCAL NOTATION SYMBOL-51;So;0;ON;;;;;N;;;;; +1D21A;GREEK VOCAL NOTATION SYMBOL-52;So;0;ON;;;;;N;;;;; +1D21B;GREEK VOCAL NOTATION SYMBOL-53;So;0;ON;;;;;N;;;;; +1D21C;GREEK VOCAL NOTATION SYMBOL-54;So;0;ON;;;;;N;;;;; +1D21D;GREEK INSTRUMENTAL NOTATION SYMBOL-1;So;0;ON;;;;;N;;;;; +1D21E;GREEK INSTRUMENTAL NOTATION SYMBOL-2;So;0;ON;;;;;N;;;;; +1D21F;GREEK INSTRUMENTAL NOTATION SYMBOL-4;So;0;ON;;;;;N;;;;; +1D220;GREEK INSTRUMENTAL NOTATION SYMBOL-5;So;0;ON;;;;;N;;;;; +1D221;GREEK INSTRUMENTAL NOTATION SYMBOL-7;So;0;ON;;;;;N;;;;; +1D222;GREEK INSTRUMENTAL NOTATION SYMBOL-8;So;0;ON;;;;;N;;;;; +1D223;GREEK INSTRUMENTAL NOTATION SYMBOL-11;So;0;ON;;;;;N;;;;; +1D224;GREEK INSTRUMENTAL NOTATION SYMBOL-12;So;0;ON;;;;;N;;;;; +1D225;GREEK INSTRUMENTAL NOTATION SYMBOL-13;So;0;ON;;;;;N;;;;; +1D226;GREEK INSTRUMENTAL NOTATION SYMBOL-14;So;0;ON;;;;;N;;;;; +1D227;GREEK INSTRUMENTAL NOTATION SYMBOL-17;So;0;ON;;;;;N;;;;; +1D228;GREEK INSTRUMENTAL NOTATION SYMBOL-18;So;0;ON;;;;;N;;;;; +1D229;GREEK INSTRUMENTAL NOTATION SYMBOL-19;So;0;ON;;;;;N;;;;; +1D22A;GREEK INSTRUMENTAL NOTATION SYMBOL-23;So;0;ON;;;;;N;;;;; +1D22B;GREEK INSTRUMENTAL NOTATION SYMBOL-24;So;0;ON;;;;;N;;;;; +1D22C;GREEK INSTRUMENTAL NOTATION SYMBOL-25;So;0;ON;;;;;N;;;;; +1D22D;GREEK INSTRUMENTAL NOTATION SYMBOL-26;So;0;ON;;;;;N;;;;; +1D22E;GREEK INSTRUMENTAL NOTATION SYMBOL-27;So;0;ON;;;;;N;;;;; +1D22F;GREEK INSTRUMENTAL NOTATION SYMBOL-29;So;0;ON;;;;;N;;;;; +1D230;GREEK INSTRUMENTAL NOTATION SYMBOL-30;So;0;ON;;;;;N;;;;; +1D231;GREEK INSTRUMENTAL NOTATION SYMBOL-32;So;0;ON;;;;;N;;;;; +1D232;GREEK INSTRUMENTAL NOTATION SYMBOL-36;So;0;ON;;;;;N;;;;; +1D233;GREEK INSTRUMENTAL NOTATION SYMBOL-37;So;0;ON;;;;;N;;;;; +1D234;GREEK INSTRUMENTAL NOTATION SYMBOL-38;So;0;ON;;;;;N;;;;; +1D235;GREEK INSTRUMENTAL NOTATION SYMBOL-39;So;0;ON;;;;;N;;;;; +1D236;GREEK INSTRUMENTAL NOTATION SYMBOL-40;So;0;ON;;;;;N;;;;; +1D237;GREEK INSTRUMENTAL NOTATION SYMBOL-42;So;0;ON;;;;;N;;;;; +1D238;GREEK INSTRUMENTAL NOTATION SYMBOL-43;So;0;ON;;;;;N;;;;; +1D239;GREEK INSTRUMENTAL NOTATION SYMBOL-45;So;0;ON;;;;;N;;;;; +1D23A;GREEK INSTRUMENTAL NOTATION SYMBOL-47;So;0;ON;;;;;N;;;;; +1D23B;GREEK INSTRUMENTAL NOTATION SYMBOL-48;So;0;ON;;;;;N;;;;; +1D23C;GREEK INSTRUMENTAL NOTATION SYMBOL-49;So;0;ON;;;;;N;;;;; +1D23D;GREEK INSTRUMENTAL NOTATION SYMBOL-50;So;0;ON;;;;;N;;;;; +1D23E;GREEK INSTRUMENTAL NOTATION SYMBOL-51;So;0;ON;;;;;N;;;;; +1D23F;GREEK INSTRUMENTAL NOTATION SYMBOL-52;So;0;ON;;;;;N;;;;; +1D240;GREEK INSTRUMENTAL NOTATION SYMBOL-53;So;0;ON;;;;;N;;;;; +1D241;GREEK INSTRUMENTAL NOTATION SYMBOL-54;So;0;ON;;;;;N;;;;; +1D242;COMBINING GREEK MUSICAL TRISEME;Mn;230;NSM;;;;;N;;;;; +1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; +1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; +1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D300;MONOGRAM FOR EARTH;So;0;ON;;;;;N;;;;; +1D301;DIGRAM FOR HEAVENLY EARTH;So;0;ON;;;;;N;;;;; +1D302;DIGRAM FOR HUMAN EARTH;So;0;ON;;;;;N;;;;; +1D303;DIGRAM FOR EARTHLY HEAVEN;So;0;ON;;;;;N;;;;; +1D304;DIGRAM FOR EARTHLY HUMAN;So;0;ON;;;;;N;;;;; +1D305;DIGRAM FOR EARTH;So;0;ON;;;;;N;;;;; +1D306;TETRAGRAM FOR CENTRE;So;0;ON;;;;;N;;;;; +1D307;TETRAGRAM FOR FULL CIRCLE;So;0;ON;;;;;N;;;;; +1D308;TETRAGRAM FOR MIRED;So;0;ON;;;;;N;;;;; +1D309;TETRAGRAM FOR BARRIER;So;0;ON;;;;;N;;;;; +1D30A;TETRAGRAM FOR KEEPING SMALL;So;0;ON;;;;;N;;;;; +1D30B;TETRAGRAM FOR CONTRARIETY;So;0;ON;;;;;N;;;;; +1D30C;TETRAGRAM FOR ASCENT;So;0;ON;;;;;N;;;;; +1D30D;TETRAGRAM FOR OPPOSITION;So;0;ON;;;;;N;;;;; +1D30E;TETRAGRAM FOR BRANCHING OUT;So;0;ON;;;;;N;;;;; +1D30F;TETRAGRAM FOR DEFECTIVENESS OR DISTORTION;So;0;ON;;;;;N;;;;; +1D310;TETRAGRAM FOR DIVERGENCE;So;0;ON;;;;;N;;;;; +1D311;TETRAGRAM FOR YOUTHFULNESS;So;0;ON;;;;;N;;;;; +1D312;TETRAGRAM FOR INCREASE;So;0;ON;;;;;N;;;;; +1D313;TETRAGRAM FOR PENETRATION;So;0;ON;;;;;N;;;;; +1D314;TETRAGRAM FOR REACH;So;0;ON;;;;;N;;;;; +1D315;TETRAGRAM FOR CONTACT;So;0;ON;;;;;N;;;;; +1D316;TETRAGRAM FOR HOLDING BACK;So;0;ON;;;;;N;;;;; +1D317;TETRAGRAM FOR WAITING;So;0;ON;;;;;N;;;;; +1D318;TETRAGRAM FOR FOLLOWING;So;0;ON;;;;;N;;;;; +1D319;TETRAGRAM FOR ADVANCE;So;0;ON;;;;;N;;;;; +1D31A;TETRAGRAM FOR RELEASE;So;0;ON;;;;;N;;;;; +1D31B;TETRAGRAM FOR RESISTANCE;So;0;ON;;;;;N;;;;; +1D31C;TETRAGRAM FOR EASE;So;0;ON;;;;;N;;;;; +1D31D;TETRAGRAM FOR JOY;So;0;ON;;;;;N;;;;; +1D31E;TETRAGRAM FOR CONTENTION;So;0;ON;;;;;N;;;;; +1D31F;TETRAGRAM FOR ENDEAVOUR;So;0;ON;;;;;N;;;;; +1D320;TETRAGRAM FOR DUTIES;So;0;ON;;;;;N;;;;; +1D321;TETRAGRAM FOR CHANGE;So;0;ON;;;;;N;;;;; +1D322;TETRAGRAM FOR DECISIVENESS;So;0;ON;;;;;N;;;;; +1D323;TETRAGRAM FOR BOLD RESOLUTION;So;0;ON;;;;;N;;;;; +1D324;TETRAGRAM FOR PACKING;So;0;ON;;;;;N;;;;; +1D325;TETRAGRAM FOR LEGION;So;0;ON;;;;;N;;;;; +1D326;TETRAGRAM FOR CLOSENESS;So;0;ON;;;;;N;;;;; +1D327;TETRAGRAM FOR KINSHIP;So;0;ON;;;;;N;;;;; +1D328;TETRAGRAM FOR GATHERING;So;0;ON;;;;;N;;;;; +1D329;TETRAGRAM FOR STRENGTH;So;0;ON;;;;;N;;;;; +1D32A;TETRAGRAM FOR PURITY;So;0;ON;;;;;N;;;;; +1D32B;TETRAGRAM FOR FULLNESS;So;0;ON;;;;;N;;;;; +1D32C;TETRAGRAM FOR RESIDENCE;So;0;ON;;;;;N;;;;; +1D32D;TETRAGRAM FOR LAW OR MODEL;So;0;ON;;;;;N;;;;; +1D32E;TETRAGRAM FOR RESPONSE;So;0;ON;;;;;N;;;;; +1D32F;TETRAGRAM FOR GOING TO MEET;So;0;ON;;;;;N;;;;; +1D330;TETRAGRAM FOR ENCOUNTERS;So;0;ON;;;;;N;;;;; +1D331;TETRAGRAM FOR STOVE;So;0;ON;;;;;N;;;;; +1D332;TETRAGRAM FOR GREATNESS;So;0;ON;;;;;N;;;;; +1D333;TETRAGRAM FOR ENLARGEMENT;So;0;ON;;;;;N;;;;; +1D334;TETRAGRAM FOR PATTERN;So;0;ON;;;;;N;;;;; +1D335;TETRAGRAM FOR RITUAL;So;0;ON;;;;;N;;;;; +1D336;TETRAGRAM FOR FLIGHT;So;0;ON;;;;;N;;;;; +1D337;TETRAGRAM FOR VASTNESS OR WASTING;So;0;ON;;;;;N;;;;; +1D338;TETRAGRAM FOR CONSTANCY;So;0;ON;;;;;N;;;;; +1D339;TETRAGRAM FOR MEASURE;So;0;ON;;;;;N;;;;; +1D33A;TETRAGRAM FOR ETERNITY;So;0;ON;;;;;N;;;;; +1D33B;TETRAGRAM FOR UNITY;So;0;ON;;;;;N;;;;; +1D33C;TETRAGRAM FOR DIMINISHMENT;So;0;ON;;;;;N;;;;; +1D33D;TETRAGRAM FOR CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1D33E;TETRAGRAM FOR GUARDEDNESS;So;0;ON;;;;;N;;;;; +1D33F;TETRAGRAM FOR GATHERING IN;So;0;ON;;;;;N;;;;; +1D340;TETRAGRAM FOR MASSING;So;0;ON;;;;;N;;;;; +1D341;TETRAGRAM FOR ACCUMULATION;So;0;ON;;;;;N;;;;; +1D342;TETRAGRAM FOR EMBELLISHMENT;So;0;ON;;;;;N;;;;; +1D343;TETRAGRAM FOR DOUBT;So;0;ON;;;;;N;;;;; +1D344;TETRAGRAM FOR WATCH;So;0;ON;;;;;N;;;;; +1D345;TETRAGRAM FOR SINKING;So;0;ON;;;;;N;;;;; +1D346;TETRAGRAM FOR INNER;So;0;ON;;;;;N;;;;; +1D347;TETRAGRAM FOR DEPARTURE;So;0;ON;;;;;N;;;;; +1D348;TETRAGRAM FOR DARKENING;So;0;ON;;;;;N;;;;; +1D349;TETRAGRAM FOR DIMMING;So;0;ON;;;;;N;;;;; +1D34A;TETRAGRAM FOR EXHAUSTION;So;0;ON;;;;;N;;;;; +1D34B;TETRAGRAM FOR SEVERANCE;So;0;ON;;;;;N;;;;; +1D34C;TETRAGRAM FOR STOPPAGE;So;0;ON;;;;;N;;;;; +1D34D;TETRAGRAM FOR HARDNESS;So;0;ON;;;;;N;;;;; +1D34E;TETRAGRAM FOR COMPLETION;So;0;ON;;;;;N;;;;; +1D34F;TETRAGRAM FOR CLOSURE;So;0;ON;;;;;N;;;;; +1D350;TETRAGRAM FOR FAILURE;So;0;ON;;;;;N;;;;; +1D351;TETRAGRAM FOR AGGRAVATION;So;0;ON;;;;;N;;;;; +1D352;TETRAGRAM FOR COMPLIANCE;So;0;ON;;;;;N;;;;; +1D353;TETRAGRAM FOR ON THE VERGE;So;0;ON;;;;;N;;;;; +1D354;TETRAGRAM FOR DIFFICULTIES;So;0;ON;;;;;N;;;;; +1D355;TETRAGRAM FOR LABOURING;So;0;ON;;;;;N;;;;; +1D356;TETRAGRAM FOR FOSTERING;So;0;ON;;;;;N;;;;; +1D360;COUNTING ROD UNIT DIGIT ONE;No;0;L;;;;1;N;;;;; +1D361;COUNTING ROD UNIT DIGIT TWO;No;0;L;;;;2;N;;;;; +1D362;COUNTING ROD UNIT DIGIT THREE;No;0;L;;;;3;N;;;;; +1D363;COUNTING ROD UNIT DIGIT FOUR;No;0;L;;;;4;N;;;;; +1D364;COUNTING ROD UNIT DIGIT FIVE;No;0;L;;;;5;N;;;;; +1D365;COUNTING ROD UNIT DIGIT SIX;No;0;L;;;;6;N;;;;; +1D366;COUNTING ROD UNIT DIGIT SEVEN;No;0;L;;;;7;N;;;;; +1D367;COUNTING ROD UNIT DIGIT EIGHT;No;0;L;;;;8;N;;;;; +1D368;COUNTING ROD UNIT DIGIT NINE;No;0;L;;;;9;N;;;;; +1D369;COUNTING ROD TENS DIGIT ONE;No;0;L;;;;10;N;;;;; +1D36A;COUNTING ROD TENS DIGIT TWO;No;0;L;;;;20;N;;;;; +1D36B;COUNTING ROD TENS DIGIT THREE;No;0;L;;;;30;N;;;;; +1D36C;COUNTING ROD TENS DIGIT FOUR;No;0;L;;;;40;N;;;;; +1D36D;COUNTING ROD TENS DIGIT FIVE;No;0;L;;;;50;N;;;;; +1D36E;COUNTING ROD TENS DIGIT SIX;No;0;L;;;;60;N;;;;; +1D36F;COUNTING ROD TENS DIGIT SEVEN;No;0;L;;;;70;N;;;;; +1D370;COUNTING ROD TENS DIGIT EIGHT;No;0;L;;;;80;N;;;;; +1D371;COUNTING ROD TENS DIGIT NINE;No;0;L;;;;90;N;;;;; +1D400;MATHEMATICAL BOLD CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D401;MATHEMATICAL BOLD CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D402;MATHEMATICAL BOLD CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D403;MATHEMATICAL BOLD CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D404;MATHEMATICAL BOLD CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D405;MATHEMATICAL BOLD CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D406;MATHEMATICAL BOLD CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D407;MATHEMATICAL BOLD CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D408;MATHEMATICAL BOLD CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D409;MATHEMATICAL BOLD CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D40A;MATHEMATICAL BOLD CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D40B;MATHEMATICAL BOLD CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D40C;MATHEMATICAL BOLD CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D40D;MATHEMATICAL BOLD CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D40E;MATHEMATICAL BOLD CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D40F;MATHEMATICAL BOLD CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D410;MATHEMATICAL BOLD CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D411;MATHEMATICAL BOLD CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D412;MATHEMATICAL BOLD CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D413;MATHEMATICAL BOLD CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D414;MATHEMATICAL BOLD CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D415;MATHEMATICAL BOLD CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D416;MATHEMATICAL BOLD CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D417;MATHEMATICAL BOLD CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D418;MATHEMATICAL BOLD CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D419;MATHEMATICAL BOLD CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D41A;MATHEMATICAL BOLD SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D41B;MATHEMATICAL BOLD SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D41C;MATHEMATICAL BOLD SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D41D;MATHEMATICAL BOLD SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D41E;MATHEMATICAL BOLD SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D41F;MATHEMATICAL BOLD SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D420;MATHEMATICAL BOLD SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D421;MATHEMATICAL BOLD SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D422;MATHEMATICAL BOLD SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D423;MATHEMATICAL BOLD SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D424;MATHEMATICAL BOLD SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D425;MATHEMATICAL BOLD SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D426;MATHEMATICAL BOLD SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D427;MATHEMATICAL BOLD SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D428;MATHEMATICAL BOLD SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D429;MATHEMATICAL BOLD SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D42A;MATHEMATICAL BOLD SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D42B;MATHEMATICAL BOLD SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D42C;MATHEMATICAL BOLD SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D42D;MATHEMATICAL BOLD SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D42E;MATHEMATICAL BOLD SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D42F;MATHEMATICAL BOLD SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D430;MATHEMATICAL BOLD SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D431;MATHEMATICAL BOLD SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D432;MATHEMATICAL BOLD SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D433;MATHEMATICAL BOLD SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D434;MATHEMATICAL ITALIC CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D435;MATHEMATICAL ITALIC CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D436;MATHEMATICAL ITALIC CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D437;MATHEMATICAL ITALIC CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D438;MATHEMATICAL ITALIC CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D439;MATHEMATICAL ITALIC CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D43A;MATHEMATICAL ITALIC CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D43B;MATHEMATICAL ITALIC CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D43C;MATHEMATICAL ITALIC CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D43D;MATHEMATICAL ITALIC CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D43E;MATHEMATICAL ITALIC CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D43F;MATHEMATICAL ITALIC CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D440;MATHEMATICAL ITALIC CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D441;MATHEMATICAL ITALIC CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D442;MATHEMATICAL ITALIC CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D443;MATHEMATICAL ITALIC CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D444;MATHEMATICAL ITALIC CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D445;MATHEMATICAL ITALIC CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D446;MATHEMATICAL ITALIC CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D447;MATHEMATICAL ITALIC CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D448;MATHEMATICAL ITALIC CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D449;MATHEMATICAL ITALIC CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D44A;MATHEMATICAL ITALIC CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D44B;MATHEMATICAL ITALIC CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D44C;MATHEMATICAL ITALIC CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D44D;MATHEMATICAL ITALIC CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D44E;MATHEMATICAL ITALIC SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D44F;MATHEMATICAL ITALIC SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D450;MATHEMATICAL ITALIC SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D451;MATHEMATICAL ITALIC SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D452;MATHEMATICAL ITALIC SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D453;MATHEMATICAL ITALIC SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D454;MATHEMATICAL ITALIC SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D456;MATHEMATICAL ITALIC SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D457;MATHEMATICAL ITALIC SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D458;MATHEMATICAL ITALIC SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D459;MATHEMATICAL ITALIC SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D45A;MATHEMATICAL ITALIC SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D45B;MATHEMATICAL ITALIC SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D45C;MATHEMATICAL ITALIC SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D45D;MATHEMATICAL ITALIC SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D45E;MATHEMATICAL ITALIC SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D45F;MATHEMATICAL ITALIC SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D460;MATHEMATICAL ITALIC SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D461;MATHEMATICAL ITALIC SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D462;MATHEMATICAL ITALIC SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D463;MATHEMATICAL ITALIC SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D464;MATHEMATICAL ITALIC SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D465;MATHEMATICAL ITALIC SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D466;MATHEMATICAL ITALIC SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D467;MATHEMATICAL ITALIC SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D468;MATHEMATICAL BOLD ITALIC CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D469;MATHEMATICAL BOLD ITALIC CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D46A;MATHEMATICAL BOLD ITALIC CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D46B;MATHEMATICAL BOLD ITALIC CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D46C;MATHEMATICAL BOLD ITALIC CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D46D;MATHEMATICAL BOLD ITALIC CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D46E;MATHEMATICAL BOLD ITALIC CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D46F;MATHEMATICAL BOLD ITALIC CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D470;MATHEMATICAL BOLD ITALIC CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D471;MATHEMATICAL BOLD ITALIC CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D472;MATHEMATICAL BOLD ITALIC CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D473;MATHEMATICAL BOLD ITALIC CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D474;MATHEMATICAL BOLD ITALIC CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D475;MATHEMATICAL BOLD ITALIC CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D476;MATHEMATICAL BOLD ITALIC CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D477;MATHEMATICAL BOLD ITALIC CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D478;MATHEMATICAL BOLD ITALIC CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D479;MATHEMATICAL BOLD ITALIC CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D47A;MATHEMATICAL BOLD ITALIC CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D47B;MATHEMATICAL BOLD ITALIC CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D47C;MATHEMATICAL BOLD ITALIC CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D47D;MATHEMATICAL BOLD ITALIC CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D47E;MATHEMATICAL BOLD ITALIC CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D47F;MATHEMATICAL BOLD ITALIC CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D480;MATHEMATICAL BOLD ITALIC CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D481;MATHEMATICAL BOLD ITALIC CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D482;MATHEMATICAL BOLD ITALIC SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D483;MATHEMATICAL BOLD ITALIC SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D484;MATHEMATICAL BOLD ITALIC SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D485;MATHEMATICAL BOLD ITALIC SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D486;MATHEMATICAL BOLD ITALIC SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D487;MATHEMATICAL BOLD ITALIC SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D488;MATHEMATICAL BOLD ITALIC SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D489;MATHEMATICAL BOLD ITALIC SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D48A;MATHEMATICAL BOLD ITALIC SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D48B;MATHEMATICAL BOLD ITALIC SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D48C;MATHEMATICAL BOLD ITALIC SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D48D;MATHEMATICAL BOLD ITALIC SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D48E;MATHEMATICAL BOLD ITALIC SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D48F;MATHEMATICAL BOLD ITALIC SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D490;MATHEMATICAL BOLD ITALIC SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D491;MATHEMATICAL BOLD ITALIC SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D492;MATHEMATICAL BOLD ITALIC SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D493;MATHEMATICAL BOLD ITALIC SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D494;MATHEMATICAL BOLD ITALIC SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D495;MATHEMATICAL BOLD ITALIC SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D496;MATHEMATICAL BOLD ITALIC SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D497;MATHEMATICAL BOLD ITALIC SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D498;MATHEMATICAL BOLD ITALIC SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D499;MATHEMATICAL BOLD ITALIC SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D49A;MATHEMATICAL BOLD ITALIC SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D49B;MATHEMATICAL BOLD ITALIC SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D49C;MATHEMATICAL SCRIPT CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D49E;MATHEMATICAL SCRIPT CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D49F;MATHEMATICAL SCRIPT CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D4A2;MATHEMATICAL SCRIPT CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D4A5;MATHEMATICAL SCRIPT CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D4A6;MATHEMATICAL SCRIPT CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D4A9;MATHEMATICAL SCRIPT CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D4AA;MATHEMATICAL SCRIPT CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D4AB;MATHEMATICAL SCRIPT CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D4AC;MATHEMATICAL SCRIPT CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D4AE;MATHEMATICAL SCRIPT CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D4AF;MATHEMATICAL SCRIPT CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D4B0;MATHEMATICAL SCRIPT CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D4B1;MATHEMATICAL SCRIPT CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D4B2;MATHEMATICAL SCRIPT CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D4B3;MATHEMATICAL SCRIPT CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D4B4;MATHEMATICAL SCRIPT CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D4B5;MATHEMATICAL SCRIPT CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D4B6;MATHEMATICAL SCRIPT SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D4B7;MATHEMATICAL SCRIPT SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D4B8;MATHEMATICAL SCRIPT SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D4B9;MATHEMATICAL SCRIPT SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D4BB;MATHEMATICAL SCRIPT SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D4BD;MATHEMATICAL SCRIPT SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D4BE;MATHEMATICAL SCRIPT SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D4BF;MATHEMATICAL SCRIPT SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D4C0;MATHEMATICAL SCRIPT SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D4C1;MATHEMATICAL SCRIPT SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D4C2;MATHEMATICAL SCRIPT SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D4C3;MATHEMATICAL SCRIPT SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D4C5;MATHEMATICAL SCRIPT SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D4C6;MATHEMATICAL SCRIPT SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D4C7;MATHEMATICAL SCRIPT SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D4C8;MATHEMATICAL SCRIPT SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D4C9;MATHEMATICAL SCRIPT SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D4CA;MATHEMATICAL SCRIPT SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D4CB;MATHEMATICAL SCRIPT SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D4CC;MATHEMATICAL SCRIPT SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D4CD;MATHEMATICAL SCRIPT SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D4CE;MATHEMATICAL SCRIPT SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D4CF;MATHEMATICAL SCRIPT SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D4D0;MATHEMATICAL BOLD SCRIPT CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D4D1;MATHEMATICAL BOLD SCRIPT CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D4D2;MATHEMATICAL BOLD SCRIPT CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D4D3;MATHEMATICAL BOLD SCRIPT CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D4D4;MATHEMATICAL BOLD SCRIPT CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D4D5;MATHEMATICAL BOLD SCRIPT CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D4D6;MATHEMATICAL BOLD SCRIPT CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D4D7;MATHEMATICAL BOLD SCRIPT CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D4D8;MATHEMATICAL BOLD SCRIPT CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D4D9;MATHEMATICAL BOLD SCRIPT CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D4DA;MATHEMATICAL BOLD SCRIPT CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D4DB;MATHEMATICAL BOLD SCRIPT CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D4DC;MATHEMATICAL BOLD SCRIPT CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D4DD;MATHEMATICAL BOLD SCRIPT CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D4DE;MATHEMATICAL BOLD SCRIPT CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D4DF;MATHEMATICAL BOLD SCRIPT CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D4E0;MATHEMATICAL BOLD SCRIPT CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D4E1;MATHEMATICAL BOLD SCRIPT CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D4E2;MATHEMATICAL BOLD SCRIPT CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D4E3;MATHEMATICAL BOLD SCRIPT CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D4E4;MATHEMATICAL BOLD SCRIPT CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D4E5;MATHEMATICAL BOLD SCRIPT CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D4E6;MATHEMATICAL BOLD SCRIPT CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D4E7;MATHEMATICAL BOLD SCRIPT CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D4E8;MATHEMATICAL BOLD SCRIPT CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D4E9;MATHEMATICAL BOLD SCRIPT CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D4EA;MATHEMATICAL BOLD SCRIPT SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D4EB;MATHEMATICAL BOLD SCRIPT SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D4EC;MATHEMATICAL BOLD SCRIPT SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D4ED;MATHEMATICAL BOLD SCRIPT SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D4EE;MATHEMATICAL BOLD SCRIPT SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D4EF;MATHEMATICAL BOLD SCRIPT SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D4F0;MATHEMATICAL BOLD SCRIPT SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D4F1;MATHEMATICAL BOLD SCRIPT SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D4F2;MATHEMATICAL BOLD SCRIPT SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D4F3;MATHEMATICAL BOLD SCRIPT SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D4F4;MATHEMATICAL BOLD SCRIPT SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D4F5;MATHEMATICAL BOLD SCRIPT SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D4F6;MATHEMATICAL BOLD SCRIPT SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D4F7;MATHEMATICAL BOLD SCRIPT SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D4F8;MATHEMATICAL BOLD SCRIPT SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D4F9;MATHEMATICAL BOLD SCRIPT SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D4FA;MATHEMATICAL BOLD SCRIPT SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D4FB;MATHEMATICAL BOLD SCRIPT SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D4FC;MATHEMATICAL BOLD SCRIPT SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D4FD;MATHEMATICAL BOLD SCRIPT SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D4FE;MATHEMATICAL BOLD SCRIPT SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D4FF;MATHEMATICAL BOLD SCRIPT SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D500;MATHEMATICAL BOLD SCRIPT SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D501;MATHEMATICAL BOLD SCRIPT SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D502;MATHEMATICAL BOLD SCRIPT SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D503;MATHEMATICAL BOLD SCRIPT SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D504;MATHEMATICAL FRAKTUR CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D505;MATHEMATICAL FRAKTUR CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D507;MATHEMATICAL FRAKTUR CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D508;MATHEMATICAL FRAKTUR CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D509;MATHEMATICAL FRAKTUR CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D50A;MATHEMATICAL FRAKTUR CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D50D;MATHEMATICAL FRAKTUR CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D50E;MATHEMATICAL FRAKTUR CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D50F;MATHEMATICAL FRAKTUR CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D510;MATHEMATICAL FRAKTUR CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D511;MATHEMATICAL FRAKTUR CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D512;MATHEMATICAL FRAKTUR CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D513;MATHEMATICAL FRAKTUR CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D514;MATHEMATICAL FRAKTUR CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D516;MATHEMATICAL FRAKTUR CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D517;MATHEMATICAL FRAKTUR CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D518;MATHEMATICAL FRAKTUR CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D519;MATHEMATICAL FRAKTUR CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D51A;MATHEMATICAL FRAKTUR CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D51B;MATHEMATICAL FRAKTUR CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D51C;MATHEMATICAL FRAKTUR CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D51E;MATHEMATICAL FRAKTUR SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D51F;MATHEMATICAL FRAKTUR SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D520;MATHEMATICAL FRAKTUR SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D521;MATHEMATICAL FRAKTUR SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D522;MATHEMATICAL FRAKTUR SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D523;MATHEMATICAL FRAKTUR SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D524;MATHEMATICAL FRAKTUR SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D525;MATHEMATICAL FRAKTUR SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D526;MATHEMATICAL FRAKTUR SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D527;MATHEMATICAL FRAKTUR SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D528;MATHEMATICAL FRAKTUR SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D529;MATHEMATICAL FRAKTUR SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D52A;MATHEMATICAL FRAKTUR SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D52B;MATHEMATICAL FRAKTUR SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D52C;MATHEMATICAL FRAKTUR SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D52D;MATHEMATICAL FRAKTUR SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D52E;MATHEMATICAL FRAKTUR SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D52F;MATHEMATICAL FRAKTUR SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D530;MATHEMATICAL FRAKTUR SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D531;MATHEMATICAL FRAKTUR SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D532;MATHEMATICAL FRAKTUR SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D533;MATHEMATICAL FRAKTUR SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D534;MATHEMATICAL FRAKTUR SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D535;MATHEMATICAL FRAKTUR SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D536;MATHEMATICAL FRAKTUR SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D537;MATHEMATICAL FRAKTUR SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D538;MATHEMATICAL DOUBLE-STRUCK CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D539;MATHEMATICAL DOUBLE-STRUCK CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D53B;MATHEMATICAL DOUBLE-STRUCK CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D53C;MATHEMATICAL DOUBLE-STRUCK CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D53D;MATHEMATICAL DOUBLE-STRUCK CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D53E;MATHEMATICAL DOUBLE-STRUCK CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D540;MATHEMATICAL DOUBLE-STRUCK CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D541;MATHEMATICAL DOUBLE-STRUCK CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D542;MATHEMATICAL DOUBLE-STRUCK CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D543;MATHEMATICAL DOUBLE-STRUCK CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D544;MATHEMATICAL DOUBLE-STRUCK CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D546;MATHEMATICAL DOUBLE-STRUCK CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D54A;MATHEMATICAL DOUBLE-STRUCK CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D54B;MATHEMATICAL DOUBLE-STRUCK CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D54C;MATHEMATICAL DOUBLE-STRUCK CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D54D;MATHEMATICAL DOUBLE-STRUCK CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D54E;MATHEMATICAL DOUBLE-STRUCK CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D54F;MATHEMATICAL DOUBLE-STRUCK CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D550;MATHEMATICAL DOUBLE-STRUCK CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D552;MATHEMATICAL DOUBLE-STRUCK SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D553;MATHEMATICAL DOUBLE-STRUCK SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D554;MATHEMATICAL DOUBLE-STRUCK SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D555;MATHEMATICAL DOUBLE-STRUCK SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D556;MATHEMATICAL DOUBLE-STRUCK SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D557;MATHEMATICAL DOUBLE-STRUCK SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D558;MATHEMATICAL DOUBLE-STRUCK SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D559;MATHEMATICAL DOUBLE-STRUCK SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D55A;MATHEMATICAL DOUBLE-STRUCK SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D55B;MATHEMATICAL DOUBLE-STRUCK SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D55C;MATHEMATICAL DOUBLE-STRUCK SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D55D;MATHEMATICAL DOUBLE-STRUCK SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D55E;MATHEMATICAL DOUBLE-STRUCK SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D55F;MATHEMATICAL DOUBLE-STRUCK SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D560;MATHEMATICAL DOUBLE-STRUCK SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D561;MATHEMATICAL DOUBLE-STRUCK SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D562;MATHEMATICAL DOUBLE-STRUCK SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D563;MATHEMATICAL DOUBLE-STRUCK SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D564;MATHEMATICAL DOUBLE-STRUCK SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D565;MATHEMATICAL DOUBLE-STRUCK SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D566;MATHEMATICAL DOUBLE-STRUCK SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D567;MATHEMATICAL DOUBLE-STRUCK SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D568;MATHEMATICAL DOUBLE-STRUCK SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D569;MATHEMATICAL DOUBLE-STRUCK SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D56A;MATHEMATICAL DOUBLE-STRUCK SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D56B;MATHEMATICAL DOUBLE-STRUCK SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D56C;MATHEMATICAL BOLD FRAKTUR CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D56D;MATHEMATICAL BOLD FRAKTUR CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D56E;MATHEMATICAL BOLD FRAKTUR CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D56F;MATHEMATICAL BOLD FRAKTUR CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D570;MATHEMATICAL BOLD FRAKTUR CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D571;MATHEMATICAL BOLD FRAKTUR CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D572;MATHEMATICAL BOLD FRAKTUR CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D573;MATHEMATICAL BOLD FRAKTUR CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D574;MATHEMATICAL BOLD FRAKTUR CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D575;MATHEMATICAL BOLD FRAKTUR CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D576;MATHEMATICAL BOLD FRAKTUR CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D577;MATHEMATICAL BOLD FRAKTUR CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D578;MATHEMATICAL BOLD FRAKTUR CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D579;MATHEMATICAL BOLD FRAKTUR CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D57A;MATHEMATICAL BOLD FRAKTUR CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D57B;MATHEMATICAL BOLD FRAKTUR CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D57C;MATHEMATICAL BOLD FRAKTUR CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D57D;MATHEMATICAL BOLD FRAKTUR CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D57E;MATHEMATICAL BOLD FRAKTUR CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D57F;MATHEMATICAL BOLD FRAKTUR CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D580;MATHEMATICAL BOLD FRAKTUR CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D581;MATHEMATICAL BOLD FRAKTUR CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D582;MATHEMATICAL BOLD FRAKTUR CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D583;MATHEMATICAL BOLD FRAKTUR CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D584;MATHEMATICAL BOLD FRAKTUR CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D585;MATHEMATICAL BOLD FRAKTUR CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D586;MATHEMATICAL BOLD FRAKTUR SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D587;MATHEMATICAL BOLD FRAKTUR SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D588;MATHEMATICAL BOLD FRAKTUR SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D589;MATHEMATICAL BOLD FRAKTUR SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D58A;MATHEMATICAL BOLD FRAKTUR SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D58B;MATHEMATICAL BOLD FRAKTUR SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D58C;MATHEMATICAL BOLD FRAKTUR SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D58D;MATHEMATICAL BOLD FRAKTUR SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D58E;MATHEMATICAL BOLD FRAKTUR SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D58F;MATHEMATICAL BOLD FRAKTUR SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D590;MATHEMATICAL BOLD FRAKTUR SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D591;MATHEMATICAL BOLD FRAKTUR SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D592;MATHEMATICAL BOLD FRAKTUR SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D593;MATHEMATICAL BOLD FRAKTUR SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D594;MATHEMATICAL BOLD FRAKTUR SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D595;MATHEMATICAL BOLD FRAKTUR SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D596;MATHEMATICAL BOLD FRAKTUR SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D597;MATHEMATICAL BOLD FRAKTUR SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D598;MATHEMATICAL BOLD FRAKTUR SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D599;MATHEMATICAL BOLD FRAKTUR SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D59A;MATHEMATICAL BOLD FRAKTUR SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D59B;MATHEMATICAL BOLD FRAKTUR SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D59C;MATHEMATICAL BOLD FRAKTUR SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D59D;MATHEMATICAL BOLD FRAKTUR SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D59E;MATHEMATICAL BOLD FRAKTUR SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D59F;MATHEMATICAL BOLD FRAKTUR SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D5A0;MATHEMATICAL SANS-SERIF CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D5A1;MATHEMATICAL SANS-SERIF CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D5A2;MATHEMATICAL SANS-SERIF CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D5A3;MATHEMATICAL SANS-SERIF CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D5A4;MATHEMATICAL SANS-SERIF CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D5A5;MATHEMATICAL SANS-SERIF CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D5A6;MATHEMATICAL SANS-SERIF CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D5A7;MATHEMATICAL SANS-SERIF CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D5A8;MATHEMATICAL SANS-SERIF CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D5A9;MATHEMATICAL SANS-SERIF CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D5AA;MATHEMATICAL SANS-SERIF CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D5AB;MATHEMATICAL SANS-SERIF CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D5AC;MATHEMATICAL SANS-SERIF CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D5AD;MATHEMATICAL SANS-SERIF CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D5AE;MATHEMATICAL SANS-SERIF CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D5AF;MATHEMATICAL SANS-SERIF CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D5B0;MATHEMATICAL SANS-SERIF CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D5B1;MATHEMATICAL SANS-SERIF CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D5B2;MATHEMATICAL SANS-SERIF CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D5B3;MATHEMATICAL SANS-SERIF CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D5B4;MATHEMATICAL SANS-SERIF CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D5B5;MATHEMATICAL SANS-SERIF CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D5B6;MATHEMATICAL SANS-SERIF CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D5B7;MATHEMATICAL SANS-SERIF CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D5B8;MATHEMATICAL SANS-SERIF CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D5B9;MATHEMATICAL SANS-SERIF CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D5BA;MATHEMATICAL SANS-SERIF SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D5BB;MATHEMATICAL SANS-SERIF SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D5BC;MATHEMATICAL SANS-SERIF SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D5BD;MATHEMATICAL SANS-SERIF SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D5BE;MATHEMATICAL SANS-SERIF SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D5BF;MATHEMATICAL SANS-SERIF SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D5C0;MATHEMATICAL SANS-SERIF SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D5C1;MATHEMATICAL SANS-SERIF SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D5C2;MATHEMATICAL SANS-SERIF SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D5C3;MATHEMATICAL SANS-SERIF SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D5C4;MATHEMATICAL SANS-SERIF SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D5C5;MATHEMATICAL SANS-SERIF SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D5C6;MATHEMATICAL SANS-SERIF SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D5C7;MATHEMATICAL SANS-SERIF SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D5C8;MATHEMATICAL SANS-SERIF SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D5C9;MATHEMATICAL SANS-SERIF SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D5CA;MATHEMATICAL SANS-SERIF SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D5CB;MATHEMATICAL SANS-SERIF SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D5CC;MATHEMATICAL SANS-SERIF SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D5CD;MATHEMATICAL SANS-SERIF SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D5CE;MATHEMATICAL SANS-SERIF SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D5CF;MATHEMATICAL SANS-SERIF SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D5D0;MATHEMATICAL SANS-SERIF SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D5D1;MATHEMATICAL SANS-SERIF SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D5D2;MATHEMATICAL SANS-SERIF SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D5D3;MATHEMATICAL SANS-SERIF SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D5D4;MATHEMATICAL SANS-SERIF BOLD CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D5D5;MATHEMATICAL SANS-SERIF BOLD CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D5D6;MATHEMATICAL SANS-SERIF BOLD CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D5D7;MATHEMATICAL SANS-SERIF BOLD CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D5D8;MATHEMATICAL SANS-SERIF BOLD CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D5D9;MATHEMATICAL SANS-SERIF BOLD CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D5DA;MATHEMATICAL SANS-SERIF BOLD CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D5DB;MATHEMATICAL SANS-SERIF BOLD CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D5DC;MATHEMATICAL SANS-SERIF BOLD CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D5DD;MATHEMATICAL SANS-SERIF BOLD CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D5DE;MATHEMATICAL SANS-SERIF BOLD CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D5DF;MATHEMATICAL SANS-SERIF BOLD CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D5E0;MATHEMATICAL SANS-SERIF BOLD CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D5E1;MATHEMATICAL SANS-SERIF BOLD CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D5E2;MATHEMATICAL SANS-SERIF BOLD CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D5E3;MATHEMATICAL SANS-SERIF BOLD CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D5E4;MATHEMATICAL SANS-SERIF BOLD CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D5E5;MATHEMATICAL SANS-SERIF BOLD CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D5E6;MATHEMATICAL SANS-SERIF BOLD CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D5E7;MATHEMATICAL SANS-SERIF BOLD CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D5E8;MATHEMATICAL SANS-SERIF BOLD CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D5E9;MATHEMATICAL SANS-SERIF BOLD CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D5EA;MATHEMATICAL SANS-SERIF BOLD CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D5EB;MATHEMATICAL SANS-SERIF BOLD CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D5EC;MATHEMATICAL SANS-SERIF BOLD CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D5ED;MATHEMATICAL SANS-SERIF BOLD CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D5EE;MATHEMATICAL SANS-SERIF BOLD SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D5EF;MATHEMATICAL SANS-SERIF BOLD SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D5F0;MATHEMATICAL SANS-SERIF BOLD SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D5F1;MATHEMATICAL SANS-SERIF BOLD SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D5F2;MATHEMATICAL SANS-SERIF BOLD SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D5F3;MATHEMATICAL SANS-SERIF BOLD SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D5F4;MATHEMATICAL SANS-SERIF BOLD SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D5F5;MATHEMATICAL SANS-SERIF BOLD SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D5F6;MATHEMATICAL SANS-SERIF BOLD SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D5F7;MATHEMATICAL SANS-SERIF BOLD SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D5F8;MATHEMATICAL SANS-SERIF BOLD SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D5F9;MATHEMATICAL SANS-SERIF BOLD SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D5FA;MATHEMATICAL SANS-SERIF BOLD SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D5FB;MATHEMATICAL SANS-SERIF BOLD SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D5FC;MATHEMATICAL SANS-SERIF BOLD SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D5FD;MATHEMATICAL SANS-SERIF BOLD SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D5FE;MATHEMATICAL SANS-SERIF BOLD SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D5FF;MATHEMATICAL SANS-SERIF BOLD SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D600;MATHEMATICAL SANS-SERIF BOLD SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D601;MATHEMATICAL SANS-SERIF BOLD SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D602;MATHEMATICAL SANS-SERIF BOLD SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D603;MATHEMATICAL SANS-SERIF BOLD SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D604;MATHEMATICAL SANS-SERIF BOLD SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D605;MATHEMATICAL SANS-SERIF BOLD SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D606;MATHEMATICAL SANS-SERIF BOLD SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D607;MATHEMATICAL SANS-SERIF BOLD SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D608;MATHEMATICAL SANS-SERIF ITALIC CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D609;MATHEMATICAL SANS-SERIF ITALIC CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D60A;MATHEMATICAL SANS-SERIF ITALIC CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D60B;MATHEMATICAL SANS-SERIF ITALIC CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D60C;MATHEMATICAL SANS-SERIF ITALIC CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D60D;MATHEMATICAL SANS-SERIF ITALIC CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D60E;MATHEMATICAL SANS-SERIF ITALIC CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D60F;MATHEMATICAL SANS-SERIF ITALIC CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D610;MATHEMATICAL SANS-SERIF ITALIC CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D611;MATHEMATICAL SANS-SERIF ITALIC CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D612;MATHEMATICAL SANS-SERIF ITALIC CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D613;MATHEMATICAL SANS-SERIF ITALIC CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D614;MATHEMATICAL SANS-SERIF ITALIC CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D615;MATHEMATICAL SANS-SERIF ITALIC CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D616;MATHEMATICAL SANS-SERIF ITALIC CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D617;MATHEMATICAL SANS-SERIF ITALIC CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D618;MATHEMATICAL SANS-SERIF ITALIC CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D619;MATHEMATICAL SANS-SERIF ITALIC CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D61A;MATHEMATICAL SANS-SERIF ITALIC CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D61B;MATHEMATICAL SANS-SERIF ITALIC CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D61C;MATHEMATICAL SANS-SERIF ITALIC CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D61D;MATHEMATICAL SANS-SERIF ITALIC CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D61E;MATHEMATICAL SANS-SERIF ITALIC CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D61F;MATHEMATICAL SANS-SERIF ITALIC CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D620;MATHEMATICAL SANS-SERIF ITALIC CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D621;MATHEMATICAL SANS-SERIF ITALIC CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D622;MATHEMATICAL SANS-SERIF ITALIC SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D623;MATHEMATICAL SANS-SERIF ITALIC SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D624;MATHEMATICAL SANS-SERIF ITALIC SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D625;MATHEMATICAL SANS-SERIF ITALIC SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D626;MATHEMATICAL SANS-SERIF ITALIC SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D627;MATHEMATICAL SANS-SERIF ITALIC SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D628;MATHEMATICAL SANS-SERIF ITALIC SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D629;MATHEMATICAL SANS-SERIF ITALIC SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D62A;MATHEMATICAL SANS-SERIF ITALIC SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D62B;MATHEMATICAL SANS-SERIF ITALIC SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D62C;MATHEMATICAL SANS-SERIF ITALIC SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D62D;MATHEMATICAL SANS-SERIF ITALIC SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D62E;MATHEMATICAL SANS-SERIF ITALIC SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D62F;MATHEMATICAL SANS-SERIF ITALIC SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D630;MATHEMATICAL SANS-SERIF ITALIC SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D631;MATHEMATICAL SANS-SERIF ITALIC SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D632;MATHEMATICAL SANS-SERIF ITALIC SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D633;MATHEMATICAL SANS-SERIF ITALIC SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D634;MATHEMATICAL SANS-SERIF ITALIC SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D635;MATHEMATICAL SANS-SERIF ITALIC SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D636;MATHEMATICAL SANS-SERIF ITALIC SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D637;MATHEMATICAL SANS-SERIF ITALIC SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D638;MATHEMATICAL SANS-SERIF ITALIC SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D639;MATHEMATICAL SANS-SERIF ITALIC SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D63A;MATHEMATICAL SANS-SERIF ITALIC SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D63B;MATHEMATICAL SANS-SERIF ITALIC SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D63C;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D63D;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D63E;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D63F;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D640;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D641;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D642;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D643;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D644;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D645;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D646;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D647;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D648;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D649;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D64A;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D64B;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D64C;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D64D;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D64E;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D64F;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D650;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D651;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D652;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D653;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D654;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D655;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D656;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D657;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D658;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D659;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D65A;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D65B;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D65C;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D65D;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D65E;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D65F;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D660;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D661;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D662;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D663;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D664;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D665;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D666;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D667;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D668;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D669;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D66A;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D66B;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D66C;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D66D;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D66E;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D66F;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D670;MATHEMATICAL MONOSPACE CAPITAL A;Lu;0;L; 0041;;;;N;;;;; +1D671;MATHEMATICAL MONOSPACE CAPITAL B;Lu;0;L; 0042;;;;N;;;;; +1D672;MATHEMATICAL MONOSPACE CAPITAL C;Lu;0;L; 0043;;;;N;;;;; +1D673;MATHEMATICAL MONOSPACE CAPITAL D;Lu;0;L; 0044;;;;N;;;;; +1D674;MATHEMATICAL MONOSPACE CAPITAL E;Lu;0;L; 0045;;;;N;;;;; +1D675;MATHEMATICAL MONOSPACE CAPITAL F;Lu;0;L; 0046;;;;N;;;;; +1D676;MATHEMATICAL MONOSPACE CAPITAL G;Lu;0;L; 0047;;;;N;;;;; +1D677;MATHEMATICAL MONOSPACE CAPITAL H;Lu;0;L; 0048;;;;N;;;;; +1D678;MATHEMATICAL MONOSPACE CAPITAL I;Lu;0;L; 0049;;;;N;;;;; +1D679;MATHEMATICAL MONOSPACE CAPITAL J;Lu;0;L; 004A;;;;N;;;;; +1D67A;MATHEMATICAL MONOSPACE CAPITAL K;Lu;0;L; 004B;;;;N;;;;; +1D67B;MATHEMATICAL MONOSPACE CAPITAL L;Lu;0;L; 004C;;;;N;;;;; +1D67C;MATHEMATICAL MONOSPACE CAPITAL M;Lu;0;L; 004D;;;;N;;;;; +1D67D;MATHEMATICAL MONOSPACE CAPITAL N;Lu;0;L; 004E;;;;N;;;;; +1D67E;MATHEMATICAL MONOSPACE CAPITAL O;Lu;0;L; 004F;;;;N;;;;; +1D67F;MATHEMATICAL MONOSPACE CAPITAL P;Lu;0;L; 0050;;;;N;;;;; +1D680;MATHEMATICAL MONOSPACE CAPITAL Q;Lu;0;L; 0051;;;;N;;;;; +1D681;MATHEMATICAL MONOSPACE CAPITAL R;Lu;0;L; 0052;;;;N;;;;; +1D682;MATHEMATICAL MONOSPACE CAPITAL S;Lu;0;L; 0053;;;;N;;;;; +1D683;MATHEMATICAL MONOSPACE CAPITAL T;Lu;0;L; 0054;;;;N;;;;; +1D684;MATHEMATICAL MONOSPACE CAPITAL U;Lu;0;L; 0055;;;;N;;;;; +1D685;MATHEMATICAL MONOSPACE CAPITAL V;Lu;0;L; 0056;;;;N;;;;; +1D686;MATHEMATICAL MONOSPACE CAPITAL W;Lu;0;L; 0057;;;;N;;;;; +1D687;MATHEMATICAL MONOSPACE CAPITAL X;Lu;0;L; 0058;;;;N;;;;; +1D688;MATHEMATICAL MONOSPACE CAPITAL Y;Lu;0;L; 0059;;;;N;;;;; +1D689;MATHEMATICAL MONOSPACE CAPITAL Z;Lu;0;L; 005A;;;;N;;;;; +1D68A;MATHEMATICAL MONOSPACE SMALL A;Ll;0;L; 0061;;;;N;;;;; +1D68B;MATHEMATICAL MONOSPACE SMALL B;Ll;0;L; 0062;;;;N;;;;; +1D68C;MATHEMATICAL MONOSPACE SMALL C;Ll;0;L; 0063;;;;N;;;;; +1D68D;MATHEMATICAL MONOSPACE SMALL D;Ll;0;L; 0064;;;;N;;;;; +1D68E;MATHEMATICAL MONOSPACE SMALL E;Ll;0;L; 0065;;;;N;;;;; +1D68F;MATHEMATICAL MONOSPACE SMALL F;Ll;0;L; 0066;;;;N;;;;; +1D690;MATHEMATICAL MONOSPACE SMALL G;Ll;0;L; 0067;;;;N;;;;; +1D691;MATHEMATICAL MONOSPACE SMALL H;Ll;0;L; 0068;;;;N;;;;; +1D692;MATHEMATICAL MONOSPACE SMALL I;Ll;0;L; 0069;;;;N;;;;; +1D693;MATHEMATICAL MONOSPACE SMALL J;Ll;0;L; 006A;;;;N;;;;; +1D694;MATHEMATICAL MONOSPACE SMALL K;Ll;0;L; 006B;;;;N;;;;; +1D695;MATHEMATICAL MONOSPACE SMALL L;Ll;0;L; 006C;;;;N;;;;; +1D696;MATHEMATICAL MONOSPACE SMALL M;Ll;0;L; 006D;;;;N;;;;; +1D697;MATHEMATICAL MONOSPACE SMALL N;Ll;0;L; 006E;;;;N;;;;; +1D698;MATHEMATICAL MONOSPACE SMALL O;Ll;0;L; 006F;;;;N;;;;; +1D699;MATHEMATICAL MONOSPACE SMALL P;Ll;0;L; 0070;;;;N;;;;; +1D69A;MATHEMATICAL MONOSPACE SMALL Q;Ll;0;L; 0071;;;;N;;;;; +1D69B;MATHEMATICAL MONOSPACE SMALL R;Ll;0;L; 0072;;;;N;;;;; +1D69C;MATHEMATICAL MONOSPACE SMALL S;Ll;0;L; 0073;;;;N;;;;; +1D69D;MATHEMATICAL MONOSPACE SMALL T;Ll;0;L; 0074;;;;N;;;;; +1D69E;MATHEMATICAL MONOSPACE SMALL U;Ll;0;L; 0075;;;;N;;;;; +1D69F;MATHEMATICAL MONOSPACE SMALL V;Ll;0;L; 0076;;;;N;;;;; +1D6A0;MATHEMATICAL MONOSPACE SMALL W;Ll;0;L; 0077;;;;N;;;;; +1D6A1;MATHEMATICAL MONOSPACE SMALL X;Ll;0;L; 0078;;;;N;;;;; +1D6A2;MATHEMATICAL MONOSPACE SMALL Y;Ll;0;L; 0079;;;;N;;;;; +1D6A3;MATHEMATICAL MONOSPACE SMALL Z;Ll;0;L; 007A;;;;N;;;;; +1D6A4;MATHEMATICAL ITALIC SMALL DOTLESS I;Ll;0;L; 0131;;;;N;;;;; +1D6A5;MATHEMATICAL ITALIC SMALL DOTLESS J;Ll;0;L; 0237;;;;N;;;;; +1D6A8;MATHEMATICAL BOLD CAPITAL ALPHA;Lu;0;L; 0391;;;;N;;;;; +1D6A9;MATHEMATICAL BOLD CAPITAL BETA;Lu;0;L; 0392;;;;N;;;;; +1D6AA;MATHEMATICAL BOLD CAPITAL GAMMA;Lu;0;L; 0393;;;;N;;;;; +1D6AB;MATHEMATICAL BOLD CAPITAL DELTA;Lu;0;L; 0394;;;;N;;;;; +1D6AC;MATHEMATICAL BOLD CAPITAL EPSILON;Lu;0;L; 0395;;;;N;;;;; +1D6AD;MATHEMATICAL BOLD CAPITAL ZETA;Lu;0;L; 0396;;;;N;;;;; +1D6AE;MATHEMATICAL BOLD CAPITAL ETA;Lu;0;L; 0397;;;;N;;;;; +1D6AF;MATHEMATICAL BOLD CAPITAL THETA;Lu;0;L; 0398;;;;N;;;;; +1D6B0;MATHEMATICAL BOLD CAPITAL IOTA;Lu;0;L; 0399;;;;N;;;;; +1D6B1;MATHEMATICAL BOLD CAPITAL KAPPA;Lu;0;L; 039A;;;;N;;;;; +1D6B2;MATHEMATICAL BOLD CAPITAL LAMDA;Lu;0;L; 039B;;;;N;;;;; +1D6B3;MATHEMATICAL BOLD CAPITAL MU;Lu;0;L; 039C;;;;N;;;;; +1D6B4;MATHEMATICAL BOLD CAPITAL NU;Lu;0;L; 039D;;;;N;;;;; +1D6B5;MATHEMATICAL BOLD CAPITAL XI;Lu;0;L; 039E;;;;N;;;;; +1D6B6;MATHEMATICAL BOLD CAPITAL OMICRON;Lu;0;L; 039F;;;;N;;;;; +1D6B7;MATHEMATICAL BOLD CAPITAL PI;Lu;0;L; 03A0;;;;N;;;;; +1D6B8;MATHEMATICAL BOLD CAPITAL RHO;Lu;0;L; 03A1;;;;N;;;;; +1D6B9;MATHEMATICAL BOLD CAPITAL THETA SYMBOL;Lu;0;L; 03F4;;;;N;;;;; +1D6BA;MATHEMATICAL BOLD CAPITAL SIGMA;Lu;0;L; 03A3;;;;N;;;;; +1D6BB;MATHEMATICAL BOLD CAPITAL TAU;Lu;0;L; 03A4;;;;N;;;;; +1D6BC;MATHEMATICAL BOLD CAPITAL UPSILON;Lu;0;L; 03A5;;;;N;;;;; +1D6BD;MATHEMATICAL BOLD CAPITAL PHI;Lu;0;L; 03A6;;;;N;;;;; +1D6BE;MATHEMATICAL BOLD CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; +1D6BF;MATHEMATICAL BOLD CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; +1D6C0;MATHEMATICAL BOLD CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; +1D6C1;MATHEMATICAL BOLD NABLA;Sm;0;L; 2207;;;;N;;;;; +1D6C2;MATHEMATICAL BOLD SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; +1D6C3;MATHEMATICAL BOLD SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; +1D6C4;MATHEMATICAL BOLD SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; +1D6C5;MATHEMATICAL BOLD SMALL DELTA;Ll;0;L; 03B4;;;;N;;;;; +1D6C6;MATHEMATICAL BOLD SMALL EPSILON;Ll;0;L; 03B5;;;;N;;;;; +1D6C7;MATHEMATICAL BOLD SMALL ZETA;Ll;0;L; 03B6;;;;N;;;;; +1D6C8;MATHEMATICAL BOLD SMALL ETA;Ll;0;L; 03B7;;;;N;;;;; +1D6C9;MATHEMATICAL BOLD SMALL THETA;Ll;0;L; 03B8;;;;N;;;;; +1D6CA;MATHEMATICAL BOLD SMALL IOTA;Ll;0;L; 03B9;;;;N;;;;; +1D6CB;MATHEMATICAL BOLD SMALL KAPPA;Ll;0;L; 03BA;;;;N;;;;; +1D6CC;MATHEMATICAL BOLD SMALL LAMDA;Ll;0;L; 03BB;;;;N;;;;; +1D6CD;MATHEMATICAL BOLD SMALL MU;Ll;0;L; 03BC;;;;N;;;;; +1D6CE;MATHEMATICAL BOLD SMALL NU;Ll;0;L; 03BD;;;;N;;;;; +1D6CF;MATHEMATICAL BOLD SMALL XI;Ll;0;L; 03BE;;;;N;;;;; +1D6D0;MATHEMATICAL BOLD SMALL OMICRON;Ll;0;L; 03BF;;;;N;;;;; +1D6D1;MATHEMATICAL BOLD SMALL PI;Ll;0;L; 03C0;;;;N;;;;; +1D6D2;MATHEMATICAL BOLD SMALL RHO;Ll;0;L; 03C1;;;;N;;;;; +1D6D3;MATHEMATICAL BOLD SMALL FINAL SIGMA;Ll;0;L; 03C2;;;;N;;;;; +1D6D4;MATHEMATICAL BOLD SMALL SIGMA;Ll;0;L; 03C3;;;;N;;;;; +1D6D5;MATHEMATICAL BOLD SMALL TAU;Ll;0;L; 03C4;;;;N;;;;; +1D6D6;MATHEMATICAL BOLD SMALL UPSILON;Ll;0;L; 03C5;;;;N;;;;; +1D6D7;MATHEMATICAL BOLD SMALL PHI;Ll;0;L; 03C6;;;;N;;;;; +1D6D8;MATHEMATICAL BOLD SMALL CHI;Ll;0;L; 03C7;;;;N;;;;; +1D6D9;MATHEMATICAL BOLD SMALL PSI;Ll;0;L; 03C8;;;;N;;;;; +1D6DA;MATHEMATICAL BOLD SMALL OMEGA;Ll;0;L; 03C9;;;;N;;;;; +1D6DB;MATHEMATICAL BOLD PARTIAL DIFFERENTIAL;Sm;0;ON; 2202;;;;Y;;;;; +1D6DC;MATHEMATICAL BOLD EPSILON SYMBOL;Ll;0;L; 03F5;;;;N;;;;; +1D6DD;MATHEMATICAL BOLD THETA SYMBOL;Ll;0;L; 03D1;;;;N;;;;; +1D6DE;MATHEMATICAL BOLD KAPPA SYMBOL;Ll;0;L; 03F0;;;;N;;;;; +1D6DF;MATHEMATICAL BOLD PHI SYMBOL;Ll;0;L; 03D5;;;;N;;;;; +1D6E0;MATHEMATICAL BOLD RHO SYMBOL;Ll;0;L; 03F1;;;;N;;;;; +1D6E1;MATHEMATICAL BOLD PI SYMBOL;Ll;0;L; 03D6;;;;N;;;;; +1D6E2;MATHEMATICAL ITALIC CAPITAL ALPHA;Lu;0;L; 0391;;;;N;;;;; +1D6E3;MATHEMATICAL ITALIC CAPITAL BETA;Lu;0;L; 0392;;;;N;;;;; +1D6E4;MATHEMATICAL ITALIC CAPITAL GAMMA;Lu;0;L; 0393;;;;N;;;;; +1D6E5;MATHEMATICAL ITALIC CAPITAL DELTA;Lu;0;L; 0394;;;;N;;;;; +1D6E6;MATHEMATICAL ITALIC CAPITAL EPSILON;Lu;0;L; 0395;;;;N;;;;; +1D6E7;MATHEMATICAL ITALIC CAPITAL ZETA;Lu;0;L; 0396;;;;N;;;;; +1D6E8;MATHEMATICAL ITALIC CAPITAL ETA;Lu;0;L; 0397;;;;N;;;;; +1D6E9;MATHEMATICAL ITALIC CAPITAL THETA;Lu;0;L; 0398;;;;N;;;;; +1D6EA;MATHEMATICAL ITALIC CAPITAL IOTA;Lu;0;L; 0399;;;;N;;;;; +1D6EB;MATHEMATICAL ITALIC CAPITAL KAPPA;Lu;0;L; 039A;;;;N;;;;; +1D6EC;MATHEMATICAL ITALIC CAPITAL LAMDA;Lu;0;L; 039B;;;;N;;;;; +1D6ED;MATHEMATICAL ITALIC CAPITAL MU;Lu;0;L; 039C;;;;N;;;;; +1D6EE;MATHEMATICAL ITALIC CAPITAL NU;Lu;0;L; 039D;;;;N;;;;; +1D6EF;MATHEMATICAL ITALIC CAPITAL XI;Lu;0;L; 039E;;;;N;;;;; +1D6F0;MATHEMATICAL ITALIC CAPITAL OMICRON;Lu;0;L; 039F;;;;N;;;;; +1D6F1;MATHEMATICAL ITALIC CAPITAL PI;Lu;0;L; 03A0;;;;N;;;;; +1D6F2;MATHEMATICAL ITALIC CAPITAL RHO;Lu;0;L; 03A1;;;;N;;;;; +1D6F3;MATHEMATICAL ITALIC CAPITAL THETA SYMBOL;Lu;0;L; 03F4;;;;N;;;;; +1D6F4;MATHEMATICAL ITALIC CAPITAL SIGMA;Lu;0;L; 03A3;;;;N;;;;; +1D6F5;MATHEMATICAL ITALIC CAPITAL TAU;Lu;0;L; 03A4;;;;N;;;;; +1D6F6;MATHEMATICAL ITALIC CAPITAL UPSILON;Lu;0;L; 03A5;;;;N;;;;; +1D6F7;MATHEMATICAL ITALIC CAPITAL PHI;Lu;0;L; 03A6;;;;N;;;;; +1D6F8;MATHEMATICAL ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; +1D6F9;MATHEMATICAL ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; +1D6FA;MATHEMATICAL ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; +1D6FB;MATHEMATICAL ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D6FC;MATHEMATICAL ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; +1D6FD;MATHEMATICAL ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; +1D6FE;MATHEMATICAL ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; +1D6FF;MATHEMATICAL ITALIC SMALL DELTA;Ll;0;L; 03B4;;;;N;;;;; +1D700;MATHEMATICAL ITALIC SMALL EPSILON;Ll;0;L; 03B5;;;;N;;;;; +1D701;MATHEMATICAL ITALIC SMALL ZETA;Ll;0;L; 03B6;;;;N;;;;; +1D702;MATHEMATICAL ITALIC SMALL ETA;Ll;0;L; 03B7;;;;N;;;;; +1D703;MATHEMATICAL ITALIC SMALL THETA;Ll;0;L; 03B8;;;;N;;;;; +1D704;MATHEMATICAL ITALIC SMALL IOTA;Ll;0;L; 03B9;;;;N;;;;; +1D705;MATHEMATICAL ITALIC SMALL KAPPA;Ll;0;L; 03BA;;;;N;;;;; +1D706;MATHEMATICAL ITALIC SMALL LAMDA;Ll;0;L; 03BB;;;;N;;;;; +1D707;MATHEMATICAL ITALIC SMALL MU;Ll;0;L; 03BC;;;;N;;;;; +1D708;MATHEMATICAL ITALIC SMALL NU;Ll;0;L; 03BD;;;;N;;;;; +1D709;MATHEMATICAL ITALIC SMALL XI;Ll;0;L; 03BE;;;;N;;;;; +1D70A;MATHEMATICAL ITALIC SMALL OMICRON;Ll;0;L; 03BF;;;;N;;;;; +1D70B;MATHEMATICAL ITALIC SMALL PI;Ll;0;L; 03C0;;;;N;;;;; +1D70C;MATHEMATICAL ITALIC SMALL RHO;Ll;0;L; 03C1;;;;N;;;;; +1D70D;MATHEMATICAL ITALIC SMALL FINAL SIGMA;Ll;0;L; 03C2;;;;N;;;;; +1D70E;MATHEMATICAL ITALIC SMALL SIGMA;Ll;0;L; 03C3;;;;N;;;;; +1D70F;MATHEMATICAL ITALIC SMALL TAU;Ll;0;L; 03C4;;;;N;;;;; +1D710;MATHEMATICAL ITALIC SMALL UPSILON;Ll;0;L; 03C5;;;;N;;;;; +1D711;MATHEMATICAL ITALIC SMALL PHI;Ll;0;L; 03C6;;;;N;;;;; +1D712;MATHEMATICAL ITALIC SMALL CHI;Ll;0;L; 03C7;;;;N;;;;; +1D713;MATHEMATICAL ITALIC SMALL PSI;Ll;0;L; 03C8;;;;N;;;;; +1D714;MATHEMATICAL ITALIC SMALL OMEGA;Ll;0;L; 03C9;;;;N;;;;; +1D715;MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL;Sm;0;ON; 2202;;;;Y;;;;; +1D716;MATHEMATICAL ITALIC EPSILON SYMBOL;Ll;0;L; 03F5;;;;N;;;;; +1D717;MATHEMATICAL ITALIC THETA SYMBOL;Ll;0;L; 03D1;;;;N;;;;; +1D718;MATHEMATICAL ITALIC KAPPA SYMBOL;Ll;0;L; 03F0;;;;N;;;;; +1D719;MATHEMATICAL ITALIC PHI SYMBOL;Ll;0;L; 03D5;;;;N;;;;; +1D71A;MATHEMATICAL ITALIC RHO SYMBOL;Ll;0;L; 03F1;;;;N;;;;; +1D71B;MATHEMATICAL ITALIC PI SYMBOL;Ll;0;L; 03D6;;;;N;;;;; +1D71C;MATHEMATICAL BOLD ITALIC CAPITAL ALPHA;Lu;0;L; 0391;;;;N;;;;; +1D71D;MATHEMATICAL BOLD ITALIC CAPITAL BETA;Lu;0;L; 0392;;;;N;;;;; +1D71E;MATHEMATICAL BOLD ITALIC CAPITAL GAMMA;Lu;0;L; 0393;;;;N;;;;; +1D71F;MATHEMATICAL BOLD ITALIC CAPITAL DELTA;Lu;0;L; 0394;;;;N;;;;; +1D720;MATHEMATICAL BOLD ITALIC CAPITAL EPSILON;Lu;0;L; 0395;;;;N;;;;; +1D721;MATHEMATICAL BOLD ITALIC CAPITAL ZETA;Lu;0;L; 0396;;;;N;;;;; +1D722;MATHEMATICAL BOLD ITALIC CAPITAL ETA;Lu;0;L; 0397;;;;N;;;;; +1D723;MATHEMATICAL BOLD ITALIC CAPITAL THETA;Lu;0;L; 0398;;;;N;;;;; +1D724;MATHEMATICAL BOLD ITALIC CAPITAL IOTA;Lu;0;L; 0399;;;;N;;;;; +1D725;MATHEMATICAL BOLD ITALIC CAPITAL KAPPA;Lu;0;L; 039A;;;;N;;;;; +1D726;MATHEMATICAL BOLD ITALIC CAPITAL LAMDA;Lu;0;L; 039B;;;;N;;;;; +1D727;MATHEMATICAL BOLD ITALIC CAPITAL MU;Lu;0;L; 039C;;;;N;;;;; +1D728;MATHEMATICAL BOLD ITALIC CAPITAL NU;Lu;0;L; 039D;;;;N;;;;; +1D729;MATHEMATICAL BOLD ITALIC CAPITAL XI;Lu;0;L; 039E;;;;N;;;;; +1D72A;MATHEMATICAL BOLD ITALIC CAPITAL OMICRON;Lu;0;L; 039F;;;;N;;;;; +1D72B;MATHEMATICAL BOLD ITALIC CAPITAL PI;Lu;0;L; 03A0;;;;N;;;;; +1D72C;MATHEMATICAL BOLD ITALIC CAPITAL RHO;Lu;0;L; 03A1;;;;N;;;;; +1D72D;MATHEMATICAL BOLD ITALIC CAPITAL THETA SYMBOL;Lu;0;L; 03F4;;;;N;;;;; +1D72E;MATHEMATICAL BOLD ITALIC CAPITAL SIGMA;Lu;0;L; 03A3;;;;N;;;;; +1D72F;MATHEMATICAL BOLD ITALIC CAPITAL TAU;Lu;0;L; 03A4;;;;N;;;;; +1D730;MATHEMATICAL BOLD ITALIC CAPITAL UPSILON;Lu;0;L; 03A5;;;;N;;;;; +1D731;MATHEMATICAL BOLD ITALIC CAPITAL PHI;Lu;0;L; 03A6;;;;N;;;;; +1D732;MATHEMATICAL BOLD ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; +1D733;MATHEMATICAL BOLD ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; +1D734;MATHEMATICAL BOLD ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; +1D735;MATHEMATICAL BOLD ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D736;MATHEMATICAL BOLD ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; +1D737;MATHEMATICAL BOLD ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; +1D738;MATHEMATICAL BOLD ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; +1D739;MATHEMATICAL BOLD ITALIC SMALL DELTA;Ll;0;L; 03B4;;;;N;;;;; +1D73A;MATHEMATICAL BOLD ITALIC SMALL EPSILON;Ll;0;L; 03B5;;;;N;;;;; +1D73B;MATHEMATICAL BOLD ITALIC SMALL ZETA;Ll;0;L; 03B6;;;;N;;;;; +1D73C;MATHEMATICAL BOLD ITALIC SMALL ETA;Ll;0;L; 03B7;;;;N;;;;; +1D73D;MATHEMATICAL BOLD ITALIC SMALL THETA;Ll;0;L; 03B8;;;;N;;;;; +1D73E;MATHEMATICAL BOLD ITALIC SMALL IOTA;Ll;0;L; 03B9;;;;N;;;;; +1D73F;MATHEMATICAL BOLD ITALIC SMALL KAPPA;Ll;0;L; 03BA;;;;N;;;;; +1D740;MATHEMATICAL BOLD ITALIC SMALL LAMDA;Ll;0;L; 03BB;;;;N;;;;; +1D741;MATHEMATICAL BOLD ITALIC SMALL MU;Ll;0;L; 03BC;;;;N;;;;; +1D742;MATHEMATICAL BOLD ITALIC SMALL NU;Ll;0;L; 03BD;;;;N;;;;; +1D743;MATHEMATICAL BOLD ITALIC SMALL XI;Ll;0;L; 03BE;;;;N;;;;; +1D744;MATHEMATICAL BOLD ITALIC SMALL OMICRON;Ll;0;L; 03BF;;;;N;;;;; +1D745;MATHEMATICAL BOLD ITALIC SMALL PI;Ll;0;L; 03C0;;;;N;;;;; +1D746;MATHEMATICAL BOLD ITALIC SMALL RHO;Ll;0;L; 03C1;;;;N;;;;; +1D747;MATHEMATICAL BOLD ITALIC SMALL FINAL SIGMA;Ll;0;L; 03C2;;;;N;;;;; +1D748;MATHEMATICAL BOLD ITALIC SMALL SIGMA;Ll;0;L; 03C3;;;;N;;;;; +1D749;MATHEMATICAL BOLD ITALIC SMALL TAU;Ll;0;L; 03C4;;;;N;;;;; +1D74A;MATHEMATICAL BOLD ITALIC SMALL UPSILON;Ll;0;L; 03C5;;;;N;;;;; +1D74B;MATHEMATICAL BOLD ITALIC SMALL PHI;Ll;0;L; 03C6;;;;N;;;;; +1D74C;MATHEMATICAL BOLD ITALIC SMALL CHI;Ll;0;L; 03C7;;;;N;;;;; +1D74D;MATHEMATICAL BOLD ITALIC SMALL PSI;Ll;0;L; 03C8;;;;N;;;;; +1D74E;MATHEMATICAL BOLD ITALIC SMALL OMEGA;Ll;0;L; 03C9;;;;N;;;;; +1D74F;MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL;Sm;0;ON; 2202;;;;Y;;;;; +1D750;MATHEMATICAL BOLD ITALIC EPSILON SYMBOL;Ll;0;L; 03F5;;;;N;;;;; +1D751;MATHEMATICAL BOLD ITALIC THETA SYMBOL;Ll;0;L; 03D1;;;;N;;;;; +1D752;MATHEMATICAL BOLD ITALIC KAPPA SYMBOL;Ll;0;L; 03F0;;;;N;;;;; +1D753;MATHEMATICAL BOLD ITALIC PHI SYMBOL;Ll;0;L; 03D5;;;;N;;;;; +1D754;MATHEMATICAL BOLD ITALIC RHO SYMBOL;Ll;0;L; 03F1;;;;N;;;;; +1D755;MATHEMATICAL BOLD ITALIC PI SYMBOL;Ll;0;L; 03D6;;;;N;;;;; +1D756;MATHEMATICAL SANS-SERIF BOLD CAPITAL ALPHA;Lu;0;L; 0391;;;;N;;;;; +1D757;MATHEMATICAL SANS-SERIF BOLD CAPITAL BETA;Lu;0;L; 0392;;;;N;;;;; +1D758;MATHEMATICAL SANS-SERIF BOLD CAPITAL GAMMA;Lu;0;L; 0393;;;;N;;;;; +1D759;MATHEMATICAL SANS-SERIF BOLD CAPITAL DELTA;Lu;0;L; 0394;;;;N;;;;; +1D75A;MATHEMATICAL SANS-SERIF BOLD CAPITAL EPSILON;Lu;0;L; 0395;;;;N;;;;; +1D75B;MATHEMATICAL SANS-SERIF BOLD CAPITAL ZETA;Lu;0;L; 0396;;;;N;;;;; +1D75C;MATHEMATICAL SANS-SERIF BOLD CAPITAL ETA;Lu;0;L; 0397;;;;N;;;;; +1D75D;MATHEMATICAL SANS-SERIF BOLD CAPITAL THETA;Lu;0;L; 0398;;;;N;;;;; +1D75E;MATHEMATICAL SANS-SERIF BOLD CAPITAL IOTA;Lu;0;L; 0399;;;;N;;;;; +1D75F;MATHEMATICAL SANS-SERIF BOLD CAPITAL KAPPA;Lu;0;L; 039A;;;;N;;;;; +1D760;MATHEMATICAL SANS-SERIF BOLD CAPITAL LAMDA;Lu;0;L; 039B;;;;N;;;;; +1D761;MATHEMATICAL SANS-SERIF BOLD CAPITAL MU;Lu;0;L; 039C;;;;N;;;;; +1D762;MATHEMATICAL SANS-SERIF BOLD CAPITAL NU;Lu;0;L; 039D;;;;N;;;;; +1D763;MATHEMATICAL SANS-SERIF BOLD CAPITAL XI;Lu;0;L; 039E;;;;N;;;;; +1D764;MATHEMATICAL SANS-SERIF BOLD CAPITAL OMICRON;Lu;0;L; 039F;;;;N;;;;; +1D765;MATHEMATICAL SANS-SERIF BOLD CAPITAL PI;Lu;0;L; 03A0;;;;N;;;;; +1D766;MATHEMATICAL SANS-SERIF BOLD CAPITAL RHO;Lu;0;L; 03A1;;;;N;;;;; +1D767;MATHEMATICAL SANS-SERIF BOLD CAPITAL THETA SYMBOL;Lu;0;L; 03F4;;;;N;;;;; +1D768;MATHEMATICAL SANS-SERIF BOLD CAPITAL SIGMA;Lu;0;L; 03A3;;;;N;;;;; +1D769;MATHEMATICAL SANS-SERIF BOLD CAPITAL TAU;Lu;0;L; 03A4;;;;N;;;;; +1D76A;MATHEMATICAL SANS-SERIF BOLD CAPITAL UPSILON;Lu;0;L; 03A5;;;;N;;;;; +1D76B;MATHEMATICAL SANS-SERIF BOLD CAPITAL PHI;Lu;0;L; 03A6;;;;N;;;;; +1D76C;MATHEMATICAL SANS-SERIF BOLD CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; +1D76D;MATHEMATICAL SANS-SERIF BOLD CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; +1D76E;MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; +1D76F;MATHEMATICAL SANS-SERIF BOLD NABLA;Sm;0;L; 2207;;;;N;;;;; +1D770;MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; +1D771;MATHEMATICAL SANS-SERIF BOLD SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; +1D772;MATHEMATICAL SANS-SERIF BOLD SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; +1D773;MATHEMATICAL SANS-SERIF BOLD SMALL DELTA;Ll;0;L; 03B4;;;;N;;;;; +1D774;MATHEMATICAL SANS-SERIF BOLD SMALL EPSILON;Ll;0;L; 03B5;;;;N;;;;; +1D775;MATHEMATICAL SANS-SERIF BOLD SMALL ZETA;Ll;0;L; 03B6;;;;N;;;;; +1D776;MATHEMATICAL SANS-SERIF BOLD SMALL ETA;Ll;0;L; 03B7;;;;N;;;;; +1D777;MATHEMATICAL SANS-SERIF BOLD SMALL THETA;Ll;0;L; 03B8;;;;N;;;;; +1D778;MATHEMATICAL SANS-SERIF BOLD SMALL IOTA;Ll;0;L; 03B9;;;;N;;;;; +1D779;MATHEMATICAL SANS-SERIF BOLD SMALL KAPPA;Ll;0;L; 03BA;;;;N;;;;; +1D77A;MATHEMATICAL SANS-SERIF BOLD SMALL LAMDA;Ll;0;L; 03BB;;;;N;;;;; +1D77B;MATHEMATICAL SANS-SERIF BOLD SMALL MU;Ll;0;L; 03BC;;;;N;;;;; +1D77C;MATHEMATICAL SANS-SERIF BOLD SMALL NU;Ll;0;L; 03BD;;;;N;;;;; +1D77D;MATHEMATICAL SANS-SERIF BOLD SMALL XI;Ll;0;L; 03BE;;;;N;;;;; +1D77E;MATHEMATICAL SANS-SERIF BOLD SMALL OMICRON;Ll;0;L; 03BF;;;;N;;;;; +1D77F;MATHEMATICAL SANS-SERIF BOLD SMALL PI;Ll;0;L; 03C0;;;;N;;;;; +1D780;MATHEMATICAL SANS-SERIF BOLD SMALL RHO;Ll;0;L; 03C1;;;;N;;;;; +1D781;MATHEMATICAL SANS-SERIF BOLD SMALL FINAL SIGMA;Ll;0;L; 03C2;;;;N;;;;; +1D782;MATHEMATICAL SANS-SERIF BOLD SMALL SIGMA;Ll;0;L; 03C3;;;;N;;;;; +1D783;MATHEMATICAL SANS-SERIF BOLD SMALL TAU;Ll;0;L; 03C4;;;;N;;;;; +1D784;MATHEMATICAL SANS-SERIF BOLD SMALL UPSILON;Ll;0;L; 03C5;;;;N;;;;; +1D785;MATHEMATICAL SANS-SERIF BOLD SMALL PHI;Ll;0;L; 03C6;;;;N;;;;; +1D786;MATHEMATICAL SANS-SERIF BOLD SMALL CHI;Ll;0;L; 03C7;;;;N;;;;; +1D787;MATHEMATICAL SANS-SERIF BOLD SMALL PSI;Ll;0;L; 03C8;;;;N;;;;; +1D788;MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA;Ll;0;L; 03C9;;;;N;;;;; +1D789;MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL;Sm;0;ON; 2202;;;;Y;;;;; +1D78A;MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL;Ll;0;L; 03F5;;;;N;;;;; +1D78B;MATHEMATICAL SANS-SERIF BOLD THETA SYMBOL;Ll;0;L; 03D1;;;;N;;;;; +1D78C;MATHEMATICAL SANS-SERIF BOLD KAPPA SYMBOL;Ll;0;L; 03F0;;;;N;;;;; +1D78D;MATHEMATICAL SANS-SERIF BOLD PHI SYMBOL;Ll;0;L; 03D5;;;;N;;;;; +1D78E;MATHEMATICAL SANS-SERIF BOLD RHO SYMBOL;Ll;0;L; 03F1;;;;N;;;;; +1D78F;MATHEMATICAL SANS-SERIF BOLD PI SYMBOL;Ll;0;L; 03D6;;;;N;;;;; +1D790;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ALPHA;Lu;0;L; 0391;;;;N;;;;; +1D791;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL BETA;Lu;0;L; 0392;;;;N;;;;; +1D792;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL GAMMA;Lu;0;L; 0393;;;;N;;;;; +1D793;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL DELTA;Lu;0;L; 0394;;;;N;;;;; +1D794;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL EPSILON;Lu;0;L; 0395;;;;N;;;;; +1D795;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ZETA;Lu;0;L; 0396;;;;N;;;;; +1D796;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ETA;Lu;0;L; 0397;;;;N;;;;; +1D797;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL THETA;Lu;0;L; 0398;;;;N;;;;; +1D798;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL IOTA;Lu;0;L; 0399;;;;N;;;;; +1D799;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL KAPPA;Lu;0;L; 039A;;;;N;;;;; +1D79A;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL LAMDA;Lu;0;L; 039B;;;;N;;;;; +1D79B;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL MU;Lu;0;L; 039C;;;;N;;;;; +1D79C;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL NU;Lu;0;L; 039D;;;;N;;;;; +1D79D;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL XI;Lu;0;L; 039E;;;;N;;;;; +1D79E;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMICRON;Lu;0;L; 039F;;;;N;;;;; +1D79F;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PI;Lu;0;L; 03A0;;;;N;;;;; +1D7A0;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL RHO;Lu;0;L; 03A1;;;;N;;;;; +1D7A1;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL THETA SYMBOL;Lu;0;L; 03F4;;;;N;;;;; +1D7A2;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL SIGMA;Lu;0;L; 03A3;;;;N;;;;; +1D7A3;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL TAU;Lu;0;L; 03A4;;;;N;;;;; +1D7A4;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL UPSILON;Lu;0;L; 03A5;;;;N;;;;; +1D7A5;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PHI;Lu;0;L; 03A6;;;;N;;;;; +1D7A6;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; +1D7A7;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; +1D7A8;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; +1D7A9;MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D7AA;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; +1D7AB;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; +1D7AC;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; +1D7AD;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL DELTA;Ll;0;L; 03B4;;;;N;;;;; +1D7AE;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL EPSILON;Ll;0;L; 03B5;;;;N;;;;; +1D7AF;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ZETA;Ll;0;L; 03B6;;;;N;;;;; +1D7B0;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ETA;Ll;0;L; 03B7;;;;N;;;;; +1D7B1;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL THETA;Ll;0;L; 03B8;;;;N;;;;; +1D7B2;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL IOTA;Ll;0;L; 03B9;;;;N;;;;; +1D7B3;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL KAPPA;Ll;0;L; 03BA;;;;N;;;;; +1D7B4;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL LAMDA;Ll;0;L; 03BB;;;;N;;;;; +1D7B5;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL MU;Ll;0;L; 03BC;;;;N;;;;; +1D7B6;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL NU;Ll;0;L; 03BD;;;;N;;;;; +1D7B7;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL XI;Ll;0;L; 03BE;;;;N;;;;; +1D7B8;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMICRON;Ll;0;L; 03BF;;;;N;;;;; +1D7B9;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL PI;Ll;0;L; 03C0;;;;N;;;;; +1D7BA;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL RHO;Ll;0;L; 03C1;;;;N;;;;; +1D7BB;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL FINAL SIGMA;Ll;0;L; 03C2;;;;N;;;;; +1D7BC;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL SIGMA;Ll;0;L; 03C3;;;;N;;;;; +1D7BD;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL TAU;Ll;0;L; 03C4;;;;N;;;;; +1D7BE;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL UPSILON;Ll;0;L; 03C5;;;;N;;;;; +1D7BF;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL PHI;Ll;0;L; 03C6;;;;N;;;;; +1D7C0;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL CHI;Ll;0;L; 03C7;;;;N;;;;; +1D7C1;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL PSI;Ll;0;L; 03C8;;;;N;;;;; +1D7C2;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA;Ll;0;L; 03C9;;;;N;;;;; +1D7C3;MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL;Sm;0;ON; 2202;;;;Y;;;;; +1D7C4;MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL;Ll;0;L; 03F5;;;;N;;;;; +1D7C5;MATHEMATICAL SANS-SERIF BOLD ITALIC THETA SYMBOL;Ll;0;L; 03D1;;;;N;;;;; +1D7C6;MATHEMATICAL SANS-SERIF BOLD ITALIC KAPPA SYMBOL;Ll;0;L; 03F0;;;;N;;;;; +1D7C7;MATHEMATICAL SANS-SERIF BOLD ITALIC PHI SYMBOL;Ll;0;L; 03D5;;;;N;;;;; +1D7C8;MATHEMATICAL SANS-SERIF BOLD ITALIC RHO SYMBOL;Ll;0;L; 03F1;;;;N;;;;; +1D7C9;MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL;Ll;0;L; 03D6;;;;N;;;;; +1D7CA;MATHEMATICAL BOLD CAPITAL DIGAMMA;Lu;0;L; 03DC;;;;N;;;;; +1D7CB;MATHEMATICAL BOLD SMALL DIGAMMA;Ll;0;L; 03DD;;;;N;;;;; +1D7CE;MATHEMATICAL BOLD DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1D7CF;MATHEMATICAL BOLD DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1D7D0;MATHEMATICAL BOLD DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1D7D1;MATHEMATICAL BOLD DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1D7D2;MATHEMATICAL BOLD DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1D7D3;MATHEMATICAL BOLD DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1D7D4;MATHEMATICAL BOLD DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1D7D5;MATHEMATICAL BOLD DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1D7D6;MATHEMATICAL BOLD DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1D7D7;MATHEMATICAL BOLD DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1D7D8;MATHEMATICAL DOUBLE-STRUCK DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1D7D9;MATHEMATICAL DOUBLE-STRUCK DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1D7DA;MATHEMATICAL DOUBLE-STRUCK DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1D7DB;MATHEMATICAL DOUBLE-STRUCK DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1D7DC;MATHEMATICAL DOUBLE-STRUCK DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1D7DD;MATHEMATICAL DOUBLE-STRUCK DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1D7DE;MATHEMATICAL DOUBLE-STRUCK DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1D7DF;MATHEMATICAL DOUBLE-STRUCK DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1D7E0;MATHEMATICAL DOUBLE-STRUCK DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1D7E1;MATHEMATICAL DOUBLE-STRUCK DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1D7E2;MATHEMATICAL SANS-SERIF DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1D7E3;MATHEMATICAL SANS-SERIF DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1D7E4;MATHEMATICAL SANS-SERIF DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1D7E5;MATHEMATICAL SANS-SERIF DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1D7E6;MATHEMATICAL SANS-SERIF DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1D7E7;MATHEMATICAL SANS-SERIF DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1D7E8;MATHEMATICAL SANS-SERIF DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1D7E9;MATHEMATICAL SANS-SERIF DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1D7EA;MATHEMATICAL SANS-SERIF DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1D7EB;MATHEMATICAL SANS-SERIF DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1D7EC;MATHEMATICAL SANS-SERIF BOLD DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1D7ED;MATHEMATICAL SANS-SERIF BOLD DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1D7EE;MATHEMATICAL SANS-SERIF BOLD DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1D7EF;MATHEMATICAL SANS-SERIF BOLD DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1D7F0;MATHEMATICAL SANS-SERIF BOLD DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1D7F1;MATHEMATICAL SANS-SERIF BOLD DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1D7F2;MATHEMATICAL SANS-SERIF BOLD DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1D7F3;MATHEMATICAL SANS-SERIF BOLD DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1D7F4;MATHEMATICAL SANS-SERIF BOLD DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1D7F5;MATHEMATICAL SANS-SERIF BOLD DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1D7F6;MATHEMATICAL MONOSPACE DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1D7F7;MATHEMATICAL MONOSPACE DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1D7F8;MATHEMATICAL MONOSPACE DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1D7F9;MATHEMATICAL MONOSPACE DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1D7FA;MATHEMATICAL MONOSPACE DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1D7FB;MATHEMATICAL MONOSPACE DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1D7FC;MATHEMATICAL MONOSPACE DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1D7FD;MATHEMATICAL MONOSPACE DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1D7FE;MATHEMATICAL MONOSPACE DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1D7FF;MATHEMATICAL MONOSPACE DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1EE00;ARABIC MATHEMATICAL ALEF;Lo;0;AL; 0627;;;;N;;;;; +1EE01;ARABIC MATHEMATICAL BEH;Lo;0;AL; 0628;;;;N;;;;; +1EE02;ARABIC MATHEMATICAL JEEM;Lo;0;AL; 062C;;;;N;;;;; +1EE03;ARABIC MATHEMATICAL DAL;Lo;0;AL; 062F;;;;N;;;;; +1EE05;ARABIC MATHEMATICAL WAW;Lo;0;AL; 0648;;;;N;;;;; +1EE06;ARABIC MATHEMATICAL ZAIN;Lo;0;AL; 0632;;;;N;;;;; +1EE07;ARABIC MATHEMATICAL HAH;Lo;0;AL; 062D;;;;N;;;;; +1EE08;ARABIC MATHEMATICAL TAH;Lo;0;AL; 0637;;;;N;;;;; +1EE09;ARABIC MATHEMATICAL YEH;Lo;0;AL; 064A;;;;N;;;;; +1EE0A;ARABIC MATHEMATICAL KAF;Lo;0;AL; 0643;;;;N;;;;; +1EE0B;ARABIC MATHEMATICAL LAM;Lo;0;AL; 0644;;;;N;;;;; +1EE0C;ARABIC MATHEMATICAL MEEM;Lo;0;AL; 0645;;;;N;;;;; +1EE0D;ARABIC MATHEMATICAL NOON;Lo;0;AL; 0646;;;;N;;;;; +1EE0E;ARABIC MATHEMATICAL SEEN;Lo;0;AL; 0633;;;;N;;;;; +1EE0F;ARABIC MATHEMATICAL AIN;Lo;0;AL; 0639;;;;N;;;;; +1EE10;ARABIC MATHEMATICAL FEH;Lo;0;AL; 0641;;;;N;;;;; +1EE11;ARABIC MATHEMATICAL SAD;Lo;0;AL; 0635;;;;N;;;;; +1EE12;ARABIC MATHEMATICAL QAF;Lo;0;AL; 0642;;;;N;;;;; +1EE13;ARABIC MATHEMATICAL REH;Lo;0;AL; 0631;;;;N;;;;; +1EE14;ARABIC MATHEMATICAL SHEEN;Lo;0;AL; 0634;;;;N;;;;; +1EE15;ARABIC MATHEMATICAL TEH;Lo;0;AL; 062A;;;;N;;;;; +1EE16;ARABIC MATHEMATICAL THEH;Lo;0;AL; 062B;;;;N;;;;; +1EE17;ARABIC MATHEMATICAL KHAH;Lo;0;AL; 062E;;;;N;;;;; +1EE18;ARABIC MATHEMATICAL THAL;Lo;0;AL; 0630;;;;N;;;;; +1EE19;ARABIC MATHEMATICAL DAD;Lo;0;AL; 0636;;;;N;;;;; +1EE1A;ARABIC MATHEMATICAL ZAH;Lo;0;AL; 0638;;;;N;;;;; +1EE1B;ARABIC MATHEMATICAL GHAIN;Lo;0;AL; 063A;;;;N;;;;; +1EE1C;ARABIC MATHEMATICAL DOTLESS BEH;Lo;0;AL; 066E;;;;N;;;;; +1EE1D;ARABIC MATHEMATICAL DOTLESS NOON;Lo;0;AL; 06BA;;;;N;;;;; +1EE1E;ARABIC MATHEMATICAL DOTLESS FEH;Lo;0;AL; 06A1;;;;N;;;;; +1EE1F;ARABIC MATHEMATICAL DOTLESS QAF;Lo;0;AL; 066F;;;;N;;;;; +1EE21;ARABIC MATHEMATICAL INITIAL BEH;Lo;0;AL; 0628;;;;N;;;;; +1EE22;ARABIC MATHEMATICAL INITIAL JEEM;Lo;0;AL; 062C;;;;N;;;;; +1EE24;ARABIC MATHEMATICAL INITIAL HEH;Lo;0;AL; 0647;;;;N;;;;; +1EE27;ARABIC MATHEMATICAL INITIAL HAH;Lo;0;AL; 062D;;;;N;;;;; +1EE29;ARABIC MATHEMATICAL INITIAL YEH;Lo;0;AL; 064A;;;;N;;;;; +1EE2A;ARABIC MATHEMATICAL INITIAL KAF;Lo;0;AL; 0643;;;;N;;;;; +1EE2B;ARABIC MATHEMATICAL INITIAL LAM;Lo;0;AL; 0644;;;;N;;;;; +1EE2C;ARABIC MATHEMATICAL INITIAL MEEM;Lo;0;AL; 0645;;;;N;;;;; +1EE2D;ARABIC MATHEMATICAL INITIAL NOON;Lo;0;AL; 0646;;;;N;;;;; +1EE2E;ARABIC MATHEMATICAL INITIAL SEEN;Lo;0;AL; 0633;;;;N;;;;; +1EE2F;ARABIC MATHEMATICAL INITIAL AIN;Lo;0;AL; 0639;;;;N;;;;; +1EE30;ARABIC MATHEMATICAL INITIAL FEH;Lo;0;AL; 0641;;;;N;;;;; +1EE31;ARABIC MATHEMATICAL INITIAL SAD;Lo;0;AL; 0635;;;;N;;;;; +1EE32;ARABIC MATHEMATICAL INITIAL QAF;Lo;0;AL; 0642;;;;N;;;;; +1EE34;ARABIC MATHEMATICAL INITIAL SHEEN;Lo;0;AL; 0634;;;;N;;;;; +1EE35;ARABIC MATHEMATICAL INITIAL TEH;Lo;0;AL; 062A;;;;N;;;;; +1EE36;ARABIC MATHEMATICAL INITIAL THEH;Lo;0;AL; 062B;;;;N;;;;; +1EE37;ARABIC MATHEMATICAL INITIAL KHAH;Lo;0;AL; 062E;;;;N;;;;; +1EE39;ARABIC MATHEMATICAL INITIAL DAD;Lo;0;AL; 0636;;;;N;;;;; +1EE3B;ARABIC MATHEMATICAL INITIAL GHAIN;Lo;0;AL; 063A;;;;N;;;;; +1EE42;ARABIC MATHEMATICAL TAILED JEEM;Lo;0;AL; 062C;;;;N;;;;; +1EE47;ARABIC MATHEMATICAL TAILED HAH;Lo;0;AL; 062D;;;;N;;;;; +1EE49;ARABIC MATHEMATICAL TAILED YEH;Lo;0;AL; 064A;;;;N;;;;; +1EE4B;ARABIC MATHEMATICAL TAILED LAM;Lo;0;AL; 0644;;;;N;;;;; +1EE4D;ARABIC MATHEMATICAL TAILED NOON;Lo;0;AL; 0646;;;;N;;;;; +1EE4E;ARABIC MATHEMATICAL TAILED SEEN;Lo;0;AL; 0633;;;;N;;;;; +1EE4F;ARABIC MATHEMATICAL TAILED AIN;Lo;0;AL; 0639;;;;N;;;;; +1EE51;ARABIC MATHEMATICAL TAILED SAD;Lo;0;AL; 0635;;;;N;;;;; +1EE52;ARABIC MATHEMATICAL TAILED QAF;Lo;0;AL; 0642;;;;N;;;;; +1EE54;ARABIC MATHEMATICAL TAILED SHEEN;Lo;0;AL; 0634;;;;N;;;;; +1EE57;ARABIC MATHEMATICAL TAILED KHAH;Lo;0;AL; 062E;;;;N;;;;; +1EE59;ARABIC MATHEMATICAL TAILED DAD;Lo;0;AL; 0636;;;;N;;;;; +1EE5B;ARABIC MATHEMATICAL TAILED GHAIN;Lo;0;AL; 063A;;;;N;;;;; +1EE5D;ARABIC MATHEMATICAL TAILED DOTLESS NOON;Lo;0;AL; 06BA;;;;N;;;;; +1EE5F;ARABIC MATHEMATICAL TAILED DOTLESS QAF;Lo;0;AL; 066F;;;;N;;;;; +1EE61;ARABIC MATHEMATICAL STRETCHED BEH;Lo;0;AL; 0628;;;;N;;;;; +1EE62;ARABIC MATHEMATICAL STRETCHED JEEM;Lo;0;AL; 062C;;;;N;;;;; +1EE64;ARABIC MATHEMATICAL STRETCHED HEH;Lo;0;AL; 0647;;;;N;;;;; +1EE67;ARABIC MATHEMATICAL STRETCHED HAH;Lo;0;AL; 062D;;;;N;;;;; +1EE68;ARABIC MATHEMATICAL STRETCHED TAH;Lo;0;AL; 0637;;;;N;;;;; +1EE69;ARABIC MATHEMATICAL STRETCHED YEH;Lo;0;AL; 064A;;;;N;;;;; +1EE6A;ARABIC MATHEMATICAL STRETCHED KAF;Lo;0;AL; 0643;;;;N;;;;; +1EE6C;ARABIC MATHEMATICAL STRETCHED MEEM;Lo;0;AL; 0645;;;;N;;;;; +1EE6D;ARABIC MATHEMATICAL STRETCHED NOON;Lo;0;AL; 0646;;;;N;;;;; +1EE6E;ARABIC MATHEMATICAL STRETCHED SEEN;Lo;0;AL; 0633;;;;N;;;;; +1EE6F;ARABIC MATHEMATICAL STRETCHED AIN;Lo;0;AL; 0639;;;;N;;;;; +1EE70;ARABIC MATHEMATICAL STRETCHED FEH;Lo;0;AL; 0641;;;;N;;;;; +1EE71;ARABIC MATHEMATICAL STRETCHED SAD;Lo;0;AL; 0635;;;;N;;;;; +1EE72;ARABIC MATHEMATICAL STRETCHED QAF;Lo;0;AL; 0642;;;;N;;;;; +1EE74;ARABIC MATHEMATICAL STRETCHED SHEEN;Lo;0;AL; 0634;;;;N;;;;; +1EE75;ARABIC MATHEMATICAL STRETCHED TEH;Lo;0;AL; 062A;;;;N;;;;; +1EE76;ARABIC MATHEMATICAL STRETCHED THEH;Lo;0;AL; 062B;;;;N;;;;; +1EE77;ARABIC MATHEMATICAL STRETCHED KHAH;Lo;0;AL; 062E;;;;N;;;;; +1EE79;ARABIC MATHEMATICAL STRETCHED DAD;Lo;0;AL; 0636;;;;N;;;;; +1EE7A;ARABIC MATHEMATICAL STRETCHED ZAH;Lo;0;AL; 0638;;;;N;;;;; +1EE7B;ARABIC MATHEMATICAL STRETCHED GHAIN;Lo;0;AL; 063A;;;;N;;;;; +1EE7C;ARABIC MATHEMATICAL STRETCHED DOTLESS BEH;Lo;0;AL; 066E;;;;N;;;;; +1EE7E;ARABIC MATHEMATICAL STRETCHED DOTLESS FEH;Lo;0;AL; 06A1;;;;N;;;;; +1EE80;ARABIC MATHEMATICAL LOOPED ALEF;Lo;0;AL; 0627;;;;N;;;;; +1EE81;ARABIC MATHEMATICAL LOOPED BEH;Lo;0;AL; 0628;;;;N;;;;; +1EE82;ARABIC MATHEMATICAL LOOPED JEEM;Lo;0;AL; 062C;;;;N;;;;; +1EE83;ARABIC MATHEMATICAL LOOPED DAL;Lo;0;AL; 062F;;;;N;;;;; +1EE84;ARABIC MATHEMATICAL LOOPED HEH;Lo;0;AL; 0647;;;;N;;;;; +1EE85;ARABIC MATHEMATICAL LOOPED WAW;Lo;0;AL; 0648;;;;N;;;;; +1EE86;ARABIC MATHEMATICAL LOOPED ZAIN;Lo;0;AL; 0632;;;;N;;;;; +1EE87;ARABIC MATHEMATICAL LOOPED HAH;Lo;0;AL; 062D;;;;N;;;;; +1EE88;ARABIC MATHEMATICAL LOOPED TAH;Lo;0;AL; 0637;;;;N;;;;; +1EE89;ARABIC MATHEMATICAL LOOPED YEH;Lo;0;AL; 064A;;;;N;;;;; +1EE8B;ARABIC MATHEMATICAL LOOPED LAM;Lo;0;AL; 0644;;;;N;;;;; +1EE8C;ARABIC MATHEMATICAL LOOPED MEEM;Lo;0;AL; 0645;;;;N;;;;; +1EE8D;ARABIC MATHEMATICAL LOOPED NOON;Lo;0;AL; 0646;;;;N;;;;; +1EE8E;ARABIC MATHEMATICAL LOOPED SEEN;Lo;0;AL; 0633;;;;N;;;;; +1EE8F;ARABIC MATHEMATICAL LOOPED AIN;Lo;0;AL; 0639;;;;N;;;;; +1EE90;ARABIC MATHEMATICAL LOOPED FEH;Lo;0;AL; 0641;;;;N;;;;; +1EE91;ARABIC MATHEMATICAL LOOPED SAD;Lo;0;AL; 0635;;;;N;;;;; +1EE92;ARABIC MATHEMATICAL LOOPED QAF;Lo;0;AL; 0642;;;;N;;;;; +1EE93;ARABIC MATHEMATICAL LOOPED REH;Lo;0;AL; 0631;;;;N;;;;; +1EE94;ARABIC MATHEMATICAL LOOPED SHEEN;Lo;0;AL; 0634;;;;N;;;;; +1EE95;ARABIC MATHEMATICAL LOOPED TEH;Lo;0;AL; 062A;;;;N;;;;; +1EE96;ARABIC MATHEMATICAL LOOPED THEH;Lo;0;AL; 062B;;;;N;;;;; +1EE97;ARABIC MATHEMATICAL LOOPED KHAH;Lo;0;AL; 062E;;;;N;;;;; +1EE98;ARABIC MATHEMATICAL LOOPED THAL;Lo;0;AL; 0630;;;;N;;;;; +1EE99;ARABIC MATHEMATICAL LOOPED DAD;Lo;0;AL; 0636;;;;N;;;;; +1EE9A;ARABIC MATHEMATICAL LOOPED ZAH;Lo;0;AL; 0638;;;;N;;;;; +1EE9B;ARABIC MATHEMATICAL LOOPED GHAIN;Lo;0;AL; 063A;;;;N;;;;; +1EEA1;ARABIC MATHEMATICAL DOUBLE-STRUCK BEH;Lo;0;AL; 0628;;;;N;;;;; +1EEA2;ARABIC MATHEMATICAL DOUBLE-STRUCK JEEM;Lo;0;AL; 062C;;;;N;;;;; +1EEA3;ARABIC MATHEMATICAL DOUBLE-STRUCK DAL;Lo;0;AL; 062F;;;;N;;;;; +1EEA5;ARABIC MATHEMATICAL DOUBLE-STRUCK WAW;Lo;0;AL; 0648;;;;N;;;;; +1EEA6;ARABIC MATHEMATICAL DOUBLE-STRUCK ZAIN;Lo;0;AL; 0632;;;;N;;;;; +1EEA7;ARABIC MATHEMATICAL DOUBLE-STRUCK HAH;Lo;0;AL; 062D;;;;N;;;;; +1EEA8;ARABIC MATHEMATICAL DOUBLE-STRUCK TAH;Lo;0;AL; 0637;;;;N;;;;; +1EEA9;ARABIC MATHEMATICAL DOUBLE-STRUCK YEH;Lo;0;AL; 064A;;;;N;;;;; +1EEAB;ARABIC MATHEMATICAL DOUBLE-STRUCK LAM;Lo;0;AL; 0644;;;;N;;;;; +1EEAC;ARABIC MATHEMATICAL DOUBLE-STRUCK MEEM;Lo;0;AL; 0645;;;;N;;;;; +1EEAD;ARABIC MATHEMATICAL DOUBLE-STRUCK NOON;Lo;0;AL; 0646;;;;N;;;;; +1EEAE;ARABIC MATHEMATICAL DOUBLE-STRUCK SEEN;Lo;0;AL; 0633;;;;N;;;;; +1EEAF;ARABIC MATHEMATICAL DOUBLE-STRUCK AIN;Lo;0;AL; 0639;;;;N;;;;; +1EEB0;ARABIC MATHEMATICAL DOUBLE-STRUCK FEH;Lo;0;AL; 0641;;;;N;;;;; +1EEB1;ARABIC MATHEMATICAL DOUBLE-STRUCK SAD;Lo;0;AL; 0635;;;;N;;;;; +1EEB2;ARABIC MATHEMATICAL DOUBLE-STRUCK QAF;Lo;0;AL; 0642;;;;N;;;;; +1EEB3;ARABIC MATHEMATICAL DOUBLE-STRUCK REH;Lo;0;AL; 0631;;;;N;;;;; +1EEB4;ARABIC MATHEMATICAL DOUBLE-STRUCK SHEEN;Lo;0;AL; 0634;;;;N;;;;; +1EEB5;ARABIC MATHEMATICAL DOUBLE-STRUCK TEH;Lo;0;AL; 062A;;;;N;;;;; +1EEB6;ARABIC MATHEMATICAL DOUBLE-STRUCK THEH;Lo;0;AL; 062B;;;;N;;;;; +1EEB7;ARABIC MATHEMATICAL DOUBLE-STRUCK KHAH;Lo;0;AL; 062E;;;;N;;;;; +1EEB8;ARABIC MATHEMATICAL DOUBLE-STRUCK THAL;Lo;0;AL; 0630;;;;N;;;;; +1EEB9;ARABIC MATHEMATICAL DOUBLE-STRUCK DAD;Lo;0;AL; 0636;;;;N;;;;; +1EEBA;ARABIC MATHEMATICAL DOUBLE-STRUCK ZAH;Lo;0;AL; 0638;;;;N;;;;; +1EEBB;ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN;Lo;0;AL; 063A;;;;N;;;;; +1EEF0;ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL;Sm;0;ON;;;;;N;;;;; +1EEF1;ARABIC MATHEMATICAL OPERATOR HAH WITH DAL;Sm;0;ON;;;;;N;;;;; +1F000;MAHJONG TILE EAST WIND;So;0;ON;;;;;N;;;;; +1F001;MAHJONG TILE SOUTH WIND;So;0;ON;;;;;N;;;;; +1F002;MAHJONG TILE WEST WIND;So;0;ON;;;;;N;;;;; +1F003;MAHJONG TILE NORTH WIND;So;0;ON;;;;;N;;;;; +1F004;MAHJONG TILE RED DRAGON;So;0;ON;;;;;N;;;;; +1F005;MAHJONG TILE GREEN DRAGON;So;0;ON;;;;;N;;;;; +1F006;MAHJONG TILE WHITE DRAGON;So;0;ON;;;;;N;;;;; +1F007;MAHJONG TILE ONE OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F008;MAHJONG TILE TWO OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F009;MAHJONG TILE THREE OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F00A;MAHJONG TILE FOUR OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F00B;MAHJONG TILE FIVE OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F00C;MAHJONG TILE SIX OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F00D;MAHJONG TILE SEVEN OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F00E;MAHJONG TILE EIGHT OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F00F;MAHJONG TILE NINE OF CHARACTERS;So;0;ON;;;;;N;;;;; +1F010;MAHJONG TILE ONE OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F011;MAHJONG TILE TWO OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F012;MAHJONG TILE THREE OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F013;MAHJONG TILE FOUR OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F014;MAHJONG TILE FIVE OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F015;MAHJONG TILE SIX OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F016;MAHJONG TILE SEVEN OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F017;MAHJONG TILE EIGHT OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F018;MAHJONG TILE NINE OF BAMBOOS;So;0;ON;;;;;N;;;;; +1F019;MAHJONG TILE ONE OF CIRCLES;So;0;ON;;;;;N;;;;; +1F01A;MAHJONG TILE TWO OF CIRCLES;So;0;ON;;;;;N;;;;; +1F01B;MAHJONG TILE THREE OF CIRCLES;So;0;ON;;;;;N;;;;; +1F01C;MAHJONG TILE FOUR OF CIRCLES;So;0;ON;;;;;N;;;;; +1F01D;MAHJONG TILE FIVE OF CIRCLES;So;0;ON;;;;;N;;;;; +1F01E;MAHJONG TILE SIX OF CIRCLES;So;0;ON;;;;;N;;;;; +1F01F;MAHJONG TILE SEVEN OF CIRCLES;So;0;ON;;;;;N;;;;; +1F020;MAHJONG TILE EIGHT OF CIRCLES;So;0;ON;;;;;N;;;;; +1F021;MAHJONG TILE NINE OF CIRCLES;So;0;ON;;;;;N;;;;; +1F022;MAHJONG TILE PLUM;So;0;ON;;;;;N;;;;; +1F023;MAHJONG TILE ORCHID;So;0;ON;;;;;N;;;;; +1F024;MAHJONG TILE BAMBOO;So;0;ON;;;;;N;;;;; +1F025;MAHJONG TILE CHRYSANTHEMUM;So;0;ON;;;;;N;;;;; +1F026;MAHJONG TILE SPRING;So;0;ON;;;;;N;;;;; +1F027;MAHJONG TILE SUMMER;So;0;ON;;;;;N;;;;; +1F028;MAHJONG TILE AUTUMN;So;0;ON;;;;;N;;;;; +1F029;MAHJONG TILE WINTER;So;0;ON;;;;;N;;;;; +1F02A;MAHJONG TILE JOKER;So;0;ON;;;;;N;;;;; +1F02B;MAHJONG TILE BACK;So;0;ON;;;;;N;;;;; +1F030;DOMINO TILE HORIZONTAL BACK;So;0;ON;;;;;N;;;;; +1F031;DOMINO TILE HORIZONTAL-00-00;So;0;ON;;;;;N;;;;; +1F032;DOMINO TILE HORIZONTAL-00-01;So;0;ON;;;;;N;;;;; +1F033;DOMINO TILE HORIZONTAL-00-02;So;0;ON;;;;;N;;;;; +1F034;DOMINO TILE HORIZONTAL-00-03;So;0;ON;;;;;N;;;;; +1F035;DOMINO TILE HORIZONTAL-00-04;So;0;ON;;;;;N;;;;; +1F036;DOMINO TILE HORIZONTAL-00-05;So;0;ON;;;;;N;;;;; +1F037;DOMINO TILE HORIZONTAL-00-06;So;0;ON;;;;;N;;;;; +1F038;DOMINO TILE HORIZONTAL-01-00;So;0;ON;;;;;N;;;;; +1F039;DOMINO TILE HORIZONTAL-01-01;So;0;ON;;;;;N;;;;; +1F03A;DOMINO TILE HORIZONTAL-01-02;So;0;ON;;;;;N;;;;; +1F03B;DOMINO TILE HORIZONTAL-01-03;So;0;ON;;;;;N;;;;; +1F03C;DOMINO TILE HORIZONTAL-01-04;So;0;ON;;;;;N;;;;; +1F03D;DOMINO TILE HORIZONTAL-01-05;So;0;ON;;;;;N;;;;; +1F03E;DOMINO TILE HORIZONTAL-01-06;So;0;ON;;;;;N;;;;; +1F03F;DOMINO TILE HORIZONTAL-02-00;So;0;ON;;;;;N;;;;; +1F040;DOMINO TILE HORIZONTAL-02-01;So;0;ON;;;;;N;;;;; +1F041;DOMINO TILE HORIZONTAL-02-02;So;0;ON;;;;;N;;;;; +1F042;DOMINO TILE HORIZONTAL-02-03;So;0;ON;;;;;N;;;;; +1F043;DOMINO TILE HORIZONTAL-02-04;So;0;ON;;;;;N;;;;; +1F044;DOMINO TILE HORIZONTAL-02-05;So;0;ON;;;;;N;;;;; +1F045;DOMINO TILE HORIZONTAL-02-06;So;0;ON;;;;;N;;;;; +1F046;DOMINO TILE HORIZONTAL-03-00;So;0;ON;;;;;N;;;;; +1F047;DOMINO TILE HORIZONTAL-03-01;So;0;ON;;;;;N;;;;; +1F048;DOMINO TILE HORIZONTAL-03-02;So;0;ON;;;;;N;;;;; +1F049;DOMINO TILE HORIZONTAL-03-03;So;0;ON;;;;;N;;;;; +1F04A;DOMINO TILE HORIZONTAL-03-04;So;0;ON;;;;;N;;;;; +1F04B;DOMINO TILE HORIZONTAL-03-05;So;0;ON;;;;;N;;;;; +1F04C;DOMINO TILE HORIZONTAL-03-06;So;0;ON;;;;;N;;;;; +1F04D;DOMINO TILE HORIZONTAL-04-00;So;0;ON;;;;;N;;;;; +1F04E;DOMINO TILE HORIZONTAL-04-01;So;0;ON;;;;;N;;;;; +1F04F;DOMINO TILE HORIZONTAL-04-02;So;0;ON;;;;;N;;;;; +1F050;DOMINO TILE HORIZONTAL-04-03;So;0;ON;;;;;N;;;;; +1F051;DOMINO TILE HORIZONTAL-04-04;So;0;ON;;;;;N;;;;; +1F052;DOMINO TILE HORIZONTAL-04-05;So;0;ON;;;;;N;;;;; +1F053;DOMINO TILE HORIZONTAL-04-06;So;0;ON;;;;;N;;;;; +1F054;DOMINO TILE HORIZONTAL-05-00;So;0;ON;;;;;N;;;;; +1F055;DOMINO TILE HORIZONTAL-05-01;So;0;ON;;;;;N;;;;; +1F056;DOMINO TILE HORIZONTAL-05-02;So;0;ON;;;;;N;;;;; +1F057;DOMINO TILE HORIZONTAL-05-03;So;0;ON;;;;;N;;;;; +1F058;DOMINO TILE HORIZONTAL-05-04;So;0;ON;;;;;N;;;;; +1F059;DOMINO TILE HORIZONTAL-05-05;So;0;ON;;;;;N;;;;; +1F05A;DOMINO TILE HORIZONTAL-05-06;So;0;ON;;;;;N;;;;; +1F05B;DOMINO TILE HORIZONTAL-06-00;So;0;ON;;;;;N;;;;; +1F05C;DOMINO TILE HORIZONTAL-06-01;So;0;ON;;;;;N;;;;; +1F05D;DOMINO TILE HORIZONTAL-06-02;So;0;ON;;;;;N;;;;; +1F05E;DOMINO TILE HORIZONTAL-06-03;So;0;ON;;;;;N;;;;; +1F05F;DOMINO TILE HORIZONTAL-06-04;So;0;ON;;;;;N;;;;; +1F060;DOMINO TILE HORIZONTAL-06-05;So;0;ON;;;;;N;;;;; +1F061;DOMINO TILE HORIZONTAL-06-06;So;0;ON;;;;;N;;;;; +1F062;DOMINO TILE VERTICAL BACK;So;0;ON;;;;;N;;;;; +1F063;DOMINO TILE VERTICAL-00-00;So;0;ON;;;;;N;;;;; +1F064;DOMINO TILE VERTICAL-00-01;So;0;ON;;;;;N;;;;; +1F065;DOMINO TILE VERTICAL-00-02;So;0;ON;;;;;N;;;;; +1F066;DOMINO TILE VERTICAL-00-03;So;0;ON;;;;;N;;;;; +1F067;DOMINO TILE VERTICAL-00-04;So;0;ON;;;;;N;;;;; +1F068;DOMINO TILE VERTICAL-00-05;So;0;ON;;;;;N;;;;; +1F069;DOMINO TILE VERTICAL-00-06;So;0;ON;;;;;N;;;;; +1F06A;DOMINO TILE VERTICAL-01-00;So;0;ON;;;;;N;;;;; +1F06B;DOMINO TILE VERTICAL-01-01;So;0;ON;;;;;N;;;;; +1F06C;DOMINO TILE VERTICAL-01-02;So;0;ON;;;;;N;;;;; +1F06D;DOMINO TILE VERTICAL-01-03;So;0;ON;;;;;N;;;;; +1F06E;DOMINO TILE VERTICAL-01-04;So;0;ON;;;;;N;;;;; +1F06F;DOMINO TILE VERTICAL-01-05;So;0;ON;;;;;N;;;;; +1F070;DOMINO TILE VERTICAL-01-06;So;0;ON;;;;;N;;;;; +1F071;DOMINO TILE VERTICAL-02-00;So;0;ON;;;;;N;;;;; +1F072;DOMINO TILE VERTICAL-02-01;So;0;ON;;;;;N;;;;; +1F073;DOMINO TILE VERTICAL-02-02;So;0;ON;;;;;N;;;;; +1F074;DOMINO TILE VERTICAL-02-03;So;0;ON;;;;;N;;;;; +1F075;DOMINO TILE VERTICAL-02-04;So;0;ON;;;;;N;;;;; +1F076;DOMINO TILE VERTICAL-02-05;So;0;ON;;;;;N;;;;; +1F077;DOMINO TILE VERTICAL-02-06;So;0;ON;;;;;N;;;;; +1F078;DOMINO TILE VERTICAL-03-00;So;0;ON;;;;;N;;;;; +1F079;DOMINO TILE VERTICAL-03-01;So;0;ON;;;;;N;;;;; +1F07A;DOMINO TILE VERTICAL-03-02;So;0;ON;;;;;N;;;;; +1F07B;DOMINO TILE VERTICAL-03-03;So;0;ON;;;;;N;;;;; +1F07C;DOMINO TILE VERTICAL-03-04;So;0;ON;;;;;N;;;;; +1F07D;DOMINO TILE VERTICAL-03-05;So;0;ON;;;;;N;;;;; +1F07E;DOMINO TILE VERTICAL-03-06;So;0;ON;;;;;N;;;;; +1F07F;DOMINO TILE VERTICAL-04-00;So;0;ON;;;;;N;;;;; +1F080;DOMINO TILE VERTICAL-04-01;So;0;ON;;;;;N;;;;; +1F081;DOMINO TILE VERTICAL-04-02;So;0;ON;;;;;N;;;;; +1F082;DOMINO TILE VERTICAL-04-03;So;0;ON;;;;;N;;;;; +1F083;DOMINO TILE VERTICAL-04-04;So;0;ON;;;;;N;;;;; +1F084;DOMINO TILE VERTICAL-04-05;So;0;ON;;;;;N;;;;; +1F085;DOMINO TILE VERTICAL-04-06;So;0;ON;;;;;N;;;;; +1F086;DOMINO TILE VERTICAL-05-00;So;0;ON;;;;;N;;;;; +1F087;DOMINO TILE VERTICAL-05-01;So;0;ON;;;;;N;;;;; +1F088;DOMINO TILE VERTICAL-05-02;So;0;ON;;;;;N;;;;; +1F089;DOMINO TILE VERTICAL-05-03;So;0;ON;;;;;N;;;;; +1F08A;DOMINO TILE VERTICAL-05-04;So;0;ON;;;;;N;;;;; +1F08B;DOMINO TILE VERTICAL-05-05;So;0;ON;;;;;N;;;;; +1F08C;DOMINO TILE VERTICAL-05-06;So;0;ON;;;;;N;;;;; +1F08D;DOMINO TILE VERTICAL-06-00;So;0;ON;;;;;N;;;;; +1F08E;DOMINO TILE VERTICAL-06-01;So;0;ON;;;;;N;;;;; +1F08F;DOMINO TILE VERTICAL-06-02;So;0;ON;;;;;N;;;;; +1F090;DOMINO TILE VERTICAL-06-03;So;0;ON;;;;;N;;;;; +1F091;DOMINO TILE VERTICAL-06-04;So;0;ON;;;;;N;;;;; +1F092;DOMINO TILE VERTICAL-06-05;So;0;ON;;;;;N;;;;; +1F093;DOMINO TILE VERTICAL-06-06;So;0;ON;;;;;N;;;;; +1F0A0;PLAYING CARD BACK;So;0;ON;;;;;N;;;;; +1F0A1;PLAYING CARD ACE OF SPADES;So;0;ON;;;;;N;;;;; +1F0A2;PLAYING CARD TWO OF SPADES;So;0;ON;;;;;N;;;;; +1F0A3;PLAYING CARD THREE OF SPADES;So;0;ON;;;;;N;;;;; +1F0A4;PLAYING CARD FOUR OF SPADES;So;0;ON;;;;;N;;;;; +1F0A5;PLAYING CARD FIVE OF SPADES;So;0;ON;;;;;N;;;;; +1F0A6;PLAYING CARD SIX OF SPADES;So;0;ON;;;;;N;;;;; +1F0A7;PLAYING CARD SEVEN OF SPADES;So;0;ON;;;;;N;;;;; +1F0A8;PLAYING CARD EIGHT OF SPADES;So;0;ON;;;;;N;;;;; +1F0A9;PLAYING CARD NINE OF SPADES;So;0;ON;;;;;N;;;;; +1F0AA;PLAYING CARD TEN OF SPADES;So;0;ON;;;;;N;;;;; +1F0AB;PLAYING CARD JACK OF SPADES;So;0;ON;;;;;N;;;;; +1F0AC;PLAYING CARD KNIGHT OF SPADES;So;0;ON;;;;;N;;;;; +1F0AD;PLAYING CARD QUEEN OF SPADES;So;0;ON;;;;;N;;;;; +1F0AE;PLAYING CARD KING OF SPADES;So;0;ON;;;;;N;;;;; +1F0B1;PLAYING CARD ACE OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B2;PLAYING CARD TWO OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B3;PLAYING CARD THREE OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B4;PLAYING CARD FOUR OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B5;PLAYING CARD FIVE OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B6;PLAYING CARD SIX OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B7;PLAYING CARD SEVEN OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B8;PLAYING CARD EIGHT OF HEARTS;So;0;ON;;;;;N;;;;; +1F0B9;PLAYING CARD NINE OF HEARTS;So;0;ON;;;;;N;;;;; +1F0BA;PLAYING CARD TEN OF HEARTS;So;0;ON;;;;;N;;;;; +1F0BB;PLAYING CARD JACK OF HEARTS;So;0;ON;;;;;N;;;;; +1F0BC;PLAYING CARD KNIGHT OF HEARTS;So;0;ON;;;;;N;;;;; +1F0BD;PLAYING CARD QUEEN OF HEARTS;So;0;ON;;;;;N;;;;; +1F0BE;PLAYING CARD KING OF HEARTS;So;0;ON;;;;;N;;;;; +1F0C1;PLAYING CARD ACE OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C2;PLAYING CARD TWO OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C3;PLAYING CARD THREE OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C4;PLAYING CARD FOUR OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C5;PLAYING CARD FIVE OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C6;PLAYING CARD SIX OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C7;PLAYING CARD SEVEN OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C8;PLAYING CARD EIGHT OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0C9;PLAYING CARD NINE OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0CA;PLAYING CARD TEN OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0CB;PLAYING CARD JACK OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0CC;PLAYING CARD KNIGHT OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0CD;PLAYING CARD QUEEN OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0CE;PLAYING CARD KING OF DIAMONDS;So;0;ON;;;;;N;;;;; +1F0CF;PLAYING CARD BLACK JOKER;So;0;ON;;;;;N;;;;; +1F0D1;PLAYING CARD ACE OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D2;PLAYING CARD TWO OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D3;PLAYING CARD THREE OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D4;PLAYING CARD FOUR OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D5;PLAYING CARD FIVE OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D6;PLAYING CARD SIX OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D7;PLAYING CARD SEVEN OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D8;PLAYING CARD EIGHT OF CLUBS;So;0;ON;;;;;N;;;;; +1F0D9;PLAYING CARD NINE OF CLUBS;So;0;ON;;;;;N;;;;; +1F0DA;PLAYING CARD TEN OF CLUBS;So;0;ON;;;;;N;;;;; +1F0DB;PLAYING CARD JACK OF CLUBS;So;0;ON;;;;;N;;;;; +1F0DC;PLAYING CARD KNIGHT OF CLUBS;So;0;ON;;;;;N;;;;; +1F0DD;PLAYING CARD QUEEN OF CLUBS;So;0;ON;;;;;N;;;;; +1F0DE;PLAYING CARD KING OF CLUBS;So;0;ON;;;;;N;;;;; +1F0DF;PLAYING CARD WHITE JOKER;So;0;ON;;;;;N;;;;; +1F100;DIGIT ZERO FULL STOP;No;0;EN; 0030 002E;;0;0;N;;;;; +1F101;DIGIT ZERO COMMA;No;0;EN; 0030 002C;;0;0;N;;;;; +1F102;DIGIT ONE COMMA;No;0;EN; 0031 002C;;1;1;N;;;;; +1F103;DIGIT TWO COMMA;No;0;EN; 0032 002C;;2;2;N;;;;; +1F104;DIGIT THREE COMMA;No;0;EN; 0033 002C;;3;3;N;;;;; +1F105;DIGIT FOUR COMMA;No;0;EN; 0034 002C;;4;4;N;;;;; +1F106;DIGIT FIVE COMMA;No;0;EN; 0035 002C;;5;5;N;;;;; +1F107;DIGIT SIX COMMA;No;0;EN; 0036 002C;;6;6;N;;;;; +1F108;DIGIT SEVEN COMMA;No;0;EN; 0037 002C;;7;7;N;;;;; +1F109;DIGIT EIGHT COMMA;No;0;EN; 0038 002C;;8;8;N;;;;; +1F10A;DIGIT NINE COMMA;No;0;EN; 0039 002C;;9;9;N;;;;; +1F110;PARENTHESIZED LATIN CAPITAL LETTER A;So;0;L; 0028 0041 0029;;;;N;;;;; +1F111;PARENTHESIZED LATIN CAPITAL LETTER B;So;0;L; 0028 0042 0029;;;;N;;;;; +1F112;PARENTHESIZED LATIN CAPITAL LETTER C;So;0;L; 0028 0043 0029;;;;N;;;;; +1F113;PARENTHESIZED LATIN CAPITAL LETTER D;So;0;L; 0028 0044 0029;;;;N;;;;; +1F114;PARENTHESIZED LATIN CAPITAL LETTER E;So;0;L; 0028 0045 0029;;;;N;;;;; +1F115;PARENTHESIZED LATIN CAPITAL LETTER F;So;0;L; 0028 0046 0029;;;;N;;;;; +1F116;PARENTHESIZED LATIN CAPITAL LETTER G;So;0;L; 0028 0047 0029;;;;N;;;;; +1F117;PARENTHESIZED LATIN CAPITAL LETTER H;So;0;L; 0028 0048 0029;;;;N;;;;; +1F118;PARENTHESIZED LATIN CAPITAL LETTER I;So;0;L; 0028 0049 0029;;;;N;;;;; +1F119;PARENTHESIZED LATIN CAPITAL LETTER J;So;0;L; 0028 004A 0029;;;;N;;;;; +1F11A;PARENTHESIZED LATIN CAPITAL LETTER K;So;0;L; 0028 004B 0029;;;;N;;;;; +1F11B;PARENTHESIZED LATIN CAPITAL LETTER L;So;0;L; 0028 004C 0029;;;;N;;;;; +1F11C;PARENTHESIZED LATIN CAPITAL LETTER M;So;0;L; 0028 004D 0029;;;;N;;;;; +1F11D;PARENTHESIZED LATIN CAPITAL LETTER N;So;0;L; 0028 004E 0029;;;;N;;;;; +1F11E;PARENTHESIZED LATIN CAPITAL LETTER O;So;0;L; 0028 004F 0029;;;;N;;;;; +1F11F;PARENTHESIZED LATIN CAPITAL LETTER P;So;0;L; 0028 0050 0029;;;;N;;;;; +1F120;PARENTHESIZED LATIN CAPITAL LETTER Q;So;0;L; 0028 0051 0029;;;;N;;;;; +1F121;PARENTHESIZED LATIN CAPITAL LETTER R;So;0;L; 0028 0052 0029;;;;N;;;;; +1F122;PARENTHESIZED LATIN CAPITAL LETTER S;So;0;L; 0028 0053 0029;;;;N;;;;; +1F123;PARENTHESIZED LATIN CAPITAL LETTER T;So;0;L; 0028 0054 0029;;;;N;;;;; +1F124;PARENTHESIZED LATIN CAPITAL LETTER U;So;0;L; 0028 0055 0029;;;;N;;;;; +1F125;PARENTHESIZED LATIN CAPITAL LETTER V;So;0;L; 0028 0056 0029;;;;N;;;;; +1F126;PARENTHESIZED LATIN CAPITAL LETTER W;So;0;L; 0028 0057 0029;;;;N;;;;; +1F127;PARENTHESIZED LATIN CAPITAL LETTER X;So;0;L; 0028 0058 0029;;;;N;;;;; +1F128;PARENTHESIZED LATIN CAPITAL LETTER Y;So;0;L; 0028 0059 0029;;;;N;;;;; +1F129;PARENTHESIZED LATIN CAPITAL LETTER Z;So;0;L; 0028 005A 0029;;;;N;;;;; +1F12A;TORTOISE SHELL BRACKETED LATIN CAPITAL LETTER S;So;0;L; 3014 0053 3015;;;;N;;;;; +1F12B;CIRCLED ITALIC LATIN CAPITAL LETTER C;So;0;L; 0043;;;;N;;;;; +1F12C;CIRCLED ITALIC LATIN CAPITAL LETTER R;So;0;L; 0052;;;;N;;;;; +1F12D;CIRCLED CD;So;0;L; 0043 0044;;;;N;;;;; +1F12E;CIRCLED WZ;So;0;L; 0057 005A;;;;N;;;;; +1F130;SQUARED LATIN CAPITAL LETTER A;So;0;L; 0041;;;;N;;;;; +1F131;SQUARED LATIN CAPITAL LETTER B;So;0;L; 0042;;;;N;;;;; +1F132;SQUARED LATIN CAPITAL LETTER C;So;0;L; 0043;;;;N;;;;; +1F133;SQUARED LATIN CAPITAL LETTER D;So;0;L; 0044;;;;N;;;;; +1F134;SQUARED LATIN CAPITAL LETTER E;So;0;L; 0045;;;;N;;;;; +1F135;SQUARED LATIN CAPITAL LETTER F;So;0;L; 0046;;;;N;;;;; +1F136;SQUARED LATIN CAPITAL LETTER G;So;0;L; 0047;;;;N;;;;; +1F137;SQUARED LATIN CAPITAL LETTER H;So;0;L; 0048;;;;N;;;;; +1F138;SQUARED LATIN CAPITAL LETTER I;So;0;L; 0049;;;;N;;;;; +1F139;SQUARED LATIN CAPITAL LETTER J;So;0;L; 004A;;;;N;;;;; +1F13A;SQUARED LATIN CAPITAL LETTER K;So;0;L; 004B;;;;N;;;;; +1F13B;SQUARED LATIN CAPITAL LETTER L;So;0;L; 004C;;;;N;;;;; +1F13C;SQUARED LATIN CAPITAL LETTER M;So;0;L; 004D;;;;N;;;;; +1F13D;SQUARED LATIN CAPITAL LETTER N;So;0;L; 004E;;;;N;;;;; +1F13E;SQUARED LATIN CAPITAL LETTER O;So;0;L; 004F;;;;N;;;;; +1F13F;SQUARED LATIN CAPITAL LETTER P;So;0;L; 0050;;;;N;;;;; +1F140;SQUARED LATIN CAPITAL LETTER Q;So;0;L; 0051;;;;N;;;;; +1F141;SQUARED LATIN CAPITAL LETTER R;So;0;L; 0052;;;;N;;;;; +1F142;SQUARED LATIN CAPITAL LETTER S;So;0;L; 0053;;;;N;;;;; +1F143;SQUARED LATIN CAPITAL LETTER T;So;0;L; 0054;;;;N;;;;; +1F144;SQUARED LATIN CAPITAL LETTER U;So;0;L; 0055;;;;N;;;;; +1F145;SQUARED LATIN CAPITAL LETTER V;So;0;L; 0056;;;;N;;;;; +1F146;SQUARED LATIN CAPITAL LETTER W;So;0;L; 0057;;;;N;;;;; +1F147;SQUARED LATIN CAPITAL LETTER X;So;0;L; 0058;;;;N;;;;; +1F148;SQUARED LATIN CAPITAL LETTER Y;So;0;L; 0059;;;;N;;;;; +1F149;SQUARED LATIN CAPITAL LETTER Z;So;0;L; 005A;;;;N;;;;; +1F14A;SQUARED HV;So;0;L; 0048 0056;;;;N;;;;; +1F14B;SQUARED MV;So;0;L; 004D 0056;;;;N;;;;; +1F14C;SQUARED SD;So;0;L; 0053 0044;;;;N;;;;; +1F14D;SQUARED SS;So;0;L; 0053 0053;;;;N;;;;; +1F14E;SQUARED PPV;So;0;L; 0050 0050 0056;;;;N;;;;; +1F14F;SQUARED WC;So;0;L; 0057 0043;;;;N;;;;; +1F150;NEGATIVE CIRCLED LATIN CAPITAL LETTER A;So;0;L;;;;;N;;;;; +1F151;NEGATIVE CIRCLED LATIN CAPITAL LETTER B;So;0;L;;;;;N;;;;; +1F152;NEGATIVE CIRCLED LATIN CAPITAL LETTER C;So;0;L;;;;;N;;;;; +1F153;NEGATIVE CIRCLED LATIN CAPITAL LETTER D;So;0;L;;;;;N;;;;; +1F154;NEGATIVE CIRCLED LATIN CAPITAL LETTER E;So;0;L;;;;;N;;;;; +1F155;NEGATIVE CIRCLED LATIN CAPITAL LETTER F;So;0;L;;;;;N;;;;; +1F156;NEGATIVE CIRCLED LATIN CAPITAL LETTER G;So;0;L;;;;;N;;;;; +1F157;NEGATIVE CIRCLED LATIN CAPITAL LETTER H;So;0;L;;;;;N;;;;; +1F158;NEGATIVE CIRCLED LATIN CAPITAL LETTER I;So;0;L;;;;;N;;;;; +1F159;NEGATIVE CIRCLED LATIN CAPITAL LETTER J;So;0;L;;;;;N;;;;; +1F15A;NEGATIVE CIRCLED LATIN CAPITAL LETTER K;So;0;L;;;;;N;;;;; +1F15B;NEGATIVE CIRCLED LATIN CAPITAL LETTER L;So;0;L;;;;;N;;;;; +1F15C;NEGATIVE CIRCLED LATIN CAPITAL LETTER M;So;0;L;;;;;N;;;;; +1F15D;NEGATIVE CIRCLED LATIN CAPITAL LETTER N;So;0;L;;;;;N;;;;; +1F15E;NEGATIVE CIRCLED LATIN CAPITAL LETTER O;So;0;L;;;;;N;;;;; +1F15F;NEGATIVE CIRCLED LATIN CAPITAL LETTER P;So;0;L;;;;;N;;;;; +1F160;NEGATIVE CIRCLED LATIN CAPITAL LETTER Q;So;0;L;;;;;N;;;;; +1F161;NEGATIVE CIRCLED LATIN CAPITAL LETTER R;So;0;L;;;;;N;;;;; +1F162;NEGATIVE CIRCLED LATIN CAPITAL LETTER S;So;0;L;;;;;N;;;;; +1F163;NEGATIVE CIRCLED LATIN CAPITAL LETTER T;So;0;L;;;;;N;;;;; +1F164;NEGATIVE CIRCLED LATIN CAPITAL LETTER U;So;0;L;;;;;N;;;;; +1F165;NEGATIVE CIRCLED LATIN CAPITAL LETTER V;So;0;L;;;;;N;;;;; +1F166;NEGATIVE CIRCLED LATIN CAPITAL LETTER W;So;0;L;;;;;N;;;;; +1F167;NEGATIVE CIRCLED LATIN CAPITAL LETTER X;So;0;L;;;;;N;;;;; +1F168;NEGATIVE CIRCLED LATIN CAPITAL LETTER Y;So;0;L;;;;;N;;;;; +1F169;NEGATIVE CIRCLED LATIN CAPITAL LETTER Z;So;0;L;;;;;N;;;;; +1F16A;RAISED MC SIGN;So;0;ON; 004D 0043;;;;N;;;;; +1F16B;RAISED MD SIGN;So;0;ON; 004D 0044;;;;N;;;;; +1F170;NEGATIVE SQUARED LATIN CAPITAL LETTER A;So;0;L;;;;;N;;;;; +1F171;NEGATIVE SQUARED LATIN CAPITAL LETTER B;So;0;L;;;;;N;;;;; +1F172;NEGATIVE SQUARED LATIN CAPITAL LETTER C;So;0;L;;;;;N;;;;; +1F173;NEGATIVE SQUARED LATIN CAPITAL LETTER D;So;0;L;;;;;N;;;;; +1F174;NEGATIVE SQUARED LATIN CAPITAL LETTER E;So;0;L;;;;;N;;;;; +1F175;NEGATIVE SQUARED LATIN CAPITAL LETTER F;So;0;L;;;;;N;;;;; +1F176;NEGATIVE SQUARED LATIN CAPITAL LETTER G;So;0;L;;;;;N;;;;; +1F177;NEGATIVE SQUARED LATIN CAPITAL LETTER H;So;0;L;;;;;N;;;;; +1F178;NEGATIVE SQUARED LATIN CAPITAL LETTER I;So;0;L;;;;;N;;;;; +1F179;NEGATIVE SQUARED LATIN CAPITAL LETTER J;So;0;L;;;;;N;;;;; +1F17A;NEGATIVE SQUARED LATIN CAPITAL LETTER K;So;0;L;;;;;N;;;;; +1F17B;NEGATIVE SQUARED LATIN CAPITAL LETTER L;So;0;L;;;;;N;;;;; +1F17C;NEGATIVE SQUARED LATIN CAPITAL LETTER M;So;0;L;;;;;N;;;;; +1F17D;NEGATIVE SQUARED LATIN CAPITAL LETTER N;So;0;L;;;;;N;;;;; +1F17E;NEGATIVE SQUARED LATIN CAPITAL LETTER O;So;0;L;;;;;N;;;;; +1F17F;NEGATIVE SQUARED LATIN CAPITAL LETTER P;So;0;L;;;;;N;;;;; +1F180;NEGATIVE SQUARED LATIN CAPITAL LETTER Q;So;0;L;;;;;N;;;;; +1F181;NEGATIVE SQUARED LATIN CAPITAL LETTER R;So;0;L;;;;;N;;;;; +1F182;NEGATIVE SQUARED LATIN CAPITAL LETTER S;So;0;L;;;;;N;;;;; +1F183;NEGATIVE SQUARED LATIN CAPITAL LETTER T;So;0;L;;;;;N;;;;; +1F184;NEGATIVE SQUARED LATIN CAPITAL LETTER U;So;0;L;;;;;N;;;;; +1F185;NEGATIVE SQUARED LATIN CAPITAL LETTER V;So;0;L;;;;;N;;;;; +1F186;NEGATIVE SQUARED LATIN CAPITAL LETTER W;So;0;L;;;;;N;;;;; +1F187;NEGATIVE SQUARED LATIN CAPITAL LETTER X;So;0;L;;;;;N;;;;; +1F188;NEGATIVE SQUARED LATIN CAPITAL LETTER Y;So;0;L;;;;;N;;;;; +1F189;NEGATIVE SQUARED LATIN CAPITAL LETTER Z;So;0;L;;;;;N;;;;; +1F18A;CROSSED NEGATIVE SQUARED LATIN CAPITAL LETTER P;So;0;L;;;;;N;;;;; +1F18B;NEGATIVE SQUARED IC;So;0;L;;;;;N;;;;; +1F18C;NEGATIVE SQUARED PA;So;0;L;;;;;N;;;;; +1F18D;NEGATIVE SQUARED SA;So;0;L;;;;;N;;;;; +1F18E;NEGATIVE SQUARED AB;So;0;L;;;;;N;;;;; +1F18F;NEGATIVE SQUARED WC;So;0;L;;;;;N;;;;; +1F190;SQUARE DJ;So;0;L; 0044 004A;;;;N;;;;; +1F191;SQUARED CL;So;0;L;;;;;N;;;;; +1F192;SQUARED COOL;So;0;L;;;;;N;;;;; +1F193;SQUARED FREE;So;0;L;;;;;N;;;;; +1F194;SQUARED ID;So;0;L;;;;;N;;;;; +1F195;SQUARED NEW;So;0;L;;;;;N;;;;; +1F196;SQUARED NG;So;0;L;;;;;N;;;;; +1F197;SQUARED OK;So;0;L;;;;;N;;;;; +1F198;SQUARED SOS;So;0;L;;;;;N;;;;; +1F199;SQUARED UP WITH EXCLAMATION MARK;So;0;L;;;;;N;;;;; +1F19A;SQUARED VS;So;0;L;;;;;N;;;;; +1F1E6;REGIONAL INDICATOR SYMBOL LETTER A;So;0;L;;;;;N;;;;; +1F1E7;REGIONAL INDICATOR SYMBOL LETTER B;So;0;L;;;;;N;;;;; +1F1E8;REGIONAL INDICATOR SYMBOL LETTER C;So;0;L;;;;;N;;;;; +1F1E9;REGIONAL INDICATOR SYMBOL LETTER D;So;0;L;;;;;N;;;;; +1F1EA;REGIONAL INDICATOR SYMBOL LETTER E;So;0;L;;;;;N;;;;; +1F1EB;REGIONAL INDICATOR SYMBOL LETTER F;So;0;L;;;;;N;;;;; +1F1EC;REGIONAL INDICATOR SYMBOL LETTER G;So;0;L;;;;;N;;;;; +1F1ED;REGIONAL INDICATOR SYMBOL LETTER H;So;0;L;;;;;N;;;;; +1F1EE;REGIONAL INDICATOR SYMBOL LETTER I;So;0;L;;;;;N;;;;; +1F1EF;REGIONAL INDICATOR SYMBOL LETTER J;So;0;L;;;;;N;;;;; +1F1F0;REGIONAL INDICATOR SYMBOL LETTER K;So;0;L;;;;;N;;;;; +1F1F1;REGIONAL INDICATOR SYMBOL LETTER L;So;0;L;;;;;N;;;;; +1F1F2;REGIONAL INDICATOR SYMBOL LETTER M;So;0;L;;;;;N;;;;; +1F1F3;REGIONAL INDICATOR SYMBOL LETTER N;So;0;L;;;;;N;;;;; +1F1F4;REGIONAL INDICATOR SYMBOL LETTER O;So;0;L;;;;;N;;;;; +1F1F5;REGIONAL INDICATOR SYMBOL LETTER P;So;0;L;;;;;N;;;;; +1F1F6;REGIONAL INDICATOR SYMBOL LETTER Q;So;0;L;;;;;N;;;;; +1F1F7;REGIONAL INDICATOR SYMBOL LETTER R;So;0;L;;;;;N;;;;; +1F1F8;REGIONAL INDICATOR SYMBOL LETTER S;So;0;L;;;;;N;;;;; +1F1F9;REGIONAL INDICATOR SYMBOL LETTER T;So;0;L;;;;;N;;;;; +1F1FA;REGIONAL INDICATOR SYMBOL LETTER U;So;0;L;;;;;N;;;;; +1F1FB;REGIONAL INDICATOR SYMBOL LETTER V;So;0;L;;;;;N;;;;; +1F1FC;REGIONAL INDICATOR SYMBOL LETTER W;So;0;L;;;;;N;;;;; +1F1FD;REGIONAL INDICATOR SYMBOL LETTER X;So;0;L;;;;;N;;;;; +1F1FE;REGIONAL INDICATOR SYMBOL LETTER Y;So;0;L;;;;;N;;;;; +1F1FF;REGIONAL INDICATOR SYMBOL LETTER Z;So;0;L;;;;;N;;;;; +1F200;SQUARE HIRAGANA HOKA;So;0;L; 307B 304B;;;;N;;;;; +1F201;SQUARED KATAKANA KOKO;So;0;L; 30B3 30B3;;;;N;;;;; +1F202;SQUARED KATAKANA SA;So;0;L; 30B5;;;;N;;;;; +1F210;SQUARED CJK UNIFIED IDEOGRAPH-624B;So;0;L; 624B;;;;N;;;;; +1F211;SQUARED CJK UNIFIED IDEOGRAPH-5B57;So;0;L; 5B57;;;;N;;;;; +1F212;SQUARED CJK UNIFIED IDEOGRAPH-53CC;So;0;L; 53CC;;;;N;;;;; +1F213;SQUARED KATAKANA DE;So;0;L; 30C7;;;;N;;;;; +1F214;SQUARED CJK UNIFIED IDEOGRAPH-4E8C;So;0;L; 4E8C;;;;N;;;;; +1F215;SQUARED CJK UNIFIED IDEOGRAPH-591A;So;0;L; 591A;;;;N;;;;; +1F216;SQUARED CJK UNIFIED IDEOGRAPH-89E3;So;0;L; 89E3;;;;N;;;;; +1F217;SQUARED CJK UNIFIED IDEOGRAPH-5929;So;0;L; 5929;;;;N;;;;; +1F218;SQUARED CJK UNIFIED IDEOGRAPH-4EA4;So;0;L; 4EA4;;;;N;;;;; +1F219;SQUARED CJK UNIFIED IDEOGRAPH-6620;So;0;L; 6620;;;;N;;;;; +1F21A;SQUARED CJK UNIFIED IDEOGRAPH-7121;So;0;L; 7121;;;;N;;;;; +1F21B;SQUARED CJK UNIFIED IDEOGRAPH-6599;So;0;L; 6599;;;;N;;;;; +1F21C;SQUARED CJK UNIFIED IDEOGRAPH-524D;So;0;L; 524D;;;;N;;;;; +1F21D;SQUARED CJK UNIFIED IDEOGRAPH-5F8C;So;0;L; 5F8C;;;;N;;;;; +1F21E;SQUARED CJK UNIFIED IDEOGRAPH-518D;So;0;L; 518D;;;;N;;;;; +1F21F;SQUARED CJK UNIFIED IDEOGRAPH-65B0;So;0;L; 65B0;;;;N;;;;; +1F220;SQUARED CJK UNIFIED IDEOGRAPH-521D;So;0;L; 521D;;;;N;;;;; +1F221;SQUARED CJK UNIFIED IDEOGRAPH-7D42;So;0;L; 7D42;;;;N;;;;; +1F222;SQUARED CJK UNIFIED IDEOGRAPH-751F;So;0;L; 751F;;;;N;;;;; +1F223;SQUARED CJK UNIFIED IDEOGRAPH-8CA9;So;0;L; 8CA9;;;;N;;;;; +1F224;SQUARED CJK UNIFIED IDEOGRAPH-58F0;So;0;L; 58F0;;;;N;;;;; +1F225;SQUARED CJK UNIFIED IDEOGRAPH-5439;So;0;L; 5439;;;;N;;;;; +1F226;SQUARED CJK UNIFIED IDEOGRAPH-6F14;So;0;L; 6F14;;;;N;;;;; +1F227;SQUARED CJK UNIFIED IDEOGRAPH-6295;So;0;L; 6295;;;;N;;;;; +1F228;SQUARED CJK UNIFIED IDEOGRAPH-6355;So;0;L; 6355;;;;N;;;;; +1F229;SQUARED CJK UNIFIED IDEOGRAPH-4E00;So;0;L; 4E00;;;;N;;;;; +1F22A;SQUARED CJK UNIFIED IDEOGRAPH-4E09;So;0;L; 4E09;;;;N;;;;; +1F22B;SQUARED CJK UNIFIED IDEOGRAPH-904A;So;0;L; 904A;;;;N;;;;; +1F22C;SQUARED CJK UNIFIED IDEOGRAPH-5DE6;So;0;L; 5DE6;;;;N;;;;; +1F22D;SQUARED CJK UNIFIED IDEOGRAPH-4E2D;So;0;L; 4E2D;;;;N;;;;; +1F22E;SQUARED CJK UNIFIED IDEOGRAPH-53F3;So;0;L; 53F3;;;;N;;;;; +1F22F;SQUARED CJK UNIFIED IDEOGRAPH-6307;So;0;L; 6307;;;;N;;;;; +1F230;SQUARED CJK UNIFIED IDEOGRAPH-8D70;So;0;L; 8D70;;;;N;;;;; +1F231;SQUARED CJK UNIFIED IDEOGRAPH-6253;So;0;L; 6253;;;;N;;;;; +1F232;SQUARED CJK UNIFIED IDEOGRAPH-7981;So;0;L; 7981;;;;N;;;;; +1F233;SQUARED CJK UNIFIED IDEOGRAPH-7A7A;So;0;L; 7A7A;;;;N;;;;; +1F234;SQUARED CJK UNIFIED IDEOGRAPH-5408;So;0;L; 5408;;;;N;;;;; +1F235;SQUARED CJK UNIFIED IDEOGRAPH-6E80;So;0;L; 6E80;;;;N;;;;; +1F236;SQUARED CJK UNIFIED IDEOGRAPH-6709;So;0;L; 6709;;;;N;;;;; +1F237;SQUARED CJK UNIFIED IDEOGRAPH-6708;So;0;L; 6708;;;;N;;;;; +1F238;SQUARED CJK UNIFIED IDEOGRAPH-7533;So;0;L; 7533;;;;N;;;;; +1F239;SQUARED CJK UNIFIED IDEOGRAPH-5272;So;0;L; 5272;;;;N;;;;; +1F23A;SQUARED CJK UNIFIED IDEOGRAPH-55B6;So;0;L; 55B6;;;;N;;;;; +1F240;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C;So;0;L; 3014 672C 3015;;;;N;;;;; +1F241;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-4E09;So;0;L; 3014 4E09 3015;;;;N;;;;; +1F242;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-4E8C;So;0;L; 3014 4E8C 3015;;;;N;;;;; +1F243;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-5B89;So;0;L; 3014 5B89 3015;;;;N;;;;; +1F244;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-70B9;So;0;L; 3014 70B9 3015;;;;N;;;;; +1F245;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6253;So;0;L; 3014 6253 3015;;;;N;;;;; +1F246;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-76D7;So;0;L; 3014 76D7 3015;;;;N;;;;; +1F247;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-52DD;So;0;L; 3014 52DD 3015;;;;N;;;;; +1F248;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557;So;0;L; 3014 6557 3015;;;;N;;;;; +1F250;CIRCLED IDEOGRAPH ADVANTAGE;So;0;L; 5F97;;;;N;;;;; +1F251;CIRCLED IDEOGRAPH ACCEPT;So;0;L; 53EF;;;;N;;;;; +1F300;CYCLONE;So;0;ON;;;;;N;;;;; +1F301;FOGGY;So;0;ON;;;;;N;;;;; +1F302;CLOSED UMBRELLA;So;0;ON;;;;;N;;;;; +1F303;NIGHT WITH STARS;So;0;ON;;;;;N;;;;; +1F304;SUNRISE OVER MOUNTAINS;So;0;ON;;;;;N;;;;; +1F305;SUNRISE;So;0;ON;;;;;N;;;;; +1F306;CITYSCAPE AT DUSK;So;0;ON;;;;;N;;;;; +1F307;SUNSET OVER BUILDINGS;So;0;ON;;;;;N;;;;; +1F308;RAINBOW;So;0;ON;;;;;N;;;;; +1F309;BRIDGE AT NIGHT;So;0;ON;;;;;N;;;;; +1F30A;WATER WAVE;So;0;ON;;;;;N;;;;; +1F30B;VOLCANO;So;0;ON;;;;;N;;;;; +1F30C;MILKY WAY;So;0;ON;;;;;N;;;;; +1F30D;EARTH GLOBE EUROPE-AFRICA;So;0;ON;;;;;N;;;;; +1F30E;EARTH GLOBE AMERICAS;So;0;ON;;;;;N;;;;; +1F30F;EARTH GLOBE ASIA-AUSTRALIA;So;0;ON;;;;;N;;;;; +1F310;GLOBE WITH MERIDIANS;So;0;ON;;;;;N;;;;; +1F311;NEW MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F312;WAXING CRESCENT MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F313;FIRST QUARTER MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F314;WAXING GIBBOUS MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F315;FULL MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F316;WANING GIBBOUS MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F317;LAST QUARTER MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F318;WANING CRESCENT MOON SYMBOL;So;0;ON;;;;;N;;;;; +1F319;CRESCENT MOON;So;0;ON;;;;;N;;;;; +1F31A;NEW MOON WITH FACE;So;0;ON;;;;;N;;;;; +1F31B;FIRST QUARTER MOON WITH FACE;So;0;ON;;;;;N;;;;; +1F31C;LAST QUARTER MOON WITH FACE;So;0;ON;;;;;N;;;;; +1F31D;FULL MOON WITH FACE;So;0;ON;;;;;N;;;;; +1F31E;SUN WITH FACE;So;0;ON;;;;;N;;;;; +1F31F;GLOWING STAR;So;0;ON;;;;;N;;;;; +1F320;SHOOTING STAR;So;0;ON;;;;;N;;;;; +1F330;CHESTNUT;So;0;ON;;;;;N;;;;; +1F331;SEEDLING;So;0;ON;;;;;N;;;;; +1F332;EVERGREEN TREE;So;0;ON;;;;;N;;;;; +1F333;DECIDUOUS TREE;So;0;ON;;;;;N;;;;; +1F334;PALM TREE;So;0;ON;;;;;N;;;;; +1F335;CACTUS;So;0;ON;;;;;N;;;;; +1F337;TULIP;So;0;ON;;;;;N;;;;; +1F338;CHERRY BLOSSOM;So;0;ON;;;;;N;;;;; +1F339;ROSE;So;0;ON;;;;;N;;;;; +1F33A;HIBISCUS;So;0;ON;;;;;N;;;;; +1F33B;SUNFLOWER;So;0;ON;;;;;N;;;;; +1F33C;BLOSSOM;So;0;ON;;;;;N;;;;; +1F33D;EAR OF MAIZE;So;0;ON;;;;;N;;;;; +1F33E;EAR OF RICE;So;0;ON;;;;;N;;;;; +1F33F;HERB;So;0;ON;;;;;N;;;;; +1F340;FOUR LEAF CLOVER;So;0;ON;;;;;N;;;;; +1F341;MAPLE LEAF;So;0;ON;;;;;N;;;;; +1F342;FALLEN LEAF;So;0;ON;;;;;N;;;;; +1F343;LEAF FLUTTERING IN WIND;So;0;ON;;;;;N;;;;; +1F344;MUSHROOM;So;0;ON;;;;;N;;;;; +1F345;TOMATO;So;0;ON;;;;;N;;;;; +1F346;AUBERGINE;So;0;ON;;;;;N;;;;; +1F347;GRAPES;So;0;ON;;;;;N;;;;; +1F348;MELON;So;0;ON;;;;;N;;;;; +1F349;WATERMELON;So;0;ON;;;;;N;;;;; +1F34A;TANGERINE;So;0;ON;;;;;N;;;;; +1F34B;LEMON;So;0;ON;;;;;N;;;;; +1F34C;BANANA;So;0;ON;;;;;N;;;;; +1F34D;PINEAPPLE;So;0;ON;;;;;N;;;;; +1F34E;RED APPLE;So;0;ON;;;;;N;;;;; +1F34F;GREEN APPLE;So;0;ON;;;;;N;;;;; +1F350;PEAR;So;0;ON;;;;;N;;;;; +1F351;PEACH;So;0;ON;;;;;N;;;;; +1F352;CHERRIES;So;0;ON;;;;;N;;;;; +1F353;STRAWBERRY;So;0;ON;;;;;N;;;;; +1F354;HAMBURGER;So;0;ON;;;;;N;;;;; +1F355;SLICE OF PIZZA;So;0;ON;;;;;N;;;;; +1F356;MEAT ON BONE;So;0;ON;;;;;N;;;;; +1F357;POULTRY LEG;So;0;ON;;;;;N;;;;; +1F358;RICE CRACKER;So;0;ON;;;;;N;;;;; +1F359;RICE BALL;So;0;ON;;;;;N;;;;; +1F35A;COOKED RICE;So;0;ON;;;;;N;;;;; +1F35B;CURRY AND RICE;So;0;ON;;;;;N;;;;; +1F35C;STEAMING BOWL;So;0;ON;;;;;N;;;;; +1F35D;SPAGHETTI;So;0;ON;;;;;N;;;;; +1F35E;BREAD;So;0;ON;;;;;N;;;;; +1F35F;FRENCH FRIES;So;0;ON;;;;;N;;;;; +1F360;ROASTED SWEET POTATO;So;0;ON;;;;;N;;;;; +1F361;DANGO;So;0;ON;;;;;N;;;;; +1F362;ODEN;So;0;ON;;;;;N;;;;; +1F363;SUSHI;So;0;ON;;;;;N;;;;; +1F364;FRIED SHRIMP;So;0;ON;;;;;N;;;;; +1F365;FISH CAKE WITH SWIRL DESIGN;So;0;ON;;;;;N;;;;; +1F366;SOFT ICE CREAM;So;0;ON;;;;;N;;;;; +1F367;SHAVED ICE;So;0;ON;;;;;N;;;;; +1F368;ICE CREAM;So;0;ON;;;;;N;;;;; +1F369;DOUGHNUT;So;0;ON;;;;;N;;;;; +1F36A;COOKIE;So;0;ON;;;;;N;;;;; +1F36B;CHOCOLATE BAR;So;0;ON;;;;;N;;;;; +1F36C;CANDY;So;0;ON;;;;;N;;;;; +1F36D;LOLLIPOP;So;0;ON;;;;;N;;;;; +1F36E;CUSTARD;So;0;ON;;;;;N;;;;; +1F36F;HONEY POT;So;0;ON;;;;;N;;;;; +1F370;SHORTCAKE;So;0;ON;;;;;N;;;;; +1F371;BENTO BOX;So;0;ON;;;;;N;;;;; +1F372;POT OF FOOD;So;0;ON;;;;;N;;;;; +1F373;COOKING;So;0;ON;;;;;N;;;;; +1F374;FORK AND KNIFE;So;0;ON;;;;;N;;;;; +1F375;TEACUP WITHOUT HANDLE;So;0;ON;;;;;N;;;;; +1F376;SAKE BOTTLE AND CUP;So;0;ON;;;;;N;;;;; +1F377;WINE GLASS;So;0;ON;;;;;N;;;;; +1F378;COCKTAIL GLASS;So;0;ON;;;;;N;;;;; +1F379;TROPICAL DRINK;So;0;ON;;;;;N;;;;; +1F37A;BEER MUG;So;0;ON;;;;;N;;;;; +1F37B;CLINKING BEER MUGS;So;0;ON;;;;;N;;;;; +1F37C;BABY BOTTLE;So;0;ON;;;;;N;;;;; +1F380;RIBBON;So;0;ON;;;;;N;;;;; +1F381;WRAPPED PRESENT;So;0;ON;;;;;N;;;;; +1F382;BIRTHDAY CAKE;So;0;ON;;;;;N;;;;; +1F383;JACK-O-LANTERN;So;0;ON;;;;;N;;;;; +1F384;CHRISTMAS TREE;So;0;ON;;;;;N;;;;; +1F385;FATHER CHRISTMAS;So;0;ON;;;;;N;;;;; +1F386;FIREWORKS;So;0;ON;;;;;N;;;;; +1F387;FIREWORK SPARKLER;So;0;ON;;;;;N;;;;; +1F388;BALLOON;So;0;ON;;;;;N;;;;; +1F389;PARTY POPPER;So;0;ON;;;;;N;;;;; +1F38A;CONFETTI BALL;So;0;ON;;;;;N;;;;; +1F38B;TANABATA TREE;So;0;ON;;;;;N;;;;; +1F38C;CROSSED FLAGS;So;0;ON;;;;;N;;;;; +1F38D;PINE DECORATION;So;0;ON;;;;;N;;;;; +1F38E;JAPANESE DOLLS;So;0;ON;;;;;N;;;;; +1F38F;CARP STREAMER;So;0;ON;;;;;N;;;;; +1F390;WIND CHIME;So;0;ON;;;;;N;;;;; +1F391;MOON VIEWING CEREMONY;So;0;ON;;;;;N;;;;; +1F392;SCHOOL SATCHEL;So;0;ON;;;;;N;;;;; +1F393;GRADUATION CAP;So;0;ON;;;;;N;;;;; +1F3A0;CAROUSEL HORSE;So;0;ON;;;;;N;;;;; +1F3A1;FERRIS WHEEL;So;0;ON;;;;;N;;;;; +1F3A2;ROLLER COASTER;So;0;ON;;;;;N;;;;; +1F3A3;FISHING POLE AND FISH;So;0;ON;;;;;N;;;;; +1F3A4;MICROPHONE;So;0;ON;;;;;N;;;;; +1F3A5;MOVIE CAMERA;So;0;ON;;;;;N;;;;; +1F3A6;CINEMA;So;0;ON;;;;;N;;;;; +1F3A7;HEADPHONE;So;0;ON;;;;;N;;;;; +1F3A8;ARTIST PALETTE;So;0;ON;;;;;N;;;;; +1F3A9;TOP HAT;So;0;ON;;;;;N;;;;; +1F3AA;CIRCUS TENT;So;0;ON;;;;;N;;;;; +1F3AB;TICKET;So;0;ON;;;;;N;;;;; +1F3AC;CLAPPER BOARD;So;0;ON;;;;;N;;;;; +1F3AD;PERFORMING ARTS;So;0;ON;;;;;N;;;;; +1F3AE;VIDEO GAME;So;0;ON;;;;;N;;;;; +1F3AF;DIRECT HIT;So;0;ON;;;;;N;;;;; +1F3B0;SLOT MACHINE;So;0;ON;;;;;N;;;;; +1F3B1;BILLIARDS;So;0;ON;;;;;N;;;;; +1F3B2;GAME DIE;So;0;ON;;;;;N;;;;; +1F3B3;BOWLING;So;0;ON;;;;;N;;;;; +1F3B4;FLOWER PLAYING CARDS;So;0;ON;;;;;N;;;;; +1F3B5;MUSICAL NOTE;So;0;ON;;;;;N;;;;; +1F3B6;MULTIPLE MUSICAL NOTES;So;0;ON;;;;;N;;;;; +1F3B7;SAXOPHONE;So;0;ON;;;;;N;;;;; +1F3B8;GUITAR;So;0;ON;;;;;N;;;;; +1F3B9;MUSICAL KEYBOARD;So;0;ON;;;;;N;;;;; +1F3BA;TRUMPET;So;0;ON;;;;;N;;;;; +1F3BB;VIOLIN;So;0;ON;;;;;N;;;;; +1F3BC;MUSICAL SCORE;So;0;ON;;;;;N;;;;; +1F3BD;RUNNING SHIRT WITH SASH;So;0;ON;;;;;N;;;;; +1F3BE;TENNIS RACQUET AND BALL;So;0;ON;;;;;N;;;;; +1F3BF;SKI AND SKI BOOT;So;0;ON;;;;;N;;;;; +1F3C0;BASKETBALL AND HOOP;So;0;ON;;;;;N;;;;; +1F3C1;CHEQUERED FLAG;So;0;ON;;;;;N;;;;; +1F3C2;SNOWBOARDER;So;0;ON;;;;;N;;;;; +1F3C3;RUNNER;So;0;ON;;;;;N;;;;; +1F3C4;SURFER;So;0;ON;;;;;N;;;;; +1F3C6;TROPHY;So;0;ON;;;;;N;;;;; +1F3C7;HORSE RACING;So;0;ON;;;;;N;;;;; +1F3C8;AMERICAN FOOTBALL;So;0;ON;;;;;N;;;;; +1F3C9;RUGBY FOOTBALL;So;0;ON;;;;;N;;;;; +1F3CA;SWIMMER;So;0;ON;;;;;N;;;;; +1F3E0;HOUSE BUILDING;So;0;ON;;;;;N;;;;; +1F3E1;HOUSE WITH GARDEN;So;0;ON;;;;;N;;;;; +1F3E2;OFFICE BUILDING;So;0;ON;;;;;N;;;;; +1F3E3;JAPANESE POST OFFICE;So;0;ON;;;;;N;;;;; +1F3E4;EUROPEAN POST OFFICE;So;0;ON;;;;;N;;;;; +1F3E5;HOSPITAL;So;0;ON;;;;;N;;;;; +1F3E6;BANK;So;0;ON;;;;;N;;;;; +1F3E7;AUTOMATED TELLER MACHINE;So;0;ON;;;;;N;;;;; +1F3E8;HOTEL;So;0;ON;;;;;N;;;;; +1F3E9;LOVE HOTEL;So;0;ON;;;;;N;;;;; +1F3EA;CONVENIENCE STORE;So;0;ON;;;;;N;;;;; +1F3EB;SCHOOL;So;0;ON;;;;;N;;;;; +1F3EC;DEPARTMENT STORE;So;0;ON;;;;;N;;;;; +1F3ED;FACTORY;So;0;ON;;;;;N;;;;; +1F3EE;IZAKAYA LANTERN;So;0;ON;;;;;N;;;;; +1F3EF;JAPANESE CASTLE;So;0;ON;;;;;N;;;;; +1F3F0;EUROPEAN CASTLE;So;0;ON;;;;;N;;;;; +1F400;RAT;So;0;ON;;;;;N;;;;; +1F401;MOUSE;So;0;ON;;;;;N;;;;; +1F402;OX;So;0;ON;;;;;N;;;;; +1F403;WATER BUFFALO;So;0;ON;;;;;N;;;;; +1F404;COW;So;0;ON;;;;;N;;;;; +1F405;TIGER;So;0;ON;;;;;N;;;;; +1F406;LEOPARD;So;0;ON;;;;;N;;;;; +1F407;RABBIT;So;0;ON;;;;;N;;;;; +1F408;CAT;So;0;ON;;;;;N;;;;; +1F409;DRAGON;So;0;ON;;;;;N;;;;; +1F40A;CROCODILE;So;0;ON;;;;;N;;;;; +1F40B;WHALE;So;0;ON;;;;;N;;;;; +1F40C;SNAIL;So;0;ON;;;;;N;;;;; +1F40D;SNAKE;So;0;ON;;;;;N;;;;; +1F40E;HORSE;So;0;ON;;;;;N;;;;; +1F40F;RAM;So;0;ON;;;;;N;;;;; +1F410;GOAT;So;0;ON;;;;;N;;;;; +1F411;SHEEP;So;0;ON;;;;;N;;;;; +1F412;MONKEY;So;0;ON;;;;;N;;;;; +1F413;ROOSTER;So;0;ON;;;;;N;;;;; +1F414;CHICKEN;So;0;ON;;;;;N;;;;; +1F415;DOG;So;0;ON;;;;;N;;;;; +1F416;PIG;So;0;ON;;;;;N;;;;; +1F417;BOAR;So;0;ON;;;;;N;;;;; +1F418;ELEPHANT;So;0;ON;;;;;N;;;;; +1F419;OCTOPUS;So;0;ON;;;;;N;;;;; +1F41A;SPIRAL SHELL;So;0;ON;;;;;N;;;;; +1F41B;BUG;So;0;ON;;;;;N;;;;; +1F41C;ANT;So;0;ON;;;;;N;;;;; +1F41D;HONEYBEE;So;0;ON;;;;;N;;;;; +1F41E;LADY BEETLE;So;0;ON;;;;;N;;;;; +1F41F;FISH;So;0;ON;;;;;N;;;;; +1F420;TROPICAL FISH;So;0;ON;;;;;N;;;;; +1F421;BLOWFISH;So;0;ON;;;;;N;;;;; +1F422;TURTLE;So;0;ON;;;;;N;;;;; +1F423;HATCHING CHICK;So;0;ON;;;;;N;;;;; +1F424;BABY CHICK;So;0;ON;;;;;N;;;;; +1F425;FRONT-FACING BABY CHICK;So;0;ON;;;;;N;;;;; +1F426;BIRD;So;0;ON;;;;;N;;;;; +1F427;PENGUIN;So;0;ON;;;;;N;;;;; +1F428;KOALA;So;0;ON;;;;;N;;;;; +1F429;POODLE;So;0;ON;;;;;N;;;;; +1F42A;DROMEDARY CAMEL;So;0;ON;;;;;N;;;;; +1F42B;BACTRIAN CAMEL;So;0;ON;;;;;N;;;;; +1F42C;DOLPHIN;So;0;ON;;;;;N;;;;; +1F42D;MOUSE FACE;So;0;ON;;;;;N;;;;; +1F42E;COW FACE;So;0;ON;;;;;N;;;;; +1F42F;TIGER FACE;So;0;ON;;;;;N;;;;; +1F430;RABBIT FACE;So;0;ON;;;;;N;;;;; +1F431;CAT FACE;So;0;ON;;;;;N;;;;; +1F432;DRAGON FACE;So;0;ON;;;;;N;;;;; +1F433;SPOUTING WHALE;So;0;ON;;;;;N;;;;; +1F434;HORSE FACE;So;0;ON;;;;;N;;;;; +1F435;MONKEY FACE;So;0;ON;;;;;N;;;;; +1F436;DOG FACE;So;0;ON;;;;;N;;;;; +1F437;PIG FACE;So;0;ON;;;;;N;;;;; +1F438;FROG FACE;So;0;ON;;;;;N;;;;; +1F439;HAMSTER FACE;So;0;ON;;;;;N;;;;; +1F43A;WOLF FACE;So;0;ON;;;;;N;;;;; +1F43B;BEAR FACE;So;0;ON;;;;;N;;;;; +1F43C;PANDA FACE;So;0;ON;;;;;N;;;;; +1F43D;PIG NOSE;So;0;ON;;;;;N;;;;; +1F43E;PAW PRINTS;So;0;ON;;;;;N;;;;; +1F440;EYES;So;0;ON;;;;;N;;;;; +1F442;EAR;So;0;ON;;;;;N;;;;; +1F443;NOSE;So;0;ON;;;;;N;;;;; +1F444;MOUTH;So;0;ON;;;;;N;;;;; +1F445;TONGUE;So;0;ON;;;;;N;;;;; +1F446;WHITE UP POINTING BACKHAND INDEX;So;0;ON;;;;;N;;;;; +1F447;WHITE DOWN POINTING BACKHAND INDEX;So;0;ON;;;;;N;;;;; +1F448;WHITE LEFT POINTING BACKHAND INDEX;So;0;ON;;;;;N;;;;; +1F449;WHITE RIGHT POINTING BACKHAND INDEX;So;0;ON;;;;;N;;;;; +1F44A;FISTED HAND SIGN;So;0;ON;;;;;N;;;;; +1F44B;WAVING HAND SIGN;So;0;ON;;;;;N;;;;; +1F44C;OK HAND SIGN;So;0;ON;;;;;N;;;;; +1F44D;THUMBS UP SIGN;So;0;ON;;;;;N;;;;; +1F44E;THUMBS DOWN SIGN;So;0;ON;;;;;N;;;;; +1F44F;CLAPPING HANDS SIGN;So;0;ON;;;;;N;;;;; +1F450;OPEN HANDS SIGN;So;0;ON;;;;;N;;;;; +1F451;CROWN;So;0;ON;;;;;N;;;;; +1F452;WOMANS HAT;So;0;ON;;;;;N;;;;; +1F453;EYEGLASSES;So;0;ON;;;;;N;;;;; +1F454;NECKTIE;So;0;ON;;;;;N;;;;; +1F455;T-SHIRT;So;0;ON;;;;;N;;;;; +1F456;JEANS;So;0;ON;;;;;N;;;;; +1F457;DRESS;So;0;ON;;;;;N;;;;; +1F458;KIMONO;So;0;ON;;;;;N;;;;; +1F459;BIKINI;So;0;ON;;;;;N;;;;; +1F45A;WOMANS CLOTHES;So;0;ON;;;;;N;;;;; +1F45B;PURSE;So;0;ON;;;;;N;;;;; +1F45C;HANDBAG;So;0;ON;;;;;N;;;;; +1F45D;POUCH;So;0;ON;;;;;N;;;;; +1F45E;MANS SHOE;So;0;ON;;;;;N;;;;; +1F45F;ATHLETIC SHOE;So;0;ON;;;;;N;;;;; +1F460;HIGH-HEELED SHOE;So;0;ON;;;;;N;;;;; +1F461;WOMANS SANDAL;So;0;ON;;;;;N;;;;; +1F462;WOMANS BOOTS;So;0;ON;;;;;N;;;;; +1F463;FOOTPRINTS;So;0;ON;;;;;N;;;;; +1F464;BUST IN SILHOUETTE;So;0;ON;;;;;N;;;;; +1F465;BUSTS IN SILHOUETTE;So;0;ON;;;;;N;;;;; +1F466;BOY;So;0;ON;;;;;N;;;;; +1F467;GIRL;So;0;ON;;;;;N;;;;; +1F468;MAN;So;0;ON;;;;;N;;;;; +1F469;WOMAN;So;0;ON;;;;;N;;;;; +1F46A;FAMILY;So;0;ON;;;;;N;;;;; +1F46B;MAN AND WOMAN HOLDING HANDS;So;0;ON;;;;;N;;;;; +1F46C;TWO MEN HOLDING HANDS;So;0;ON;;;;;N;;;;; +1F46D;TWO WOMEN HOLDING HANDS;So;0;ON;;;;;N;;;;; +1F46E;POLICE OFFICER;So;0;ON;;;;;N;;;;; +1F46F;WOMAN WITH BUNNY EARS;So;0;ON;;;;;N;;;;; +1F470;BRIDE WITH VEIL;So;0;ON;;;;;N;;;;; +1F471;PERSON WITH BLOND HAIR;So;0;ON;;;;;N;;;;; +1F472;MAN WITH GUA PI MAO;So;0;ON;;;;;N;;;;; +1F473;MAN WITH TURBAN;So;0;ON;;;;;N;;;;; +1F474;OLDER MAN;So;0;ON;;;;;N;;;;; +1F475;OLDER WOMAN;So;0;ON;;;;;N;;;;; +1F476;BABY;So;0;ON;;;;;N;;;;; +1F477;CONSTRUCTION WORKER;So;0;ON;;;;;N;;;;; +1F478;PRINCESS;So;0;ON;;;;;N;;;;; +1F479;JAPANESE OGRE;So;0;ON;;;;;N;;;;; +1F47A;JAPANESE GOBLIN;So;0;ON;;;;;N;;;;; +1F47B;GHOST;So;0;ON;;;;;N;;;;; +1F47C;BABY ANGEL;So;0;ON;;;;;N;;;;; +1F47D;EXTRATERRESTRIAL ALIEN;So;0;ON;;;;;N;;;;; +1F47E;ALIEN MONSTER;So;0;ON;;;;;N;;;;; +1F47F;IMP;So;0;ON;;;;;N;;;;; +1F480;SKULL;So;0;ON;;;;;N;;;;; +1F481;INFORMATION DESK PERSON;So;0;ON;;;;;N;;;;; +1F482;GUARDSMAN;So;0;ON;;;;;N;;;;; +1F483;DANCER;So;0;ON;;;;;N;;;;; +1F484;LIPSTICK;So;0;ON;;;;;N;;;;; +1F485;NAIL POLISH;So;0;ON;;;;;N;;;;; +1F486;FACE MASSAGE;So;0;ON;;;;;N;;;;; +1F487;HAIRCUT;So;0;ON;;;;;N;;;;; +1F488;BARBER POLE;So;0;ON;;;;;N;;;;; +1F489;SYRINGE;So;0;ON;;;;;N;;;;; +1F48A;PILL;So;0;ON;;;;;N;;;;; +1F48B;KISS MARK;So;0;ON;;;;;N;;;;; +1F48C;LOVE LETTER;So;0;ON;;;;;N;;;;; +1F48D;RING;So;0;ON;;;;;N;;;;; +1F48E;GEM STONE;So;0;ON;;;;;N;;;;; +1F48F;KISS;So;0;ON;;;;;N;;;;; +1F490;BOUQUET;So;0;ON;;;;;N;;;;; +1F491;COUPLE WITH HEART;So;0;ON;;;;;N;;;;; +1F492;WEDDING;So;0;ON;;;;;N;;;;; +1F493;BEATING HEART;So;0;ON;;;;;N;;;;; +1F494;BROKEN HEART;So;0;ON;;;;;N;;;;; +1F495;TWO HEARTS;So;0;ON;;;;;N;;;;; +1F496;SPARKLING HEART;So;0;ON;;;;;N;;;;; +1F497;GROWING HEART;So;0;ON;;;;;N;;;;; +1F498;HEART WITH ARROW;So;0;ON;;;;;N;;;;; +1F499;BLUE HEART;So;0;ON;;;;;N;;;;; +1F49A;GREEN HEART;So;0;ON;;;;;N;;;;; +1F49B;YELLOW HEART;So;0;ON;;;;;N;;;;; +1F49C;PURPLE HEART;So;0;ON;;;;;N;;;;; +1F49D;HEART WITH RIBBON;So;0;ON;;;;;N;;;;; +1F49E;REVOLVING HEARTS;So;0;ON;;;;;N;;;;; +1F49F;HEART DECORATION;So;0;ON;;;;;N;;;;; +1F4A0;DIAMOND SHAPE WITH A DOT INSIDE;So;0;ON;;;;;N;;;;; +1F4A1;ELECTRIC LIGHT BULB;So;0;ON;;;;;N;;;;; +1F4A2;ANGER SYMBOL;So;0;ON;;;;;N;;;;; +1F4A3;BOMB;So;0;ON;;;;;N;;;;; +1F4A4;SLEEPING SYMBOL;So;0;ON;;;;;N;;;;; +1F4A5;COLLISION SYMBOL;So;0;ON;;;;;N;;;;; +1F4A6;SPLASHING SWEAT SYMBOL;So;0;ON;;;;;N;;;;; +1F4A7;DROPLET;So;0;ON;;;;;N;;;;; +1F4A8;DASH SYMBOL;So;0;ON;;;;;N;;;;; +1F4A9;PILE OF POO;So;0;ON;;;;;N;;;;; +1F4AA;FLEXED BICEPS;So;0;ON;;;;;N;;;;; +1F4AB;DIZZY SYMBOL;So;0;ON;;;;;N;;;;; +1F4AC;SPEECH BALLOON;So;0;ON;;;;;N;;;;; +1F4AD;THOUGHT BALLOON;So;0;ON;;;;;N;;;;; +1F4AE;WHITE FLOWER;So;0;ON;;;;;N;;;;; +1F4AF;HUNDRED POINTS SYMBOL;So;0;ON;;;;;N;;;;; +1F4B0;MONEY BAG;So;0;ON;;;;;N;;;;; +1F4B1;CURRENCY EXCHANGE;So;0;ON;;;;;N;;;;; +1F4B2;HEAVY DOLLAR SIGN;So;0;ON;;;;;N;;;;; +1F4B3;CREDIT CARD;So;0;ON;;;;;N;;;;; +1F4B4;BANKNOTE WITH YEN SIGN;So;0;ON;;;;;N;;;;; +1F4B5;BANKNOTE WITH DOLLAR SIGN;So;0;ON;;;;;N;;;;; +1F4B6;BANKNOTE WITH EURO SIGN;So;0;ON;;;;;N;;;;; +1F4B7;BANKNOTE WITH POUND SIGN;So;0;ON;;;;;N;;;;; +1F4B8;MONEY WITH WINGS;So;0;ON;;;;;N;;;;; +1F4B9;CHART WITH UPWARDS TREND AND YEN SIGN;So;0;ON;;;;;N;;;;; +1F4BA;SEAT;So;0;ON;;;;;N;;;;; +1F4BB;PERSONAL COMPUTER;So;0;ON;;;;;N;;;;; +1F4BC;BRIEFCASE;So;0;ON;;;;;N;;;;; +1F4BD;MINIDISC;So;0;ON;;;;;N;;;;; +1F4BE;FLOPPY DISK;So;0;ON;;;;;N;;;;; +1F4BF;OPTICAL DISC;So;0;ON;;;;;N;;;;; +1F4C0;DVD;So;0;ON;;;;;N;;;;; +1F4C1;FILE FOLDER;So;0;ON;;;;;N;;;;; +1F4C2;OPEN FILE FOLDER;So;0;ON;;;;;N;;;;; +1F4C3;PAGE WITH CURL;So;0;ON;;;;;N;;;;; +1F4C4;PAGE FACING UP;So;0;ON;;;;;N;;;;; +1F4C5;CALENDAR;So;0;ON;;;;;N;;;;; +1F4C6;TEAR-OFF CALENDAR;So;0;ON;;;;;N;;;;; +1F4C7;CARD INDEX;So;0;ON;;;;;N;;;;; +1F4C8;CHART WITH UPWARDS TREND;So;0;ON;;;;;N;;;;; +1F4C9;CHART WITH DOWNWARDS TREND;So;0;ON;;;;;N;;;;; +1F4CA;BAR CHART;So;0;ON;;;;;N;;;;; +1F4CB;CLIPBOARD;So;0;ON;;;;;N;;;;; +1F4CC;PUSHPIN;So;0;ON;;;;;N;;;;; +1F4CD;ROUND PUSHPIN;So;0;ON;;;;;N;;;;; +1F4CE;PAPERCLIP;So;0;ON;;;;;N;;;;; +1F4CF;STRAIGHT RULER;So;0;ON;;;;;N;;;;; +1F4D0;TRIANGULAR RULER;So;0;ON;;;;;N;;;;; +1F4D1;BOOKMARK TABS;So;0;ON;;;;;N;;;;; +1F4D2;LEDGER;So;0;ON;;;;;N;;;;; +1F4D3;NOTEBOOK;So;0;ON;;;;;N;;;;; +1F4D4;NOTEBOOK WITH DECORATIVE COVER;So;0;ON;;;;;N;;;;; +1F4D5;CLOSED BOOK;So;0;ON;;;;;N;;;;; +1F4D6;OPEN BOOK;So;0;ON;;;;;N;;;;; +1F4D7;GREEN BOOK;So;0;ON;;;;;N;;;;; +1F4D8;BLUE BOOK;So;0;ON;;;;;N;;;;; +1F4D9;ORANGE BOOK;So;0;ON;;;;;N;;;;; +1F4DA;BOOKS;So;0;ON;;;;;N;;;;; +1F4DB;NAME BADGE;So;0;ON;;;;;N;;;;; +1F4DC;SCROLL;So;0;ON;;;;;N;;;;; +1F4DD;MEMO;So;0;ON;;;;;N;;;;; +1F4DE;TELEPHONE RECEIVER;So;0;ON;;;;;N;;;;; +1F4DF;PAGER;So;0;ON;;;;;N;;;;; +1F4E0;FAX MACHINE;So;0;ON;;;;;N;;;;; +1F4E1;SATELLITE ANTENNA;So;0;ON;;;;;N;;;;; +1F4E2;PUBLIC ADDRESS LOUDSPEAKER;So;0;ON;;;;;N;;;;; +1F4E3;CHEERING MEGAPHONE;So;0;ON;;;;;N;;;;; +1F4E4;OUTBOX TRAY;So;0;ON;;;;;N;;;;; +1F4E5;INBOX TRAY;So;0;ON;;;;;N;;;;; +1F4E6;PACKAGE;So;0;ON;;;;;N;;;;; +1F4E7;E-MAIL SYMBOL;So;0;ON;;;;;N;;;;; +1F4E8;INCOMING ENVELOPE;So;0;ON;;;;;N;;;;; +1F4E9;ENVELOPE WITH DOWNWARDS ARROW ABOVE;So;0;ON;;;;;N;;;;; +1F4EA;CLOSED MAILBOX WITH LOWERED FLAG;So;0;ON;;;;;N;;;;; +1F4EB;CLOSED MAILBOX WITH RAISED FLAG;So;0;ON;;;;;N;;;;; +1F4EC;OPEN MAILBOX WITH RAISED FLAG;So;0;ON;;;;;N;;;;; +1F4ED;OPEN MAILBOX WITH LOWERED FLAG;So;0;ON;;;;;N;;;;; +1F4EE;POSTBOX;So;0;ON;;;;;N;;;;; +1F4EF;POSTAL HORN;So;0;ON;;;;;N;;;;; +1F4F0;NEWSPAPER;So;0;ON;;;;;N;;;;; +1F4F1;MOBILE PHONE;So;0;ON;;;;;N;;;;; +1F4F2;MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT;So;0;ON;;;;;N;;;;; +1F4F3;VIBRATION MODE;So;0;ON;;;;;N;;;;; +1F4F4;MOBILE PHONE OFF;So;0;ON;;;;;N;;;;; +1F4F5;NO MOBILE PHONES;So;0;ON;;;;;N;;;;; +1F4F6;ANTENNA WITH BARS;So;0;ON;;;;;N;;;;; +1F4F7;CAMERA;So;0;ON;;;;;N;;;;; +1F4F9;VIDEO CAMERA;So;0;ON;;;;;N;;;;; +1F4FA;TELEVISION;So;0;ON;;;;;N;;;;; +1F4FB;RADIO;So;0;ON;;;;;N;;;;; +1F4FC;VIDEOCASSETTE;So;0;ON;;;;;N;;;;; +1F500;TWISTED RIGHTWARDS ARROWS;So;0;ON;;;;;N;;;;; +1F501;CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE ARROWS;So;0;ON;;;;;N;;;;; +1F502;CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE ARROWS WITH CIRCLED ONE OVERLAY;So;0;ON;;;;;N;;;;; +1F503;CLOCKWISE DOWNWARDS AND UPWARDS OPEN CIRCLE ARROWS;So;0;ON;;;;;N;;;;; +1F504;ANTICLOCKWISE DOWNWARDS AND UPWARDS OPEN CIRCLE ARROWS;So;0;ON;;;;;N;;;;; +1F505;LOW BRIGHTNESS SYMBOL;So;0;ON;;;;;N;;;;; +1F506;HIGH BRIGHTNESS SYMBOL;So;0;ON;;;;;N;;;;; +1F507;SPEAKER WITH CANCELLATION STROKE;So;0;ON;;;;;N;;;;; +1F508;SPEAKER;So;0;ON;;;;;N;;;;; +1F509;SPEAKER WITH ONE SOUND WAVE;So;0;ON;;;;;N;;;;; +1F50A;SPEAKER WITH THREE SOUND WAVES;So;0;ON;;;;;N;;;;; +1F50B;BATTERY;So;0;ON;;;;;N;;;;; +1F50C;ELECTRIC PLUG;So;0;ON;;;;;N;;;;; +1F50D;LEFT-POINTING MAGNIFYING GLASS;So;0;ON;;;;;N;;;;; +1F50E;RIGHT-POINTING MAGNIFYING GLASS;So;0;ON;;;;;N;;;;; +1F50F;LOCK WITH INK PEN;So;0;ON;;;;;N;;;;; +1F510;CLOSED LOCK WITH KEY;So;0;ON;;;;;N;;;;; +1F511;KEY;So;0;ON;;;;;N;;;;; +1F512;LOCK;So;0;ON;;;;;N;;;;; +1F513;OPEN LOCK;So;0;ON;;;;;N;;;;; +1F514;BELL;So;0;ON;;;;;N;;;;; +1F515;BELL WITH CANCELLATION STROKE;So;0;ON;;;;;N;;;;; +1F516;BOOKMARK;So;0;ON;;;;;N;;;;; +1F517;LINK SYMBOL;So;0;ON;;;;;N;;;;; +1F518;RADIO BUTTON;So;0;ON;;;;;N;;;;; +1F519;BACK WITH LEFTWARDS ARROW ABOVE;So;0;ON;;;;;N;;;;; +1F51A;END WITH LEFTWARDS ARROW ABOVE;So;0;ON;;;;;N;;;;; +1F51B;ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW ABOVE;So;0;ON;;;;;N;;;;; +1F51C;SOON WITH RIGHTWARDS ARROW ABOVE;So;0;ON;;;;;N;;;;; +1F51D;TOP WITH UPWARDS ARROW ABOVE;So;0;ON;;;;;N;;;;; +1F51E;NO ONE UNDER EIGHTEEN SYMBOL;So;0;ON;;;;;N;;;;; +1F51F;KEYCAP TEN;So;0;ON;;;;;N;;;;; +1F520;INPUT SYMBOL FOR LATIN CAPITAL LETTERS;So;0;ON;;;;;N;;;;; +1F521;INPUT SYMBOL FOR LATIN SMALL LETTERS;So;0;ON;;;;;N;;;;; +1F522;INPUT SYMBOL FOR NUMBERS;So;0;ON;;;;;N;;;;; +1F523;INPUT SYMBOL FOR SYMBOLS;So;0;ON;;;;;N;;;;; +1F524;INPUT SYMBOL FOR LATIN LETTERS;So;0;ON;;;;;N;;;;; +1F525;FIRE;So;0;ON;;;;;N;;;;; +1F526;ELECTRIC TORCH;So;0;ON;;;;;N;;;;; +1F527;WRENCH;So;0;ON;;;;;N;;;;; +1F528;HAMMER;So;0;ON;;;;;N;;;;; +1F529;NUT AND BOLT;So;0;ON;;;;;N;;;;; +1F52A;HOCHO;So;0;ON;;;;;N;;;;; +1F52B;PISTOL;So;0;ON;;;;;N;;;;; +1F52C;MICROSCOPE;So;0;ON;;;;;N;;;;; +1F52D;TELESCOPE;So;0;ON;;;;;N;;;;; +1F52E;CRYSTAL BALL;So;0;ON;;;;;N;;;;; +1F52F;SIX POINTED STAR WITH MIDDLE DOT;So;0;ON;;;;;N;;;;; +1F530;JAPANESE SYMBOL FOR BEGINNER;So;0;ON;;;;;N;;;;; +1F531;TRIDENT EMBLEM;So;0;ON;;;;;N;;;;; +1F532;BLACK SQUARE BUTTON;So;0;ON;;;;;N;;;;; +1F533;WHITE SQUARE BUTTON;So;0;ON;;;;;N;;;;; +1F534;LARGE RED CIRCLE;So;0;ON;;;;;N;;;;; +1F535;LARGE BLUE CIRCLE;So;0;ON;;;;;N;;;;; +1F536;LARGE ORANGE DIAMOND;So;0;ON;;;;;N;;;;; +1F537;LARGE BLUE DIAMOND;So;0;ON;;;;;N;;;;; +1F538;SMALL ORANGE DIAMOND;So;0;ON;;;;;N;;;;; +1F539;SMALL BLUE DIAMOND;So;0;ON;;;;;N;;;;; +1F53A;UP-POINTING RED TRIANGLE;So;0;ON;;;;;N;;;;; +1F53B;DOWN-POINTING RED TRIANGLE;So;0;ON;;;;;N;;;;; +1F53C;UP-POINTING SMALL RED TRIANGLE;So;0;ON;;;;;N;;;;; +1F53D;DOWN-POINTING SMALL RED TRIANGLE;So;0;ON;;;;;N;;;;; +1F540;CIRCLED CROSS POMMEE;So;0;ON;;;;;N;;;;; +1F541;CROSS POMMEE WITH HALF-CIRCLE BELOW;So;0;ON;;;;;N;;;;; +1F542;CROSS POMMEE;So;0;ON;;;;;N;;;;; +1F543;NOTCHED LEFT SEMICIRCLE WITH THREE DOTS;So;0;ON;;;;;N;;;;; +1F550;CLOCK FACE ONE OCLOCK;So;0;ON;;;;;N;;;;; +1F551;CLOCK FACE TWO OCLOCK;So;0;ON;;;;;N;;;;; +1F552;CLOCK FACE THREE OCLOCK;So;0;ON;;;;;N;;;;; +1F553;CLOCK FACE FOUR OCLOCK;So;0;ON;;;;;N;;;;; +1F554;CLOCK FACE FIVE OCLOCK;So;0;ON;;;;;N;;;;; +1F555;CLOCK FACE SIX OCLOCK;So;0;ON;;;;;N;;;;; +1F556;CLOCK FACE SEVEN OCLOCK;So;0;ON;;;;;N;;;;; +1F557;CLOCK FACE EIGHT OCLOCK;So;0;ON;;;;;N;;;;; +1F558;CLOCK FACE NINE OCLOCK;So;0;ON;;;;;N;;;;; +1F559;CLOCK FACE TEN OCLOCK;So;0;ON;;;;;N;;;;; +1F55A;CLOCK FACE ELEVEN OCLOCK;So;0;ON;;;;;N;;;;; +1F55B;CLOCK FACE TWELVE OCLOCK;So;0;ON;;;;;N;;;;; +1F55C;CLOCK FACE ONE-THIRTY;So;0;ON;;;;;N;;;;; +1F55D;CLOCK FACE TWO-THIRTY;So;0;ON;;;;;N;;;;; +1F55E;CLOCK FACE THREE-THIRTY;So;0;ON;;;;;N;;;;; +1F55F;CLOCK FACE FOUR-THIRTY;So;0;ON;;;;;N;;;;; +1F560;CLOCK FACE FIVE-THIRTY;So;0;ON;;;;;N;;;;; +1F561;CLOCK FACE SIX-THIRTY;So;0;ON;;;;;N;;;;; +1F562;CLOCK FACE SEVEN-THIRTY;So;0;ON;;;;;N;;;;; +1F563;CLOCK FACE EIGHT-THIRTY;So;0;ON;;;;;N;;;;; +1F564;CLOCK FACE NINE-THIRTY;So;0;ON;;;;;N;;;;; +1F565;CLOCK FACE TEN-THIRTY;So;0;ON;;;;;N;;;;; +1F566;CLOCK FACE ELEVEN-THIRTY;So;0;ON;;;;;N;;;;; +1F567;CLOCK FACE TWELVE-THIRTY;So;0;ON;;;;;N;;;;; +1F5FB;MOUNT FUJI;So;0;ON;;;;;N;;;;; +1F5FC;TOKYO TOWER;So;0;ON;;;;;N;;;;; +1F5FD;STATUE OF LIBERTY;So;0;ON;;;;;N;;;;; +1F5FE;SILHOUETTE OF JAPAN;So;0;ON;;;;;N;;;;; +1F5FF;MOYAI;So;0;ON;;;;;N;;;;; +1F600;GRINNING FACE;So;0;ON;;;;;N;;;;; +1F601;GRINNING FACE WITH SMILING EYES;So;0;ON;;;;;N;;;;; +1F602;FACE WITH TEARS OF JOY;So;0;ON;;;;;N;;;;; +1F603;SMILING FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1F604;SMILING FACE WITH OPEN MOUTH AND SMILING EYES;So;0;ON;;;;;N;;;;; +1F605;SMILING FACE WITH OPEN MOUTH AND COLD SWEAT;So;0;ON;;;;;N;;;;; +1F606;SMILING FACE WITH OPEN MOUTH AND TIGHTLY-CLOSED EYES;So;0;ON;;;;;N;;;;; +1F607;SMILING FACE WITH HALO;So;0;ON;;;;;N;;;;; +1F608;SMILING FACE WITH HORNS;So;0;ON;;;;;N;;;;; +1F609;WINKING FACE;So;0;ON;;;;;N;;;;; +1F60A;SMILING FACE WITH SMILING EYES;So;0;ON;;;;;N;;;;; +1F60B;FACE SAVOURING DELICIOUS FOOD;So;0;ON;;;;;N;;;;; +1F60C;RELIEVED FACE;So;0;ON;;;;;N;;;;; +1F60D;SMILING FACE WITH HEART-SHAPED EYES;So;0;ON;;;;;N;;;;; +1F60E;SMILING FACE WITH SUNGLASSES;So;0;ON;;;;;N;;;;; +1F60F;SMIRKING FACE;So;0;ON;;;;;N;;;;; +1F610;NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1F611;EXPRESSIONLESS FACE;So;0;ON;;;;;N;;;;; +1F612;UNAMUSED FACE;So;0;ON;;;;;N;;;;; +1F613;FACE WITH COLD SWEAT;So;0;ON;;;;;N;;;;; +1F614;PENSIVE FACE;So;0;ON;;;;;N;;;;; +1F615;CONFUSED FACE;So;0;ON;;;;;N;;;;; +1F616;CONFOUNDED FACE;So;0;ON;;;;;N;;;;; +1F617;KISSING FACE;So;0;ON;;;;;N;;;;; +1F618;FACE THROWING A KISS;So;0;ON;;;;;N;;;;; +1F619;KISSING FACE WITH SMILING EYES;So;0;ON;;;;;N;;;;; +1F61A;KISSING FACE WITH CLOSED EYES;So;0;ON;;;;;N;;;;; +1F61B;FACE WITH STUCK-OUT TONGUE;So;0;ON;;;;;N;;;;; +1F61C;FACE WITH STUCK-OUT TONGUE AND WINKING EYE;So;0;ON;;;;;N;;;;; +1F61D;FACE WITH STUCK-OUT TONGUE AND TIGHTLY-CLOSED EYES;So;0;ON;;;;;N;;;;; +1F61E;DISAPPOINTED FACE;So;0;ON;;;;;N;;;;; +1F61F;WORRIED FACE;So;0;ON;;;;;N;;;;; +1F620;ANGRY FACE;So;0;ON;;;;;N;;;;; +1F621;POUTING FACE;So;0;ON;;;;;N;;;;; +1F622;CRYING FACE;So;0;ON;;;;;N;;;;; +1F623;PERSEVERING FACE;So;0;ON;;;;;N;;;;; +1F624;FACE WITH LOOK OF TRIUMPH;So;0;ON;;;;;N;;;;; +1F625;DISAPPOINTED BUT RELIEVED FACE;So;0;ON;;;;;N;;;;; +1F626;FROWNING FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1F627;ANGUISHED FACE;So;0;ON;;;;;N;;;;; +1F628;FEARFUL FACE;So;0;ON;;;;;N;;;;; +1F629;WEARY FACE;So;0;ON;;;;;N;;;;; +1F62A;SLEEPY FACE;So;0;ON;;;;;N;;;;; +1F62B;TIRED FACE;So;0;ON;;;;;N;;;;; +1F62C;GRIMACING FACE;So;0;ON;;;;;N;;;;; +1F62D;LOUDLY CRYING FACE;So;0;ON;;;;;N;;;;; +1F62E;FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1F62F;HUSHED FACE;So;0;ON;;;;;N;;;;; +1F630;FACE WITH OPEN MOUTH AND COLD SWEAT;So;0;ON;;;;;N;;;;; +1F631;FACE SCREAMING IN FEAR;So;0;ON;;;;;N;;;;; +1F632;ASTONISHED FACE;So;0;ON;;;;;N;;;;; +1F633;FLUSHED FACE;So;0;ON;;;;;N;;;;; +1F634;SLEEPING FACE;So;0;ON;;;;;N;;;;; +1F635;DIZZY FACE;So;0;ON;;;;;N;;;;; +1F636;FACE WITHOUT MOUTH;So;0;ON;;;;;N;;;;; +1F637;FACE WITH MEDICAL MASK;So;0;ON;;;;;N;;;;; +1F638;GRINNING CAT FACE WITH SMILING EYES;So;0;ON;;;;;N;;;;; +1F639;CAT FACE WITH TEARS OF JOY;So;0;ON;;;;;N;;;;; +1F63A;SMILING CAT FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1F63B;SMILING CAT FACE WITH HEART-SHAPED EYES;So;0;ON;;;;;N;;;;; +1F63C;CAT FACE WITH WRY SMILE;So;0;ON;;;;;N;;;;; +1F63D;KISSING CAT FACE WITH CLOSED EYES;So;0;ON;;;;;N;;;;; +1F63E;POUTING CAT FACE;So;0;ON;;;;;N;;;;; +1F63F;CRYING CAT FACE;So;0;ON;;;;;N;;;;; +1F640;WEARY CAT FACE;So;0;ON;;;;;N;;;;; +1F645;FACE WITH NO GOOD GESTURE;So;0;ON;;;;;N;;;;; +1F646;FACE WITH OK GESTURE;So;0;ON;;;;;N;;;;; +1F647;PERSON BOWING DEEPLY;So;0;ON;;;;;N;;;;; +1F648;SEE-NO-EVIL MONKEY;So;0;ON;;;;;N;;;;; +1F649;HEAR-NO-EVIL MONKEY;So;0;ON;;;;;N;;;;; +1F64A;SPEAK-NO-EVIL MONKEY;So;0;ON;;;;;N;;;;; +1F64B;HAPPY PERSON RAISING ONE HAND;So;0;ON;;;;;N;;;;; +1F64C;PERSON RAISING BOTH HANDS IN CELEBRATION;So;0;ON;;;;;N;;;;; +1F64D;PERSON FROWNING;So;0;ON;;;;;N;;;;; +1F64E;PERSON WITH POUTING FACE;So;0;ON;;;;;N;;;;; +1F64F;PERSON WITH FOLDED HANDS;So;0;ON;;;;;N;;;;; +1F680;ROCKET;So;0;ON;;;;;N;;;;; +1F681;HELICOPTER;So;0;ON;;;;;N;;;;; +1F682;STEAM LOCOMOTIVE;So;0;ON;;;;;N;;;;; +1F683;RAILWAY CAR;So;0;ON;;;;;N;;;;; +1F684;HIGH-SPEED TRAIN;So;0;ON;;;;;N;;;;; +1F685;HIGH-SPEED TRAIN WITH BULLET NOSE;So;0;ON;;;;;N;;;;; +1F686;TRAIN;So;0;ON;;;;;N;;;;; +1F687;METRO;So;0;ON;;;;;N;;;;; +1F688;LIGHT RAIL;So;0;ON;;;;;N;;;;; +1F689;STATION;So;0;ON;;;;;N;;;;; +1F68A;TRAM;So;0;ON;;;;;N;;;;; +1F68B;TRAM CAR;So;0;ON;;;;;N;;;;; +1F68C;BUS;So;0;ON;;;;;N;;;;; +1F68D;ONCOMING BUS;So;0;ON;;;;;N;;;;; +1F68E;TROLLEYBUS;So;0;ON;;;;;N;;;;; +1F68F;BUS STOP;So;0;ON;;;;;N;;;;; +1F690;MINIBUS;So;0;ON;;;;;N;;;;; +1F691;AMBULANCE;So;0;ON;;;;;N;;;;; +1F692;FIRE ENGINE;So;0;ON;;;;;N;;;;; +1F693;POLICE CAR;So;0;ON;;;;;N;;;;; +1F694;ONCOMING POLICE CAR;So;0;ON;;;;;N;;;;; +1F695;TAXI;So;0;ON;;;;;N;;;;; +1F696;ONCOMING TAXI;So;0;ON;;;;;N;;;;; +1F697;AUTOMOBILE;So;0;ON;;;;;N;;;;; +1F698;ONCOMING AUTOMOBILE;So;0;ON;;;;;N;;;;; +1F699;RECREATIONAL VEHICLE;So;0;ON;;;;;N;;;;; +1F69A;DELIVERY TRUCK;So;0;ON;;;;;N;;;;; +1F69B;ARTICULATED LORRY;So;0;ON;;;;;N;;;;; +1F69C;TRACTOR;So;0;ON;;;;;N;;;;; +1F69D;MONORAIL;So;0;ON;;;;;N;;;;; +1F69E;MOUNTAIN RAILWAY;So;0;ON;;;;;N;;;;; +1F69F;SUSPENSION RAILWAY;So;0;ON;;;;;N;;;;; +1F6A0;MOUNTAIN CABLEWAY;So;0;ON;;;;;N;;;;; +1F6A1;AERIAL TRAMWAY;So;0;ON;;;;;N;;;;; +1F6A2;SHIP;So;0;ON;;;;;N;;;;; +1F6A3;ROWBOAT;So;0;ON;;;;;N;;;;; +1F6A4;SPEEDBOAT;So;0;ON;;;;;N;;;;; +1F6A5;HORIZONTAL TRAFFIC LIGHT;So;0;ON;;;;;N;;;;; +1F6A6;VERTICAL TRAFFIC LIGHT;So;0;ON;;;;;N;;;;; +1F6A7;CONSTRUCTION SIGN;So;0;ON;;;;;N;;;;; +1F6A8;POLICE CARS REVOLVING LIGHT;So;0;ON;;;;;N;;;;; +1F6A9;TRIANGULAR FLAG ON POST;So;0;ON;;;;;N;;;;; +1F6AA;DOOR;So;0;ON;;;;;N;;;;; +1F6AB;NO ENTRY SIGN;So;0;ON;;;;;N;;;;; +1F6AC;SMOKING SYMBOL;So;0;ON;;;;;N;;;;; +1F6AD;NO SMOKING SYMBOL;So;0;ON;;;;;N;;;;; +1F6AE;PUT LITTER IN ITS PLACE SYMBOL;So;0;ON;;;;;N;;;;; +1F6AF;DO NOT LITTER SYMBOL;So;0;ON;;;;;N;;;;; +1F6B0;POTABLE WATER SYMBOL;So;0;ON;;;;;N;;;;; +1F6B1;NON-POTABLE WATER SYMBOL;So;0;ON;;;;;N;;;;; +1F6B2;BICYCLE;So;0;ON;;;;;N;;;;; +1F6B3;NO BICYCLES;So;0;ON;;;;;N;;;;; +1F6B4;BICYCLIST;So;0;ON;;;;;N;;;;; +1F6B5;MOUNTAIN BICYCLIST;So;0;ON;;;;;N;;;;; +1F6B6;PEDESTRIAN;So;0;ON;;;;;N;;;;; +1F6B7;NO PEDESTRIANS;So;0;ON;;;;;N;;;;; +1F6B8;CHILDREN CROSSING;So;0;ON;;;;;N;;;;; +1F6B9;MENS SYMBOL;So;0;ON;;;;;N;;;;; +1F6BA;WOMENS SYMBOL;So;0;ON;;;;;N;;;;; +1F6BB;RESTROOM;So;0;ON;;;;;N;;;;; +1F6BC;BABY SYMBOL;So;0;ON;;;;;N;;;;; +1F6BD;TOILET;So;0;ON;;;;;N;;;;; +1F6BE;WATER CLOSET;So;0;ON;;;;;N;;;;; +1F6BF;SHOWER;So;0;ON;;;;;N;;;;; +1F6C0;BATH;So;0;ON;;;;;N;;;;; +1F6C1;BATHTUB;So;0;ON;;;;;N;;;;; +1F6C2;PASSPORT CONTROL;So;0;ON;;;;;N;;;;; +1F6C3;CUSTOMS;So;0;ON;;;;;N;;;;; +1F6C4;BAGGAGE CLAIM;So;0;ON;;;;;N;;;;; +1F6C5;LEFT LUGGAGE;So;0;ON;;;;;N;;;;; +1F700;ALCHEMICAL SYMBOL FOR QUINTESSENCE;So;0;ON;;;;;N;;;;; +1F701;ALCHEMICAL SYMBOL FOR AIR;So;0;ON;;;;;N;;;;; +1F702;ALCHEMICAL SYMBOL FOR FIRE;So;0;ON;;;;;N;;;;; +1F703;ALCHEMICAL SYMBOL FOR EARTH;So;0;ON;;;;;N;;;;; +1F704;ALCHEMICAL SYMBOL FOR WATER;So;0;ON;;;;;N;;;;; +1F705;ALCHEMICAL SYMBOL FOR AQUAFORTIS;So;0;ON;;;;;N;;;;; +1F706;ALCHEMICAL SYMBOL FOR AQUA REGIA;So;0;ON;;;;;N;;;;; +1F707;ALCHEMICAL SYMBOL FOR AQUA REGIA-2;So;0;ON;;;;;N;;;;; +1F708;ALCHEMICAL SYMBOL FOR AQUA VITAE;So;0;ON;;;;;N;;;;; +1F709;ALCHEMICAL SYMBOL FOR AQUA VITAE-2;So;0;ON;;;;;N;;;;; +1F70A;ALCHEMICAL SYMBOL FOR VINEGAR;So;0;ON;;;;;N;;;;; +1F70B;ALCHEMICAL SYMBOL FOR VINEGAR-2;So;0;ON;;;;;N;;;;; +1F70C;ALCHEMICAL SYMBOL FOR VINEGAR-3;So;0;ON;;;;;N;;;;; +1F70D;ALCHEMICAL SYMBOL FOR SULFUR;So;0;ON;;;;;N;;;;; +1F70E;ALCHEMICAL SYMBOL FOR PHILOSOPHERS SULFUR;So;0;ON;;;;;N;;;;; +1F70F;ALCHEMICAL SYMBOL FOR BLACK SULFUR;So;0;ON;;;;;N;;;;; +1F710;ALCHEMICAL SYMBOL FOR MERCURY SUBLIMATE;So;0;ON;;;;;N;;;;; +1F711;ALCHEMICAL SYMBOL FOR MERCURY SUBLIMATE-2;So;0;ON;;;;;N;;;;; +1F712;ALCHEMICAL SYMBOL FOR MERCURY SUBLIMATE-3;So;0;ON;;;;;N;;;;; +1F713;ALCHEMICAL SYMBOL FOR CINNABAR;So;0;ON;;;;;N;;;;; +1F714;ALCHEMICAL SYMBOL FOR SALT;So;0;ON;;;;;N;;;;; +1F715;ALCHEMICAL SYMBOL FOR NITRE;So;0;ON;;;;;N;;;;; +1F716;ALCHEMICAL SYMBOL FOR VITRIOL;So;0;ON;;;;;N;;;;; +1F717;ALCHEMICAL SYMBOL FOR VITRIOL-2;So;0;ON;;;;;N;;;;; +1F718;ALCHEMICAL SYMBOL FOR ROCK SALT;So;0;ON;;;;;N;;;;; +1F719;ALCHEMICAL SYMBOL FOR ROCK SALT-2;So;0;ON;;;;;N;;;;; +1F71A;ALCHEMICAL SYMBOL FOR GOLD;So;0;ON;;;;;N;;;;; +1F71B;ALCHEMICAL SYMBOL FOR SILVER;So;0;ON;;;;;N;;;;; +1F71C;ALCHEMICAL SYMBOL FOR IRON ORE;So;0;ON;;;;;N;;;;; +1F71D;ALCHEMICAL SYMBOL FOR IRON ORE-2;So;0;ON;;;;;N;;;;; +1F71E;ALCHEMICAL SYMBOL FOR CROCUS OF IRON;So;0;ON;;;;;N;;;;; +1F71F;ALCHEMICAL SYMBOL FOR REGULUS OF IRON;So;0;ON;;;;;N;;;;; +1F720;ALCHEMICAL SYMBOL FOR COPPER ORE;So;0;ON;;;;;N;;;;; +1F721;ALCHEMICAL SYMBOL FOR IRON-COPPER ORE;So;0;ON;;;;;N;;;;; +1F722;ALCHEMICAL SYMBOL FOR SUBLIMATE OF COPPER;So;0;ON;;;;;N;;;;; +1F723;ALCHEMICAL SYMBOL FOR CROCUS OF COPPER;So;0;ON;;;;;N;;;;; +1F724;ALCHEMICAL SYMBOL FOR CROCUS OF COPPER-2;So;0;ON;;;;;N;;;;; +1F725;ALCHEMICAL SYMBOL FOR COPPER ANTIMONIATE;So;0;ON;;;;;N;;;;; +1F726;ALCHEMICAL SYMBOL FOR SALT OF COPPER ANTIMONIATE;So;0;ON;;;;;N;;;;; +1F727;ALCHEMICAL SYMBOL FOR SUBLIMATE OF SALT OF COPPER;So;0;ON;;;;;N;;;;; +1F728;ALCHEMICAL SYMBOL FOR VERDIGRIS;So;0;ON;;;;;N;;;;; +1F729;ALCHEMICAL SYMBOL FOR TIN ORE;So;0;ON;;;;;N;;;;; +1F72A;ALCHEMICAL SYMBOL FOR LEAD ORE;So;0;ON;;;;;N;;;;; +1F72B;ALCHEMICAL SYMBOL FOR ANTIMONY ORE;So;0;ON;;;;;N;;;;; +1F72C;ALCHEMICAL SYMBOL FOR SUBLIMATE OF ANTIMONY;So;0;ON;;;;;N;;;;; +1F72D;ALCHEMICAL SYMBOL FOR SALT OF ANTIMONY;So;0;ON;;;;;N;;;;; +1F72E;ALCHEMICAL SYMBOL FOR SUBLIMATE OF SALT OF ANTIMONY;So;0;ON;;;;;N;;;;; +1F72F;ALCHEMICAL SYMBOL FOR VINEGAR OF ANTIMONY;So;0;ON;;;;;N;;;;; +1F730;ALCHEMICAL SYMBOL FOR REGULUS OF ANTIMONY;So;0;ON;;;;;N;;;;; +1F731;ALCHEMICAL SYMBOL FOR REGULUS OF ANTIMONY-2;So;0;ON;;;;;N;;;;; +1F732;ALCHEMICAL SYMBOL FOR REGULUS;So;0;ON;;;;;N;;;;; +1F733;ALCHEMICAL SYMBOL FOR REGULUS-2;So;0;ON;;;;;N;;;;; +1F734;ALCHEMICAL SYMBOL FOR REGULUS-3;So;0;ON;;;;;N;;;;; +1F735;ALCHEMICAL SYMBOL FOR REGULUS-4;So;0;ON;;;;;N;;;;; +1F736;ALCHEMICAL SYMBOL FOR ALKALI;So;0;ON;;;;;N;;;;; +1F737;ALCHEMICAL SYMBOL FOR ALKALI-2;So;0;ON;;;;;N;;;;; +1F738;ALCHEMICAL SYMBOL FOR MARCASITE;So;0;ON;;;;;N;;;;; +1F739;ALCHEMICAL SYMBOL FOR SAL-AMMONIAC;So;0;ON;;;;;N;;;;; +1F73A;ALCHEMICAL SYMBOL FOR ARSENIC;So;0;ON;;;;;N;;;;; +1F73B;ALCHEMICAL SYMBOL FOR REALGAR;So;0;ON;;;;;N;;;;; +1F73C;ALCHEMICAL SYMBOL FOR REALGAR-2;So;0;ON;;;;;N;;;;; +1F73D;ALCHEMICAL SYMBOL FOR AURIPIGMENT;So;0;ON;;;;;N;;;;; +1F73E;ALCHEMICAL SYMBOL FOR BISMUTH ORE;So;0;ON;;;;;N;;;;; +1F73F;ALCHEMICAL SYMBOL FOR TARTAR;So;0;ON;;;;;N;;;;; +1F740;ALCHEMICAL SYMBOL FOR TARTAR-2;So;0;ON;;;;;N;;;;; +1F741;ALCHEMICAL SYMBOL FOR QUICK LIME;So;0;ON;;;;;N;;;;; +1F742;ALCHEMICAL SYMBOL FOR BORAX;So;0;ON;;;;;N;;;;; +1F743;ALCHEMICAL SYMBOL FOR BORAX-2;So;0;ON;;;;;N;;;;; +1F744;ALCHEMICAL SYMBOL FOR BORAX-3;So;0;ON;;;;;N;;;;; +1F745;ALCHEMICAL SYMBOL FOR ALUM;So;0;ON;;;;;N;;;;; +1F746;ALCHEMICAL SYMBOL FOR OIL;So;0;ON;;;;;N;;;;; +1F747;ALCHEMICAL SYMBOL FOR SPIRIT;So;0;ON;;;;;N;;;;; +1F748;ALCHEMICAL SYMBOL FOR TINCTURE;So;0;ON;;;;;N;;;;; +1F749;ALCHEMICAL SYMBOL FOR GUM;So;0;ON;;;;;N;;;;; +1F74A;ALCHEMICAL SYMBOL FOR WAX;So;0;ON;;;;;N;;;;; +1F74B;ALCHEMICAL SYMBOL FOR POWDER;So;0;ON;;;;;N;;;;; +1F74C;ALCHEMICAL SYMBOL FOR CALX;So;0;ON;;;;;N;;;;; +1F74D;ALCHEMICAL SYMBOL FOR TUTTY;So;0;ON;;;;;N;;;;; +1F74E;ALCHEMICAL SYMBOL FOR CAPUT MORTUUM;So;0;ON;;;;;N;;;;; +1F74F;ALCHEMICAL SYMBOL FOR SCEPTER OF JOVE;So;0;ON;;;;;N;;;;; +1F750;ALCHEMICAL SYMBOL FOR CADUCEUS;So;0;ON;;;;;N;;;;; +1F751;ALCHEMICAL SYMBOL FOR TRIDENT;So;0;ON;;;;;N;;;;; +1F752;ALCHEMICAL SYMBOL FOR STARRED TRIDENT;So;0;ON;;;;;N;;;;; +1F753;ALCHEMICAL SYMBOL FOR LODESTONE;So;0;ON;;;;;N;;;;; +1F754;ALCHEMICAL SYMBOL FOR SOAP;So;0;ON;;;;;N;;;;; +1F755;ALCHEMICAL SYMBOL FOR URINE;So;0;ON;;;;;N;;;;; +1F756;ALCHEMICAL SYMBOL FOR HORSE DUNG;So;0;ON;;;;;N;;;;; +1F757;ALCHEMICAL SYMBOL FOR ASHES;So;0;ON;;;;;N;;;;; +1F758;ALCHEMICAL SYMBOL FOR POT ASHES;So;0;ON;;;;;N;;;;; +1F759;ALCHEMICAL SYMBOL FOR BRICK;So;0;ON;;;;;N;;;;; +1F75A;ALCHEMICAL SYMBOL FOR POWDERED BRICK;So;0;ON;;;;;N;;;;; +1F75B;ALCHEMICAL SYMBOL FOR AMALGAM;So;0;ON;;;;;N;;;;; +1F75C;ALCHEMICAL SYMBOL FOR STRATUM SUPER STRATUM;So;0;ON;;;;;N;;;;; +1F75D;ALCHEMICAL SYMBOL FOR STRATUM SUPER STRATUM-2;So;0;ON;;;;;N;;;;; +1F75E;ALCHEMICAL SYMBOL FOR SUBLIMATION;So;0;ON;;;;;N;;;;; +1F75F;ALCHEMICAL SYMBOL FOR PRECIPITATE;So;0;ON;;;;;N;;;;; +1F760;ALCHEMICAL SYMBOL FOR DISTILL;So;0;ON;;;;;N;;;;; +1F761;ALCHEMICAL SYMBOL FOR DISSOLVE;So;0;ON;;;;;N;;;;; +1F762;ALCHEMICAL SYMBOL FOR DISSOLVE-2;So;0;ON;;;;;N;;;;; +1F763;ALCHEMICAL SYMBOL FOR PURIFY;So;0;ON;;;;;N;;;;; +1F764;ALCHEMICAL SYMBOL FOR PUTREFACTION;So;0;ON;;;;;N;;;;; +1F765;ALCHEMICAL SYMBOL FOR CRUCIBLE;So;0;ON;;;;;N;;;;; +1F766;ALCHEMICAL SYMBOL FOR CRUCIBLE-2;So;0;ON;;;;;N;;;;; +1F767;ALCHEMICAL SYMBOL FOR CRUCIBLE-3;So;0;ON;;;;;N;;;;; +1F768;ALCHEMICAL SYMBOL FOR CRUCIBLE-4;So;0;ON;;;;;N;;;;; +1F769;ALCHEMICAL SYMBOL FOR CRUCIBLE-5;So;0;ON;;;;;N;;;;; +1F76A;ALCHEMICAL SYMBOL FOR ALEMBIC;So;0;ON;;;;;N;;;;; +1F76B;ALCHEMICAL SYMBOL FOR BATH OF MARY;So;0;ON;;;;;N;;;;; +1F76C;ALCHEMICAL SYMBOL FOR BATH OF VAPOURS;So;0;ON;;;;;N;;;;; +1F76D;ALCHEMICAL SYMBOL FOR RETORT;So;0;ON;;;;;N;;;;; +1F76E;ALCHEMICAL SYMBOL FOR HOUR;So;0;ON;;;;;N;;;;; +1F76F;ALCHEMICAL SYMBOL FOR NIGHT;So;0;ON;;;;;N;;;;; +1F770;ALCHEMICAL SYMBOL FOR DAY-NIGHT;So;0;ON;;;;;N;;;;; +1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;; +1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;; +1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;; +20000;;Lo;0;L;;;;;N;;;;; +2A6D6;;Lo;0;L;;;;;N;;;;; +2A700;;Lo;0;L;;;;;N;;;;; +2B734;;Lo;0;L;;;;;N;;;;; +2B740;;Lo;0;L;;;;;N;;;;; +2B81D;;Lo;0;L;;;;;N;;;;; +2F800;CJK COMPATIBILITY IDEOGRAPH-2F800;Lo;0;L;4E3D;;;;N;;;;; +2F801;CJK COMPATIBILITY IDEOGRAPH-2F801;Lo;0;L;4E38;;;;N;;;;; +2F802;CJK COMPATIBILITY IDEOGRAPH-2F802;Lo;0;L;4E41;;;;N;;;;; +2F803;CJK COMPATIBILITY IDEOGRAPH-2F803;Lo;0;L;20122;;;;N;;;;; +2F804;CJK COMPATIBILITY IDEOGRAPH-2F804;Lo;0;L;4F60;;;;N;;;;; +2F805;CJK COMPATIBILITY IDEOGRAPH-2F805;Lo;0;L;4FAE;;;;N;;;;; +2F806;CJK COMPATIBILITY IDEOGRAPH-2F806;Lo;0;L;4FBB;;;;N;;;;; +2F807;CJK COMPATIBILITY IDEOGRAPH-2F807;Lo;0;L;5002;;;;N;;;;; +2F808;CJK COMPATIBILITY IDEOGRAPH-2F808;Lo;0;L;507A;;;;N;;;;; +2F809;CJK COMPATIBILITY IDEOGRAPH-2F809;Lo;0;L;5099;;;;N;;;;; +2F80A;CJK COMPATIBILITY IDEOGRAPH-2F80A;Lo;0;L;50E7;;;;N;;;;; +2F80B;CJK COMPATIBILITY IDEOGRAPH-2F80B;Lo;0;L;50CF;;;;N;;;;; +2F80C;CJK COMPATIBILITY IDEOGRAPH-2F80C;Lo;0;L;349E;;;;N;;;;; +2F80D;CJK COMPATIBILITY IDEOGRAPH-2F80D;Lo;0;L;2063A;;;;N;;;;; +2F80E;CJK COMPATIBILITY IDEOGRAPH-2F80E;Lo;0;L;514D;;;;N;;;;; +2F80F;CJK COMPATIBILITY IDEOGRAPH-2F80F;Lo;0;L;5154;;;;N;;;;; +2F810;CJK COMPATIBILITY IDEOGRAPH-2F810;Lo;0;L;5164;;;;N;;;;; +2F811;CJK COMPATIBILITY IDEOGRAPH-2F811;Lo;0;L;5177;;;;N;;;;; +2F812;CJK COMPATIBILITY IDEOGRAPH-2F812;Lo;0;L;2051C;;;;N;;;;; +2F813;CJK COMPATIBILITY IDEOGRAPH-2F813;Lo;0;L;34B9;;;;N;;;;; +2F814;CJK COMPATIBILITY IDEOGRAPH-2F814;Lo;0;L;5167;;;;N;;;;; +2F815;CJK COMPATIBILITY IDEOGRAPH-2F815;Lo;0;L;518D;;;;N;;;;; +2F816;CJK COMPATIBILITY IDEOGRAPH-2F816;Lo;0;L;2054B;;;;N;;;;; +2F817;CJK COMPATIBILITY IDEOGRAPH-2F817;Lo;0;L;5197;;;;N;;;;; +2F818;CJK COMPATIBILITY IDEOGRAPH-2F818;Lo;0;L;51A4;;;;N;;;;; +2F819;CJK COMPATIBILITY IDEOGRAPH-2F819;Lo;0;L;4ECC;;;;N;;;;; +2F81A;CJK COMPATIBILITY IDEOGRAPH-2F81A;Lo;0;L;51AC;;;;N;;;;; +2F81B;CJK COMPATIBILITY IDEOGRAPH-2F81B;Lo;0;L;51B5;;;;N;;;;; +2F81C;CJK COMPATIBILITY IDEOGRAPH-2F81C;Lo;0;L;291DF;;;;N;;;;; +2F81D;CJK COMPATIBILITY IDEOGRAPH-2F81D;Lo;0;L;51F5;;;;N;;;;; +2F81E;CJK COMPATIBILITY IDEOGRAPH-2F81E;Lo;0;L;5203;;;;N;;;;; +2F81F;CJK COMPATIBILITY IDEOGRAPH-2F81F;Lo;0;L;34DF;;;;N;;;;; +2F820;CJK COMPATIBILITY IDEOGRAPH-2F820;Lo;0;L;523B;;;;N;;;;; +2F821;CJK COMPATIBILITY IDEOGRAPH-2F821;Lo;0;L;5246;;;;N;;;;; +2F822;CJK COMPATIBILITY IDEOGRAPH-2F822;Lo;0;L;5272;;;;N;;;;; +2F823;CJK COMPATIBILITY IDEOGRAPH-2F823;Lo;0;L;5277;;;;N;;;;; +2F824;CJK COMPATIBILITY IDEOGRAPH-2F824;Lo;0;L;3515;;;;N;;;;; +2F825;CJK COMPATIBILITY IDEOGRAPH-2F825;Lo;0;L;52C7;;;;N;;;;; +2F826;CJK COMPATIBILITY IDEOGRAPH-2F826;Lo;0;L;52C9;;;;N;;;;; +2F827;CJK COMPATIBILITY IDEOGRAPH-2F827;Lo;0;L;52E4;;;;N;;;;; +2F828;CJK COMPATIBILITY IDEOGRAPH-2F828;Lo;0;L;52FA;;;;N;;;;; +2F829;CJK COMPATIBILITY IDEOGRAPH-2F829;Lo;0;L;5305;;;;N;;;;; +2F82A;CJK COMPATIBILITY IDEOGRAPH-2F82A;Lo;0;L;5306;;;;N;;;;; +2F82B;CJK COMPATIBILITY IDEOGRAPH-2F82B;Lo;0;L;5317;;;;N;;;;; +2F82C;CJK COMPATIBILITY IDEOGRAPH-2F82C;Lo;0;L;5349;;;;N;;;;; +2F82D;CJK COMPATIBILITY IDEOGRAPH-2F82D;Lo;0;L;5351;;;;N;;;;; +2F82E;CJK COMPATIBILITY IDEOGRAPH-2F82E;Lo;0;L;535A;;;;N;;;;; +2F82F;CJK COMPATIBILITY IDEOGRAPH-2F82F;Lo;0;L;5373;;;;N;;;;; +2F830;CJK COMPATIBILITY IDEOGRAPH-2F830;Lo;0;L;537D;;;;N;;;;; +2F831;CJK COMPATIBILITY IDEOGRAPH-2F831;Lo;0;L;537F;;;;N;;;;; +2F832;CJK COMPATIBILITY IDEOGRAPH-2F832;Lo;0;L;537F;;;;N;;;;; +2F833;CJK COMPATIBILITY IDEOGRAPH-2F833;Lo;0;L;537F;;;;N;;;;; +2F834;CJK COMPATIBILITY IDEOGRAPH-2F834;Lo;0;L;20A2C;;;;N;;;;; +2F835;CJK COMPATIBILITY IDEOGRAPH-2F835;Lo;0;L;7070;;;;N;;;;; +2F836;CJK COMPATIBILITY IDEOGRAPH-2F836;Lo;0;L;53CA;;;;N;;;;; +2F837;CJK COMPATIBILITY IDEOGRAPH-2F837;Lo;0;L;53DF;;;;N;;;;; +2F838;CJK COMPATIBILITY IDEOGRAPH-2F838;Lo;0;L;20B63;;;;N;;;;; +2F839;CJK COMPATIBILITY IDEOGRAPH-2F839;Lo;0;L;53EB;;;;N;;;;; +2F83A;CJK COMPATIBILITY IDEOGRAPH-2F83A;Lo;0;L;53F1;;;;N;;;;; +2F83B;CJK COMPATIBILITY IDEOGRAPH-2F83B;Lo;0;L;5406;;;;N;;;;; +2F83C;CJK COMPATIBILITY IDEOGRAPH-2F83C;Lo;0;L;549E;;;;N;;;;; +2F83D;CJK COMPATIBILITY IDEOGRAPH-2F83D;Lo;0;L;5438;;;;N;;;;; +2F83E;CJK COMPATIBILITY IDEOGRAPH-2F83E;Lo;0;L;5448;;;;N;;;;; +2F83F;CJK COMPATIBILITY IDEOGRAPH-2F83F;Lo;0;L;5468;;;;N;;;;; +2F840;CJK COMPATIBILITY IDEOGRAPH-2F840;Lo;0;L;54A2;;;;N;;;;; +2F841;CJK COMPATIBILITY IDEOGRAPH-2F841;Lo;0;L;54F6;;;;N;;;;; +2F842;CJK COMPATIBILITY IDEOGRAPH-2F842;Lo;0;L;5510;;;;N;;;;; +2F843;CJK COMPATIBILITY IDEOGRAPH-2F843;Lo;0;L;5553;;;;N;;;;; +2F844;CJK COMPATIBILITY IDEOGRAPH-2F844;Lo;0;L;5563;;;;N;;;;; +2F845;CJK COMPATIBILITY IDEOGRAPH-2F845;Lo;0;L;5584;;;;N;;;;; +2F846;CJK COMPATIBILITY IDEOGRAPH-2F846;Lo;0;L;5584;;;;N;;;;; +2F847;CJK COMPATIBILITY IDEOGRAPH-2F847;Lo;0;L;5599;;;;N;;;;; +2F848;CJK COMPATIBILITY IDEOGRAPH-2F848;Lo;0;L;55AB;;;;N;;;;; +2F849;CJK COMPATIBILITY IDEOGRAPH-2F849;Lo;0;L;55B3;;;;N;;;;; +2F84A;CJK COMPATIBILITY IDEOGRAPH-2F84A;Lo;0;L;55C2;;;;N;;;;; +2F84B;CJK COMPATIBILITY IDEOGRAPH-2F84B;Lo;0;L;5716;;;;N;;;;; +2F84C;CJK COMPATIBILITY IDEOGRAPH-2F84C;Lo;0;L;5606;;;;N;;;;; +2F84D;CJK COMPATIBILITY IDEOGRAPH-2F84D;Lo;0;L;5717;;;;N;;;;; +2F84E;CJK COMPATIBILITY IDEOGRAPH-2F84E;Lo;0;L;5651;;;;N;;;;; +2F84F;CJK COMPATIBILITY IDEOGRAPH-2F84F;Lo;0;L;5674;;;;N;;;;; +2F850;CJK COMPATIBILITY IDEOGRAPH-2F850;Lo;0;L;5207;;;;N;;;;; +2F851;CJK COMPATIBILITY IDEOGRAPH-2F851;Lo;0;L;58EE;;;;N;;;;; +2F852;CJK COMPATIBILITY IDEOGRAPH-2F852;Lo;0;L;57CE;;;;N;;;;; +2F853;CJK COMPATIBILITY IDEOGRAPH-2F853;Lo;0;L;57F4;;;;N;;;;; +2F854;CJK COMPATIBILITY IDEOGRAPH-2F854;Lo;0;L;580D;;;;N;;;;; +2F855;CJK COMPATIBILITY IDEOGRAPH-2F855;Lo;0;L;578B;;;;N;;;;; +2F856;CJK COMPATIBILITY IDEOGRAPH-2F856;Lo;0;L;5832;;;;N;;;;; +2F857;CJK COMPATIBILITY IDEOGRAPH-2F857;Lo;0;L;5831;;;;N;;;;; +2F858;CJK COMPATIBILITY IDEOGRAPH-2F858;Lo;0;L;58AC;;;;N;;;;; +2F859;CJK COMPATIBILITY IDEOGRAPH-2F859;Lo;0;L;214E4;;;;N;;;;; +2F85A;CJK COMPATIBILITY IDEOGRAPH-2F85A;Lo;0;L;58F2;;;;N;;;;; +2F85B;CJK COMPATIBILITY IDEOGRAPH-2F85B;Lo;0;L;58F7;;;;N;;;;; +2F85C;CJK COMPATIBILITY IDEOGRAPH-2F85C;Lo;0;L;5906;;;;N;;;;; +2F85D;CJK COMPATIBILITY IDEOGRAPH-2F85D;Lo;0;L;591A;;;;N;;;;; +2F85E;CJK COMPATIBILITY IDEOGRAPH-2F85E;Lo;0;L;5922;;;;N;;;;; +2F85F;CJK COMPATIBILITY IDEOGRAPH-2F85F;Lo;0;L;5962;;;;N;;;;; +2F860;CJK COMPATIBILITY IDEOGRAPH-2F860;Lo;0;L;216A8;;;;N;;;;; +2F861;CJK COMPATIBILITY IDEOGRAPH-2F861;Lo;0;L;216EA;;;;N;;;;; +2F862;CJK COMPATIBILITY IDEOGRAPH-2F862;Lo;0;L;59EC;;;;N;;;;; +2F863;CJK COMPATIBILITY IDEOGRAPH-2F863;Lo;0;L;5A1B;;;;N;;;;; +2F864;CJK COMPATIBILITY IDEOGRAPH-2F864;Lo;0;L;5A27;;;;N;;;;; +2F865;CJK COMPATIBILITY IDEOGRAPH-2F865;Lo;0;L;59D8;;;;N;;;;; +2F866;CJK COMPATIBILITY IDEOGRAPH-2F866;Lo;0;L;5A66;;;;N;;;;; +2F867;CJK COMPATIBILITY IDEOGRAPH-2F867;Lo;0;L;36EE;;;;N;;;;; +2F868;CJK COMPATIBILITY IDEOGRAPH-2F868;Lo;0;L;36FC;;;;N;;;;; +2F869;CJK COMPATIBILITY IDEOGRAPH-2F869;Lo;0;L;5B08;;;;N;;;;; +2F86A;CJK COMPATIBILITY IDEOGRAPH-2F86A;Lo;0;L;5B3E;;;;N;;;;; +2F86B;CJK COMPATIBILITY IDEOGRAPH-2F86B;Lo;0;L;5B3E;;;;N;;;;; +2F86C;CJK COMPATIBILITY IDEOGRAPH-2F86C;Lo;0;L;219C8;;;;N;;;;; +2F86D;CJK COMPATIBILITY IDEOGRAPH-2F86D;Lo;0;L;5BC3;;;;N;;;;; +2F86E;CJK COMPATIBILITY IDEOGRAPH-2F86E;Lo;0;L;5BD8;;;;N;;;;; +2F86F;CJK COMPATIBILITY IDEOGRAPH-2F86F;Lo;0;L;5BE7;;;;N;;;;; +2F870;CJK COMPATIBILITY IDEOGRAPH-2F870;Lo;0;L;5BF3;;;;N;;;;; +2F871;CJK COMPATIBILITY IDEOGRAPH-2F871;Lo;0;L;21B18;;;;N;;;;; +2F872;CJK COMPATIBILITY IDEOGRAPH-2F872;Lo;0;L;5BFF;;;;N;;;;; +2F873;CJK COMPATIBILITY IDEOGRAPH-2F873;Lo;0;L;5C06;;;;N;;;;; +2F874;CJK COMPATIBILITY IDEOGRAPH-2F874;Lo;0;L;5F53;;;;N;;;;; +2F875;CJK COMPATIBILITY IDEOGRAPH-2F875;Lo;0;L;5C22;;;;N;;;;; +2F876;CJK COMPATIBILITY IDEOGRAPH-2F876;Lo;0;L;3781;;;;N;;;;; +2F877;CJK COMPATIBILITY IDEOGRAPH-2F877;Lo;0;L;5C60;;;;N;;;;; +2F878;CJK COMPATIBILITY IDEOGRAPH-2F878;Lo;0;L;5C6E;;;;N;;;;; +2F879;CJK COMPATIBILITY IDEOGRAPH-2F879;Lo;0;L;5CC0;;;;N;;;;; +2F87A;CJK COMPATIBILITY IDEOGRAPH-2F87A;Lo;0;L;5C8D;;;;N;;;;; +2F87B;CJK COMPATIBILITY IDEOGRAPH-2F87B;Lo;0;L;21DE4;;;;N;;;;; +2F87C;CJK COMPATIBILITY IDEOGRAPH-2F87C;Lo;0;L;5D43;;;;N;;;;; +2F87D;CJK COMPATIBILITY IDEOGRAPH-2F87D;Lo;0;L;21DE6;;;;N;;;;; +2F87E;CJK COMPATIBILITY IDEOGRAPH-2F87E;Lo;0;L;5D6E;;;;N;;;;; +2F87F;CJK COMPATIBILITY IDEOGRAPH-2F87F;Lo;0;L;5D6B;;;;N;;;;; +2F880;CJK COMPATIBILITY IDEOGRAPH-2F880;Lo;0;L;5D7C;;;;N;;;;; +2F881;CJK COMPATIBILITY IDEOGRAPH-2F881;Lo;0;L;5DE1;;;;N;;;;; +2F882;CJK COMPATIBILITY IDEOGRAPH-2F882;Lo;0;L;5DE2;;;;N;;;;; +2F883;CJK COMPATIBILITY IDEOGRAPH-2F883;Lo;0;L;382F;;;;N;;;;; +2F884;CJK COMPATIBILITY IDEOGRAPH-2F884;Lo;0;L;5DFD;;;;N;;;;; +2F885;CJK COMPATIBILITY IDEOGRAPH-2F885;Lo;0;L;5E28;;;;N;;;;; +2F886;CJK COMPATIBILITY IDEOGRAPH-2F886;Lo;0;L;5E3D;;;;N;;;;; +2F887;CJK COMPATIBILITY IDEOGRAPH-2F887;Lo;0;L;5E69;;;;N;;;;; +2F888;CJK COMPATIBILITY IDEOGRAPH-2F888;Lo;0;L;3862;;;;N;;;;; +2F889;CJK COMPATIBILITY IDEOGRAPH-2F889;Lo;0;L;22183;;;;N;;;;; +2F88A;CJK COMPATIBILITY IDEOGRAPH-2F88A;Lo;0;L;387C;;;;N;;;;; +2F88B;CJK COMPATIBILITY IDEOGRAPH-2F88B;Lo;0;L;5EB0;;;;N;;;;; +2F88C;CJK COMPATIBILITY IDEOGRAPH-2F88C;Lo;0;L;5EB3;;;;N;;;;; +2F88D;CJK COMPATIBILITY IDEOGRAPH-2F88D;Lo;0;L;5EB6;;;;N;;;;; +2F88E;CJK COMPATIBILITY IDEOGRAPH-2F88E;Lo;0;L;5ECA;;;;N;;;;; +2F88F;CJK COMPATIBILITY IDEOGRAPH-2F88F;Lo;0;L;2A392;;;;N;;;;; +2F890;CJK COMPATIBILITY IDEOGRAPH-2F890;Lo;0;L;5EFE;;;9;N;;;;; +2F891;CJK COMPATIBILITY IDEOGRAPH-2F891;Lo;0;L;22331;;;;N;;;;; +2F892;CJK COMPATIBILITY IDEOGRAPH-2F892;Lo;0;L;22331;;;;N;;;;; +2F893;CJK COMPATIBILITY IDEOGRAPH-2F893;Lo;0;L;8201;;;;N;;;;; +2F894;CJK COMPATIBILITY IDEOGRAPH-2F894;Lo;0;L;5F22;;;;N;;;;; +2F895;CJK COMPATIBILITY IDEOGRAPH-2F895;Lo;0;L;5F22;;;;N;;;;; +2F896;CJK COMPATIBILITY IDEOGRAPH-2F896;Lo;0;L;38C7;;;;N;;;;; +2F897;CJK COMPATIBILITY IDEOGRAPH-2F897;Lo;0;L;232B8;;;;N;;;;; +2F898;CJK COMPATIBILITY IDEOGRAPH-2F898;Lo;0;L;261DA;;;;N;;;;; +2F899;CJK COMPATIBILITY IDEOGRAPH-2F899;Lo;0;L;5F62;;;;N;;;;; +2F89A;CJK COMPATIBILITY IDEOGRAPH-2F89A;Lo;0;L;5F6B;;;;N;;;;; +2F89B;CJK COMPATIBILITY IDEOGRAPH-2F89B;Lo;0;L;38E3;;;;N;;;;; +2F89C;CJK COMPATIBILITY IDEOGRAPH-2F89C;Lo;0;L;5F9A;;;;N;;;;; +2F89D;CJK COMPATIBILITY IDEOGRAPH-2F89D;Lo;0;L;5FCD;;;;N;;;;; +2F89E;CJK COMPATIBILITY IDEOGRAPH-2F89E;Lo;0;L;5FD7;;;;N;;;;; +2F89F;CJK COMPATIBILITY IDEOGRAPH-2F89F;Lo;0;L;5FF9;;;;N;;;;; +2F8A0;CJK COMPATIBILITY IDEOGRAPH-2F8A0;Lo;0;L;6081;;;;N;;;;; +2F8A1;CJK COMPATIBILITY IDEOGRAPH-2F8A1;Lo;0;L;393A;;;;N;;;;; +2F8A2;CJK COMPATIBILITY IDEOGRAPH-2F8A2;Lo;0;L;391C;;;;N;;;;; +2F8A3;CJK COMPATIBILITY IDEOGRAPH-2F8A3;Lo;0;L;6094;;;;N;;;;; +2F8A4;CJK COMPATIBILITY IDEOGRAPH-2F8A4;Lo;0;L;226D4;;;;N;;;;; +2F8A5;CJK COMPATIBILITY IDEOGRAPH-2F8A5;Lo;0;L;60C7;;;;N;;;;; +2F8A6;CJK COMPATIBILITY IDEOGRAPH-2F8A6;Lo;0;L;6148;;;;N;;;;; +2F8A7;CJK COMPATIBILITY IDEOGRAPH-2F8A7;Lo;0;L;614C;;;;N;;;;; +2F8A8;CJK COMPATIBILITY IDEOGRAPH-2F8A8;Lo;0;L;614E;;;;N;;;;; +2F8A9;CJK COMPATIBILITY IDEOGRAPH-2F8A9;Lo;0;L;614C;;;;N;;;;; +2F8AA;CJK COMPATIBILITY IDEOGRAPH-2F8AA;Lo;0;L;617A;;;;N;;;;; +2F8AB;CJK COMPATIBILITY IDEOGRAPH-2F8AB;Lo;0;L;618E;;;;N;;;;; +2F8AC;CJK COMPATIBILITY IDEOGRAPH-2F8AC;Lo;0;L;61B2;;;;N;;;;; +2F8AD;CJK COMPATIBILITY IDEOGRAPH-2F8AD;Lo;0;L;61A4;;;;N;;;;; +2F8AE;CJK COMPATIBILITY IDEOGRAPH-2F8AE;Lo;0;L;61AF;;;;N;;;;; +2F8AF;CJK COMPATIBILITY IDEOGRAPH-2F8AF;Lo;0;L;61DE;;;;N;;;;; +2F8B0;CJK COMPATIBILITY IDEOGRAPH-2F8B0;Lo;0;L;61F2;;;;N;;;;; +2F8B1;CJK COMPATIBILITY IDEOGRAPH-2F8B1;Lo;0;L;61F6;;;;N;;;;; +2F8B2;CJK COMPATIBILITY IDEOGRAPH-2F8B2;Lo;0;L;6210;;;;N;;;;; +2F8B3;CJK COMPATIBILITY IDEOGRAPH-2F8B3;Lo;0;L;621B;;;;N;;;;; +2F8B4;CJK COMPATIBILITY IDEOGRAPH-2F8B4;Lo;0;L;625D;;;;N;;;;; +2F8B5;CJK COMPATIBILITY IDEOGRAPH-2F8B5;Lo;0;L;62B1;;;;N;;;;; +2F8B6;CJK COMPATIBILITY IDEOGRAPH-2F8B6;Lo;0;L;62D4;;;;N;;;;; +2F8B7;CJK COMPATIBILITY IDEOGRAPH-2F8B7;Lo;0;L;6350;;;;N;;;;; +2F8B8;CJK COMPATIBILITY IDEOGRAPH-2F8B8;Lo;0;L;22B0C;;;;N;;;;; +2F8B9;CJK COMPATIBILITY IDEOGRAPH-2F8B9;Lo;0;L;633D;;;;N;;;;; +2F8BA;CJK COMPATIBILITY IDEOGRAPH-2F8BA;Lo;0;L;62FC;;;;N;;;;; +2F8BB;CJK COMPATIBILITY IDEOGRAPH-2F8BB;Lo;0;L;6368;;;;N;;;;; +2F8BC;CJK COMPATIBILITY IDEOGRAPH-2F8BC;Lo;0;L;6383;;;;N;;;;; +2F8BD;CJK COMPATIBILITY IDEOGRAPH-2F8BD;Lo;0;L;63E4;;;;N;;;;; +2F8BE;CJK COMPATIBILITY IDEOGRAPH-2F8BE;Lo;0;L;22BF1;;;;N;;;;; +2F8BF;CJK COMPATIBILITY IDEOGRAPH-2F8BF;Lo;0;L;6422;;;;N;;;;; +2F8C0;CJK COMPATIBILITY IDEOGRAPH-2F8C0;Lo;0;L;63C5;;;;N;;;;; +2F8C1;CJK COMPATIBILITY IDEOGRAPH-2F8C1;Lo;0;L;63A9;;;;N;;;;; +2F8C2;CJK COMPATIBILITY IDEOGRAPH-2F8C2;Lo;0;L;3A2E;;;;N;;;;; +2F8C3;CJK COMPATIBILITY IDEOGRAPH-2F8C3;Lo;0;L;6469;;;;N;;;;; +2F8C4;CJK COMPATIBILITY IDEOGRAPH-2F8C4;Lo;0;L;647E;;;;N;;;;; +2F8C5;CJK COMPATIBILITY IDEOGRAPH-2F8C5;Lo;0;L;649D;;;;N;;;;; +2F8C6;CJK COMPATIBILITY IDEOGRAPH-2F8C6;Lo;0;L;6477;;;;N;;;;; +2F8C7;CJK COMPATIBILITY IDEOGRAPH-2F8C7;Lo;0;L;3A6C;;;;N;;;;; +2F8C8;CJK COMPATIBILITY IDEOGRAPH-2F8C8;Lo;0;L;654F;;;;N;;;;; +2F8C9;CJK COMPATIBILITY IDEOGRAPH-2F8C9;Lo;0;L;656C;;;;N;;;;; +2F8CA;CJK COMPATIBILITY IDEOGRAPH-2F8CA;Lo;0;L;2300A;;;;N;;;;; +2F8CB;CJK COMPATIBILITY IDEOGRAPH-2F8CB;Lo;0;L;65E3;;;;N;;;;; +2F8CC;CJK COMPATIBILITY IDEOGRAPH-2F8CC;Lo;0;L;66F8;;;;N;;;;; +2F8CD;CJK COMPATIBILITY IDEOGRAPH-2F8CD;Lo;0;L;6649;;;;N;;;;; +2F8CE;CJK COMPATIBILITY IDEOGRAPH-2F8CE;Lo;0;L;3B19;;;;N;;;;; +2F8CF;CJK COMPATIBILITY IDEOGRAPH-2F8CF;Lo;0;L;6691;;;;N;;;;; +2F8D0;CJK COMPATIBILITY IDEOGRAPH-2F8D0;Lo;0;L;3B08;;;;N;;;;; +2F8D1;CJK COMPATIBILITY IDEOGRAPH-2F8D1;Lo;0;L;3AE4;;;;N;;;;; +2F8D2;CJK COMPATIBILITY IDEOGRAPH-2F8D2;Lo;0;L;5192;;;;N;;;;; +2F8D3;CJK COMPATIBILITY IDEOGRAPH-2F8D3;Lo;0;L;5195;;;;N;;;;; +2F8D4;CJK COMPATIBILITY IDEOGRAPH-2F8D4;Lo;0;L;6700;;;;N;;;;; +2F8D5;CJK COMPATIBILITY IDEOGRAPH-2F8D5;Lo;0;L;669C;;;;N;;;;; +2F8D6;CJK COMPATIBILITY IDEOGRAPH-2F8D6;Lo;0;L;80AD;;;;N;;;;; +2F8D7;CJK COMPATIBILITY IDEOGRAPH-2F8D7;Lo;0;L;43D9;;;;N;;;;; +2F8D8;CJK COMPATIBILITY IDEOGRAPH-2F8D8;Lo;0;L;6717;;;;N;;;;; +2F8D9;CJK COMPATIBILITY IDEOGRAPH-2F8D9;Lo;0;L;671B;;;;N;;;;; +2F8DA;CJK COMPATIBILITY IDEOGRAPH-2F8DA;Lo;0;L;6721;;;;N;;;;; +2F8DB;CJK COMPATIBILITY IDEOGRAPH-2F8DB;Lo;0;L;675E;;;;N;;;;; +2F8DC;CJK COMPATIBILITY IDEOGRAPH-2F8DC;Lo;0;L;6753;;;;N;;;;; +2F8DD;CJK COMPATIBILITY IDEOGRAPH-2F8DD;Lo;0;L;233C3;;;;N;;;;; +2F8DE;CJK COMPATIBILITY IDEOGRAPH-2F8DE;Lo;0;L;3B49;;;;N;;;;; +2F8DF;CJK COMPATIBILITY IDEOGRAPH-2F8DF;Lo;0;L;67FA;;;;N;;;;; +2F8E0;CJK COMPATIBILITY IDEOGRAPH-2F8E0;Lo;0;L;6785;;;;N;;;;; +2F8E1;CJK COMPATIBILITY IDEOGRAPH-2F8E1;Lo;0;L;6852;;;;N;;;;; +2F8E2;CJK COMPATIBILITY IDEOGRAPH-2F8E2;Lo;0;L;6885;;;;N;;;;; +2F8E3;CJK COMPATIBILITY IDEOGRAPH-2F8E3;Lo;0;L;2346D;;;;N;;;;; +2F8E4;CJK COMPATIBILITY IDEOGRAPH-2F8E4;Lo;0;L;688E;;;;N;;;;; +2F8E5;CJK COMPATIBILITY IDEOGRAPH-2F8E5;Lo;0;L;681F;;;;N;;;;; +2F8E6;CJK COMPATIBILITY IDEOGRAPH-2F8E6;Lo;0;L;6914;;;;N;;;;; +2F8E7;CJK COMPATIBILITY IDEOGRAPH-2F8E7;Lo;0;L;3B9D;;;;N;;;;; +2F8E8;CJK COMPATIBILITY IDEOGRAPH-2F8E8;Lo;0;L;6942;;;;N;;;;; +2F8E9;CJK COMPATIBILITY IDEOGRAPH-2F8E9;Lo;0;L;69A3;;;;N;;;;; +2F8EA;CJK COMPATIBILITY IDEOGRAPH-2F8EA;Lo;0;L;69EA;;;;N;;;;; +2F8EB;CJK COMPATIBILITY IDEOGRAPH-2F8EB;Lo;0;L;6AA8;;;;N;;;;; +2F8EC;CJK COMPATIBILITY IDEOGRAPH-2F8EC;Lo;0;L;236A3;;;;N;;;;; +2F8ED;CJK COMPATIBILITY IDEOGRAPH-2F8ED;Lo;0;L;6ADB;;;;N;;;;; +2F8EE;CJK COMPATIBILITY IDEOGRAPH-2F8EE;Lo;0;L;3C18;;;;N;;;;; +2F8EF;CJK COMPATIBILITY IDEOGRAPH-2F8EF;Lo;0;L;6B21;;;;N;;;;; +2F8F0;CJK COMPATIBILITY IDEOGRAPH-2F8F0;Lo;0;L;238A7;;;;N;;;;; +2F8F1;CJK COMPATIBILITY IDEOGRAPH-2F8F1;Lo;0;L;6B54;;;;N;;;;; +2F8F2;CJK COMPATIBILITY IDEOGRAPH-2F8F2;Lo;0;L;3C4E;;;;N;;;;; +2F8F3;CJK COMPATIBILITY IDEOGRAPH-2F8F3;Lo;0;L;6B72;;;;N;;;;; +2F8F4;CJK COMPATIBILITY IDEOGRAPH-2F8F4;Lo;0;L;6B9F;;;;N;;;;; +2F8F5;CJK COMPATIBILITY IDEOGRAPH-2F8F5;Lo;0;L;6BBA;;;;N;;;;; +2F8F6;CJK COMPATIBILITY IDEOGRAPH-2F8F6;Lo;0;L;6BBB;;;;N;;;;; +2F8F7;CJK COMPATIBILITY IDEOGRAPH-2F8F7;Lo;0;L;23A8D;;;;N;;;;; +2F8F8;CJK COMPATIBILITY IDEOGRAPH-2F8F8;Lo;0;L;21D0B;;;;N;;;;; +2F8F9;CJK COMPATIBILITY IDEOGRAPH-2F8F9;Lo;0;L;23AFA;;;;N;;;;; +2F8FA;CJK COMPATIBILITY IDEOGRAPH-2F8FA;Lo;0;L;6C4E;;;;N;;;;; +2F8FB;CJK COMPATIBILITY IDEOGRAPH-2F8FB;Lo;0;L;23CBC;;;;N;;;;; +2F8FC;CJK COMPATIBILITY IDEOGRAPH-2F8FC;Lo;0;L;6CBF;;;;N;;;;; +2F8FD;CJK COMPATIBILITY IDEOGRAPH-2F8FD;Lo;0;L;6CCD;;;;N;;;;; +2F8FE;CJK COMPATIBILITY IDEOGRAPH-2F8FE;Lo;0;L;6C67;;;;N;;;;; +2F8FF;CJK COMPATIBILITY IDEOGRAPH-2F8FF;Lo;0;L;6D16;;;;N;;;;; +2F900;CJK COMPATIBILITY IDEOGRAPH-2F900;Lo;0;L;6D3E;;;;N;;;;; +2F901;CJK COMPATIBILITY IDEOGRAPH-2F901;Lo;0;L;6D77;;;;N;;;;; +2F902;CJK COMPATIBILITY IDEOGRAPH-2F902;Lo;0;L;6D41;;;;N;;;;; +2F903;CJK COMPATIBILITY IDEOGRAPH-2F903;Lo;0;L;6D69;;;;N;;;;; +2F904;CJK COMPATIBILITY IDEOGRAPH-2F904;Lo;0;L;6D78;;;;N;;;;; +2F905;CJK COMPATIBILITY IDEOGRAPH-2F905;Lo;0;L;6D85;;;;N;;;;; +2F906;CJK COMPATIBILITY IDEOGRAPH-2F906;Lo;0;L;23D1E;;;;N;;;;; +2F907;CJK COMPATIBILITY IDEOGRAPH-2F907;Lo;0;L;6D34;;;;N;;;;; +2F908;CJK COMPATIBILITY IDEOGRAPH-2F908;Lo;0;L;6E2F;;;;N;;;;; +2F909;CJK COMPATIBILITY IDEOGRAPH-2F909;Lo;0;L;6E6E;;;;N;;;;; +2F90A;CJK COMPATIBILITY IDEOGRAPH-2F90A;Lo;0;L;3D33;;;;N;;;;; +2F90B;CJK COMPATIBILITY IDEOGRAPH-2F90B;Lo;0;L;6ECB;;;;N;;;;; +2F90C;CJK COMPATIBILITY IDEOGRAPH-2F90C;Lo;0;L;6EC7;;;;N;;;;; +2F90D;CJK COMPATIBILITY IDEOGRAPH-2F90D;Lo;0;L;23ED1;;;;N;;;;; +2F90E;CJK COMPATIBILITY IDEOGRAPH-2F90E;Lo;0;L;6DF9;;;;N;;;;; +2F90F;CJK COMPATIBILITY IDEOGRAPH-2F90F;Lo;0;L;6F6E;;;;N;;;;; +2F910;CJK COMPATIBILITY IDEOGRAPH-2F910;Lo;0;L;23F5E;;;;N;;;;; +2F911;CJK COMPATIBILITY IDEOGRAPH-2F911;Lo;0;L;23F8E;;;;N;;;;; +2F912;CJK COMPATIBILITY IDEOGRAPH-2F912;Lo;0;L;6FC6;;;;N;;;;; +2F913;CJK COMPATIBILITY IDEOGRAPH-2F913;Lo;0;L;7039;;;;N;;;;; +2F914;CJK COMPATIBILITY IDEOGRAPH-2F914;Lo;0;L;701E;;;;N;;;;; +2F915;CJK COMPATIBILITY IDEOGRAPH-2F915;Lo;0;L;701B;;;;N;;;;; +2F916;CJK COMPATIBILITY IDEOGRAPH-2F916;Lo;0;L;3D96;;;;N;;;;; +2F917;CJK COMPATIBILITY IDEOGRAPH-2F917;Lo;0;L;704A;;;;N;;;;; +2F918;CJK COMPATIBILITY IDEOGRAPH-2F918;Lo;0;L;707D;;;;N;;;;; +2F919;CJK COMPATIBILITY IDEOGRAPH-2F919;Lo;0;L;7077;;;;N;;;;; +2F91A;CJK COMPATIBILITY IDEOGRAPH-2F91A;Lo;0;L;70AD;;;;N;;;;; +2F91B;CJK COMPATIBILITY IDEOGRAPH-2F91B;Lo;0;L;20525;;;;N;;;;; +2F91C;CJK COMPATIBILITY IDEOGRAPH-2F91C;Lo;0;L;7145;;;;N;;;;; +2F91D;CJK COMPATIBILITY IDEOGRAPH-2F91D;Lo;0;L;24263;;;;N;;;;; +2F91E;CJK COMPATIBILITY IDEOGRAPH-2F91E;Lo;0;L;719C;;;;N;;;;; +2F91F;CJK COMPATIBILITY IDEOGRAPH-2F91F;Lo;0;L;243AB;;;;N;;;;; +2F920;CJK COMPATIBILITY IDEOGRAPH-2F920;Lo;0;L;7228;;;;N;;;;; +2F921;CJK COMPATIBILITY IDEOGRAPH-2F921;Lo;0;L;7235;;;;N;;;;; +2F922;CJK COMPATIBILITY IDEOGRAPH-2F922;Lo;0;L;7250;;;;N;;;;; +2F923;CJK COMPATIBILITY IDEOGRAPH-2F923;Lo;0;L;24608;;;;N;;;;; +2F924;CJK COMPATIBILITY IDEOGRAPH-2F924;Lo;0;L;7280;;;;N;;;;; +2F925;CJK COMPATIBILITY IDEOGRAPH-2F925;Lo;0;L;7295;;;;N;;;;; +2F926;CJK COMPATIBILITY IDEOGRAPH-2F926;Lo;0;L;24735;;;;N;;;;; +2F927;CJK COMPATIBILITY IDEOGRAPH-2F927;Lo;0;L;24814;;;;N;;;;; +2F928;CJK COMPATIBILITY IDEOGRAPH-2F928;Lo;0;L;737A;;;;N;;;;; +2F929;CJK COMPATIBILITY IDEOGRAPH-2F929;Lo;0;L;738B;;;;N;;;;; +2F92A;CJK COMPATIBILITY IDEOGRAPH-2F92A;Lo;0;L;3EAC;;;;N;;;;; +2F92B;CJK COMPATIBILITY IDEOGRAPH-2F92B;Lo;0;L;73A5;;;;N;;;;; +2F92C;CJK COMPATIBILITY IDEOGRAPH-2F92C;Lo;0;L;3EB8;;;;N;;;;; +2F92D;CJK COMPATIBILITY IDEOGRAPH-2F92D;Lo;0;L;3EB8;;;;N;;;;; +2F92E;CJK COMPATIBILITY IDEOGRAPH-2F92E;Lo;0;L;7447;;;;N;;;;; +2F92F;CJK COMPATIBILITY IDEOGRAPH-2F92F;Lo;0;L;745C;;;;N;;;;; +2F930;CJK COMPATIBILITY IDEOGRAPH-2F930;Lo;0;L;7471;;;;N;;;;; +2F931;CJK COMPATIBILITY IDEOGRAPH-2F931;Lo;0;L;7485;;;;N;;;;; +2F932;CJK COMPATIBILITY IDEOGRAPH-2F932;Lo;0;L;74CA;;;;N;;;;; +2F933;CJK COMPATIBILITY IDEOGRAPH-2F933;Lo;0;L;3F1B;;;;N;;;;; +2F934;CJK COMPATIBILITY IDEOGRAPH-2F934;Lo;0;L;7524;;;;N;;;;; +2F935;CJK COMPATIBILITY IDEOGRAPH-2F935;Lo;0;L;24C36;;;;N;;;;; +2F936;CJK COMPATIBILITY IDEOGRAPH-2F936;Lo;0;L;753E;;;;N;;;;; +2F937;CJK COMPATIBILITY IDEOGRAPH-2F937;Lo;0;L;24C92;;;;N;;;;; +2F938;CJK COMPATIBILITY IDEOGRAPH-2F938;Lo;0;L;7570;;;;N;;;;; +2F939;CJK COMPATIBILITY IDEOGRAPH-2F939;Lo;0;L;2219F;;;;N;;;;; +2F93A;CJK COMPATIBILITY IDEOGRAPH-2F93A;Lo;0;L;7610;;;;N;;;;; +2F93B;CJK COMPATIBILITY IDEOGRAPH-2F93B;Lo;0;L;24FA1;;;;N;;;;; +2F93C;CJK COMPATIBILITY IDEOGRAPH-2F93C;Lo;0;L;24FB8;;;;N;;;;; +2F93D;CJK COMPATIBILITY IDEOGRAPH-2F93D;Lo;0;L;25044;;;;N;;;;; +2F93E;CJK COMPATIBILITY IDEOGRAPH-2F93E;Lo;0;L;3FFC;;;;N;;;;; +2F93F;CJK COMPATIBILITY IDEOGRAPH-2F93F;Lo;0;L;4008;;;;N;;;;; +2F940;CJK COMPATIBILITY IDEOGRAPH-2F940;Lo;0;L;76F4;;;;N;;;;; +2F941;CJK COMPATIBILITY IDEOGRAPH-2F941;Lo;0;L;250F3;;;;N;;;;; +2F942;CJK COMPATIBILITY IDEOGRAPH-2F942;Lo;0;L;250F2;;;;N;;;;; +2F943;CJK COMPATIBILITY IDEOGRAPH-2F943;Lo;0;L;25119;;;;N;;;;; +2F944;CJK COMPATIBILITY IDEOGRAPH-2F944;Lo;0;L;25133;;;;N;;;;; +2F945;CJK COMPATIBILITY IDEOGRAPH-2F945;Lo;0;L;771E;;;;N;;;;; +2F946;CJK COMPATIBILITY IDEOGRAPH-2F946;Lo;0;L;771F;;;;N;;;;; +2F947;CJK COMPATIBILITY IDEOGRAPH-2F947;Lo;0;L;771F;;;;N;;;;; +2F948;CJK COMPATIBILITY IDEOGRAPH-2F948;Lo;0;L;774A;;;;N;;;;; +2F949;CJK COMPATIBILITY IDEOGRAPH-2F949;Lo;0;L;4039;;;;N;;;;; +2F94A;CJK COMPATIBILITY IDEOGRAPH-2F94A;Lo;0;L;778B;;;;N;;;;; +2F94B;CJK COMPATIBILITY IDEOGRAPH-2F94B;Lo;0;L;4046;;;;N;;;;; +2F94C;CJK COMPATIBILITY IDEOGRAPH-2F94C;Lo;0;L;4096;;;;N;;;;; +2F94D;CJK COMPATIBILITY IDEOGRAPH-2F94D;Lo;0;L;2541D;;;;N;;;;; +2F94E;CJK COMPATIBILITY IDEOGRAPH-2F94E;Lo;0;L;784E;;;;N;;;;; +2F94F;CJK COMPATIBILITY IDEOGRAPH-2F94F;Lo;0;L;788C;;;;N;;;;; +2F950;CJK COMPATIBILITY IDEOGRAPH-2F950;Lo;0;L;78CC;;;;N;;;;; +2F951;CJK COMPATIBILITY IDEOGRAPH-2F951;Lo;0;L;40E3;;;;N;;;;; +2F952;CJK COMPATIBILITY IDEOGRAPH-2F952;Lo;0;L;25626;;;;N;;;;; +2F953;CJK COMPATIBILITY IDEOGRAPH-2F953;Lo;0;L;7956;;;;N;;;;; +2F954;CJK COMPATIBILITY IDEOGRAPH-2F954;Lo;0;L;2569A;;;;N;;;;; +2F955;CJK COMPATIBILITY IDEOGRAPH-2F955;Lo;0;L;256C5;;;;N;;;;; +2F956;CJK COMPATIBILITY IDEOGRAPH-2F956;Lo;0;L;798F;;;;N;;;;; +2F957;CJK COMPATIBILITY IDEOGRAPH-2F957;Lo;0;L;79EB;;;;N;;;;; +2F958;CJK COMPATIBILITY IDEOGRAPH-2F958;Lo;0;L;412F;;;;N;;;;; +2F959;CJK COMPATIBILITY IDEOGRAPH-2F959;Lo;0;L;7A40;;;;N;;;;; +2F95A;CJK COMPATIBILITY IDEOGRAPH-2F95A;Lo;0;L;7A4A;;;;N;;;;; +2F95B;CJK COMPATIBILITY IDEOGRAPH-2F95B;Lo;0;L;7A4F;;;;N;;;;; +2F95C;CJK COMPATIBILITY IDEOGRAPH-2F95C;Lo;0;L;2597C;;;;N;;;;; +2F95D;CJK COMPATIBILITY IDEOGRAPH-2F95D;Lo;0;L;25AA7;;;;N;;;;; +2F95E;CJK COMPATIBILITY IDEOGRAPH-2F95E;Lo;0;L;25AA7;;;;N;;;;; +2F95F;CJK COMPATIBILITY IDEOGRAPH-2F95F;Lo;0;L;7AEE;;;;N;;;;; +2F960;CJK COMPATIBILITY IDEOGRAPH-2F960;Lo;0;L;4202;;;;N;;;;; +2F961;CJK COMPATIBILITY IDEOGRAPH-2F961;Lo;0;L;25BAB;;;;N;;;;; +2F962;CJK COMPATIBILITY IDEOGRAPH-2F962;Lo;0;L;7BC6;;;;N;;;;; +2F963;CJK COMPATIBILITY IDEOGRAPH-2F963;Lo;0;L;7BC9;;;;N;;;;; +2F964;CJK COMPATIBILITY IDEOGRAPH-2F964;Lo;0;L;4227;;;;N;;;;; +2F965;CJK COMPATIBILITY IDEOGRAPH-2F965;Lo;0;L;25C80;;;;N;;;;; +2F966;CJK COMPATIBILITY IDEOGRAPH-2F966;Lo;0;L;7CD2;;;;N;;;;; +2F967;CJK COMPATIBILITY IDEOGRAPH-2F967;Lo;0;L;42A0;;;;N;;;;; +2F968;CJK COMPATIBILITY IDEOGRAPH-2F968;Lo;0;L;7CE8;;;;N;;;;; +2F969;CJK COMPATIBILITY IDEOGRAPH-2F969;Lo;0;L;7CE3;;;;N;;;;; +2F96A;CJK COMPATIBILITY IDEOGRAPH-2F96A;Lo;0;L;7D00;;;;N;;;;; +2F96B;CJK COMPATIBILITY IDEOGRAPH-2F96B;Lo;0;L;25F86;;;;N;;;;; +2F96C;CJK COMPATIBILITY IDEOGRAPH-2F96C;Lo;0;L;7D63;;;;N;;;;; +2F96D;CJK COMPATIBILITY IDEOGRAPH-2F96D;Lo;0;L;4301;;;;N;;;;; +2F96E;CJK COMPATIBILITY IDEOGRAPH-2F96E;Lo;0;L;7DC7;;;;N;;;;; +2F96F;CJK COMPATIBILITY IDEOGRAPH-2F96F;Lo;0;L;7E02;;;;N;;;;; +2F970;CJK COMPATIBILITY IDEOGRAPH-2F970;Lo;0;L;7E45;;;;N;;;;; +2F971;CJK COMPATIBILITY IDEOGRAPH-2F971;Lo;0;L;4334;;;;N;;;;; +2F972;CJK COMPATIBILITY IDEOGRAPH-2F972;Lo;0;L;26228;;;;N;;;;; +2F973;CJK COMPATIBILITY IDEOGRAPH-2F973;Lo;0;L;26247;;;;N;;;;; +2F974;CJK COMPATIBILITY IDEOGRAPH-2F974;Lo;0;L;4359;;;;N;;;;; +2F975;CJK COMPATIBILITY IDEOGRAPH-2F975;Lo;0;L;262D9;;;;N;;;;; +2F976;CJK COMPATIBILITY IDEOGRAPH-2F976;Lo;0;L;7F7A;;;;N;;;;; +2F977;CJK COMPATIBILITY IDEOGRAPH-2F977;Lo;0;L;2633E;;;;N;;;;; +2F978;CJK COMPATIBILITY IDEOGRAPH-2F978;Lo;0;L;7F95;;;;N;;;;; +2F979;CJK COMPATIBILITY IDEOGRAPH-2F979;Lo;0;L;7FFA;;;;N;;;;; +2F97A;CJK COMPATIBILITY IDEOGRAPH-2F97A;Lo;0;L;8005;;;;N;;;;; +2F97B;CJK COMPATIBILITY IDEOGRAPH-2F97B;Lo;0;L;264DA;;;;N;;;;; +2F97C;CJK COMPATIBILITY IDEOGRAPH-2F97C;Lo;0;L;26523;;;;N;;;;; +2F97D;CJK COMPATIBILITY IDEOGRAPH-2F97D;Lo;0;L;8060;;;;N;;;;; +2F97E;CJK COMPATIBILITY IDEOGRAPH-2F97E;Lo;0;L;265A8;;;;N;;;;; +2F97F;CJK COMPATIBILITY IDEOGRAPH-2F97F;Lo;0;L;8070;;;;N;;;;; +2F980;CJK COMPATIBILITY IDEOGRAPH-2F980;Lo;0;L;2335F;;;;N;;;;; +2F981;CJK COMPATIBILITY IDEOGRAPH-2F981;Lo;0;L;43D5;;;;N;;;;; +2F982;CJK COMPATIBILITY IDEOGRAPH-2F982;Lo;0;L;80B2;;;;N;;;;; +2F983;CJK COMPATIBILITY IDEOGRAPH-2F983;Lo;0;L;8103;;;;N;;;;; +2F984;CJK COMPATIBILITY IDEOGRAPH-2F984;Lo;0;L;440B;;;;N;;;;; +2F985;CJK COMPATIBILITY IDEOGRAPH-2F985;Lo;0;L;813E;;;;N;;;;; +2F986;CJK COMPATIBILITY IDEOGRAPH-2F986;Lo;0;L;5AB5;;;;N;;;;; +2F987;CJK COMPATIBILITY IDEOGRAPH-2F987;Lo;0;L;267A7;;;;N;;;;; +2F988;CJK COMPATIBILITY IDEOGRAPH-2F988;Lo;0;L;267B5;;;;N;;;;; +2F989;CJK COMPATIBILITY IDEOGRAPH-2F989;Lo;0;L;23393;;;;N;;;;; +2F98A;CJK COMPATIBILITY IDEOGRAPH-2F98A;Lo;0;L;2339C;;;;N;;;;; +2F98B;CJK COMPATIBILITY IDEOGRAPH-2F98B;Lo;0;L;8201;;;;N;;;;; +2F98C;CJK COMPATIBILITY IDEOGRAPH-2F98C;Lo;0;L;8204;;;;N;;;;; +2F98D;CJK COMPATIBILITY IDEOGRAPH-2F98D;Lo;0;L;8F9E;;;;N;;;;; +2F98E;CJK COMPATIBILITY IDEOGRAPH-2F98E;Lo;0;L;446B;;;;N;;;;; +2F98F;CJK COMPATIBILITY IDEOGRAPH-2F98F;Lo;0;L;8291;;;;N;;;;; +2F990;CJK COMPATIBILITY IDEOGRAPH-2F990;Lo;0;L;828B;;;;N;;;;; +2F991;CJK COMPATIBILITY IDEOGRAPH-2F991;Lo;0;L;829D;;;;N;;;;; +2F992;CJK COMPATIBILITY IDEOGRAPH-2F992;Lo;0;L;52B3;;;;N;;;;; +2F993;CJK COMPATIBILITY IDEOGRAPH-2F993;Lo;0;L;82B1;;;;N;;;;; +2F994;CJK COMPATIBILITY IDEOGRAPH-2F994;Lo;0;L;82B3;;;;N;;;;; +2F995;CJK COMPATIBILITY IDEOGRAPH-2F995;Lo;0;L;82BD;;;;N;;;;; +2F996;CJK COMPATIBILITY IDEOGRAPH-2F996;Lo;0;L;82E6;;;;N;;;;; +2F997;CJK COMPATIBILITY IDEOGRAPH-2F997;Lo;0;L;26B3C;;;;N;;;;; +2F998;CJK COMPATIBILITY IDEOGRAPH-2F998;Lo;0;L;82E5;;;;N;;;;; +2F999;CJK COMPATIBILITY IDEOGRAPH-2F999;Lo;0;L;831D;;;;N;;;;; +2F99A;CJK COMPATIBILITY IDEOGRAPH-2F99A;Lo;0;L;8363;;;;N;;;;; +2F99B;CJK COMPATIBILITY IDEOGRAPH-2F99B;Lo;0;L;83AD;;;;N;;;;; +2F99C;CJK COMPATIBILITY IDEOGRAPH-2F99C;Lo;0;L;8323;;;;N;;;;; +2F99D;CJK COMPATIBILITY IDEOGRAPH-2F99D;Lo;0;L;83BD;;;;N;;;;; +2F99E;CJK COMPATIBILITY IDEOGRAPH-2F99E;Lo;0;L;83E7;;;;N;;;;; +2F99F;CJK COMPATIBILITY IDEOGRAPH-2F99F;Lo;0;L;8457;;;;N;;;;; +2F9A0;CJK COMPATIBILITY IDEOGRAPH-2F9A0;Lo;0;L;8353;;;;N;;;;; +2F9A1;CJK COMPATIBILITY IDEOGRAPH-2F9A1;Lo;0;L;83CA;;;;N;;;;; +2F9A2;CJK COMPATIBILITY IDEOGRAPH-2F9A2;Lo;0;L;83CC;;;;N;;;;; +2F9A3;CJK COMPATIBILITY IDEOGRAPH-2F9A3;Lo;0;L;83DC;;;;N;;;;; +2F9A4;CJK COMPATIBILITY IDEOGRAPH-2F9A4;Lo;0;L;26C36;;;;N;;;;; +2F9A5;CJK COMPATIBILITY IDEOGRAPH-2F9A5;Lo;0;L;26D6B;;;;N;;;;; +2F9A6;CJK COMPATIBILITY IDEOGRAPH-2F9A6;Lo;0;L;26CD5;;;;N;;;;; +2F9A7;CJK COMPATIBILITY IDEOGRAPH-2F9A7;Lo;0;L;452B;;;;N;;;;; +2F9A8;CJK COMPATIBILITY IDEOGRAPH-2F9A8;Lo;0;L;84F1;;;;N;;;;; +2F9A9;CJK COMPATIBILITY IDEOGRAPH-2F9A9;Lo;0;L;84F3;;;;N;;;;; +2F9AA;CJK COMPATIBILITY IDEOGRAPH-2F9AA;Lo;0;L;8516;;;;N;;;;; +2F9AB;CJK COMPATIBILITY IDEOGRAPH-2F9AB;Lo;0;L;273CA;;;;N;;;;; +2F9AC;CJK COMPATIBILITY IDEOGRAPH-2F9AC;Lo;0;L;8564;;;;N;;;;; +2F9AD;CJK COMPATIBILITY IDEOGRAPH-2F9AD;Lo;0;L;26F2C;;;;N;;;;; +2F9AE;CJK COMPATIBILITY IDEOGRAPH-2F9AE;Lo;0;L;455D;;;;N;;;;; +2F9AF;CJK COMPATIBILITY IDEOGRAPH-2F9AF;Lo;0;L;4561;;;;N;;;;; +2F9B0;CJK COMPATIBILITY IDEOGRAPH-2F9B0;Lo;0;L;26FB1;;;;N;;;;; +2F9B1;CJK COMPATIBILITY IDEOGRAPH-2F9B1;Lo;0;L;270D2;;;;N;;;;; +2F9B2;CJK COMPATIBILITY IDEOGRAPH-2F9B2;Lo;0;L;456B;;;;N;;;;; +2F9B3;CJK COMPATIBILITY IDEOGRAPH-2F9B3;Lo;0;L;8650;;;;N;;;;; +2F9B4;CJK COMPATIBILITY IDEOGRAPH-2F9B4;Lo;0;L;865C;;;;N;;;;; +2F9B5;CJK COMPATIBILITY IDEOGRAPH-2F9B5;Lo;0;L;8667;;;;N;;;;; +2F9B6;CJK COMPATIBILITY IDEOGRAPH-2F9B6;Lo;0;L;8669;;;;N;;;;; +2F9B7;CJK COMPATIBILITY IDEOGRAPH-2F9B7;Lo;0;L;86A9;;;;N;;;;; +2F9B8;CJK COMPATIBILITY IDEOGRAPH-2F9B8;Lo;0;L;8688;;;;N;;;;; +2F9B9;CJK COMPATIBILITY IDEOGRAPH-2F9B9;Lo;0;L;870E;;;;N;;;;; +2F9BA;CJK COMPATIBILITY IDEOGRAPH-2F9BA;Lo;0;L;86E2;;;;N;;;;; +2F9BB;CJK COMPATIBILITY IDEOGRAPH-2F9BB;Lo;0;L;8779;;;;N;;;;; +2F9BC;CJK COMPATIBILITY IDEOGRAPH-2F9BC;Lo;0;L;8728;;;;N;;;;; +2F9BD;CJK COMPATIBILITY IDEOGRAPH-2F9BD;Lo;0;L;876B;;;;N;;;;; +2F9BE;CJK COMPATIBILITY IDEOGRAPH-2F9BE;Lo;0;L;8786;;;;N;;;;; +2F9BF;CJK COMPATIBILITY IDEOGRAPH-2F9BF;Lo;0;L;45D7;;;;N;;;;; +2F9C0;CJK COMPATIBILITY IDEOGRAPH-2F9C0;Lo;0;L;87E1;;;;N;;;;; +2F9C1;CJK COMPATIBILITY IDEOGRAPH-2F9C1;Lo;0;L;8801;;;;N;;;;; +2F9C2;CJK COMPATIBILITY IDEOGRAPH-2F9C2;Lo;0;L;45F9;;;;N;;;;; +2F9C3;CJK COMPATIBILITY IDEOGRAPH-2F9C3;Lo;0;L;8860;;;;N;;;;; +2F9C4;CJK COMPATIBILITY IDEOGRAPH-2F9C4;Lo;0;L;8863;;;;N;;;;; +2F9C5;CJK COMPATIBILITY IDEOGRAPH-2F9C5;Lo;0;L;27667;;;;N;;;;; +2F9C6;CJK COMPATIBILITY IDEOGRAPH-2F9C6;Lo;0;L;88D7;;;;N;;;;; +2F9C7;CJK COMPATIBILITY IDEOGRAPH-2F9C7;Lo;0;L;88DE;;;;N;;;;; +2F9C8;CJK COMPATIBILITY IDEOGRAPH-2F9C8;Lo;0;L;4635;;;;N;;;;; +2F9C9;CJK COMPATIBILITY IDEOGRAPH-2F9C9;Lo;0;L;88FA;;;;N;;;;; +2F9CA;CJK COMPATIBILITY IDEOGRAPH-2F9CA;Lo;0;L;34BB;;;;N;;;;; +2F9CB;CJK COMPATIBILITY IDEOGRAPH-2F9CB;Lo;0;L;278AE;;;;N;;;;; +2F9CC;CJK COMPATIBILITY IDEOGRAPH-2F9CC;Lo;0;L;27966;;;;N;;;;; +2F9CD;CJK COMPATIBILITY IDEOGRAPH-2F9CD;Lo;0;L;46BE;;;;N;;;;; +2F9CE;CJK COMPATIBILITY IDEOGRAPH-2F9CE;Lo;0;L;46C7;;;;N;;;;; +2F9CF;CJK COMPATIBILITY IDEOGRAPH-2F9CF;Lo;0;L;8AA0;;;;N;;;;; +2F9D0;CJK COMPATIBILITY IDEOGRAPH-2F9D0;Lo;0;L;8AED;;;;N;;;;; +2F9D1;CJK COMPATIBILITY IDEOGRAPH-2F9D1;Lo;0;L;8B8A;;;;N;;;;; +2F9D2;CJK COMPATIBILITY IDEOGRAPH-2F9D2;Lo;0;L;8C55;;;;N;;;;; +2F9D3;CJK COMPATIBILITY IDEOGRAPH-2F9D3;Lo;0;L;27CA8;;;;N;;;;; +2F9D4;CJK COMPATIBILITY IDEOGRAPH-2F9D4;Lo;0;L;8CAB;;;;N;;;;; +2F9D5;CJK COMPATIBILITY IDEOGRAPH-2F9D5;Lo;0;L;8CC1;;;;N;;;;; +2F9D6;CJK COMPATIBILITY IDEOGRAPH-2F9D6;Lo;0;L;8D1B;;;;N;;;;; +2F9D7;CJK COMPATIBILITY IDEOGRAPH-2F9D7;Lo;0;L;8D77;;;;N;;;;; +2F9D8;CJK COMPATIBILITY IDEOGRAPH-2F9D8;Lo;0;L;27F2F;;;;N;;;;; +2F9D9;CJK COMPATIBILITY IDEOGRAPH-2F9D9;Lo;0;L;20804;;;;N;;;;; +2F9DA;CJK COMPATIBILITY IDEOGRAPH-2F9DA;Lo;0;L;8DCB;;;;N;;;;; +2F9DB;CJK COMPATIBILITY IDEOGRAPH-2F9DB;Lo;0;L;8DBC;;;;N;;;;; +2F9DC;CJK COMPATIBILITY IDEOGRAPH-2F9DC;Lo;0;L;8DF0;;;;N;;;;; +2F9DD;CJK COMPATIBILITY IDEOGRAPH-2F9DD;Lo;0;L;208DE;;;;N;;;;; +2F9DE;CJK COMPATIBILITY IDEOGRAPH-2F9DE;Lo;0;L;8ED4;;;;N;;;;; +2F9DF;CJK COMPATIBILITY IDEOGRAPH-2F9DF;Lo;0;L;8F38;;;;N;;;;; +2F9E0;CJK COMPATIBILITY IDEOGRAPH-2F9E0;Lo;0;L;285D2;;;;N;;;;; +2F9E1;CJK COMPATIBILITY IDEOGRAPH-2F9E1;Lo;0;L;285ED;;;;N;;;;; +2F9E2;CJK COMPATIBILITY IDEOGRAPH-2F9E2;Lo;0;L;9094;;;;N;;;;; +2F9E3;CJK COMPATIBILITY IDEOGRAPH-2F9E3;Lo;0;L;90F1;;;;N;;;;; +2F9E4;CJK COMPATIBILITY IDEOGRAPH-2F9E4;Lo;0;L;9111;;;;N;;;;; +2F9E5;CJK COMPATIBILITY IDEOGRAPH-2F9E5;Lo;0;L;2872E;;;;N;;;;; +2F9E6;CJK COMPATIBILITY IDEOGRAPH-2F9E6;Lo;0;L;911B;;;;N;;;;; +2F9E7;CJK COMPATIBILITY IDEOGRAPH-2F9E7;Lo;0;L;9238;;;;N;;;;; +2F9E8;CJK COMPATIBILITY IDEOGRAPH-2F9E8;Lo;0;L;92D7;;;;N;;;;; +2F9E9;CJK COMPATIBILITY IDEOGRAPH-2F9E9;Lo;0;L;92D8;;;;N;;;;; +2F9EA;CJK COMPATIBILITY IDEOGRAPH-2F9EA;Lo;0;L;927C;;;;N;;;;; +2F9EB;CJK COMPATIBILITY IDEOGRAPH-2F9EB;Lo;0;L;93F9;;;;N;;;;; +2F9EC;CJK COMPATIBILITY IDEOGRAPH-2F9EC;Lo;0;L;9415;;;;N;;;;; +2F9ED;CJK COMPATIBILITY IDEOGRAPH-2F9ED;Lo;0;L;28BFA;;;;N;;;;; +2F9EE;CJK COMPATIBILITY IDEOGRAPH-2F9EE;Lo;0;L;958B;;;;N;;;;; +2F9EF;CJK COMPATIBILITY IDEOGRAPH-2F9EF;Lo;0;L;4995;;;;N;;;;; +2F9F0;CJK COMPATIBILITY IDEOGRAPH-2F9F0;Lo;0;L;95B7;;;;N;;;;; +2F9F1;CJK COMPATIBILITY IDEOGRAPH-2F9F1;Lo;0;L;28D77;;;;N;;;;; +2F9F2;CJK COMPATIBILITY IDEOGRAPH-2F9F2;Lo;0;L;49E6;;;;N;;;;; +2F9F3;CJK COMPATIBILITY IDEOGRAPH-2F9F3;Lo;0;L;96C3;;;;N;;;;; +2F9F4;CJK COMPATIBILITY IDEOGRAPH-2F9F4;Lo;0;L;5DB2;;;;N;;;;; +2F9F5;CJK COMPATIBILITY IDEOGRAPH-2F9F5;Lo;0;L;9723;;;;N;;;;; +2F9F6;CJK COMPATIBILITY IDEOGRAPH-2F9F6;Lo;0;L;29145;;;;N;;;;; +2F9F7;CJK COMPATIBILITY IDEOGRAPH-2F9F7;Lo;0;L;2921A;;;;N;;;;; +2F9F8;CJK COMPATIBILITY IDEOGRAPH-2F9F8;Lo;0;L;4A6E;;;;N;;;;; +2F9F9;CJK COMPATIBILITY IDEOGRAPH-2F9F9;Lo;0;L;4A76;;;;N;;;;; +2F9FA;CJK COMPATIBILITY IDEOGRAPH-2F9FA;Lo;0;L;97E0;;;;N;;;;; +2F9FB;CJK COMPATIBILITY IDEOGRAPH-2F9FB;Lo;0;L;2940A;;;;N;;;;; +2F9FC;CJK COMPATIBILITY IDEOGRAPH-2F9FC;Lo;0;L;4AB2;;;;N;;;;; +2F9FD;CJK COMPATIBILITY IDEOGRAPH-2F9FD;Lo;0;L;29496;;;;N;;;;; +2F9FE;CJK COMPATIBILITY IDEOGRAPH-2F9FE;Lo;0;L;980B;;;;N;;;;; +2F9FF;CJK COMPATIBILITY IDEOGRAPH-2F9FF;Lo;0;L;980B;;;;N;;;;; +2FA00;CJK COMPATIBILITY IDEOGRAPH-2FA00;Lo;0;L;9829;;;;N;;;;; +2FA01;CJK COMPATIBILITY IDEOGRAPH-2FA01;Lo;0;L;295B6;;;;N;;;;; +2FA02;CJK COMPATIBILITY IDEOGRAPH-2FA02;Lo;0;L;98E2;;;;N;;;;; +2FA03;CJK COMPATIBILITY IDEOGRAPH-2FA03;Lo;0;L;4B33;;;;N;;;;; +2FA04;CJK COMPATIBILITY IDEOGRAPH-2FA04;Lo;0;L;9929;;;;N;;;;; +2FA05;CJK COMPATIBILITY IDEOGRAPH-2FA05;Lo;0;L;99A7;;;;N;;;;; +2FA06;CJK COMPATIBILITY IDEOGRAPH-2FA06;Lo;0;L;99C2;;;;N;;;;; +2FA07;CJK COMPATIBILITY IDEOGRAPH-2FA07;Lo;0;L;99FE;;;;N;;;;; +2FA08;CJK COMPATIBILITY IDEOGRAPH-2FA08;Lo;0;L;4BCE;;;;N;;;;; +2FA09;CJK COMPATIBILITY IDEOGRAPH-2FA09;Lo;0;L;29B30;;;;N;;;;; +2FA0A;CJK COMPATIBILITY IDEOGRAPH-2FA0A;Lo;0;L;9B12;;;;N;;;;; +2FA0B;CJK COMPATIBILITY IDEOGRAPH-2FA0B;Lo;0;L;9C40;;;;N;;;;; +2FA0C;CJK COMPATIBILITY IDEOGRAPH-2FA0C;Lo;0;L;9CFD;;;;N;;;;; +2FA0D;CJK COMPATIBILITY IDEOGRAPH-2FA0D;Lo;0;L;4CCE;;;;N;;;;; +2FA0E;CJK COMPATIBILITY IDEOGRAPH-2FA0E;Lo;0;L;4CED;;;;N;;;;; +2FA0F;CJK COMPATIBILITY IDEOGRAPH-2FA0F;Lo;0;L;9D67;;;;N;;;;; +2FA10;CJK COMPATIBILITY IDEOGRAPH-2FA10;Lo;0;L;2A0CE;;;;N;;;;; +2FA11;CJK COMPATIBILITY IDEOGRAPH-2FA11;Lo;0;L;4CF8;;;;N;;;;; +2FA12;CJK COMPATIBILITY IDEOGRAPH-2FA12;Lo;0;L;2A105;;;;N;;;;; +2FA13;CJK COMPATIBILITY IDEOGRAPH-2FA13;Lo;0;L;2A20E;;;;N;;;;; +2FA14;CJK COMPATIBILITY IDEOGRAPH-2FA14;Lo;0;L;2A291;;;;N;;;;; +2FA15;CJK COMPATIBILITY IDEOGRAPH-2FA15;Lo;0;L;9EBB;;;;N;;;;; +2FA16;CJK COMPATIBILITY IDEOGRAPH-2FA16;Lo;0;L;4D56;;;;N;;;;; +2FA17;CJK COMPATIBILITY IDEOGRAPH-2FA17;Lo;0;L;9EF9;;;;N;;;;; +2FA18;CJK COMPATIBILITY IDEOGRAPH-2FA18;Lo;0;L;9EFE;;;;N;;;;; +2FA19;CJK COMPATIBILITY IDEOGRAPH-2FA19;Lo;0;L;9F05;;;;N;;;;; +2FA1A;CJK COMPATIBILITY IDEOGRAPH-2FA1A;Lo;0;L;9F0F;;;;N;;;;; +2FA1B;CJK COMPATIBILITY IDEOGRAPH-2FA1B;Lo;0;L;9F16;;;;N;;;;; +2FA1C;CJK COMPATIBILITY IDEOGRAPH-2FA1C;Lo;0;L;9F3B;;;;N;;;;; +2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;; +E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; +E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; +E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; +E0022;TAG QUOTATION MARK;Cf;0;BN;;;;;N;;;;; +E0023;TAG NUMBER SIGN;Cf;0;BN;;;;;N;;;;; +E0024;TAG DOLLAR SIGN;Cf;0;BN;;;;;N;;;;; +E0025;TAG PERCENT SIGN;Cf;0;BN;;;;;N;;;;; +E0026;TAG AMPERSAND;Cf;0;BN;;;;;N;;;;; +E0027;TAG APOSTROPHE;Cf;0;BN;;;;;N;;;;; +E0028;TAG LEFT PARENTHESIS;Cf;0;BN;;;;;N;;;;; +E0029;TAG RIGHT PARENTHESIS;Cf;0;BN;;;;;N;;;;; +E002A;TAG ASTERISK;Cf;0;BN;;;;;N;;;;; +E002B;TAG PLUS SIGN;Cf;0;BN;;;;;N;;;;; +E002C;TAG COMMA;Cf;0;BN;;;;;N;;;;; +E002D;TAG HYPHEN-MINUS;Cf;0;BN;;;;;N;;;;; +E002E;TAG FULL STOP;Cf;0;BN;;;;;N;;;;; +E002F;TAG SOLIDUS;Cf;0;BN;;;;;N;;;;; +E0030;TAG DIGIT ZERO;Cf;0;BN;;;;;N;;;;; +E0031;TAG DIGIT ONE;Cf;0;BN;;;;;N;;;;; +E0032;TAG DIGIT TWO;Cf;0;BN;;;;;N;;;;; +E0033;TAG DIGIT THREE;Cf;0;BN;;;;;N;;;;; +E0034;TAG DIGIT FOUR;Cf;0;BN;;;;;N;;;;; +E0035;TAG DIGIT FIVE;Cf;0;BN;;;;;N;;;;; +E0036;TAG DIGIT SIX;Cf;0;BN;;;;;N;;;;; +E0037;TAG DIGIT SEVEN;Cf;0;BN;;;;;N;;;;; +E0038;TAG DIGIT EIGHT;Cf;0;BN;;;;;N;;;;; +E0039;TAG DIGIT NINE;Cf;0;BN;;;;;N;;;;; +E003A;TAG COLON;Cf;0;BN;;;;;N;;;;; +E003B;TAG SEMICOLON;Cf;0;BN;;;;;N;;;;; +E003C;TAG LESS-THAN SIGN;Cf;0;BN;;;;;N;;;;; +E003D;TAG EQUALS SIGN;Cf;0;BN;;;;;N;;;;; +E003E;TAG GREATER-THAN SIGN;Cf;0;BN;;;;;N;;;;; +E003F;TAG QUESTION MARK;Cf;0;BN;;;;;N;;;;; +E0040;TAG COMMERCIAL AT;Cf;0;BN;;;;;N;;;;; +E0041;TAG LATIN CAPITAL LETTER A;Cf;0;BN;;;;;N;;;;; +E0042;TAG LATIN CAPITAL LETTER B;Cf;0;BN;;;;;N;;;;; +E0043;TAG LATIN CAPITAL LETTER C;Cf;0;BN;;;;;N;;;;; +E0044;TAG LATIN CAPITAL LETTER D;Cf;0;BN;;;;;N;;;;; +E0045;TAG LATIN CAPITAL LETTER E;Cf;0;BN;;;;;N;;;;; +E0046;TAG LATIN CAPITAL LETTER F;Cf;0;BN;;;;;N;;;;; +E0047;TAG LATIN CAPITAL LETTER G;Cf;0;BN;;;;;N;;;;; +E0048;TAG LATIN CAPITAL LETTER H;Cf;0;BN;;;;;N;;;;; +E0049;TAG LATIN CAPITAL LETTER I;Cf;0;BN;;;;;N;;;;; +E004A;TAG LATIN CAPITAL LETTER J;Cf;0;BN;;;;;N;;;;; +E004B;TAG LATIN CAPITAL LETTER K;Cf;0;BN;;;;;N;;;;; +E004C;TAG LATIN CAPITAL LETTER L;Cf;0;BN;;;;;N;;;;; +E004D;TAG LATIN CAPITAL LETTER M;Cf;0;BN;;;;;N;;;;; +E004E;TAG LATIN CAPITAL LETTER N;Cf;0;BN;;;;;N;;;;; +E004F;TAG LATIN CAPITAL LETTER O;Cf;0;BN;;;;;N;;;;; +E0050;TAG LATIN CAPITAL LETTER P;Cf;0;BN;;;;;N;;;;; +E0051;TAG LATIN CAPITAL LETTER Q;Cf;0;BN;;;;;N;;;;; +E0052;TAG LATIN CAPITAL LETTER R;Cf;0;BN;;;;;N;;;;; +E0053;TAG LATIN CAPITAL LETTER S;Cf;0;BN;;;;;N;;;;; +E0054;TAG LATIN CAPITAL LETTER T;Cf;0;BN;;;;;N;;;;; +E0055;TAG LATIN CAPITAL LETTER U;Cf;0;BN;;;;;N;;;;; +E0056;TAG LATIN CAPITAL LETTER V;Cf;0;BN;;;;;N;;;;; +E0057;TAG LATIN CAPITAL LETTER W;Cf;0;BN;;;;;N;;;;; +E0058;TAG LATIN CAPITAL LETTER X;Cf;0;BN;;;;;N;;;;; +E0059;TAG LATIN CAPITAL LETTER Y;Cf;0;BN;;;;;N;;;;; +E005A;TAG LATIN CAPITAL LETTER Z;Cf;0;BN;;;;;N;;;;; +E005B;TAG LEFT SQUARE BRACKET;Cf;0;BN;;;;;N;;;;; +E005C;TAG REVERSE SOLIDUS;Cf;0;BN;;;;;N;;;;; +E005D;TAG RIGHT SQUARE BRACKET;Cf;0;BN;;;;;N;;;;; +E005E;TAG CIRCUMFLEX ACCENT;Cf;0;BN;;;;;N;;;;; +E005F;TAG LOW LINE;Cf;0;BN;;;;;N;;;;; +E0060;TAG GRAVE ACCENT;Cf;0;BN;;;;;N;;;;; +E0061;TAG LATIN SMALL LETTER A;Cf;0;BN;;;;;N;;;;; +E0062;TAG LATIN SMALL LETTER B;Cf;0;BN;;;;;N;;;;; +E0063;TAG LATIN SMALL LETTER C;Cf;0;BN;;;;;N;;;;; +E0064;TAG LATIN SMALL LETTER D;Cf;0;BN;;;;;N;;;;; +E0065;TAG LATIN SMALL LETTER E;Cf;0;BN;;;;;N;;;;; +E0066;TAG LATIN SMALL LETTER F;Cf;0;BN;;;;;N;;;;; +E0067;TAG LATIN SMALL LETTER G;Cf;0;BN;;;;;N;;;;; +E0068;TAG LATIN SMALL LETTER H;Cf;0;BN;;;;;N;;;;; +E0069;TAG LATIN SMALL LETTER I;Cf;0;BN;;;;;N;;;;; +E006A;TAG LATIN SMALL LETTER J;Cf;0;BN;;;;;N;;;;; +E006B;TAG LATIN SMALL LETTER K;Cf;0;BN;;;;;N;;;;; +E006C;TAG LATIN SMALL LETTER L;Cf;0;BN;;;;;N;;;;; +E006D;TAG LATIN SMALL LETTER M;Cf;0;BN;;;;;N;;;;; +E006E;TAG LATIN SMALL LETTER N;Cf;0;BN;;;;;N;;;;; +E006F;TAG LATIN SMALL LETTER O;Cf;0;BN;;;;;N;;;;; +E0070;TAG LATIN SMALL LETTER P;Cf;0;BN;;;;;N;;;;; +E0071;TAG LATIN SMALL LETTER Q;Cf;0;BN;;;;;N;;;;; +E0072;TAG LATIN SMALL LETTER R;Cf;0;BN;;;;;N;;;;; +E0073;TAG LATIN SMALL LETTER S;Cf;0;BN;;;;;N;;;;; +E0074;TAG LATIN SMALL LETTER T;Cf;0;BN;;;;;N;;;;; +E0075;TAG LATIN SMALL LETTER U;Cf;0;BN;;;;;N;;;;; +E0076;TAG LATIN SMALL LETTER V;Cf;0;BN;;;;;N;;;;; +E0077;TAG LATIN SMALL LETTER W;Cf;0;BN;;;;;N;;;;; +E0078;TAG LATIN SMALL LETTER X;Cf;0;BN;;;;;N;;;;; +E0079;TAG LATIN SMALL LETTER Y;Cf;0;BN;;;;;N;;;;; +E007A;TAG LATIN SMALL LETTER Z;Cf;0;BN;;;;;N;;;;; +E007B;TAG LEFT CURLY BRACKET;Cf;0;BN;;;;;N;;;;; +E007C;TAG VERTICAL LINE;Cf;0;BN;;;;;N;;;;; +E007D;TAG RIGHT CURLY BRACKET;Cf;0;BN;;;;;N;;;;; +E007E;TAG TILDE;Cf;0;BN;;;;;N;;;;; +E007F;CANCEL TAG;Cf;0;BN;;;;;N;;;;; +E0100;VARIATION SELECTOR-17;Mn;0;NSM;;;;;N;;;;; +E0101;VARIATION SELECTOR-18;Mn;0;NSM;;;;;N;;;;; +E0102;VARIATION SELECTOR-19;Mn;0;NSM;;;;;N;;;;; +E0103;VARIATION SELECTOR-20;Mn;0;NSM;;;;;N;;;;; +E0104;VARIATION SELECTOR-21;Mn;0;NSM;;;;;N;;;;; +E0105;VARIATION SELECTOR-22;Mn;0;NSM;;;;;N;;;;; +E0106;VARIATION SELECTOR-23;Mn;0;NSM;;;;;N;;;;; +E0107;VARIATION SELECTOR-24;Mn;0;NSM;;;;;N;;;;; +E0108;VARIATION SELECTOR-25;Mn;0;NSM;;;;;N;;;;; +E0109;VARIATION SELECTOR-26;Mn;0;NSM;;;;;N;;;;; +E010A;VARIATION SELECTOR-27;Mn;0;NSM;;;;;N;;;;; +E010B;VARIATION SELECTOR-28;Mn;0;NSM;;;;;N;;;;; +E010C;VARIATION SELECTOR-29;Mn;0;NSM;;;;;N;;;;; +E010D;VARIATION SELECTOR-30;Mn;0;NSM;;;;;N;;;;; +E010E;VARIATION SELECTOR-31;Mn;0;NSM;;;;;N;;;;; +E010F;VARIATION SELECTOR-32;Mn;0;NSM;;;;;N;;;;; +E0110;VARIATION SELECTOR-33;Mn;0;NSM;;;;;N;;;;; +E0111;VARIATION SELECTOR-34;Mn;0;NSM;;;;;N;;;;; +E0112;VARIATION SELECTOR-35;Mn;0;NSM;;;;;N;;;;; +E0113;VARIATION SELECTOR-36;Mn;0;NSM;;;;;N;;;;; +E0114;VARIATION SELECTOR-37;Mn;0;NSM;;;;;N;;;;; +E0115;VARIATION SELECTOR-38;Mn;0;NSM;;;;;N;;;;; +E0116;VARIATION SELECTOR-39;Mn;0;NSM;;;;;N;;;;; +E0117;VARIATION SELECTOR-40;Mn;0;NSM;;;;;N;;;;; +E0118;VARIATION SELECTOR-41;Mn;0;NSM;;;;;N;;;;; +E0119;VARIATION SELECTOR-42;Mn;0;NSM;;;;;N;;;;; +E011A;VARIATION SELECTOR-43;Mn;0;NSM;;;;;N;;;;; +E011B;VARIATION SELECTOR-44;Mn;0;NSM;;;;;N;;;;; +E011C;VARIATION SELECTOR-45;Mn;0;NSM;;;;;N;;;;; +E011D;VARIATION SELECTOR-46;Mn;0;NSM;;;;;N;;;;; +E011E;VARIATION SELECTOR-47;Mn;0;NSM;;;;;N;;;;; +E011F;VARIATION SELECTOR-48;Mn;0;NSM;;;;;N;;;;; +E0120;VARIATION SELECTOR-49;Mn;0;NSM;;;;;N;;;;; +E0121;VARIATION SELECTOR-50;Mn;0;NSM;;;;;N;;;;; +E0122;VARIATION SELECTOR-51;Mn;0;NSM;;;;;N;;;;; +E0123;VARIATION SELECTOR-52;Mn;0;NSM;;;;;N;;;;; +E0124;VARIATION SELECTOR-53;Mn;0;NSM;;;;;N;;;;; +E0125;VARIATION SELECTOR-54;Mn;0;NSM;;;;;N;;;;; +E0126;VARIATION SELECTOR-55;Mn;0;NSM;;;;;N;;;;; +E0127;VARIATION SELECTOR-56;Mn;0;NSM;;;;;N;;;;; +E0128;VARIATION SELECTOR-57;Mn;0;NSM;;;;;N;;;;; +E0129;VARIATION SELECTOR-58;Mn;0;NSM;;;;;N;;;;; +E012A;VARIATION SELECTOR-59;Mn;0;NSM;;;;;N;;;;; +E012B;VARIATION SELECTOR-60;Mn;0;NSM;;;;;N;;;;; +E012C;VARIATION SELECTOR-61;Mn;0;NSM;;;;;N;;;;; +E012D;VARIATION SELECTOR-62;Mn;0;NSM;;;;;N;;;;; +E012E;VARIATION SELECTOR-63;Mn;0;NSM;;;;;N;;;;; +E012F;VARIATION SELECTOR-64;Mn;0;NSM;;;;;N;;;;; +E0130;VARIATION SELECTOR-65;Mn;0;NSM;;;;;N;;;;; +E0131;VARIATION SELECTOR-66;Mn;0;NSM;;;;;N;;;;; +E0132;VARIATION SELECTOR-67;Mn;0;NSM;;;;;N;;;;; +E0133;VARIATION SELECTOR-68;Mn;0;NSM;;;;;N;;;;; +E0134;VARIATION SELECTOR-69;Mn;0;NSM;;;;;N;;;;; +E0135;VARIATION SELECTOR-70;Mn;0;NSM;;;;;N;;;;; +E0136;VARIATION SELECTOR-71;Mn;0;NSM;;;;;N;;;;; +E0137;VARIATION SELECTOR-72;Mn;0;NSM;;;;;N;;;;; +E0138;VARIATION SELECTOR-73;Mn;0;NSM;;;;;N;;;;; +E0139;VARIATION SELECTOR-74;Mn;0;NSM;;;;;N;;;;; +E013A;VARIATION SELECTOR-75;Mn;0;NSM;;;;;N;;;;; +E013B;VARIATION SELECTOR-76;Mn;0;NSM;;;;;N;;;;; +E013C;VARIATION SELECTOR-77;Mn;0;NSM;;;;;N;;;;; +E013D;VARIATION SELECTOR-78;Mn;0;NSM;;;;;N;;;;; +E013E;VARIATION SELECTOR-79;Mn;0;NSM;;;;;N;;;;; +E013F;VARIATION SELECTOR-80;Mn;0;NSM;;;;;N;;;;; +E0140;VARIATION SELECTOR-81;Mn;0;NSM;;;;;N;;;;; +E0141;VARIATION SELECTOR-82;Mn;0;NSM;;;;;N;;;;; +E0142;VARIATION SELECTOR-83;Mn;0;NSM;;;;;N;;;;; +E0143;VARIATION SELECTOR-84;Mn;0;NSM;;;;;N;;;;; +E0144;VARIATION SELECTOR-85;Mn;0;NSM;;;;;N;;;;; +E0145;VARIATION SELECTOR-86;Mn;0;NSM;;;;;N;;;;; +E0146;VARIATION SELECTOR-87;Mn;0;NSM;;;;;N;;;;; +E0147;VARIATION SELECTOR-88;Mn;0;NSM;;;;;N;;;;; +E0148;VARIATION SELECTOR-89;Mn;0;NSM;;;;;N;;;;; +E0149;VARIATION SELECTOR-90;Mn;0;NSM;;;;;N;;;;; +E014A;VARIATION SELECTOR-91;Mn;0;NSM;;;;;N;;;;; +E014B;VARIATION SELECTOR-92;Mn;0;NSM;;;;;N;;;;; +E014C;VARIATION SELECTOR-93;Mn;0;NSM;;;;;N;;;;; +E014D;VARIATION SELECTOR-94;Mn;0;NSM;;;;;N;;;;; +E014E;VARIATION SELECTOR-95;Mn;0;NSM;;;;;N;;;;; +E014F;VARIATION SELECTOR-96;Mn;0;NSM;;;;;N;;;;; +E0150;VARIATION SELECTOR-97;Mn;0;NSM;;;;;N;;;;; +E0151;VARIATION SELECTOR-98;Mn;0;NSM;;;;;N;;;;; +E0152;VARIATION SELECTOR-99;Mn;0;NSM;;;;;N;;;;; +E0153;VARIATION SELECTOR-100;Mn;0;NSM;;;;;N;;;;; +E0154;VARIATION SELECTOR-101;Mn;0;NSM;;;;;N;;;;; +E0155;VARIATION SELECTOR-102;Mn;0;NSM;;;;;N;;;;; +E0156;VARIATION SELECTOR-103;Mn;0;NSM;;;;;N;;;;; +E0157;VARIATION SELECTOR-104;Mn;0;NSM;;;;;N;;;;; +E0158;VARIATION SELECTOR-105;Mn;0;NSM;;;;;N;;;;; +E0159;VARIATION SELECTOR-106;Mn;0;NSM;;;;;N;;;;; +E015A;VARIATION SELECTOR-107;Mn;0;NSM;;;;;N;;;;; +E015B;VARIATION SELECTOR-108;Mn;0;NSM;;;;;N;;;;; +E015C;VARIATION SELECTOR-109;Mn;0;NSM;;;;;N;;;;; +E015D;VARIATION SELECTOR-110;Mn;0;NSM;;;;;N;;;;; +E015E;VARIATION SELECTOR-111;Mn;0;NSM;;;;;N;;;;; +E015F;VARIATION SELECTOR-112;Mn;0;NSM;;;;;N;;;;; +E0160;VARIATION SELECTOR-113;Mn;0;NSM;;;;;N;;;;; +E0161;VARIATION SELECTOR-114;Mn;0;NSM;;;;;N;;;;; +E0162;VARIATION SELECTOR-115;Mn;0;NSM;;;;;N;;;;; +E0163;VARIATION SELECTOR-116;Mn;0;NSM;;;;;N;;;;; +E0164;VARIATION SELECTOR-117;Mn;0;NSM;;;;;N;;;;; +E0165;VARIATION SELECTOR-118;Mn;0;NSM;;;;;N;;;;; +E0166;VARIATION SELECTOR-119;Mn;0;NSM;;;;;N;;;;; +E0167;VARIATION SELECTOR-120;Mn;0;NSM;;;;;N;;;;; +E0168;VARIATION SELECTOR-121;Mn;0;NSM;;;;;N;;;;; +E0169;VARIATION SELECTOR-122;Mn;0;NSM;;;;;N;;;;; +E016A;VARIATION SELECTOR-123;Mn;0;NSM;;;;;N;;;;; +E016B;VARIATION SELECTOR-124;Mn;0;NSM;;;;;N;;;;; +E016C;VARIATION SELECTOR-125;Mn;0;NSM;;;;;N;;;;; +E016D;VARIATION SELECTOR-126;Mn;0;NSM;;;;;N;;;;; +E016E;VARIATION SELECTOR-127;Mn;0;NSM;;;;;N;;;;; +E016F;VARIATION SELECTOR-128;Mn;0;NSM;;;;;N;;;;; +E0170;VARIATION SELECTOR-129;Mn;0;NSM;;;;;N;;;;; +E0171;VARIATION SELECTOR-130;Mn;0;NSM;;;;;N;;;;; +E0172;VARIATION SELECTOR-131;Mn;0;NSM;;;;;N;;;;; +E0173;VARIATION SELECTOR-132;Mn;0;NSM;;;;;N;;;;; +E0174;VARIATION SELECTOR-133;Mn;0;NSM;;;;;N;;;;; +E0175;VARIATION SELECTOR-134;Mn;0;NSM;;;;;N;;;;; +E0176;VARIATION SELECTOR-135;Mn;0;NSM;;;;;N;;;;; +E0177;VARIATION SELECTOR-136;Mn;0;NSM;;;;;N;;;;; +E0178;VARIATION SELECTOR-137;Mn;0;NSM;;;;;N;;;;; +E0179;VARIATION SELECTOR-138;Mn;0;NSM;;;;;N;;;;; +E017A;VARIATION SELECTOR-139;Mn;0;NSM;;;;;N;;;;; +E017B;VARIATION SELECTOR-140;Mn;0;NSM;;;;;N;;;;; +E017C;VARIATION SELECTOR-141;Mn;0;NSM;;;;;N;;;;; +E017D;VARIATION SELECTOR-142;Mn;0;NSM;;;;;N;;;;; +E017E;VARIATION SELECTOR-143;Mn;0;NSM;;;;;N;;;;; +E017F;VARIATION SELECTOR-144;Mn;0;NSM;;;;;N;;;;; +E0180;VARIATION SELECTOR-145;Mn;0;NSM;;;;;N;;;;; +E0181;VARIATION SELECTOR-146;Mn;0;NSM;;;;;N;;;;; +E0182;VARIATION SELECTOR-147;Mn;0;NSM;;;;;N;;;;; +E0183;VARIATION SELECTOR-148;Mn;0;NSM;;;;;N;;;;; +E0184;VARIATION SELECTOR-149;Mn;0;NSM;;;;;N;;;;; +E0185;VARIATION SELECTOR-150;Mn;0;NSM;;;;;N;;;;; +E0186;VARIATION SELECTOR-151;Mn;0;NSM;;;;;N;;;;; +E0187;VARIATION SELECTOR-152;Mn;0;NSM;;;;;N;;;;; +E0188;VARIATION SELECTOR-153;Mn;0;NSM;;;;;N;;;;; +E0189;VARIATION SELECTOR-154;Mn;0;NSM;;;;;N;;;;; +E018A;VARIATION SELECTOR-155;Mn;0;NSM;;;;;N;;;;; +E018B;VARIATION SELECTOR-156;Mn;0;NSM;;;;;N;;;;; +E018C;VARIATION SELECTOR-157;Mn;0;NSM;;;;;N;;;;; +E018D;VARIATION SELECTOR-158;Mn;0;NSM;;;;;N;;;;; +E018E;VARIATION SELECTOR-159;Mn;0;NSM;;;;;N;;;;; +E018F;VARIATION SELECTOR-160;Mn;0;NSM;;;;;N;;;;; +E0190;VARIATION SELECTOR-161;Mn;0;NSM;;;;;N;;;;; +E0191;VARIATION SELECTOR-162;Mn;0;NSM;;;;;N;;;;; +E0192;VARIATION SELECTOR-163;Mn;0;NSM;;;;;N;;;;; +E0193;VARIATION SELECTOR-164;Mn;0;NSM;;;;;N;;;;; +E0194;VARIATION SELECTOR-165;Mn;0;NSM;;;;;N;;;;; +E0195;VARIATION SELECTOR-166;Mn;0;NSM;;;;;N;;;;; +E0196;VARIATION SELECTOR-167;Mn;0;NSM;;;;;N;;;;; +E0197;VARIATION SELECTOR-168;Mn;0;NSM;;;;;N;;;;; +E0198;VARIATION SELECTOR-169;Mn;0;NSM;;;;;N;;;;; +E0199;VARIATION SELECTOR-170;Mn;0;NSM;;;;;N;;;;; +E019A;VARIATION SELECTOR-171;Mn;0;NSM;;;;;N;;;;; +E019B;VARIATION SELECTOR-172;Mn;0;NSM;;;;;N;;;;; +E019C;VARIATION SELECTOR-173;Mn;0;NSM;;;;;N;;;;; +E019D;VARIATION SELECTOR-174;Mn;0;NSM;;;;;N;;;;; +E019E;VARIATION SELECTOR-175;Mn;0;NSM;;;;;N;;;;; +E019F;VARIATION SELECTOR-176;Mn;0;NSM;;;;;N;;;;; +E01A0;VARIATION SELECTOR-177;Mn;0;NSM;;;;;N;;;;; +E01A1;VARIATION SELECTOR-178;Mn;0;NSM;;;;;N;;;;; +E01A2;VARIATION SELECTOR-179;Mn;0;NSM;;;;;N;;;;; +E01A3;VARIATION SELECTOR-180;Mn;0;NSM;;;;;N;;;;; +E01A4;VARIATION SELECTOR-181;Mn;0;NSM;;;;;N;;;;; +E01A5;VARIATION SELECTOR-182;Mn;0;NSM;;;;;N;;;;; +E01A6;VARIATION SELECTOR-183;Mn;0;NSM;;;;;N;;;;; +E01A7;VARIATION SELECTOR-184;Mn;0;NSM;;;;;N;;;;; +E01A8;VARIATION SELECTOR-185;Mn;0;NSM;;;;;N;;;;; +E01A9;VARIATION SELECTOR-186;Mn;0;NSM;;;;;N;;;;; +E01AA;VARIATION SELECTOR-187;Mn;0;NSM;;;;;N;;;;; +E01AB;VARIATION SELECTOR-188;Mn;0;NSM;;;;;N;;;;; +E01AC;VARIATION SELECTOR-189;Mn;0;NSM;;;;;N;;;;; +E01AD;VARIATION SELECTOR-190;Mn;0;NSM;;;;;N;;;;; +E01AE;VARIATION SELECTOR-191;Mn;0;NSM;;;;;N;;;;; +E01AF;VARIATION SELECTOR-192;Mn;0;NSM;;;;;N;;;;; +E01B0;VARIATION SELECTOR-193;Mn;0;NSM;;;;;N;;;;; +E01B1;VARIATION SELECTOR-194;Mn;0;NSM;;;;;N;;;;; +E01B2;VARIATION SELECTOR-195;Mn;0;NSM;;;;;N;;;;; +E01B3;VARIATION SELECTOR-196;Mn;0;NSM;;;;;N;;;;; +E01B4;VARIATION SELECTOR-197;Mn;0;NSM;;;;;N;;;;; +E01B5;VARIATION SELECTOR-198;Mn;0;NSM;;;;;N;;;;; +E01B6;VARIATION SELECTOR-199;Mn;0;NSM;;;;;N;;;;; +E01B7;VARIATION SELECTOR-200;Mn;0;NSM;;;;;N;;;;; +E01B8;VARIATION SELECTOR-201;Mn;0;NSM;;;;;N;;;;; +E01B9;VARIATION SELECTOR-202;Mn;0;NSM;;;;;N;;;;; +E01BA;VARIATION SELECTOR-203;Mn;0;NSM;;;;;N;;;;; +E01BB;VARIATION SELECTOR-204;Mn;0;NSM;;;;;N;;;;; +E01BC;VARIATION SELECTOR-205;Mn;0;NSM;;;;;N;;;;; +E01BD;VARIATION SELECTOR-206;Mn;0;NSM;;;;;N;;;;; +E01BE;VARIATION SELECTOR-207;Mn;0;NSM;;;;;N;;;;; +E01BF;VARIATION SELECTOR-208;Mn;0;NSM;;;;;N;;;;; +E01C0;VARIATION SELECTOR-209;Mn;0;NSM;;;;;N;;;;; +E01C1;VARIATION SELECTOR-210;Mn;0;NSM;;;;;N;;;;; +E01C2;VARIATION SELECTOR-211;Mn;0;NSM;;;;;N;;;;; +E01C3;VARIATION SELECTOR-212;Mn;0;NSM;;;;;N;;;;; +E01C4;VARIATION SELECTOR-213;Mn;0;NSM;;;;;N;;;;; +E01C5;VARIATION SELECTOR-214;Mn;0;NSM;;;;;N;;;;; +E01C6;VARIATION SELECTOR-215;Mn;0;NSM;;;;;N;;;;; +E01C7;VARIATION SELECTOR-216;Mn;0;NSM;;;;;N;;;;; +E01C8;VARIATION SELECTOR-217;Mn;0;NSM;;;;;N;;;;; +E01C9;VARIATION SELECTOR-218;Mn;0;NSM;;;;;N;;;;; +E01CA;VARIATION SELECTOR-219;Mn;0;NSM;;;;;N;;;;; +E01CB;VARIATION SELECTOR-220;Mn;0;NSM;;;;;N;;;;; +E01CC;VARIATION SELECTOR-221;Mn;0;NSM;;;;;N;;;;; +E01CD;VARIATION SELECTOR-222;Mn;0;NSM;;;;;N;;;;; +E01CE;VARIATION SELECTOR-223;Mn;0;NSM;;;;;N;;;;; +E01CF;VARIATION SELECTOR-224;Mn;0;NSM;;;;;N;;;;; +E01D0;VARIATION SELECTOR-225;Mn;0;NSM;;;;;N;;;;; +E01D1;VARIATION SELECTOR-226;Mn;0;NSM;;;;;N;;;;; +E01D2;VARIATION SELECTOR-227;Mn;0;NSM;;;;;N;;;;; +E01D3;VARIATION SELECTOR-228;Mn;0;NSM;;;;;N;;;;; +E01D4;VARIATION SELECTOR-229;Mn;0;NSM;;;;;N;;;;; +E01D5;VARIATION SELECTOR-230;Mn;0;NSM;;;;;N;;;;; +E01D6;VARIATION SELECTOR-231;Mn;0;NSM;;;;;N;;;;; +E01D7;VARIATION SELECTOR-232;Mn;0;NSM;;;;;N;;;;; +E01D8;VARIATION SELECTOR-233;Mn;0;NSM;;;;;N;;;;; +E01D9;VARIATION SELECTOR-234;Mn;0;NSM;;;;;N;;;;; +E01DA;VARIATION SELECTOR-235;Mn;0;NSM;;;;;N;;;;; +E01DB;VARIATION SELECTOR-236;Mn;0;NSM;;;;;N;;;;; +E01DC;VARIATION SELECTOR-237;Mn;0;NSM;;;;;N;;;;; +E01DD;VARIATION SELECTOR-238;Mn;0;NSM;;;;;N;;;;; +E01DE;VARIATION SELECTOR-239;Mn;0;NSM;;;;;N;;;;; +E01DF;VARIATION SELECTOR-240;Mn;0;NSM;;;;;N;;;;; +E01E0;VARIATION SELECTOR-241;Mn;0;NSM;;;;;N;;;;; +E01E1;VARIATION SELECTOR-242;Mn;0;NSM;;;;;N;;;;; +E01E2;VARIATION SELECTOR-243;Mn;0;NSM;;;;;N;;;;; +E01E3;VARIATION SELECTOR-244;Mn;0;NSM;;;;;N;;;;; +E01E4;VARIATION SELECTOR-245;Mn;0;NSM;;;;;N;;;;; +E01E5;VARIATION SELECTOR-246;Mn;0;NSM;;;;;N;;;;; +E01E6;VARIATION SELECTOR-247;Mn;0;NSM;;;;;N;;;;; +E01E7;VARIATION SELECTOR-248;Mn;0;NSM;;;;;N;;;;; +E01E8;VARIATION SELECTOR-249;Mn;0;NSM;;;;;N;;;;; +E01E9;VARIATION SELECTOR-250;Mn;0;NSM;;;;;N;;;;; +E01EA;VARIATION SELECTOR-251;Mn;0;NSM;;;;;N;;;;; +E01EB;VARIATION SELECTOR-252;Mn;0;NSM;;;;;N;;;;; +E01EC;VARIATION SELECTOR-253;Mn;0;NSM;;;;;N;;;;; +E01ED;VARIATION SELECTOR-254;Mn;0;NSM;;;;;N;;;;; +E01EE;VARIATION SELECTOR-255;Mn;0;NSM;;;;;N;;;;; +E01EF;VARIATION SELECTOR-256;Mn;0;NSM;;;;;N;;;;; +F0000;;Co;0;L;;;;;N;;;;; +FFFFD;;Co;0;L;;;;;N;;;;; +100000;;Co;0;L;;;;;N;;;;; +10FFFD;;Co;0;L;;;;;N;;;;; diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl new file mode 100644 index 0000000000000000000000000000000000000000..1306629da82a51dd6cee23512e6c33113294f67b --- /dev/null +++ b/ext/fts3/unicode/mkunicode.tcl @@ -0,0 +1,987 @@ + +source [file join [file dirname [info script]] parseunicode.tcl] + +proc print_rd {map} { + global tl_lookup_table + set aChar [list] + set lRange [list] + + set nRange 1 + set iFirst [lindex $map 0 0] + set cPrev [lindex $map 0 1] + set fPrev [lindex $map 0 2] + + foreach m [lrange $map 1 end] { + foreach {i c f} $m {} + + if {$cPrev == $c && $fPrev==$f} { + for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} { + if {[info exists tl_lookup_table($j)]==0} break + } + + if {$j==$i} { + set nNew [expr {(1 + $i - $iFirst)}] + if {$nNew<=8} { + set nRange $nNew + continue + } + } + } + + lappend lRange [list $iFirst $nRange] + lappend aChar $cPrev + lappend aFlag $fPrev + + set iFirst $i + set cPrev $c + set fPrev $f + set nRange 1 + } + lappend lRange [list $iFirst $nRange] + lappend aChar $cPrev + lappend aFlag $fPrev + + puts "/*" + puts "** If the argument is a codepoint corresponding to a lowercase letter" + puts "** in the ASCII range with a diacritic added, return the codepoint" + puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN" + puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER" + puts "** E\"). The resuls of passing a codepoint that corresponds to an" + puts "** uppercase letter are undefined." + puts "*/" + puts "static int ${::remove_diacritic}(int c, int bComplex)\{" + puts " unsigned short aDia\[\] = \{" + puts -nonewline " 0, " + set i 1 + foreach r $lRange { + foreach {iCode nRange} $r {} + if {($i % 8)==0} {puts "" ; puts -nonewline " " } + incr i + + puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]] + puts -nonewline ", " + } + puts "" + puts " \};" + puts "#define HIBIT ((unsigned char)0x80)" + puts " unsigned char aChar\[\] = \{" + puts -nonewline " '\\0', " + set i 1 + foreach c $aChar f $aFlag { + if { $f } { + set str "'$c'|HIBIT, " + } else { + set str "'$c', " + } + if {$c == ""} { set str "'\\0', " } + + if {($i % 6)==0} {puts "" ; puts -nonewline " " } + incr i + puts -nonewline "$str" + } + puts "" + puts " \};" + puts { + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);} + puts "\}" +} + +proc print_isdiacritic {zFunc map} { + + set lCode [list] + foreach m $map { + foreach {code char flag} $m {} + if {$flag} continue + if {$code && $char == ""} { lappend lCode $code } + } + set lCode [lsort -integer $lCode] + set iFirst [lindex $lCode 0] + set iLast [lindex $lCode end] + + set i1 0 + set i2 0 + + foreach c $lCode { + set i [expr $c - $iFirst] + if {$i < 32} { + set i1 [expr {$i1 | (1<<$i)}] + } else { + set i2 [expr {$i2 | (1<<($i-32))}] + } + } + + puts "/*" + puts "** Return true if the argument interpreted as a unicode codepoint" + puts "** is a diacritical modifier character." + puts "*/" + puts "int ${zFunc}\(int c)\{" + puts " unsigned int mask0 = [format "0x%08X" $i1];" + puts " unsigned int mask1 = [format "0x%08X" $i2];" + + puts " if( c<$iFirst || c>$iLast ) return 0;" + puts " return (c < $iFirst+32) ?" + puts " (mask0 & ((unsigned int)1 << (c-$iFirst))) :" + puts " (mask1 & ((unsigned int)1 << (c-$iFirst-32)));" + puts "\}" +} + + +#------------------------------------------------------------------------- + +proc an_load_separator_ranges {} { + global unicodedata.txt + set lSep [an_load_unicodedata_text ${unicodedata.txt}] + unset -nocomplain iFirst + unset -nocomplain nRange + set lRange [list] + foreach sep $lSep { + if {0==[info exists iFirst]} { + set iFirst $sep + set nRange 1 + } elseif { $sep == ($iFirst+$nRange) } { + incr nRange + } else { + lappend lRange [list $iFirst $nRange] + set iFirst $sep + set nRange 1 + } + } + lappend lRange [list $iFirst $nRange] + set lRange +} + +proc an_print_range_array {lRange} { + set iFirstMax 0 + set nRangeMax 0 + foreach range $lRange { + foreach {iFirst nRange} $range {} + if {$iFirst > $iFirstMax} {set iFirstMax $iFirst} + if {$nRange > $nRangeMax} {set nRangeMax $nRange} + } + if {$iFirstMax >= (1<<22)} {error "first-max is too large for format"} + if {$nRangeMax >= (1<<10)} {error "range-max is too large for format"} + + puts -nonewline " " + puts [string trim { + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + }] + puts -nonewline " static const unsigned int aEntry\[\] = \{" + set i 0 + foreach range $lRange { + foreach {iFirst nRange} $range {} + set u32 [format "0x%08X" [expr ($iFirst<<10) + $nRange]] + + if {($i % 5)==0} {puts "" ; puts -nonewline " "} + puts -nonewline " $u32," + incr i + } + puts "" + puts " \};" +} + +proc an_print_ascii_bitmap {lRange} { + foreach range $lRange { + foreach {iFirst nRange} $range {} + for {set i $iFirst} {$i < ($iFirst+$nRange)} {incr i} { + if {$i<=127} { set a($i) 1 } + } + } + + set aAscii [list 0 0 0 0] + foreach key [array names a] { + set idx [expr $key >> 5] + lset aAscii $idx [expr [lindex $aAscii $idx] | (1 << ($key&0x001F))] + } + + puts " static const unsigned int aAscii\[4\] = \{" + puts -nonewline " " + foreach v $aAscii { puts -nonewline [format " 0x%08X," $v] } + puts "" + puts " \};" +} + +proc print_isalnum {zFunc lRange} { + puts "/*" + puts "** Return true if the argument corresponds to a unicode codepoint" + puts "** classified as either a letter or a number. Otherwise false." + puts "**" + puts "** The results are undefined if the value passed to this function" + puts "** is less than zero." + puts "*/" + puts "int ${zFunc}\(int c)\{" + an_print_range_array $lRange + an_print_ascii_bitmap $lRange + puts { + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1;} + puts "\}" +} + +proc print_test_isalnum {zFunc lRange} { + foreach range $lRange { + foreach {iFirst nRange} $range {} + for {set i $iFirst} {$i < ($iFirst+$nRange)} {incr i} { set a($i) 1 } + } + + puts "static int isalnum_test(int *piCode)\{" + puts -nonewline " unsigned char aAlnum\[\] = \{" + for {set i 0} {$i < 70000} {incr i} { + if {($i % 32)==0} { puts "" ; puts -nonewline " " } + set bFlag [expr ![info exists a($i)]] + puts -nonewline "${bFlag}," + } + puts "" + puts " \};" + + puts -nonewline " int aLargeSep\[\] = \{" + set i 0 + foreach iSep [lsort -integer [array names a]] { + if {$iSep<70000} continue + if {($i % 8)==0} { puts "" ; puts -nonewline " " } + puts -nonewline " $iSep," + incr i + } + puts "" + puts " \};" + puts -nonewline " int aLargeOther\[\] = \{" + set i 0 + foreach iSep [lsort -integer [array names a]] { + if {$iSep<70000} continue + if {[info exists a([expr $iSep-1])]==0} { + if {($i % 8)==0} { puts "" ; puts -nonewline " " } + puts -nonewline " [expr $iSep-1]," + incr i + } + if {[info exists a([expr $iSep+1])]==0} { + if {($i % 8)==0} { puts "" ; puts -nonewline " " } + puts -nonewline " [expr $iSep+1]," + incr i + } + } + puts "" + puts " \};" + + puts [subst -nocommands { + int i; + for(i=0; i>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + }] + puts " static const struct TableEntry \{" + puts " unsigned short iCode;" + puts " unsigned char flags;" + puts " unsigned char nRange;" + puts " \} aEntry\[\] = \{" +} + +proc tl_print_table_entry {togglevar entry liOff} { + upvar $togglevar t + foreach {iFirst nIncr nRange nOff} $entry {} + + if {$iFirst > (1<<16)} { return 1 } + + if {[info exists t]==0} {set t 0} + if {$t==0} { puts -nonewline " " } + + set flags 0 + if {$nIncr==2} { set flags 1 ; set nRange [expr $nRange * 2]} + if {$nOff<0} { incr nOff [expr (1<<16)] } + + set idx [lsearch $liOff $nOff] + if {$idx<0} {error "malfunction generating aiOff"} + set flags [expr $flags + $idx*2] + + set txt "{$iFirst, $flags, $nRange}," + if {$t==2} { + puts $txt + } else { + puts -nonewline [format "% -23s" $txt] + } + set t [expr ($t+1)%3] + + return 0 +} + +proc tl_print_table_footer {togglevar} { + upvar $togglevar t + if {$t!=0} {puts ""} + puts " \};" +} + +proc tl_print_if_entry {entry} { + foreach {iFirst nIncr nRange nOff} $entry {} + if {$nIncr==2} {error "tl_print_if_entry needs improvement!"} + + puts " else if( c>=$iFirst && c<[expr $iFirst+$nRange] )\{" + puts " ret = c + $nOff;" + puts " \}" +} + +proc tl_generate_ioff_table {lRecord} { + foreach entry $lRecord { + foreach {iFirst nIncr nRange iOff} $entry {} + if {$iOff<0} { incr iOff [expr (1<<16)] } + if {[info exists a($iOff)]} continue + set a($iOff) 1 + } + + set liOff [lsort -integer [array names a]] + if {[llength $liOff]>128} { error "Too many distinct ioffs" } + return $liOff +} + +proc tl_print_ioff_table {liOff} { + puts -nonewline " static const unsigned short aiOff\[\] = \{" + set i 0 + foreach off $liOff { + if {($i % 8)==0} {puts "" ; puts -nonewline " "} + puts -nonewline [format "% -7s" "$off,"] + incr i + } + puts "" + puts " \};" + +} + +proc print_fold {zFunc} { + + set lRecord [tl_create_records] + + set lHigh [list] + puts "/*" + puts "** Interpret the argument as a unicode codepoint. If the codepoint" + puts "** is an upper case character that has a lower case equivalent," + puts "** return the codepoint corresponding to the lower case version." + puts "** Otherwise, return a copy of the argument." + puts "**" + puts "** The results are undefined if the value passed to this function" + puts "** is less than zero." + puts "*/" + puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{" + + set liOff [tl_generate_ioff_table $lRecord] + tl_print_table_header + foreach entry $lRecord { + if {[tl_print_table_entry toggle $entry $liOff]} { + lappend lHigh $entry + } + } + tl_print_table_footer toggle + tl_print_ioff_table $liOff + + puts [subst -nocommands { + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( eRemoveDiacritic ){ + ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2); + } + } + }] + + foreach entry $lHigh { + tl_print_if_entry $entry + } + + puts "" + puts " return ret;" + puts "\}" +} + +proc code {txt} { + set txt [string trimright $txt] + set txt [string trimleft $txt "\n"] + set n [expr {[string length $txt] - [string length [string trim $txt]]}] + set ret "" + foreach L [split $txt "\n"] { + append ret "[string range $L $n end]\n" + } + return [uplevel "subst -nocommands {$ret}"] +} + +proc intarray {lInt} { + set ret "" + set n [llength $lInt] + for {set i 0} {$i < $n} {incr i 10} { + append ret "\n " + foreach int [lrange $lInt $i [expr $i+9]] { + append ret [format "%-7s" "$int, "] + } + } + append ret "\n " + set ret +} + +proc categories_switch {Cvar first lSecond} { + upvar $Cvar C + set ret "" + append ret "case '$first':\n" + append ret " switch( zCat\[1\] ){\n" + foreach s $lSecond { + append ret " case '$s': aArray\[$C($first$s)\] = 1; break;\n" + } + append ret " case '*': \n" + foreach s $lSecond { + append ret " aArray\[$C($first$s)\] = 1;\n" + } + append ret " break;\n" + append ret " default: return 1;" + append ret " }\n" + append ret " break;\n" +} + +# Argument is a list. Each element of which is itself a list of two elements: +# +# * the codepoint +# * the category +# +# List elements are sorted in order of codepoint. +# +proc print_categories {lMap} { + set categories { + Cc Cf Cn Cs + Ll Lm Lo Lt Lu + Mc Me Mn + Nd Nl No + Pc Pd Pe Pf Pi Po Ps + Sc Sk Sm So + Zl Zp Zs + + LC Co + } + + for {set i 0} {$i < [llength $categories]} {incr i} { + set C([lindex $categories $i]) [expr 1+$i] + } + + set caseC [categories_switch C C {c f n s o}] + set caseL [categories_switch C L {l m o t u C}] + set caseM [categories_switch C M {c e n}] + set caseN [categories_switch C N {d l o}] + set caseP [categories_switch C P {c d e f i o s}] + set caseS [categories_switch C S {c k m o}] + set caseZ [categories_switch C Z {l p s}] + + set nCat [expr [llength [array names C]] + 1] + puts [code { + int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ + aArray[0] = 1; + switch( zCat[0] ){ + $caseC + $caseL + $caseM + $caseN + $caseP + $caseS + $caseZ + + default: + return 1; + } + return 0; + } + }] + + set nRepeat 0 + set first [lindex $lMap 0 0] + set class [lindex $lMap 0 1] + set prev -1 + + set CASE(0) "Lu" + set CASE(1) "Ll" + + foreach m $lMap { + foreach {codepoint cl} $m {} + set codepoint [expr "0x$codepoint"] + if {$codepoint>=(1<<20)} continue + + set bNew 0 + if {$codepoint!=($prev+1)} { + set bNew 1 + } elseif { + $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01])) + } { + incr nRepeat + } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} { + set class LC + incr nRepeat + } else { + set bNew 1 + } + if {$bNew} { + lappend lEntries [list $first $class $nRepeat] + set nRepeat 1 + set first $codepoint + set class $cl + } + set prev $codepoint + } + if {$nRepeat>0} { + lappend lEntries [list $first $class $nRepeat] + } + + set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] + set aMap [list] + foreach e $lEntries { + foreach {cp class nRepeat} $e {} + set block [expr ($cp>>16)] + if {$block>0 && [lindex $aBlock $block]==0} { + for {set i 1} {$i<=$block} {incr i} { + if {[lindex $aBlock $i]==0} { + lset aBlock $i [llength $aMap] + } + } + } + lappend aMap [expr {$cp & 0xFFFF}] + lappend aData [expr {($nRepeat << 5) + $C($class)}] + } + for {set i 1} {$i<[llength $aBlock]} {incr i} { + if {[lindex $aBlock $i]==0} { + lset aBlock $i [llength $aMap] + } + } + + set aBlockArray [intarray $aBlock] + set aMapArray [intarray $aMap] + set aDataArray [intarray $aData] + puts [code { + static u16 aFts5UnicodeBlock[] = {$aBlockArray}; + static u16 aFts5UnicodeMap[] = {$aMapArray}; + static u16 aFts5UnicodeData[] = {$aDataArray}; + + int sqlite3Fts5UnicodeCategory(u32 iCode) { + int iRes = -1; + int iHi; + int iLo; + int ret; + u16 iKey; + + if( iCode>=(1<<20) ){ + return 0; + } + iLo = aFts5UnicodeBlock[(iCode>>16)]; + iHi = aFts5UnicodeBlock[1+(iCode>>16)]; + iKey = (iCode & 0xFFFF); + while( iHi>iLo ){ + int iTest = (iHi + iLo) / 2; + assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest; + } + } + + if( iRes<0 ) return 0; + if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0; + ret = aFts5UnicodeData[iRes] & 0x1F; + if( ret!=$C(LC) ) return ret; + return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu); + } + + void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ + int i = 0; + int iTbl = 0; + while( i<128 ){ + int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; + int n = (aFts5UnicodeData[iTbl] >> 5) + i; + for(; i<128 && i" + puts "" +} + +proc print_test_main {} { + puts "" + puts "#include " + puts "" + puts "int main(int argc, char **argv)\{" + puts " int r1, r2, r3;" + puts " int code;" + puts " r3 = 0;" + puts " r1 = isalnum_test(&code);" + puts " if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);" + puts " else printf(\"isalnum(): test passed\\n\");" + puts " r2 = fold_test(&code);" + puts " if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);" + puts " else printf(\"fold(): test passed\\n\");" + if {$::generate_fts5_code} { + puts " r3 = categories_test(&code);" + puts " if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);" + puts " else printf(\"categories(): test passed\\n\");" + } + puts " return (r1 || r2 || r3);" + puts "\}" +} + +# Proces the command line arguments. Exit early if they are not to +# our liking. +# +proc usage {} { + puts -nonewline stderr "Usage: $::argv0 ?-test? ?-fts5? " + puts stderr " " + exit 1 +} +if {[llength $argv]<2} usage +set unicodedata.txt [lindex $argv end] +set casefolding.txt [lindex $argv end-1] + +set remove_diacritic remove_diacritic +set generate_test_code 0 +set generate_fts5_code 0 +set function_prefix "sqlite3Fts" +for {set i 0} {$i < [llength $argv]-2} {incr i} { + switch -- [lindex $argv $i] { + -test { + set generate_test_code 1 + } + -fts5 { + set function_prefix sqlite3Fts5 + set generate_fts5_code 1 + set remove_diacritic fts5_remove_diacritic + } + default { + usage + } + } +} + +print_fileheader + +if {$::generate_test_code} { + puts "typedef unsigned short int u16;" + puts "typedef unsigned char u8;" + puts "#include " +} + +# Print the isalnum() function to stdout. +# +set lRange [an_load_separator_ranges] +if {$generate_fts5_code==0} { + print_isalnum ${function_prefix}UnicodeIsalnum $lRange +} + +# Leave a gap between the two generated C functions. +# +puts "" +puts "" + +# Load the fold data. This is used by the [rd_XXX] commands +# as well as [print_fold]. +tl_load_casefolding_txt ${casefolding.txt} + +set mappings [rd_load_unicodedata_text ${unicodedata.txt}] +print_rd $mappings +puts "" +puts "" +print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings +puts "" +puts "" + +# Print the fold() function to stdout. +# +print_fold ${function_prefix}UnicodeFold + +if {$generate_fts5_code} { + puts "" + puts "" + print_categories [cc_load_unicodedata_text ${unicodedata.txt}] +} + +# Print the test routines and main() function to stdout, if -test +# was specified. +# +if {$::generate_test_code} { + if {$generate_fts5_code==0} { + print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange + } + print_fold_test ${function_prefix}UnicodeFold $mappings + print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}] + print_test_main +} + +if {$generate_fts5_code} { + # no-op +} else { + puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */" + puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */" +} diff --git a/ext/fts3/unicode/parseunicode.tcl b/ext/fts3/unicode/parseunicode.tcl new file mode 100644 index 0000000000000000000000000000000000000000..7c246a4a0925670669e81093951760047f7b7b05 --- /dev/null +++ b/ext/fts3/unicode/parseunicode.tcl @@ -0,0 +1,205 @@ + +#-------------------------------------------------------------------------- +# Parameter $zName must be a path to the file UnicodeData.txt. This command +# reads the file and returns a list of mappings required to remove all +# diacritical marks from a unicode string. Each mapping is itself a list +# consisting of two elements - the unicode codepoint and the single ASCII +# character that it should be replaced with, or an empty string if the +# codepoint should simply be removed from the input. Examples: +# +# { 224 a 0 } (replace codepoint 224 to "a") +# { 769 "" 0 } (remove codepoint 769 from input) +# +# Mappings are only returned for non-upper case codepoints. It is assumed +# that the input has already been folded to lower case. +# +# The third value in the list is always either 0 or 1. 0 if the +# UnicodeData.txt file maps the codepoint to a single ASCII character and +# a diacritic, or 1 if the mapping is indirect. For example, consider the +# two entries: +# +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8 +# +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a +# diacritic). The second is an indirect mapping, as it maps to the +# first codepoint plus 0302 (a diacritic). +# +proc rd_load_unicodedata_text {zName} { + global tl_lookup_table + + set fd [open $zName] + set lField { + code + character_name + general_category + canonical_combining_classes + bidirectional_category + character_decomposition_mapping + decimal_digit_value + digit_value + numeric_value + mirrored + unicode_1_name + iso10646_comment_field + uppercase_mapping + lowercase_mapping + titlecase_mapping + } + set lRet [list] + + while { ![eof $fd] } { + set line [gets $fd] + if {$line == ""} continue + + set fields [split $line ";"] + if {[llength $fields] != [llength $lField]} { error "parse error: $line" } + foreach $lField $fields {} + if { [llength $character_decomposition_mapping]!=2 + || [string is xdigit [lindex $character_decomposition_mapping 0]]==0 + } { + continue + } + + set iCode [expr "0x$code"] + set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"] + set iDia [expr "0x[lindex $character_decomposition_mapping 1]"] + + # Filter out upper-case characters, as they will be mapped to their + # lower-case equivalents before this data is used. + if {[info exists tl_lookup_table($iCode)]} continue + + # Check if this is an indirect mapping. If so, set bIndirect to true + # and change $iAscii to the indirectly mappped ASCII character. + set bIndirect 0 + if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} { + set iAscii $mapping($iAscii) + set bIndirect 1 + } + + if { ($iAscii >= 97 && $iAscii <= 122) + || ($iAscii >= 65 && $iAscii <= 90) + } { + lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect] + set mapping($iCode) $iAscii + set dia($iDia) 1 + } + } + + foreach d [array names dia] { + lappend lRet [list $d "" 0] + } + set lRet [lsort -integer -index 0 $lRet] + + close $fd + set lRet +} + +#------------------------------------------------------------------------- +# Parameter $zName must be a path to the file UnicodeData.txt. This command +# reads the file and returns a list of codepoints (integers). The list +# contains all codepoints in the UnicodeData.txt assigned to any "General +# Category" that is not a "Letter" or "Number". +# +proc an_load_unicodedata_text {zName} { + set fd [open $zName] + set lField { + code + character_name + general_category + canonical_combining_classes + bidirectional_category + character_decomposition_mapping + decimal_digit_value + digit_value + numeric_value + mirrored + unicode_1_name + iso10646_comment_field + uppercase_mapping + lowercase_mapping + titlecase_mapping + } + set lRet [list] + + while { ![eof $fd] } { + set line [gets $fd] + if {$line == ""} continue + + set fields [split $line ";"] + if {[llength $fields] != [llength $lField]} { error "parse error: $line" } + foreach $lField $fields {} + + set iCode [expr "0x$code"] + set bAlnum [expr { + [lsearch {L N} [string range $general_category 0 0]] >= 0 + || $general_category=="Co" + }] + + if { !$bAlnum } { lappend lRet $iCode } + } + + close $fd + set lRet +} + +proc tl_load_casefolding_txt {zName} { + global tl_lookup_table + + set fd [open $zName] + while { ![eof $fd] } { + set line [gets $fd] + if {[string range $line 0 0] == "#"} continue + if {$line == ""} continue + + foreach x {a b c d} {unset -nocomplain $x} + foreach {a b c d} [split $line ";"] {} + + set a2 [list] + set c2 [list] + foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] } + foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] } + set b [string trim $b] + set d [string trim $d] + + if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 } + } +} + +proc cc_load_unicodedata_text {zName} { + set fd [open $zName] + set lField { + code + character_name + general_category + canonical_combining_classes + bidirectional_category + character_decomposition_mapping + decimal_digit_value + digit_value + numeric_value + mirrored + unicode_1_name + iso10646_comment_field + uppercase_mapping + lowercase_mapping + titlecase_mapping + } + set lRet [list] + + while { ![eof $fd] } { + set line [gets $fd] + if {$line == ""} continue + + set fields [split $line ";"] + if {[llength $fields] != [llength $lField]} { error "parse error: $line" } + foreach $lField $fields {} + + lappend lRet [list $code $general_category] + } + + close $fd + set lRet +} + + diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl new file mode 100644 index 0000000000000000000000000000000000000000..6ee71c262c98b9cdb94906a7325b72b26f232ad3 --- /dev/null +++ b/ext/fts5/extract_api_docs.tcl @@ -0,0 +1,257 @@ +# +# 2014 August 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#-------------------------------------------------------------------------- +# +# This script extracts the documentation for the API used by fts5 auxiliary +# functions from header file fts5.h. It outputs html text on stdout that +# is included in the documentation on the web. +# + +set ::fts5_docs_output "" +if {[info commands hd_putsnl]==""} { + if {[llength $argv]>0} { set ::extract_api_docs_mode [lindex $argv 0] } + proc output {text} { + puts $text + } +} else { + proc output {text} { + append ::fts5_docs_output "$text\n" + } +} +if {[info exists ::extract_api_docs_mode]==0} {set ::extract_api_docs_mode api} + + +set input_file [file join [file dir [info script]] fts5.h] +set fd [open $input_file] +set data [read $fd] +close $fd + + +# Argument $data is the entire text of the fts5.h file. This function +# extracts the definition of the Fts5ExtensionApi structure from it and +# returns a key/value list of structure member names and definitions. i.e. +# +# iVersion {int iVersion} xUserData {void *(*xUserData)(Fts5Context*)} ... +# +proc get_struct_members {data} { + + # Extract the structure definition from the fts5.h file. + regexp "struct Fts5ExtensionApi {(.*?)};" $data -> defn + + # Remove all comments from the structure definition + regsub -all {/[*].*?[*]/} $defn {} defn2 + + set res [list] + foreach member [split $defn2 {;}] { + + set member [string trim $member] + if {$member!=""} { + catch { set name [lindex $member end] } + regexp {.*?[(][*]([^)]*)[)]} $member -> name + lappend res $name $member + } + } + + set res +} + +proc get_struct_docs {data names} { + # Extract the structure definition from the fts5.h file. + regexp {EXTENSION API FUNCTIONS(.*?)[*]/} $data -> docs + + set current_doc "" + set current_header "" + + foreach line [split $docs "\n"] { + regsub {[*]*} $line {} line + if {[regexp {^ } $line]} { + append current_doc "$line\n" + } elseif {[string trim $line]==""} { + if {$current_header!=""} { append current_doc "\n" } + } else { + if {$current_doc != ""} { + lappend res $current_header $current_doc + set current_doc "" + } + set subject n/a + regexp {^ *([[:alnum:]_]*)} $line -> subject + if {[lsearch $names $subject]>=0} { + set current_header $subject + } else { + set current_header [string trim $line] + } + } + } + + if {$current_doc != ""} { + lappend res $current_header $current_doc + } + + set res +} + +proc get_tokenizer_docs {data} { + regexp {(xCreate:.*?)[*]/} $data -> docs + + set res "
\n" + foreach line [split [string trim $docs] "\n"] { + regexp {[*][*](.*)} $line -> line + if {[regexp {^ ?x.*:} $line]} { + append res "
$line

\n" + continue + } + if {[regexp {FTS5_TOKENIZER} $line]} { + set line

+ } + if {[regexp {SYNONYM SUPPORT} $line]} { + set line "

Synonym Support

" + } + if {[string trim $line] == ""} { + append res "

\n" + } else { + append res "$line\n" + } + } + + set res +} + +proc get_api_docs {data} { + # Initialize global array M as a map from Fts5StructureApi member name + # to member definition. i.e. + # + # iVersion -> {int iVersion} + # xUserData -> {void *(*xUserData)(Fts5Context*)} + # ... + # + array set M [get_struct_members $data] + + # Initialize global list D as a map from section name to documentation + # text. Most (all?) section names are structure member names. + # + set D [get_struct_docs $data [array names M]] + + output "

" + foreach {sub docs} $D { + if {[info exists M($sub)]} { + set hdr $M($sub) + set link " id=$sub" + } else { + set link "" + } + + #output "
" + #set style "padding-left:6ex;font-size:1.4em;display:block" + #output "
$hdr
" + + regsub -line {^ *[)]} $hdr ")" hdr + output "
" + output "$hdr
" + + set mode "" + set margin " style=margin-top:0.1em" + foreach line [split [string trim $docs] "\n"] { + if {[string trim $line]==""} { + if {$mode != ""} {output ""} + set mode "" + } elseif {$mode == ""} { + if {[regexp {^ } $line]} { + set mode codeblock + } else { + set mode p + } + output "<$mode$margin>" + set margin "" + } + output $line + } + if {$mode != ""} {output ""} + output "
" + } + output "
" +} + +proc get_fts5_struct {data start end} { + set res "" + set bOut 0 + foreach line [split $data "\n"] { + if {$bOut==0} { + if {[regexp $start $line]} { + set bOut 1 + } + } + + if {$bOut} { + append res "$line\n" + } + + if {$bOut} { + if {[regexp $end $line]} { + set bOut 0 + } + } + } + + set map [list /* /* */ */] + string map $map $res +} + +proc main {data} { + switch $::extract_api_docs_mode { + fts5_api { + output [get_fts5_struct $data "typedef struct fts5_api" "^\};"] + } + + fts5_tokenizer { + output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"] + output [get_fts5_struct $data \ + "Flags that may be passed as the third argument to xTokenize()" \ + "#define FTS5_TOKEN_COLOCATED" + ] + } + + fts5_extension { + output [get_fts5_struct $data "typedef.*Fts5ExtensionApi" "^.;"] + } + + Fts5ExtensionApi { + set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"] + set map [list] + set lKey [list] + foreach {k v} [get_struct_members $data] { + if {[string match x* $k]==0} continue + lappend lKey $k + } + foreach k [lsort -decr $lKey] { lappend map $k "$k" } + output [string map $map $struct] + } + + api { + get_api_docs $data + } + + tokenizer_api { + output [get_tokenizer_docs $data] + } + + default { + } + } +} +main $data + +set ::fts5_docs_output + + + + + diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h new file mode 100644 index 0000000000000000000000000000000000000000..907ea232a42b9e58a3f394543f41f0ddf21e54e2 --- /dev/null +++ b/ext/fts5/fts5.h @@ -0,0 +1,754 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. +** +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension function's +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API may be slow in some cases if the token identified by parameters +** iIdx and iToken matched a prefix token in the query. In most cases, the +** first call to this API for each prefix token in the query is forced +** to scan the portion of the full-text index that matches the prefix +** token to collect the extra data required by this API. If the prefix +** token matches a large number of token instances in the document set, +** this may be a performance problem. +** +** If the user knows in advance that a query may use this API for a +** prefix token, FTS5 may be configured to collect all required data as part +** of the initial querying of the full-text index, avoiding the second scan +** entirely. This also causes prefix queries that do not use this API to +** run more slowly and use more memory. FTS5 may be configured in this way +** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] +** option, or on a per-query basis using the +** [fts5_insttoken | fts5_insttoken()] user function. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 4 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer_v2 object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The third argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
  • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
  • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
  • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
  • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
+** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
    +**
  • There is no "iVersion" field, and +**
  • The xTokenize() method does not take a locale argument. +**
+** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is substituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h new file mode 100644 index 0000000000000000000000000000000000000000..832f9ad477ed65ce4369edc9c5930ba47139b96c --- /dev/null +++ b/ext/fts5/fts5Int.h @@ -0,0 +1,946 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTS5INT_H +#define _FTS5INT_H + +#include "fts5.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#include +#include + +#ifndef SQLITE_AMALGAMATION + +typedef unsigned char u8; +typedef unsigned int u32; +typedef unsigned short u16; +typedef short i16; +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; + +#ifndef ArraySize +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#endif + +#define testcase(x) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +#define MIN(x,y) (((x) < (y)) ? (x) : (y)) +#define MAX(x,y) (((x) > (y)) ? (x) : (y)) + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +*/ +# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +/* The uptr type is an unsigned integer large enough to hold a pointer +*/ +#if defined(HAVE_STDINT_H) + typedef uintptr_t uptr; +#elif SQLITE_PTRSIZE==4 + typedef u32 uptr; +#else + typedef u64 uptr; +#endif + +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) +#else +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) +#endif + +#endif + +/* Truncate very long tokens to this many bytes. Hard limit is +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset +** field that occurs at the start of each leaf page (see fts5_index.c). */ +#define FTS5_MAX_TOKEN_SIZE 32768 + +/* +** Maximum number of prefix indexes on single FTS5 table. This must be +** less than 32. If it is set to anything large than that, an #error +** directive in fts5_index.c will cause the build to fail. +*/ +#define FTS5_MAX_PREFIX_INDEXES 31 + +/* +** Maximum segments permitted in a single index +*/ +#define FTS5_MAX_SEGMENT 2000 + +#define FTS5_DEFAULT_NEARDIST 10 +#define FTS5_DEFAULT_RANK "bm25" + +/* Name of rank and rowid columns */ +#define FTS5_RANK_NAME "rank" +#define FTS5_ROWID_NAME "rowid" + +#ifdef SQLITE_DEBUG +# define FTS5_CORRUPT sqlite3Fts5Corrupt() +int sqlite3Fts5Corrupt(void); +#else +# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB +#endif + +/* +** The assert_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +extern int sqlite3_fts5_may_be_corrupt; +# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) +#else +# define assert_nc(x) assert(x) +#endif + +/* +** A version of memcmp() that does not cause asan errors if one of the pointer +** parameters is NULL and the number of bytes to compare is zero. +*/ +#define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n))) + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef UNUSED_PARAM2 +# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y) +#endif + +typedef struct Fts5Global Fts5Global; +typedef struct Fts5Colset Fts5Colset; + +/* If a NEAR() clump or phrase may only match a specific set of columns, +** then an object of the following type is used to record the set of columns. +** Each entry in the aiCol[] array is a column that may be matched. +** +** This object is used by fts5_expr.c and fts5_index.c. +*/ +struct Fts5Colset { + int nCol; + int aiCol[1]; +}; + + + +/************************************************************************** +** Interface to code in fts5_config.c. fts5_config.c contains contains code +** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. +*/ + +typedef struct Fts5Config Fts5Config; +typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; + +struct Fts5TokenizerConfig { + Fts5Tokenizer *pTok; + fts5_tokenizer_v2 *pApi2; + fts5_tokenizer *pApi1; + const char **azArg; + int nArg; + int ePattern; /* FTS_PATTERN_XXX constant */ + const char *pLocale; /* Current locale to use */ + int nLocale; /* Size of pLocale in bytes */ +}; + +/* +** An instance of the following structure encodes all information that can +** be gleaned from the CREATE VIRTUAL TABLE statement. +** +** And all information loaded from the %_config table. +** +** nAutomerge: +** The minimum number of segments that an auto-merge operation should +** attempt to merge together. A value of 1 sets the object to use the +** compile time default. Zero disables auto-merge altogether. +** +** bContentlessDelete: +** True if the contentless_delete option was present in the CREATE +** VIRTUAL TABLE statement. +** +** zContent: +** +** zContentRowid: +** The value of the content_rowid= option, if one was specified. Or +** the string "rowid" otherwise. This text is not quoted - if it is +** used as part of an SQL statement it needs to be quoted appropriately. +** +** zContentExprlist: +** +** pzErrmsg: +** This exists in order to allow the fts5_index.c module to return a +** decent error message if it encounters a file-format version it does +** not understand. +** +** bColumnsize: +** True if the %_docsize table is created. +** +** bPrefixIndex: +** This is only used for debugging. If set to false, any prefix indexes +** are ignored. This value is configured using: +** +** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); +** +** bLocale: +** Set to true if locale=1 was specified when the table was created. +*/ +struct Fts5Config { + sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* Global fts5 object for handle db */ + char *zDb; /* Database holding FTS index (e.g. "main") */ + char *zName; /* Name of FTS index */ + int nCol; /* Number of columns */ + char **azCol; /* Column names */ + u8 *abUnindexed; /* True for unindexed columns */ + int nPrefix; /* Number of prefix indexes */ + int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ + int eContent; /* An FTS5_CONTENT value */ + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ + int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */ + char *zContent; /* content table */ + char *zContentRowid; /* "content_rowid=" option value */ + int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int bTokendata; /* "tokendata=" option value (dflt==0) */ + int bLocale; /* "locale=" option value (dflt==0) */ + int eDetail; /* FTS5_DETAIL_XXX value */ + char *zContentExprlist; + Fts5TokenizerConfig t; + int bLock; /* True when table is preparing statement */ + + + /* Values loaded from the %_config table */ + int iVersion; /* fts5 file format 'version' */ + int iCookie; /* Incremented when %_config is modified */ + int pgsz; /* Approximate page size used in %_data */ + int nAutomerge; /* 'automerge' setting */ + int nCrisisMerge; /* Maximum allowed segments per level */ + int nUsermerge; /* 'usermerge' setting */ + int nHashSize; /* Bytes of memory for in-memory hash */ + char *zRank; /* Name of rank function */ + char *zRankArgs; /* Arguments to rank function */ + int bSecureDelete; /* 'secure-delete' */ + int nDeleteMerge; /* 'deletemerge' */ + int bPrefixInsttoken; /* 'prefix-insttoken' */ + + /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ + char **pzErrmsg; + +#ifdef SQLITE_DEBUG + int bPrefixIndex; /* True to use prefix-indexes */ +#endif +}; + +/* Current expected value of %_config table 'version' field. And +** the expected version if the 'secure-delete' option has ever been +** set on the table. */ +#define FTS5_CURRENT_VERSION 4 +#define FTS5_CURRENT_VERSION_SECUREDELETE 5 + +#define FTS5_CONTENT_NORMAL 0 +#define FTS5_CONTENT_NONE 1 +#define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_CONTENT_UNINDEXED 3 + +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 + +#define FTS5_PATTERN_NONE 0 +#define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */ +#define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */ + +int sqlite3Fts5ConfigParse( + Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** +); +void sqlite3Fts5ConfigFree(Fts5Config*); + +int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); + +int sqlite3Fts5Tokenize( + Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ +); + +void sqlite3Fts5Dequote(char *z); + +/* Load the contents of the %_config table */ +int sqlite3Fts5ConfigLoad(Fts5Config*, int); + +/* Set the value of a single config attribute */ +int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); + +int sqlite3Fts5ConfigParseRank(const char*, char**, char**); + +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...); + +/* +** End of interface to code in fts5_config.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_buffer.c. +*/ + +/* +** Buffer object for the incremental building of string data. +*/ +typedef struct Fts5Buffer Fts5Buffer; +struct Fts5Buffer { + u8 *p; + int n; + int nSpace; +}; + +int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); +void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); +void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*); +void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); +void sqlite3Fts5BufferFree(Fts5Buffer*); +void sqlite3Fts5BufferZero(Fts5Buffer*); +void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); +void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); + +char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); + +#define fts5BufferZero(x) sqlite3Fts5BufferZero(x) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c) +#define fts5BufferFree(a) sqlite3Fts5BufferFree(a) +#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) +#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) + +#define fts5BufferGrow(pRc,pBuf,nn) ( \ + (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \ + sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ +) + +/* Write and decode big-endian 32-bit integer values */ +void sqlite3Fts5Put32(u8*, int); +int sqlite3Fts5Get32(const u8*); + +#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF) +#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) + +typedef struct Fts5PoslistReader Fts5PoslistReader; +struct Fts5PoslistReader { + /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ + const u8 *a; /* Position list to iterate through */ + int n; /* Size of buffer at a[] in bytes */ + int i; /* Current offset in a[] */ + + u8 bFlag; /* For client use (any custom purpose) */ + + /* Output variables */ + u8 bEof; /* Set to true at EOF */ + i64 iPos; /* (iCol<<32) + iPos */ +}; +int sqlite3Fts5PoslistReaderInit( + const u8 *a, int n, /* Poslist buffer to iterate through */ + Fts5PoslistReader *pIter /* Iterator object to initialize */ +); +int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); + +typedef struct Fts5PoslistWriter Fts5PoslistWriter; +struct Fts5PoslistWriter { + i64 iPrev; +}; +int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); +void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64); + +int sqlite3Fts5PoslistNext64( + const u8 *a, int n, /* Buffer containing poslist */ + int *pi, /* IN/OUT: Offset within a[] */ + i64 *piOff /* IN/OUT: Current offset */ +); + +/* Malloc utility */ +void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte); +char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); + +/* Character set tests (like isspace(), isalpha() etc.) */ +int sqlite3Fts5IsBareword(char t); + + +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ +typedef struct Fts5Termset Fts5Termset; +int sqlite3Fts5TermsetNew(Fts5Termset**); +int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); +void sqlite3Fts5TermsetFree(Fts5Termset*); + +/* +** End of interface to code in fts5_buffer.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_index.c. fts5_index.c contains contains code +** to access the data stored in the %_data table. +*/ + +typedef struct Fts5Index Fts5Index; +typedef struct Fts5IndexIter Fts5IndexIter; + +struct Fts5IndexIter { + i64 iRowid; + const u8 *pData; + int nData; + u8 bEof; +}; + +#define sqlite3Fts5IterEof(x) ((x)->bEof) + +/* +** Values used as part of the flags argument passed to IndexQuery(). +*/ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ + +/* The following are used internally by the fts5_index.c module. They are +** defined here only to make it easier to avoid clashes with the flags +** above. */ +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080 +#define FTS5INDEX_QUERY_SCANONETERM 0x0100 + +/* +** Create/destroy an Fts5Index object. +*/ +int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); +int sqlite3Fts5IndexClose(Fts5Index *p); + +/* +** Return a simple checksum value based on the arguments. +*/ +u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +); + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +); + +/* +** Open a new iterator to iterate though all rowids that match the +** specified token or token prefix. +*/ +int sqlite3Fts5IndexQuery( + Fts5Index *p, /* FTS index to query */ + const char *pToken, int nToken, /* Token (or prefix) to query for */ + int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ +); + +/* +** The various operations on open token or token prefix iterators opened +** using sqlite3Fts5IndexQuery(). +*/ +int sqlite3Fts5IterNext(Fts5IndexIter*); +int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); + +/* +** Close an iterator opened by sqlite3Fts5IndexQuery(). +*/ +void sqlite3Fts5IterClose(Fts5IndexIter*); + +/* +** Close the reader blob handle, if it is open. +*/ +void sqlite3Fts5IndexCloseReader(Fts5Index*); + +/* +** This interface is used by the fts5vocab module. +*/ +const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); +int sqlite3Fts5IterNextScan(Fts5IndexIter*); +void *sqlite3Fts5StructureRef(Fts5Index*); +void sqlite3Fts5StructureRelease(void*); +int sqlite3Fts5StructureTest(Fts5Index*, void*); + +/* +** Used by xInstToken(): +*/ +int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +); + +/* +** Insert or remove data to or from the index. Each time a document is +** added to or removed from the index, this function is called one or more +** times. +** +** For an insert, it must be called once for each token in the new document. +** If the operation is a delete, it must be called (at least) once for each +** unique token in the document with an iCol value less than zero. The iPos +** argument is ignored for a delete. +*/ +int sqlite3Fts5IndexWrite( + Fts5Index *p, /* Index to write to */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +); + +/* +** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to +** document iDocid. +*/ +int sqlite3Fts5IndexBeginWrite( + Fts5Index *p, /* Index to write to */ + int bDelete, /* True if current operation is a delete */ + i64 iDocid /* Docid to add or remove data from */ +); + +/* +** Flush any data stored in the in-memory hash tables to the database. +** Also close any open blob handles. +*/ +int sqlite3Fts5IndexSync(Fts5Index *p); + +/* +** Discard any data stored in the in-memory hash tables. Do not write it +** to the database. Additionally, assume that the contents of the %_data +** table may have changed on disk. So any in-memory caches of %_data +** records must be invalidated. +*/ +int sqlite3Fts5IndexRollback(Fts5Index *p); + +/* +** Get or set the "averages" values. +*/ +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); + +/* +** Functions called by the storage module as part of integrity-check. +*/ +int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum); + +/* +** Called during virtual module initialization to register UDF +** fts5_decode() with SQLite +*/ +int sqlite3Fts5IndexInit(sqlite3*); + +int sqlite3Fts5IndexSetCookie(Fts5Index*, int); + +/* +** Return the total number of entries read from the %_data table by +** this connection since it was created. +*/ +int sqlite3Fts5IndexReads(Fts5Index *p); + +int sqlite3Fts5IndexReinit(Fts5Index *p); +int sqlite3Fts5IndexOptimize(Fts5Index *p); +int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); +int sqlite3Fts5IndexReset(Fts5Index *p); + +int sqlite3Fts5IndexLoadConfig(Fts5Index *p); + +int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); +int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); + +void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); + +/* Used to populate hash tables for xInstToken in detail=none/column mode. */ +int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff +); + +/* +** End of interface to code in fts5_index.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_varint.c. +*/ +int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v); +int sqlite3Fts5GetVarintLen(u32 iVal); +u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); +int sqlite3Fts5PutVarint(unsigned char *p, u64 v); + +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b)) +#define fts5GetVarint sqlite3Fts5GetVarint + +#define fts5FastGetVarint32(a, iOff, nVal) { \ + nVal = (a)[iOff++]; \ + if( nVal & 0x80 ){ \ + iOff--; \ + iOff += fts5GetVarint32(&(a)[iOff], nVal); \ + } \ +} + + +/* +** End of interface to code in fts5_varint.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to code in fts5_main.c. +*/ + +/* +** Virtual-table object. +*/ +typedef struct Fts5Table Fts5Table; +struct Fts5Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts5Config *pConfig; /* Virtual table configuration */ + Fts5Index *pIndex; /* Full-text index */ +}; + +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig); + +Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); + +int sqlite3Fts5FlushToDisk(Fts5Table*); + +void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc); + +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); +int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, + const char **ppText, int *pnText, const char **ppLoc, int *pnLoc +); + +/* +** End of interface to code in fts5.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_hash.c. +*/ +typedef struct Fts5Hash Fts5Hash; + +/* +** Create a hash table, free a hash table. +*/ +int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); +void sqlite3Fts5HashFree(Fts5Hash*); + +int sqlite3Fts5HashWrite( + Fts5Hash*, + i64 iRowid, /* Rowid for this entry */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + char bByte, + const char *pToken, int nToken /* Token to add or remove to or from index */ +); + +/* +** Empty (but do not delete) a hash table. +*/ +void sqlite3Fts5HashClear(Fts5Hash*); + +/* +** Return true if the hash is empty, false otherwise. +*/ +int sqlite3Fts5HashIsEmpty(Fts5Hash*); + +int sqlite3Fts5HashQuery( + Fts5Hash*, /* Hash table to query */ + int nPre, + const char *pTerm, int nTerm, /* Query term */ + void **ppObj, /* OUT: Pointer to doclist for pTerm */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +); + +int sqlite3Fts5HashScanInit( + Fts5Hash*, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +); +void sqlite3Fts5HashScanNext(Fts5Hash*); +int sqlite3Fts5HashScanEof(Fts5Hash*); +void sqlite3Fts5HashScanEntry(Fts5Hash *, + const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ + const u8 **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +); + + + +/* +** End of interface to code in fts5_hash.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_storage.c. fts5_storage.c contains contains +** code to access the data stored in the %_content and %_docsize tables. +*/ + +#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */ +#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */ +#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */ + +typedef struct Fts5Storage Fts5Storage; + +int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**); +int sqlite3Fts5StorageClose(Fts5Storage *p); +int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); + +int sqlite3Fts5DropAll(Fts5Config*); +int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); + +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int); +int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*); +int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); + +int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg); + +int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); +void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); + +int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); +int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); +int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); + +int sqlite3Fts5StorageSync(Fts5Storage *p); +int sqlite3Fts5StorageRollback(Fts5Storage *p); + +int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, const char*, sqlite3_value*, int +); + +int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); +int sqlite3Fts5StorageRebuild(Fts5Storage *p); +int sqlite3Fts5StorageOptimize(Fts5Storage *p); +int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); +int sqlite3Fts5StorageReset(Fts5Storage *p); + +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*); +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel); + +/* +** End of interface to code in fts5_storage.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to code in fts5_expr.c. +*/ +typedef struct Fts5Expr Fts5Expr; +typedef struct Fts5ExprNode Fts5ExprNode; +typedef struct Fts5Parse Fts5Parse; +typedef struct Fts5Token Fts5Token; +typedef struct Fts5ExprPhrase Fts5ExprPhrase; +typedef struct Fts5ExprNearset Fts5ExprNearset; + +struct Fts5Token { + const char *p; /* Token text (not NULL terminated) */ + int n; /* Size of buffer p in bytes */ +}; + +/* Parse a MATCH expression. */ +int sqlite3Fts5ExprNew( + Fts5Config *pConfig, + int bPhraseToAnd, + int iCol, /* Column on LHS of MATCH operator */ + const char *zExpr, + Fts5Expr **ppNew, + char **pzErr +); +int sqlite3Fts5ExprPattern( + Fts5Config *pConfig, + int bGlob, + int iCol, + const char *zText, + Fts5Expr **pp +); + +/* +** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc); +** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr); +** rc = sqlite3Fts5ExprNext(pExpr) +** ){ +** // The document with rowid iRowid matches the expression! +** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); +** } +*/ +int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); +int sqlite3Fts5ExprEof(Fts5Expr*); +i64 sqlite3Fts5ExprRowid(Fts5Expr*); + +void sqlite3Fts5ExprFree(Fts5Expr*); +int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2); + +/* Called during startup to register a UDF with SQLite */ +int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); + +int sqlite3Fts5ExprPhraseCount(Fts5Expr*); +int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); +int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); + +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int +); +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); + +int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); + +int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); + +int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); +int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*); +void sqlite3Fts5ExprClearTokens(Fts5Expr*); + +/******************************************* +** The fts5_expr.c API above this point is used by the other hand-written +** C code in this module. The interfaces below this point are called by +** the parser code in fts5parse.y. */ + +void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); + +Fts5ExprNode *sqlite3Fts5ParseNode( + Fts5Parse *pParse, + int eType, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight, + Fts5ExprNearset *pNear +); + +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight +); + +Fts5ExprPhrase *sqlite3Fts5ParseTerm( + Fts5Parse *pParse, + Fts5ExprPhrase *pPhrase, + Fts5Token *pToken, + int bPrefix +); + +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*); + +Fts5ExprNearset *sqlite3Fts5ParseNearset( + Fts5Parse*, + Fts5ExprNearset*, + Fts5ExprPhrase* +); + +Fts5Colset *sqlite3Fts5ParseColset( + Fts5Parse*, + Fts5Colset*, + Fts5Token * +); + +void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); +void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); +void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); + +void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*); +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); +void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); +void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); + +/* +** End of interface to code in fts5_expr.c. +**************************************************************************/ + + + +/************************************************************************** +** Interface to code in fts5_aux.c. +*/ + +int sqlite3Fts5AuxInit(fts5_api*); +/* +** End of interface to code in fts5_aux.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_tokenizer.c. +*/ + +int sqlite3Fts5TokenizerInit(fts5_api*); +int sqlite3Fts5TokenizerPattern( + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), + Fts5Tokenizer *pTok +); +int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*); +/* +** End of interface to code in fts5_tokenizer.c. +**************************************************************************/ + +/************************************************************************** +** Interface to code in fts5_vocab.c. +*/ + +int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); + +/* +** End of interface to code in fts5_vocab.c. +**************************************************************************/ + + +/************************************************************************** +** Interface to automatically generated code in fts5_unicode2.c. +*/ +int sqlite3Fts5UnicodeIsdiacritic(int c); +int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); + +int sqlite3Fts5UnicodeCatParse(const char*, u8*); +int sqlite3Fts5UnicodeCategory(u32 iCode); +void sqlite3Fts5UnicodeAscii(u8*, u8*); +/* +** End of interface to code in fts5_unicode2.c. +**************************************************************************/ + +#endif diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c new file mode 100644 index 0000000000000000000000000000000000000000..ad578156da6355670abef96efd9e8dcdc2be7dcf --- /dev/null +++ b/ext/fts5/fts5_aux.c @@ -0,0 +1,821 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + +#include "fts5Int.h" +#include /* amalgamator: keep */ + +/* +** Object used to iterate through all "coalesced phrase instances" in +** a single column of the current row. If the phrase instances in the +** column being considered do not overlap, this object simply iterates +** through them. Or, if they do overlap (share one or more tokens in +** common), each set of overlapping instances is treated as a single +** match. See documentation for the highlight() auxiliary function for +** details. +** +** Usage is: +** +** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter); +** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter); +** rc = fts5CInstIterNext(&iter) +** ){ +** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd); +** } +** +*/ +typedef struct CInstIter CInstIter; +struct CInstIter { + const Fts5ExtensionApi *pApi; /* API offered by current FTS version */ + Fts5Context *pFts; /* First arg to pass to pApi functions */ + int iCol; /* Column to search */ + int iInst; /* Next phrase instance index */ + int nInst; /* Total number of phrase instances */ + + /* Output variables */ + int iStart; /* First token in coalesced phrase instance */ + int iEnd; /* Last token in coalesced phrase instance */ +}; + +/* +** Advance the iterator to the next coalesced phrase instance. Return +** an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +static int fts5CInstIterNext(CInstIter *pIter){ + int rc = SQLITE_OK; + pIter->iStart = -1; + pIter->iEnd = -1; + + while( rc==SQLITE_OK && pIter->iInstnInst ){ + int ip; int ic; int io; + rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + if( ic==pIter->iCol ){ + int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip); + if( pIter->iStart<0 ){ + pIter->iStart = io; + pIter->iEnd = iEnd; + }else if( io<=pIter->iEnd ){ + if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd; + }else{ + break; + } + } + pIter->iInst++; + } + } + + return rc; +} + +/* +** Initialize the iterator object indicated by the final parameter to +** iterate through coalesced phrase instances in column iCol. +*/ +static int fts5CInstIterInit( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + int iCol, + CInstIter *pIter +){ + int rc; + + memset(pIter, 0, sizeof(CInstIter)); + pIter->pApi = pApi; + pIter->pFts = pFts; + pIter->iCol = iCol; + rc = pApi->xInstCount(pFts, &pIter->nInst); + + if( rc==SQLITE_OK ){ + rc = fts5CInstIterNext(pIter); + } + + return rc; +} + + + +/************************************************************************* +** Start of highlight() implementation. +*/ +typedef struct HighlightContext HighlightContext; +struct HighlightContext { + /* Constant parameters to fts5HighlightCb() */ + int iRangeStart; /* First token to include */ + int iRangeEnd; /* If non-zero, last token to include */ + const char *zOpen; /* Opening highlight */ + const char *zClose; /* Closing highlight */ + const char *zIn; /* Input text */ + int nIn; /* Size of input text in bytes */ + + /* Variables modified by fts5HighlightCb() */ + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iOff; /* Have copied up to this offset in zIn[] */ + int bOpen; /* True if highlight is open */ + char *zOut; /* Output value */ +}; + +/* +** Append text to the HighlightContext output string - p->zOut. Argument +** z points to a buffer containing n bytes of text to append. If n is +** negative, everything up until the first '\0' is appended to the output. +** +** If *pRc is set to any value other than SQLITE_OK when this function is +** called, it is a no-op. If an error (i.e. an OOM condition) is encountered, +** *pRc is set to an error code before returning. +*/ +static void fts5HighlightAppend( + int *pRc, + HighlightContext *p, + const char *z, int n +){ + if( *pRc==SQLITE_OK && z ){ + if( n<0 ) n = (int)strlen(z); + p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z); + if( p->zOut==0 ) *pRc = SQLITE_NOMEM; + } +} + +/* +** Tokenizer callback used by implementation of highlight() function. +*/ +static int fts5HighlightCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start byte offset of token */ + int iEndOff /* End byte offset of token */ +){ + HighlightContext *p = (HighlightContext*)pContext; + int rc = SQLITE_OK; + int iPos; + + UNUSED_PARAM2(pToken, nToken); + + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; + iPos = p->iPos++; + + if( p->iRangeEnd>=0 ){ + if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; + if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; + } + + /* If the parenthesis is open, and this token is not part of the current + ** phrase, and the starting byte offset of this token is past the point + ** that has currently been copied into the output buffer, close the + ** parenthesis. */ + if( p->bOpen + && (iPos<=p->iter.iStart || p->iter.iStart<0) + && iStartOff>p->iOff + ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } + + /* If this is the start of a new phrase, and the highlight is not open: + ** + ** * copy text from the input up to the start of the phrase, and + ** * open the highlight. + */ + if( iPos==p->iter.iStart && p->bOpen==0 ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); + fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->iOff = iStartOff; + p->bOpen = 1; + } + + if( iPos==p->iter.iEnd ){ + if( p->bOpen==0 ){ + assert( p->iRangeEnd>=0 ); + fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->bOpen = 1; + } + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + + if( rc==SQLITE_OK ){ + rc = fts5CInstIterNext(&p->iter); + } + } + + if( iPos==p->iRangeEnd ){ + if( p->bOpen ){ + if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + } + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + } + + return rc; +} + + +/* +** Implementation of highlight() function. +*/ +static void fts5HighlightFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + HighlightContext ctx; + int rc; + int iCol; + + if( nVal!=3 ){ + const char *zErr = "wrong number of arguments to function highlight()"; + sqlite3_result_error(pCtx, zErr, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + memset(&ctx, 0, sizeof(HighlightContext)); + ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); + ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; + rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); + if( rc==SQLITE_RANGE ){ + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = SQLITE_OK; + }else if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ + if( rc==SQLITE_OK ){ + rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); + } + + if( rc==SQLITE_OK ){ + rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb + ); + } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); + } + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); + + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + } + sqlite3_free(ctx.zOut); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} +/* +** End of highlight() implementation. +**************************************************************************/ + +/* +** Context object passed to the fts5SentenceFinderCb() function. +*/ +typedef struct Fts5SFinder Fts5SFinder; +struct Fts5SFinder { + int iPos; /* Current token position */ + int nFirstAlloc; /* Allocated size of aFirst[] */ + int nFirst; /* Number of entries in aFirst[] */ + int *aFirst; /* Array of first token in each sentence */ + const char *zDoc; /* Document being tokenized */ +}; + +/* +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an +** error occurs. +*/ +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ + if( p->nFirstAlloc==p->nFirst ){ + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; + int *aNew; + + aNew = (int*)sqlite3_realloc64(p->aFirst, nNew*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + p->aFirst = aNew; + p->nFirstAlloc = nNew; + } + p->aFirst[p->nFirst++] = iAdd; + return SQLITE_OK; +} + +/* +** This function is an xTokenize() callback used by the auxiliary snippet() +** function. Its job is to identify tokens that are the first in a sentence. +** For each such token, an entry is added to the SFinder.aFirst[] array. +*/ +static int fts5SentenceFinderCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + int rc = SQLITE_OK; + + UNUSED_PARAM2(pToken, nToken); + UNUSED_PARAM(iEndOff); + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + Fts5SFinder *p = (Fts5SFinder*)pContext; + if( p->iPos>0 ){ + int i; + char c = 0; + for(i=iStartOff-1; i>=0; i--){ + c = p->zDoc[i]; + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; + } + if( i!=iStartOff-1 && (c=='.' || c==':') ){ + rc = fts5SentenceFinderAdd(p, p->iPos); + } + }else{ + rc = fts5SentenceFinderAdd(p, 0); + } + p->iPos++; + } + return rc; +} + +static int fts5SnippetScore( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + int nDocsize, /* Size of column in tokens */ + unsigned char *aSeen, /* Array with one element per query phrase */ + int iCol, /* Column to score */ + int iPos, /* Starting offset to score */ + int nToken, /* Max tokens per snippet */ + int *pnScore, /* OUT: Score */ + int *piPos /* OUT: Adjusted offset */ +){ + int rc; + int i; + int ip = 0; + int ic = 0; + int iOff = 0; + int iFirst = -1; + int nInst; + int nScore = 0; + int iLast = 0; + sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken; + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; ixInst(pFts, i, &ip, &ic, &iOff); + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOffxPhraseSize(pFts, ip); + } + } + + *pnScore = nScore; + if( piPos ){ + sqlite3_int64 iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; + if( iAdj<0 ) iAdj = 0; + *piPos = (int)iAdj; + } + + return rc; +} + +/* +** Return the value in pVal interpreted as utf-8 text. Except, if pVal +** contains a NULL value, return a pointer to a static string zero +** bytes in length instead of a NULL pointer. +*/ +static const char *fts5ValueToText(sqlite3_value *pVal){ + const char *zRet = (const char*)sqlite3_value_text(pVal); + return zRet ? zRet : ""; +} + +/* +** Implementation of snippet() function. +*/ +static void fts5SnippetFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + HighlightContext ctx; + int rc = SQLITE_OK; /* Return code */ + int iCol; /* 1st argument to snippet() */ + const char *zEllips; /* 4th argument to snippet() */ + int nToken; /* 5th argument to snippet() */ + int nInst = 0; /* Number of instance matches this row */ + int i; /* Used to iterate through instances */ + int nPhrase; /* Number of phrases in query */ + unsigned char *aSeen; /* Array of "seen instance" flags */ + int iBestCol; /* Column containing best snippet */ + int iBestStart = 0; /* First token of best snippet */ + int nBestScore = 0; /* Score of best snippet */ + int nColSize = 0; /* Total size of iBestCol in tokens */ + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ + int nCol; + + if( nVal!=5 ){ + const char *zErr = "wrong number of arguments to function snippet()"; + sqlite3_result_error(pCtx, zErr, -1); + return; + } + + nCol = pApi->xColumnCount(pFts); + memset(&ctx, 0, sizeof(HighlightContext)); + iCol = sqlite3_value_int(apVal[0]); + ctx.zOpen = fts5ValueToText(apVal[1]); + ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; + zEllips = fts5ValueToText(apVal[3]); + nToken = sqlite3_value_int(apVal[4]); + + iBestCol = (iCol>=0 ? iCol : 0); + nPhrase = pApi->xPhraseCount(pFts); + aSeen = sqlite3_malloc(nPhrase); + if( aSeen==0 ){ + rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + rc = pApi->xInstCount(pFts, &nInst); + } + + memset(&sFinder, 0, sizeof(Fts5SFinder)); + for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize_v2(pFts, + sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb + ); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnSize(pFts, i, &nDocsize); + if( rc!=SQLITE_OK ) break; + + for(ii=0; rc==SQLITE_OK && iixInst(pFts, ii, &ip, &ic, &io); + if( ic!=i ) continue; + if( io>nDocsize ) rc = FTS5_CORRUPT; + if( rc!=SQLITE_OK ) continue; + memset(aSeen, 0, nPhrase); + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, + io, nToken, &nScore, &iAdj + ); + if( rc==SQLITE_OK && nScore>nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = iAdj; + nColSize = nDocsize; + } + + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ + for(jj=0; jj<(sFinder.nFirst-1); jj++){ + if( sFinder.aFirst[jj+1]>io ) break; + } + + if( sFinder.aFirst[jj]nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = sFinder.aFirst[jj]; + nColSize = nDocsize; + } + } + } + } + } + } + + if( rc==SQLITE_OK ){ + rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); + } + if( rc==SQLITE_OK && nColSize==0 ){ + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); + } + if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iBestCol */ + int nLoc = 0; /* Bytes in pLoc */ + + if( rc==SQLITE_OK ){ + rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); + } + + ctx.iRangeStart = iBestStart; + ctx.iRangeEnd = iBestStart + nToken - 1; + + if( iBestStart>0 ){ + fts5HighlightAppend(&rc, &ctx, zEllips, -1); + } + + /* Advance iterator ctx.iter so that it points to the first coalesced + ** phrase instance at or following position iBestStart. */ + while( ctx.iter.iStart>=0 && ctx.iter.iStartxColumnLocale(pFts, iBestCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb + ); + } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); + } + if( ctx.iRangeEnd>=(nColSize-1) ){ + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); + }else{ + fts5HighlightAppend(&rc, &ctx, zEllips, -1); + } + } + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + sqlite3_free(ctx.zOut); + sqlite3_free(aSeen); + sqlite3_free(sFinder.aFirst); +} + +/************************************************************************/ + +/* +** The first time the bm25() function is called for a query, an instance +** of the following structure is allocated and populated. +*/ +typedef struct Fts5Bm25Data Fts5Bm25Data; +struct Fts5Bm25Data { + int nPhrase; /* Number of phrases in query */ + double avgdl; /* Average number of tokens in each row */ + double *aIDF; /* IDF for each phrase */ + double *aFreq; /* Array used to calculate phrase freq. */ +}; + +/* +** Callback used by fts5Bm25GetData() to count the number of rows in the +** table matched by each individual phrase within the query. +*/ +static int fts5CountCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData /* Pointer to sqlite3_int64 variable */ +){ + sqlite3_int64 *pn = (sqlite3_int64*)pUserData; + UNUSED_PARAM2(pApi, pFts); + (*pn)++; + return SQLITE_OK; +} + +/* +** Set *ppData to point to the Fts5Bm25Data object for the current query. +** If the object has not already been allocated, allocate and populate it +** now. +*/ +static int fts5Bm25GetData( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts5Bm25Data *p; /* Object to return */ + + p = (Fts5Bm25Data*)pApi->xGetAuxdata(pFts, 0); + if( p==0 ){ + int nPhrase; /* Number of phrases in query */ + sqlite3_int64 nRow = 0; /* Number of rows in table */ + sqlite3_int64 nToken = 0; /* Number of tokens in table */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + int i; + + /* Allocate the Fts5Bm25Data object */ + nPhrase = pApi->xPhraseCount(pFts); + nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double); + p = (Fts5Bm25Data*)sqlite3_malloc64(nByte); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(p, 0, (size_t)nByte); + p->nPhrase = nPhrase; + p->aIDF = (double*)&p[1]; + p->aFreq = &p->aIDF[nPhrase]; + } + + /* Calculate the average document length for this FTS5 table */ + if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow); + assert( rc!=SQLITE_OK || nRow>0 ); + if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken); + if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow; + + /* Calculate an IDF for each phrase in the query */ + for(i=0; rc==SQLITE_OK && ixQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb); + if( rc==SQLITE_OK ){ + /* Calculate the IDF (Inverse Document Frequency) for phrase i. + ** This is done using the standard BM25 formula as found on wikipedia: + ** + ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) ) + ** + ** where "N" is the total number of documents in the set and nHit + ** is the number that contain at least one instance of the phrase + ** under consideration. + ** + ** The problem with this is that if (N < 2*nHit), the IDF is + ** negative. Which is undesirable. So the mimimum allowable IDF is + ** (1e-6) - roughly the same as a term that appears in just over + ** half of set of 5,000,000 documents. */ + double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); + if( idf<=0.0 ) idf = 1e-6; + p->aIDF[i] = idf; + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } + if( rc!=SQLITE_OK ) p = 0; + } + *ppData = p; + return rc; +} + +/* +** Implementation of bm25() function. +*/ +static void fts5Bm25Function( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const double k1 = 1.2; /* Constant "k1" from BM25 formula */ + const double b = 0.75; /* Constant "b" from BM25 formula */ + int rc; /* Error code */ + double score = 0.0; /* SQL function return value */ + Fts5Bm25Data *pData; /* Values allocated/calculated once only */ + int i; /* Iterator variable */ + int nInst = 0; /* Value returned by xInstCount() */ + double D = 0.0; /* Total number of tokens in row */ + double *aFreq = 0; /* Array of phrase freq. for current row */ + + /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation) + ** for each phrase in the query for the current row. */ + rc = fts5Bm25GetData(pApi, pFts, &pData); + if( rc==SQLITE_OK ){ + aFreq = pData->aFreq; + memset(aFreq, 0, sizeof(double) * pData->nPhrase); + rc = pApi->xInstCount(pFts, &nInst); + } + for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0; + aFreq[ip] += w; + } + } + + /* Figure out the total size of the current row in tokens. */ + if( rc==SQLITE_OK ){ + int nTok; + rc = pApi->xColumnSize(pFts, -1, &nTok); + D = (double)nTok; + } + + /* Determine and return the BM25 score for the current row. Or, if an + ** error has occurred, throw an exception. */ + if( rc==SQLITE_OK ){ + for(i=0; inPhrase; i++){ + score += pData->aIDF[i] * ( + ( aFreq[i] * (k1 + 1.0) ) / + ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) ) + ); + } + sqlite3_result_double(pCtx, -1.0 * score); + }else{ + sqlite3_result_error_code(pCtx, rc); + } +} + +/* +** Implementation of fts5_get_locale() function. +*/ +static void fts5GetLocaleFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + int iCol = 0; + int eType = 0; + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + /* xColumnLocale() must be available */ + assert( pApi->iVersion>=4 ); + + if( nVal!=1 ){ + const char *z = "wrong number of arguments to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + eType = sqlite3_value_numeric_type(apVal[0]); + if( eType!=SQLITE_INTEGER ){ + const char *z = "non-integer argument passed to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ + sqlite3_result_error_code(pCtx, SQLITE_RANGE); + return; + } + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + return; + } + + sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT); +} + +int sqlite3Fts5AuxInit(fts5_api *pApi){ + struct Builtin { + const char *zFunc; /* Function name (nul-terminated) */ + void *pUserData; /* User-data pointer */ + fts5_extension_function xFunc;/* Callback function */ + void (*xDestroy)(void*); /* Destructor function */ + } aBuiltin [] = { + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 }, + }; + int rc = SQLITE_OK; /* Return code */ + int i; /* To iterate through builtin functions */ + + for(i=0; rc==SQLITE_OK && ixCreateFunction(pApi, + aBuiltin[i].zFunc, + aBuiltin[i].pUserData, + aBuiltin[i].xFunc, + aBuiltin[i].xDestroy + ); + } + + return rc; +} diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c new file mode 100644 index 0000000000000000000000000000000000000000..891ef0203aff1c6fb2cf17bc93490ca9b5f24433 --- /dev/null +++ b/ext/fts5/fts5_buffer.c @@ -0,0 +1,411 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + + +#include "fts5Int.h" + +int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ + if( (u32)pBuf->nSpacenSpace ? pBuf->nSpace : 64; + u8 *pNew; + while( nNewp, nNew); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return 1; + }else{ + pBuf->nSpace = (int)nNew; + pBuf->p = pNew; + } + } + return 0; +} + + +/* +** Encode value iVal as an SQLite varint and append it to the buffer object +** pBuf. If an OOM error occurs, set the error code in p. +*/ +void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ + if( fts5BufferGrow(pRc, pBuf, 9) ) return; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal); +} + +void sqlite3Fts5Put32(u8 *aBuf, int iVal){ + aBuf[0] = (iVal>>24) & 0x00FF; + aBuf[1] = (iVal>>16) & 0x00FF; + aBuf[2] = (iVal>> 8) & 0x00FF; + aBuf[3] = (iVal>> 0) & 0x00FF; +} + +int sqlite3Fts5Get32(const u8 *aBuf){ + return (int)((((u32)aBuf[0])<<24) + (aBuf[1]<<16) + (aBuf[2]<<8) + aBuf[3]); +} + +/* +** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +void sqlite3Fts5BufferAppendBlob( + int *pRc, + Fts5Buffer *pBuf, + u32 nData, + const u8 *pData +){ + if( nData ){ + if( fts5BufferGrow(pRc, pBuf, nData) ) return; + assert( pBuf->p!=0 ); + memcpy(&pBuf->p[pBuf->n], pData, nData); + pBuf->n += nData; + } +} + +/* +** Append the nul-terminated string zStr to the buffer pBuf. This function +** ensures that the byte following the buffer data is set to 0x00, even +** though this byte is not included in the pBuf->n count. +*/ +void sqlite3Fts5BufferAppendString( + int *pRc, + Fts5Buffer *pBuf, + const char *zStr +){ + int nStr = (int)strlen(zStr); + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr); + pBuf->n--; +} + +/* +** Argument zFmt is a printf() style format string. This function performs +** the printf() style processing, then appends the results to buffer pBuf. +** +** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte +** following the buffer data is set to 0x00, even though this byte is not +** included in the pBuf->n count. +*/ +void sqlite3Fts5BufferAppendPrintf( + int *pRc, + Fts5Buffer *pBuf, + char *zFmt, ... +){ + if( *pRc==SQLITE_OK ){ + char *zTmp; + va_list ap; + va_start(ap, zFmt); + zTmp = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + + if( zTmp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp); + sqlite3_free(zTmp); + } + } +} + +char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zRet==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return zRet; +} + + +/* +** Free any buffer allocated by pBuf. Zero the structure before returning. +*/ +void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ + sqlite3_free(pBuf->p); + memset(pBuf, 0, sizeof(Fts5Buffer)); +} + +/* +** Zero the contents of the buffer object. But do not free the associated +** memory allocation. +*/ +void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ + pBuf->n = 0; +} + +/* +** Set the buffer to contain nData/pData. If an OOM error occurs, leave an +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +void sqlite3Fts5BufferSet( + int *pRc, + Fts5Buffer *pBuf, + int nData, + const u8 *pData +){ + pBuf->n = 0; + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData); +} + +int sqlite3Fts5PoslistNext64( + const u8 *a, int n, /* Buffer containing poslist */ + int *pi, /* IN/OUT: Offset within a[] */ + i64 *piOff /* IN/OUT: Current offset */ +){ + int i = *pi; + assert( a!=0 || i==0 ); + if( i>=n ){ + /* EOF */ + *piOff = -1; + return 1; + }else{ + i64 iOff = *piOff; + u32 iVal; + assert( a!=0 ); + fts5FastGetVarint32(a, i, iVal); + if( iVal<=1 ){ + if( iVal==0 ){ + *pi = i; + return 0; + } + fts5FastGetVarint32(a, i, iVal); + iOff = ((i64)iVal) << 32; + assert( iOff>=0 ); + fts5FastGetVarint32(a, i, iVal); + if( iVal<2 ){ + /* This is a corrupt record. So stop parsing it here. */ + *piOff = -1; + return 1; + } + *piOff = iOff + ((iVal-2) & 0x7FFFFFFF); + }else{ + *piOff = (iOff & (i64)0x7FFFFFFF<<32)+((iOff + (iVal-2)) & 0x7FFFFFFF); + } + *pi = i; + assert_nc( *piOff>=iOff ); + return 0; + } +} + + +/* +** Advance the iterator object passed as the only argument. Return true +** if the iterator reaches EOF, or false otherwise. +*/ +int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){ + if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){ + pIter->bEof = 1; + } + return pIter->bEof; +} + +int sqlite3Fts5PoslistReaderInit( + const u8 *a, int n, /* Poslist buffer to iterate through */ + Fts5PoslistReader *pIter /* Iterator object to initialize */ +){ + memset(pIter, 0, sizeof(*pIter)); + pIter->a = a; + pIter->n = n; + sqlite3Fts5PoslistReaderNext(pIter); + return pIter->bEof; +} + +/* +** Append position iPos to the position list being accumulated in buffer +** pBuf, which must be already be large enough to hold the new data. +** The previous position written to this list is *piPrev. *piPrev is set +** to iPos before returning. +*/ +void sqlite3Fts5PoslistSafeAppend( + Fts5Buffer *pBuf, + i64 *piPrev, + i64 iPos +){ + if( iPos>=*piPrev ){ + static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; + if( (iPos & colmask) != (*piPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + *piPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); + *piPrev = iPos; + } +} + +int sqlite3Fts5PoslistWriterAppend( + Fts5Buffer *pBuf, + Fts5PoslistWriter *pWriter, + i64 iPos +){ + int rc = 0; /* Initialized only to suppress erroneous warning from Clang */ + if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc; + sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos); + return SQLITE_OK; +} + +void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + if( nByte>0 ) *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, (size_t)nByte); + } + } + return pRet; +} + +/* +** Return a nul-terminated copy of the string indicated by pIn. If nIn +** is non-negative, then it is the length of the string in bytes. Otherwise, +** the length of the string is determined using strlen(). +** +** It is the responsibility of the caller to eventually free the returned +** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. +*/ +char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + if( nIn<0 ){ + nIn = (int)strlen(pIn); + } + zRet = (char*)sqlite3_malloc(nIn+1); + if( zRet ){ + memcpy(zRet, pIn, nIn); + zRet[nIn] = '\0'; + }else{ + *pRc = SQLITE_NOMEM; + } + } + return zRet; +} + + +/* +** Return true if character 't' may be part of an FTS5 bareword, or false +** otherwise. Characters that may be part of barewords: +** +** * All non-ASCII characters, +** * The 52 upper and lower case ASCII characters, and +** * The 10 integer ASCII characters. +** * The underscore character "_" (0x5F). +** * The unicode "subsitute" character (0x1A). +*/ +int sqlite3Fts5IsBareword(char t){ + u8 aBareword[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ + }; + + return (t & 0x80) || aBareword[(int)t]; +} + + +/************************************************************************* +*/ +typedef struct Fts5TermsetEntry Fts5TermsetEntry; +struct Fts5TermsetEntry { + char *pTerm; + int nTerm; + int iIdx; /* Index (main or aPrefix[] entry) */ + Fts5TermsetEntry *pNext; +}; + +struct Fts5Termset { + Fts5TermsetEntry *apHash[512]; +}; + +int sqlite3Fts5TermsetNew(Fts5Termset **pp){ + int rc = SQLITE_OK; + *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); + return rc; +} + +int sqlite3Fts5TermsetAdd( + Fts5Termset *p, + int iIdx, + const char *pTerm, int nTerm, + int *pbPresent +){ + int rc = SQLITE_OK; + *pbPresent = 0; + if( p ){ + int i; + u32 hash = 13; + Fts5TermsetEntry *pEntry; + + /* Calculate a hash value for this term. This is the same hash checksum + ** used by the fts5_hash.c module. This is not important for correct + ** operation of the module, but is necessary to ensure that some tests + ** designed to produce hash table collisions really do work. */ + for(i=nTerm-1; i>=0; i--){ + hash = (hash << 3) ^ hash ^ pTerm[i]; + } + hash = (hash << 3) ^ hash ^ iIdx; + hash = hash % ArraySize(p->apHash); + + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ + if( pEntry->iIdx==iIdx + && pEntry->nTerm==nTerm + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 + ){ + *pbPresent = 1; + break; + } + } + + if( pEntry==0 ){ + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); + if( pEntry ){ + pEntry->pTerm = (char*)&pEntry[1]; + pEntry->nTerm = nTerm; + pEntry->iIdx = iIdx; + memcpy(pEntry->pTerm, pTerm, nTerm); + pEntry->pNext = p->apHash[hash]; + p->apHash[hash] = pEntry; + } + } + } + + return rc; +} + +void sqlite3Fts5TermsetFree(Fts5Termset *p){ + if( p ){ + u32 i; + for(i=0; iapHash); i++){ + Fts5TermsetEntry *pEntry = p->apHash[i]; + while( pEntry ){ + Fts5TermsetEntry *pDel = pEntry; + pEntry = pEntry->pNext; + sqlite3_free(pDel); + } + } + sqlite3_free(p); + } +} diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c new file mode 100644 index 0000000000000000000000000000000000000000..eea82b046deda92b4444cb9c0dce59a9b77b21b5 --- /dev/null +++ b/ext/fts5/fts5_config.c @@ -0,0 +1,1127 @@ +/* +** 2014 Jun 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + + +#include "fts5Int.h" + +#define FTS5_DEFAULT_PAGE_SIZE 4050 +#define FTS5_DEFAULT_AUTOMERGE 4 +#define FTS5_DEFAULT_USERMERGE 4 +#define FTS5_DEFAULT_CRISISMERGE 16 +#define FTS5_DEFAULT_HASHSIZE (1024*1024) + +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ + +/* Maximum allowed page size */ +#define FTS5_MAX_PAGE_SIZE (64*1024) + +static int fts5_iswhitespace(char x){ + return (x==' '); +} + +static int fts5_isopenquote(char x){ + return (x=='"' || x=='\'' || x=='[' || x=='`'); +} + +/* +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in +** the string that is not a white-space character. +*/ +static const char *fts5ConfigSkipWhitespace(const char *pIn){ + const char *p = pIn; + if( p ){ + while( fts5_iswhitespace(*p) ){ p++; } + } + return p; +} + +/* +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in +** the string that is not a "bareword" character. +*/ +static const char *fts5ConfigSkipBareword(const char *pIn){ + const char *p = pIn; + while ( sqlite3Fts5IsBareword(*p) ) p++; + if( p==pIn ) p = 0; + return p; +} + +static int fts5_isdigit(char a){ + return (a>='0' && a<='9'); +} + + + +static const char *fts5ConfigSkipLiteral(const char *pIn){ + const char *p = pIn; + switch( *p ){ + case 'n': case 'N': + if( sqlite3_strnicmp("null", p, 4)==0 ){ + p = &p[4]; + }else{ + p = 0; + } + break; + + case 'x': case 'X': + p++; + if( *p=='\'' ){ + p++; + while( (*p>='a' && *p<='f') + || (*p>='A' && *p<='F') + || (*p>='0' && *p<='9') + ){ + p++; + } + if( *p=='\'' && 0==((p-pIn)%2) ){ + p++; + }else{ + p = 0; + } + }else{ + p = 0; + } + break; + + case '\'': + p++; + while( p ){ + if( *p=='\'' ){ + p++; + if( *p!='\'' ) break; + } + p++; + if( *p==0 ) p = 0; + } + break; + + default: + /* maybe a number */ + if( *p=='+' || *p=='-' ) p++; + while( fts5_isdigit(*p) ) p++; + + /* At this point, if the literal was an integer, the parse is + ** finished. Or, if it is a floating point value, it may continue + ** with either a decimal point or an 'E' character. */ + if( *p=='.' && fts5_isdigit(p[1]) ){ + p += 2; + while( fts5_isdigit(*p) ) p++; + } + if( p==pIn ) p = 0; + + break; + } + + return p; +} + +/* +** The first character of the string pointed to by argument z is guaranteed +** to be an open-quote character (see function fts5_isopenquote()). +** +** This function searches for the corresponding close-quote character within +** the string and, if found, dequotes the string in place and adds a new +** nul-terminator byte. +** +** If the close-quote is found, the value returned is the byte offset of +** the character immediately following it. Or, if the close-quote is not +** found, -1 is returned. If -1 is returned, the buffer is left in an +** undefined state. +*/ +static int fts5Dequote(char *z){ + char q; + int iIn = 1; + int iOut = 0; + q = z[0]; + + /* Set stack variable q to the close-quote character */ + assert( q=='[' || q=='\'' || q=='"' || q=='`' ); + if( q=='[' ) q = ']'; + + while( z[iIn] ){ + if( z[iIn]==q ){ + if( z[iIn+1]!=q ){ + /* Character iIn was the close quote. */ + iIn++; + break; + }else{ + /* Character iIn and iIn+1 form an escaped quote character. Skip + ** the input cursor past both and copy a single quote character + ** to the output buffer. */ + iIn += 2; + z[iOut++] = q; + } + }else{ + z[iOut++] = z[iIn++]; + } + } + + z[iOut] = '\0'; + return iIn; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +void sqlite3Fts5Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + assert( 0==fts5_iswhitespace(z[0]) ); + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + fts5Dequote(z); + } +} + + +struct Fts5Enum { + const char *zName; + int eVal; +}; +typedef struct Fts5Enum Fts5Enum; + +static int fts5ConfigSetEnum( + const Fts5Enum *aEnum, + const char *zEnum, + int *peVal +){ + int nEnum = (int)strlen(zEnum); + int i; + int iVal = -1; + + for(i=0; aEnum[i].zName; i++){ + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ + if( iVal>=0 ) return SQLITE_ERROR; + iVal = aEnum[i].eVal; + } + } + + *peVal = iVal; + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; +} + +/* +** Parse a "special" CREATE VIRTUAL TABLE directive and update +** configuration object pConfig as appropriate. +** +** If successful, object pConfig is updated and SQLITE_OK returned. If +** an error occurs, an SQLite error code is returned and an error message +** may be left in *pzErr. It is the responsibility of the caller to +** eventually free any such error message using sqlite3_free(). +*/ +static int fts5ConfigParseSpecial( + Fts5Config *pConfig, /* Configuration object to update */ + const char *zCmd, /* Special command to parse */ + const char *zArg, /* Argument to parse */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + int nCmd = (int)strlen(zCmd); + + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ + const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; + const char *p; + int bFirst = 1; + if( pConfig->aPrefix==0 ){ + pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte); + if( rc ) return rc; + } + + p = zArg; + while( 1 ){ + int nPre = 0; + + while( p[0]==' ' ) p++; + if( bFirst==0 && p[0]==',' ){ + p++; + while( p[0]==' ' ) p++; + }else if( p[0]=='\0' ){ + break; + } + if( p[0]<'0' || p[0]>'9' ){ + *pzErr = sqlite3_mprintf("malformed prefix=... directive"); + rc = SQLITE_ERROR; + break; + } + + if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){ + *pzErr = sqlite3_mprintf( + "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES + ); + rc = SQLITE_ERROR; + break; + } + + while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ + nPre = nPre*10 + (p[0] - '0'); + p++; + } + + if( nPre<=0 || nPre>=1000 ){ + *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); + rc = SQLITE_ERROR; + break; + } + + pConfig->aPrefix[pConfig->nPrefix] = nPre; + pConfig->nPrefix++; + bFirst = 0; + } + assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES ); + return rc; + } + + if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ + const char *p = (const char*)zArg; + sqlite3_int64 nArg = strlen(zArg) + 1; + char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg); + + if( azArg ){ + char *pSpace = (char*)&azArg[nArg]; + if( pConfig->t.azArg ){ + *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); + rc = SQLITE_ERROR; + }else{ + for(nArg=0; p && *p; nArg++){ + const char *p2 = fts5ConfigSkipWhitespace(p); + if( *p2=='\'' ){ + p = fts5ConfigSkipLiteral(p2); + }else{ + p = fts5ConfigSkipBareword(p2); + } + if( p ){ + memcpy(pSpace, p2, p-p2); + azArg[nArg] = pSpace; + sqlite3Fts5Dequote(pSpace); + pSpace += (p - p2) + 1; + p = fts5ConfigSkipWhitespace(p); + } + } + if( p==0 ){ + *pzErr = sqlite3_mprintf("parse error in tokenize directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->t.azArg = (const char**)azArg; + pConfig->t.nArg = nArg; + azArg = 0; + } + } + } + sqlite3_free(azArg); + + return rc; + } + + if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + *pzErr = sqlite3_mprintf("multiple content=... directives"); + rc = SQLITE_ERROR; + }else{ + if( zArg[0] ){ + pConfig->eContent = FTS5_CONTENT_EXTERNAL; + pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg); + }else{ + pConfig->eContent = FTS5_CONTENT_NONE; + } + } + return rc; + } + + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessDelete = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessUnindexed = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ + if( pConfig->zContentRowid ){ + *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); + rc = SQLITE_ERROR; + }else{ + pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1); + } + return rc; + } + + if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bColumnsize = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed locale=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bLocale = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ + const Fts5Enum aDetail[] = { + { "none", FTS5_DETAIL_NONE }, + { "full", FTS5_DETAIL_FULL }, + { "columns", FTS5_DETAIL_COLUMNS }, + { 0, 0 } + }; + + if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ + *pzErr = sqlite3_mprintf("malformed detail=... directive"); + } + return rc; + } + + if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed tokendata=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bTokendata = (zArg[0]=='1'); + } + return rc; + } + + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); + return SQLITE_ERROR; +} + +/* +** Gobble up the first bareword or quoted word from the input buffer zIn. +** Return a pointer to the character immediately following the last in +** the gobbled word if successful, or a NULL pointer otherwise (failed +** to find close-quote character). +** +** Before returning, set pzOut to point to a new buffer containing a +** nul-terminated, dequoted copy of the gobbled word. If the word was +** quoted, *pbQuoted is also set to 1 before returning. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op (NULL is returned). Otherwise, if an OOM occurs within this +** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not* +** set if a parse error (failed to find close quote) occurs. +*/ +static const char *fts5ConfigGobbleWord( + int *pRc, /* IN/OUT: Error code */ + const char *zIn, /* Buffer to gobble string/bareword from */ + char **pzOut, /* OUT: malloc'd buffer containing str/bw */ + int *pbQuoted /* OUT: Set to true if dequoting required */ +){ + const char *zRet = 0; + + sqlite3_int64 nIn = strlen(zIn); + char *zOut = sqlite3_malloc64(nIn+1); + + assert( *pRc==SQLITE_OK ); + *pbQuoted = 0; + *pzOut = 0; + + if( zOut==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memcpy(zOut, zIn, (size_t)(nIn+1)); + if( fts5_isopenquote(zOut[0]) ){ + int ii = fts5Dequote(zOut); + zRet = &zIn[ii]; + *pbQuoted = 1; + }else{ + zRet = fts5ConfigSkipBareword(zIn); + if( zRet ){ + zOut[zRet-zIn] = '\0'; + } + } + } + + if( zRet==0 ){ + sqlite3_free(zOut); + }else{ + *pzOut = zOut; + } + + return zRet; +} + +static int fts5ConfigParseColumn( + Fts5Config *p, + char *zCol, + char *zArg, + char **pzErr, + int *pbUnindexed +){ + int rc = SQLITE_OK; + if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) + || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) + ){ + *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol); + rc = SQLITE_ERROR; + }else if( zArg ){ + if( 0==sqlite3_stricmp(zArg, "unindexed") ){ + p->abUnindexed[p->nCol] = 1; + *pbUnindexed = 1; + }else{ + *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); + rc = SQLITE_ERROR; + } + } + + p->azCol[p->nCol++] = zCol; + return rc; +} + +/* +** Populate the Fts5Config.zContentExprlist string. +*/ +static int fts5ConfigMakeExprlist(Fts5Config *p){ + int i; + int rc = SQLITE_OK; + Fts5Buffer buf = {0, 0, 0}; + + sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); + if( p->eContent!=FTS5_CONTENT_NONE ){ + assert( p->eContent==FTS5_CONTENT_EXTERNAL + || p->eContent==FTS5_CONTENT_NORMAL + || p->eContent==FTS5_CONTENT_UNINDEXED + ); + for(i=0; inCol; i++){ + if( p->eContent==FTS5_CONTENT_EXTERNAL ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); + }else if( p->eContent==FTS5_CONTENT_NORMAL || p->abUnindexed[i] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } + if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){ + for(i=0; inCol; i++){ + if( p->abUnindexed[i]==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } + + assert( p->zContentExprlist==0 ); + p->zContentExprlist = (char*)buf.p; + return rc; +} + +/* +** Arguments nArg/azArg contain the string arguments passed to the xCreate +** or xConnect method of the virtual table. This function attempts to +** allocate an instance of Fts5Config containing the results of parsing +** those arguments. +** +** If successful, SQLITE_OK is returned and *ppOut is set to point to the +** new Fts5Config object. If an error occurs, an SQLite error code is +** returned, *ppOut is set to NULL and an error message may be left in +** *pzErr. It is the responsibility of the caller to eventually free any +** such error message using sqlite3_free(). +*/ +int sqlite3Fts5ConfigParse( + Fts5Global *pGlobal, + sqlite3 *db, + int nArg, /* Number of arguments */ + const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */ + Fts5Config **ppOut, /* OUT: Results of parse */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pRet; /* New object to return */ + int i; + sqlite3_int64 nByte; + int bUnindexed = 0; /* True if there are one or more UNINDEXED */ + + *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + if( pRet==0 ) return SQLITE_NOMEM; + memset(pRet, 0, sizeof(Fts5Config)); + pRet->pGlobal = pGlobal; + pRet->db = db; + pRet->iCookie = -1; + + nByte = nArg * (sizeof(char*) + sizeof(u8)); + pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); + pRet->abUnindexed = pRet->azCol ? (u8*)&pRet->azCol[nArg] : 0; + pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); + pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); + pRet->bColumnsize = 1; + pRet->eDetail = FTS5_DETAIL_FULL; +#ifdef SQLITE_DEBUG + pRet->bPrefixIndex = 1; +#endif + if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ + *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); + rc = SQLITE_ERROR; + } + + assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK ); + for(i=3; rc==SQLITE_OK && ibContentlessDelete + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_delete=1 if columnsize=0 is not present. + ** + ** This restriction may be removed at some point. + */ + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 is incompatible with columnsize=0" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_unindexed=1 if the table is actually a + ** contentless one. + */ + if( rc==SQLITE_OK + && pRet->bContentlessUnindexed + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_unindexed=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* If no zContent option was specified, fill in the default values. */ + if( rc==SQLITE_OK && pRet->zContent==0 ){ + const char *zTail = 0; + assert( pRet->eContent==FTS5_CONTENT_NORMAL + || pRet->eContent==FTS5_CONTENT_NONE + ); + if( pRet->eContent==FTS5_CONTENT_NORMAL ){ + zTail = "content"; + }else if( bUnindexed && pRet->bContentlessUnindexed ){ + pRet->eContent = FTS5_CONTENT_UNINDEXED; + zTail = "content"; + }else if( pRet->bColumnsize ){ + zTail = "docsize"; + } + + if( zTail ){ + pRet->zContent = sqlite3Fts5Mprintf( + &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail + ); + } + } + + if( rc==SQLITE_OK && pRet->zContentRowid==0 ){ + pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1); + } + + /* Formulate the zContentExprlist text */ + if( rc==SQLITE_OK ){ + rc = fts5ConfigMakeExprlist(pRet); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts5ConfigFree(pRet); + *ppOut = 0; + } + return rc; +} + +/* +** Free the configuration object passed as the only argument. +*/ +void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ + if( pConfig ){ + int i; + if( pConfig->t.pTok ){ + if( pConfig->t.pApi1 ){ + pConfig->t.pApi1->xDelete(pConfig->t.pTok); + }else{ + pConfig->t.pApi2->xDelete(pConfig->t.pTok); + } + } + sqlite3_free((char*)pConfig->t.azArg); + sqlite3_free(pConfig->zDb); + sqlite3_free(pConfig->zName); + for(i=0; inCol; i++){ + sqlite3_free(pConfig->azCol[i]); + } + sqlite3_free(pConfig->azCol); + sqlite3_free(pConfig->aPrefix); + sqlite3_free(pConfig->zRank); + sqlite3_free(pConfig->zRankArgs); + sqlite3_free(pConfig->zContent); + sqlite3_free(pConfig->zContentRowid); + sqlite3_free(pConfig->zContentExprlist); + sqlite3_free(pConfig); + } +} + +/* +** Call sqlite3_declare_vtab() based on the contents of the configuration +** object passed as the only argument. Return SQLITE_OK if successful, or +** an SQLite error code if an error occurs. +*/ +int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ + int i; + int rc = SQLITE_OK; + char *zSql; + + zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x("); + for(i=0; zSql && inCol; i++){ + const char *zSep = (i==0?"":", "); + zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]); + } + zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)", + zSql, pConfig->zName, FTS5_RANK_NAME + ); + + assert( zSql || rc==SQLITE_NOMEM ); + if( zSql ){ + rc = sqlite3_declare_vtab(pConfig->db, zSql); + sqlite3_free(zSql); + } + + return rc; +} + +/* +** Tokenize the text passed via the second and third arguments. +** +** The callback is invoked once for each token in the input text. The +** arguments passed to it are, in order: +** +** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize() +** const char *pToken // Pointer to buffer containing token +** int nToken // Size of token in bytes +** int iStart // Byte offset of start of token within input text +** int iEnd // Byte offset of end of token within input text +** int iPos // Position of token in input (first token is 0) +** +** If the callback returns a non-zero value the tokenization is abandoned +** and no further callbacks are issued. +** +** This function returns SQLITE_OK if successful or an SQLite error code +** if an error occurs. If the tokenization was abandoned early because +** the callback returned SQLITE_DONE, this is not an error and this function +** still returns SQLITE_OK. Or, if the tokenization was abandoned early +** because the callback returned another non-zero value, it is assumed +** to be an SQLite error code and returned to the caller. +*/ +int sqlite3Fts5Tokenize( + Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ +){ + int rc = SQLITE_OK; + if( pText ){ + if( pConfig->t.pTok==0 ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } + if( rc==SQLITE_OK ){ + if( pConfig->t.pApi1 ){ + rc = pConfig->t.pApi1->xTokenize( + pConfig->t.pTok, pCtx, flags, pText, nText, xToken + ); + }else{ + rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, + pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken + ); + } + } + } + return rc; +} + +/* +** Argument pIn points to the first character in what is expected to be +** a comma-separated list of SQL literals followed by a ')' character. +** If it actually is this, return a pointer to the ')'. Otherwise, return +** NULL to indicate a parse error. +*/ +static const char *fts5ConfigSkipArgs(const char *pIn){ + const char *p = pIn; + + while( 1 ){ + p = fts5ConfigSkipWhitespace(p); + p = fts5ConfigSkipLiteral(p); + p = fts5ConfigSkipWhitespace(p); + if( p==0 || *p==')' ) break; + if( *p!=',' ){ + p = 0; + break; + } + p++; + } + + return p; +} + +/* +** Parameter zIn contains a rank() function specification. The format of +** this is: +** +** + Bareword (function name) +** + Open parenthesis - "(" +** + Zero or more SQL literals in a comma separated list +** + Close parenthesis - ")" +*/ +int sqlite3Fts5ConfigParseRank( + const char *zIn, /* Input string */ + char **pzRank, /* OUT: Rank function name */ + char **pzRankArgs /* OUT: Rank function arguments */ +){ + const char *p = zIn; + const char *pRank; + char *zRank = 0; + char *zRankArgs = 0; + int rc = SQLITE_OK; + + *pzRank = 0; + *pzRankArgs = 0; + + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + p = fts5ConfigSkipWhitespace(p); + pRank = p; + p = fts5ConfigSkipBareword(p); + + if( p ){ + zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank); + if( zRank ) memcpy(zRank, pRank, p-pRank); + }else{ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + p = fts5ConfigSkipWhitespace(p); + if( *p!='(' ) rc = SQLITE_ERROR; + p++; + } + if( rc==SQLITE_OK ){ + const char *pArgs; + p = fts5ConfigSkipWhitespace(p); + pArgs = p; + if( *p!=')' ){ + p = fts5ConfigSkipArgs(p); + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs); + if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs); + } + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRank); + assert( zRankArgs==0 ); + }else{ + *pzRank = zRank; + *pzRankArgs = zRankArgs; + } + return rc; +} + +int sqlite3Fts5ConfigSetValue( + Fts5Config *pConfig, + const char *zKey, + sqlite3_value *pVal, + int *pbBadkey +){ + int rc = SQLITE_OK; + + if( 0==sqlite3_stricmp(zKey, "pgsz") ){ + int pgsz = 0; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + pgsz = sqlite3_value_int(pVal); + } + if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE ){ + *pbBadkey = 1; + }else{ + pConfig->pgsz = pgsz; + } + } + + else if( 0==sqlite3_stricmp(zKey, "hashsize") ){ + int nHashSize = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nHashSize = sqlite3_value_int(pVal); + } + if( nHashSize<=0 ){ + *pbBadkey = 1; + }else{ + pConfig->nHashSize = nHashSize; + } + } + + else if( 0==sqlite3_stricmp(zKey, "automerge") ){ + int nAutomerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nAutomerge = sqlite3_value_int(pVal); + } + if( nAutomerge<0 || nAutomerge>64 ){ + *pbBadkey = 1; + }else{ + if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nAutomerge = nAutomerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "usermerge") ){ + int nUsermerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nUsermerge = sqlite3_value_int(pVal); + } + if( nUsermerge<2 || nUsermerge>16 ){ + *pbBadkey = 1; + }else{ + pConfig->nUsermerge = nUsermerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ + int nCrisisMerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nCrisisMerge = sqlite3_value_int(pVal); + } + if( nCrisisMerge<0 ){ + *pbBadkey = 1; + }else{ + if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + if( nCrisisMerge>=FTS5_MAX_SEGMENT ) nCrisisMerge = FTS5_MAX_SEGMENT-1; + pConfig->nCrisisMerge = nCrisisMerge; + } + } + + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ + int nVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nVal = sqlite3_value_int(pVal); + }else{ + *pbBadkey = 1; + } + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; + if( nVal>100 ) nVal = 0; + pConfig->nDeleteMerge = nVal; + } + + else if( 0==sqlite3_stricmp(zKey, "rank") ){ + const char *zIn = (const char*)sqlite3_value_text(pVal); + char *zRank; + char *zRankArgs; + rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs); + if( rc==SQLITE_OK ){ + sqlite3_free(pConfig->zRank); + sqlite3_free(pConfig->zRankArgs); + pConfig->zRank = zRank; + pConfig->zRankArgs = zRankArgs; + }else if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + *pbBadkey = 1; + } + } + + else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bSecureDelete = (bVal ? 1 : 0); + } + } + + else if( 0==sqlite3_stricmp(zKey, "insttoken") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bPrefixInsttoken = (bVal ? 1 : 0); + } + + }else{ + *pbBadkey = 1; + } + return rc; +} + +/* +** Load the contents of the %_config table into memory. +*/ +int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ + const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; + char *zSql; + sqlite3_stmt *p = 0; + int rc = SQLITE_OK; + int iVersion = 0; + + /* Set default values */ + pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; + pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; + pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; + + zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); + sqlite3_free(zSql); + } + + assert( rc==SQLITE_OK || p==0 ); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(p) ){ + const char *zK = (const char*)sqlite3_column_text(p, 0); + sqlite3_value *pVal = sqlite3_column_value(p, 1); + if( 0==sqlite3_stricmp(zK, "version") ){ + iVersion = sqlite3_value_int(pVal); + }else{ + int bDummy = 0; + sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy); + } + } + rc = sqlite3_finalize(p); + } + + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE + ); + }else{ + pConfig->iVersion = iVersion; + } + + if( rc==SQLITE_OK ){ + pConfig->iCookie = iCookie; + } + return rc; +} + +/* +** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer +** containing the error message created using printf() style formatting +** string zFmt and its trailing arguments. +*/ +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ + va_list ap; /* ... printf arguments */ + char *zMsg = 0; + + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + if( pConfig->pzErrmsg ){ + assert( *pConfig->pzErrmsg==0 ); + *pConfig->pzErrmsg = zMsg; + }else{ + sqlite3_free(zMsg); + } + + va_end(ap); +} + + diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c new file mode 100644 index 0000000000000000000000000000000000000000..877c3f79c531f79886fff0fe4ba0a31a81bf3ed3 --- /dev/null +++ b/ext/fts5/fts5_expr.c @@ -0,0 +1,3266 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + + +#include "fts5Int.h" +#include "fts5parse.h" + +#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH +# define SQLITE_FTS5_MAX_EXPR_DEPTH 256 +#endif + +/* +** All token types in the generated fts5parse.h file are greater than 0. +*/ +#define FTS5_EOF 0 + +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + +typedef struct Fts5ExprTerm Fts5ExprTerm; + +/* +** Functions generated by lemon from fts5parse.y. +*/ +void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64)); +void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*)); +void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); +#ifndef NDEBUG +#include +void sqlite3Fts5ParserTrace(FILE*, char*); +#endif +int sqlite3Fts5ParserFallback(int); + + +struct Fts5Expr { + Fts5Index *pIndex; + Fts5Config *pConfig; + Fts5ExprNode *pRoot; + int bDesc; /* Iterate in descending rowid order */ + int nPhrase; /* Number of phrases in expression */ + Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ +}; + +/* +** eType: +** Expression node type. Usually one of: +** +** FTS5_AND (nChild, apChild valid) +** FTS5_OR (nChild, apChild valid) +** FTS5_NOT (nChild, apChild valid) +** FTS5_STRING (pNear valid) +** FTS5_TERM (pNear valid) +** +** An expression node with eType==0 may also exist. It always matches zero +** rows. This is created when a phrase containing no tokens is parsed. +** e.g. "". +** +** iHeight: +** Distance from this node to furthest leaf. This is always 0 for nodes +** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one +** greater than the largest child value. +*/ +struct Fts5ExprNode { + int eType; /* Node type */ + int bEof; /* True at EOF */ + int bNomatch; /* True if entry is not a match */ + int iHeight; /* Distance to tree leaf nodes */ + + /* Next method for this node. */ + int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); + + i64 iRowid; /* Current rowid */ + Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ + + /* Child nodes. For a NOT node, this array always contains 2 entries. For + ** AND or OR nodes, it contains 2 or more entries. */ + int nChild; /* Number of child nodes */ + Fts5ExprNode *apChild[1]; /* Array of child nodes */ +}; + +#define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) + +/* +** Invoke the xNext method of an Fts5ExprNode object. This macro should be +** used as if it has the same signature as the xNext() methods themselves. +*/ +#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d)) + +/* +** An instance of the following structure represents a single search term +** or term prefix. +*/ +struct Fts5ExprTerm { + u8 bPrefix; /* True for a prefix term */ + u8 bFirst; /* True if token must be first in column */ + char *pTerm; /* Term data */ + int nQueryTerm; /* Effective size of term in bytes */ + int nFullTerm; /* Size of term in bytes incl. tokendata */ + Fts5IndexIter *pIter; /* Iterator for this term */ + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ +}; + +/* +** A phrase. One or more terms that must appear in a contiguous sequence +** within a document for it to match. +*/ +struct Fts5ExprPhrase { + Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ + Fts5Buffer poslist; /* Current position list */ + int nTerm; /* Number of entries in aTerm[] */ + Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ +}; + +/* +** One or more phrases that must appear within a certain token distance of +** each other within each matching document. +*/ +struct Fts5ExprNearset { + int nNear; /* NEAR parameter */ + Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ + int nPhrase; /* Number of entries in aPhrase[] array */ + Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ +}; + + +/* +** Parse context. +*/ +struct Fts5Parse { + Fts5Config *pConfig; + char *zErr; + int rc; + int nPhrase; /* Size of apPhrase array */ + Fts5ExprPhrase **apPhrase; /* Array of all phrases */ + Fts5ExprNode *pExpr; /* Result of a successful parse */ + int bPhraseToAnd; /* Convert "a+b" to "a AND b" */ +}; + +/* +** Check that the Fts5ExprNode.iHeight variables are set correctly in +** the expression tree passed as the only argument. +*/ +#ifndef NDEBUG +static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){ + if( rc==SQLITE_OK ){ + if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){ + assert( p->iHeight==0 ); + }else{ + int ii; + int iMaxChild = 0; + for(ii=0; iinChild; ii++){ + Fts5ExprNode *pChild = p->apChild[ii]; + iMaxChild = MAX(iMaxChild, pChild->iHeight); + assert_expr_depth_ok(SQLITE_OK, pChild); + } + assert( p->iHeight==iMaxChild+1 ); + } + } +} +#else +# define assert_expr_depth_ok(rc, p) +#endif + +void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pParse->rc==SQLITE_OK ){ + assert( pParse->zErr==0 ); + pParse->zErr = sqlite3_vmprintf(zFmt, ap); + pParse->rc = SQLITE_ERROR; + } + va_end(ap); +} + +static int fts5ExprIsspace(char t){ + return t==' ' || t=='\t' || t=='\n' || t=='\r'; +} + +/* +** Read the first token from the nul-terminated string at *pz. +*/ +static int fts5ExprGetToken( + Fts5Parse *pParse, + const char **pz, /* IN/OUT: Pointer into buffer */ + Fts5Token *pToken +){ + const char *z = *pz; + int tok; + + /* Skip past any whitespace */ + while( fts5ExprIsspace(*z) ) z++; + + pToken->p = z; + pToken->n = 1; + switch( *z ){ + case '(': tok = FTS5_LP; break; + case ')': tok = FTS5_RP; break; + case '{': tok = FTS5_LCP; break; + case '}': tok = FTS5_RCP; break; + case ':': tok = FTS5_COLON; break; + case ',': tok = FTS5_COMMA; break; + case '+': tok = FTS5_PLUS; break; + case '*': tok = FTS5_STAR; break; + case '-': tok = FTS5_MINUS; break; + case '^': tok = FTS5_CARET; break; + case '\0': tok = FTS5_EOF; break; + + case '"': { + const char *z2; + tok = FTS5_STRING; + + for(z2=&z[1]; 1; z2++){ + if( z2[0]=='"' ){ + z2++; + if( z2[0]!='"' ) break; + } + if( z2[0]=='\0' ){ + sqlite3Fts5ParseError(pParse, "unterminated string"); + return FTS5_EOF; + } + } + pToken->n = (z2 - z); + break; + } + + default: { + const char *z2; + if( sqlite3Fts5IsBareword(z[0])==0 ){ + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); + return FTS5_EOF; + } + tok = FTS5_STRING; + for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); + pToken->n = (z2 - z); + if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR; + if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT; + if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND; + break; + } + } + + *pz = &pToken->p[pToken->n]; + return tok; +} + +static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc64((sqlite3_int64)t);} +static void fts5ParseFree(void *p){ sqlite3_free(p); } + +int sqlite3Fts5ExprNew( + Fts5Config *pConfig, /* FTS5 Configuration */ + int bPhraseToAnd, + int iCol, + const char *zExpr, /* Expression text */ + Fts5Expr **ppNew, + char **pzErr +){ + Fts5Parse sParse; + Fts5Token token; + const char *z = zExpr; + int t; /* Next token type */ + void *pEngine; + Fts5Expr *pNew; + + *ppNew = 0; + *pzErr = 0; + memset(&sParse, 0, sizeof(sParse)); + sParse.bPhraseToAnd = bPhraseToAnd; + pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc); + if( pEngine==0 ){ return SQLITE_NOMEM; } + sParse.pConfig = pConfig; + + do { + t = fts5ExprGetToken(&sParse, &z, &token); + sqlite3Fts5Parser(pEngine, t, token, &sParse); + }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); + sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + + assert( sParse.pExpr || sParse.rc!=SQLITE_OK ); + assert_expr_depth_ok(sParse.rc, sParse.pExpr); + + /* If the LHS of the MATCH expression was a user column, apply the + ** implicit column-filter. */ + if( sParse.rc==SQLITE_OK && iColnCol ){ + int n = sizeof(Fts5Colset); + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); + if( pColset ){ + pColset->nCol = 1; + pColset->aiCol[0] = iCol; + sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset); + } + } + + assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); + if( sParse.rc==SQLITE_OK ){ + *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + if( pNew==0 ){ + sParse.rc = SQLITE_NOMEM; + sqlite3Fts5ParseNodeFree(sParse.pExpr); + }else{ + pNew->pRoot = sParse.pExpr; + pNew->pIndex = 0; + pNew->pConfig = pConfig; + pNew->apExprPhrase = sParse.apPhrase; + pNew->nPhrase = sParse.nPhrase; + pNew->bDesc = 0; + sParse.apPhrase = 0; + } + }else{ + sqlite3Fts5ParseNodeFree(sParse.pExpr); + } + + sqlite3_free(sParse.apPhrase); + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } + return sParse.rc; +} + +/* +** Assuming that buffer z is at least nByte bytes in size and contains a +** valid utf-8 string, return the number of characters in the string. +*/ +static int fts5ExprCountChar(const char *z, int nByte){ + int nRet = 0; + int ii; + for(ii=0; ii=3 ){ + int jj; + zExpr[iOut++] = '"'; + for(jj=iFirst; jj0 ){ + int bAnd = 0; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + bAnd = 1; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + iCol = pConfig->nCol; + } + } + zExpr[iOut] = '\0'; + rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg); + }else{ + *pp = 0; + } + sqlite3_free(zExpr); + } + + return rc; +} + +/* +** Free the expression node object passed as the only argument. +*/ +void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){ + if( p ){ + int i; + for(i=0; inChild; i++){ + sqlite3Fts5ParseNodeFree(p->apChild[i]); + } + sqlite3Fts5ParseNearsetFree(p->pNear); + sqlite3_free(p); + } +} + +/* +** Free the expression object passed as the only argument. +*/ +void sqlite3Fts5ExprFree(Fts5Expr *p){ + if( p ){ + sqlite3Fts5ParseNodeFree(p->pRoot); + sqlite3_free(p->apExprPhrase); + sqlite3_free(p); + } +} + +int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ + Fts5Parse sParse; + memset(&sParse, 0, sizeof(sParse)); + + if( *pp1 && p2 ){ + Fts5Expr *p1 = *pp1; + int nPhrase = p1->nPhrase + p2->nPhrase; + + p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0); + p2->pRoot = 0; + + if( sParse.rc==SQLITE_OK ){ + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) + ); + if( ap==0 ){ + sParse.rc = SQLITE_NOMEM; + }else{ + int i; + memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*)); + for(i=0; inPhrase; i++){ + ap[i] = p2->apExprPhrase[i]; + } + p1->nPhrase = nPhrase; + p1->apExprPhrase = ap; + } + } + sqlite3_free(p2->apExprPhrase); + sqlite3_free(p2); + }else if( p2 ){ + *pp1 = p2; + } + + return sParse.rc; +} + +/* +** Argument pTerm must be a synonym iterator. Return the current rowid +** that it points to. +*/ +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; + int bRetValid = 0; + Fts5ExprTerm *p; + + assert( pTerm ); + assert( pTerm->pSynonym ); + assert( bDesc==0 || bDesc==1 ); + for(p=pTerm; p; p=p->pSynonym){ + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + i64 iRowid = p->pIter->iRowid; + if( bRetValid==0 || (bDesc!=(iRowidpSynonym ); + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ + if( pIter->nData==0 ) continue; + if( nIter==nAlloc ){ + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc64(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + if( aIter!=aStatic ) sqlite3_free(aIter); + aIter = aNew; + } + sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; + } + } + + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + i64 iPrev = -1; + fts5BufferZero(pBuf); + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; ip; + *pn = pBuf->n; + } + } + + synonym_poslist_out: + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + + +/* +** All individual term iterators in pPhrase are guaranteed to be valid and +** pointing to the same rowid when this function is called. This function +** checks if the current rowid really is a match, and if so populates +** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch +** is set to true if this is really a match, or false otherwise. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if the current rowid is +** not a match. +*/ +static int fts5ExprPhraseIsMatch( + Fts5ExprNode *pNode, /* Node pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbMatch /* OUT: Set to true if really a match */ +){ + Fts5PoslistWriter writer = {0}; + Fts5PoslistReader aStatic[4]; + Fts5PoslistReader *aIter = aStatic; + int i; + int rc = SQLITE_OK; + int bFirst = pPhrase->aTerm[0].bFirst; + + fts5BufferZero(&pPhrase->poslist); + + /* If the aStatic[] array is not large enough, allocate a large array + ** using sqlite3_malloc(). This approach could be improved upon. */ + if( pPhrase->nTerm>ArraySize(aStatic) ){ + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; + aIter = (Fts5PoslistReader*)sqlite3_malloc64(nByte); + if( !aIter ) return SQLITE_NOMEM; + } + memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); + + /* Initialize a term iterator for each term in the phrase */ + for(i=0; inTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + int n = 0; + int bFlag = 0; + u8 *a = 0; + if( pTerm->pSynonym ){ + Fts5Buffer buf = {0, 0, 0}; + rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n); + if( rc ){ + sqlite3_free(a); + goto ismatch_out; + } + if( a==buf.p ) bFlag = 1; + }else{ + a = (u8*)pTerm->pIter->pData; + n = pTerm->pIter->nData; + } + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); + aIter[i].bFlag = (u8)bFlag; + if( aIter[i].bEof ) goto ismatch_out; + } + + while( 1 ){ + int bMatch; + i64 iPos = aIter[0].iPos; + do { + bMatch = 1; + for(i=0; inTerm; i++){ + Fts5PoslistReader *pPos = &aIter[i]; + i64 iAdj = iPos + i; + if( pPos->iPos!=iAdj ){ + bMatch = 0; + while( pPos->iPosiPos>iAdj ) iPos = pPos->iPos-i; + } + } + }while( bMatch==0 ); + + /* Append position iPos to the output */ + if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; + } + + for(i=0; inTerm; i++){ + if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; + } + } + + ismatch_out: + *pbMatch = (pPhrase->poslist.n>0); + for(i=0; inTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + +typedef struct Fts5LookaheadReader Fts5LookaheadReader; +struct Fts5LookaheadReader { + const u8 *a; /* Buffer containing position list */ + int n; /* Size of buffer a[] in bytes */ + int i; /* Current offset in position list */ + i64 iPos; /* Current position */ + i64 iLookahead; /* Next position */ +}; + +#define FTS5_LOOKAHEAD_EOF (((i64)1) << 62) + +static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){ + p->iPos = p->iLookahead; + if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){ + p->iLookahead = FTS5_LOOKAHEAD_EOF; + } + return (p->iPos==FTS5_LOOKAHEAD_EOF); +} + +static int fts5LookaheadReaderInit( + const u8 *a, int n, /* Buffer to read position list from */ + Fts5LookaheadReader *p /* Iterator object to initialize */ +){ + memset(p, 0, sizeof(Fts5LookaheadReader)); + p->a = a; + p->n = n; + fts5LookaheadReaderNext(p); + return fts5LookaheadReaderNext(p); +} + +typedef struct Fts5NearTrimmer Fts5NearTrimmer; +struct Fts5NearTrimmer { + Fts5LookaheadReader reader; /* Input iterator */ + Fts5PoslistWriter writer; /* Writer context */ + Fts5Buffer *pOut; /* Output poslist */ +}; + +/* +** The near-set object passed as the first argument contains more than +** one phrase. All phrases currently point to the same row. The +** Fts5ExprPhrase.poslist buffers are populated accordingly. This function +** tests if the current row contains instances of each phrase sufficiently +** close together to meet the NEAR constraint. Non-zero is returned if it +** does, or zero otherwise. +** +** If in/out parameter (*pRc) is set to other than SQLITE_OK when this +** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM) +** occurs within this function (*pRc) is set accordingly before returning. +** The return value is undefined in both these cases. +** +** If no error occurs and non-zero (a match) is returned, the position-list +** of each phrase object is edited to contain only those entries that +** meet the constraint before returning. +*/ +static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ + Fts5NearTrimmer aStatic[4]; + Fts5NearTrimmer *a = aStatic; + Fts5ExprPhrase **apPhrase = pNear->apPhrase; + + int i; + int rc = *pRc; + int bMatch; + + assert( pNear->nPhrase>1 ); + + /* If the aStatic[] array is not large enough, allocate a large array + ** using sqlite3_malloc(). This approach could be improved upon. */ + if( pNear->nPhrase>ArraySize(aStatic) ){ + sqlite3_int64 nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; + a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); + }else{ + memset(aStatic, 0, sizeof(aStatic)); + } + if( rc!=SQLITE_OK ){ + *pRc = rc; + return 0; + } + + /* Initialize a lookahead iterator for each phrase. After passing the + ** buffer and buffer size to the lookaside-reader init function, zero + ** the phrase poslist buffer. The new poslist for the phrase (containing + ** the same entries as the original with some entries removed on account + ** of the NEAR constraint) is written over the original even as it is + ** being read. This is safe as the entries for the new poslist are a + ** subset of the old, so it is not possible for data yet to be read to + ** be overwritten. */ + for(i=0; inPhrase; i++){ + Fts5Buffer *pPoslist = &apPhrase[i]->poslist; + fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader); + pPoslist->n = 0; + a[i].pOut = pPoslist; + } + + while( 1 ){ + int iAdv; + i64 iMin; + i64 iMax; + + /* This block advances the phrase iterators until they point to a set of + ** entries that together comprise a match. */ + iMax = a[0].reader.iPos; + do { + bMatch = 1; + for(i=0; inPhrase; i++){ + Fts5LookaheadReader *pPos = &a[i].reader; + iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear; + if( pPos->iPosiPos>iMax ){ + bMatch = 0; + while( pPos->iPosiPos>iMax ) iMax = pPos->iPos; + } + } + }while( bMatch==0 ); + + /* Add an entry to each output position list */ + for(i=0; inPhrase; i++){ + i64 iPos = a[i].reader.iPos; + Fts5PoslistWriter *pWriter = &a[i].writer; + if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){ + sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos); + } + } + + iAdv = 0; + iMin = a[0].reader.iLookahead; + for(i=0; inPhrase; i++){ + if( a[i].reader.iLookahead < iMin ){ + iMin = a[i].reader.iLookahead; + iAdv = i; + } + } + if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out; + } + + ismatch_out: { + int bRet = a[0].pOut->n>0; + *pRc = rc; + if( a!=aStatic ) sqlite3_free(a); + return bRet; + } +} + +/* +** Advance iterator pIter until it points to a value equal to or laster +** than the initial value of *piLast. If this means the iterator points +** to a value laster than *piLast, update *piLast to the new lastest value. +** +** If the iterator reaches EOF, set *pbEof to true before returning. If +** an error occurs, set *pRc to an error code. If either *pbEof or *pRc +** are set, return a non-zero value. Otherwise, return zero. +*/ +static int fts5ExprAdvanceto( + Fts5IndexIter *pIter, /* Iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc, /* OUT: Error code */ + int *pbEof /* OUT: Set to true if EOF */ +){ + i64 iLast = *piLast; + i64 iRowid; + + iRowid = pIter->iRowid; + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastiRowid; + assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); + } + *piLast = iRowid; + + return 0; +} + +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc /* OUT: Error code */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + int bEof = 0; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = p->pIter->iRowid; + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pRc = rc; + bEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); + } + return bEof; +} + + +static int fts5ExprNearTest( + int *pRc, + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ +){ + Fts5ExprNearset *pNear = pNode->pNear; + int rc = *pRc; + + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprTerm *pTerm; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + pPhrase->poslist.n = 0; + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + Fts5IndexIter *pIter = pTerm->pIter; + if( sqlite3Fts5IterEof(pIter)==0 ){ + if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){ + pPhrase->poslist.n = 1; + } + } + } + return pPhrase->poslist.n; + }else{ + int i; + + /* Check that each phrase in the nearset matches the current row. + ** Populate the pPhrase->poslist buffers at the same time. If any + ** phrase is not a match, break out of the loop early. */ + for(i=0; rc==SQLITE_OK && inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym + || pNear->pColset || pPhrase->aTerm[0].bFirst + ){ + int bMatch = 0; + rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); + if( bMatch==0 ) break; + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData); + } + } + + *pRc = rc; + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ + return 1; + } + return 0; + } +} + + +/* +** Initialize all term iterators in the pNear object. If any term is found +** to match no documents at all, return immediately without initializing any +** further iterators. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. It is not considered an error if some term matches zero +** documents. +*/ +static int fts5ExprNearInitAll( + Fts5Expr *pExpr, + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + int i; + + assert( pNode->bNomatch==0 ); + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + }else{ + int j; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bHit = 0; + + for(p=pTerm; p; p=p->pSynonym){ + int rc; + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->pTerm, p->nQueryTerm, + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + pNear->pColset, + &p->pIter + ); + assert( (rc==SQLITE_OK)==(p->pIter!=0) ); + if( rc!=SQLITE_OK ) return rc; + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + bHit = 1; + } + } + + if( bHit==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + } + } + } + } + + pNode->bEof = 0; + return SQLITE_OK; +} + +/* +** If pExpr is an ASC iterator, this function returns a value with the +** same sign as: +** +** (iLhs - iRhs) +** +** Otherwise, if this is a DESC iterator, the opposite is returned: +** +** (iRhs - iLhs) +*/ +static int fts5RowidCmp( + Fts5Expr *pExpr, + i64 iLhs, + i64 iRhs +){ + assert( pExpr->bDesc==0 || pExpr->bDesc==1 ); + if( pExpr->bDesc==0 ){ + if( iLhs iRhs); + }else{ + if( iLhs>iRhs ) return -1; + return (iLhs < iRhs); + } +} + +static void fts5ExprSetEof(Fts5ExprNode *pNode){ + int i; + pNode->bEof = 1; + pNode->bNomatch = 0; + for(i=0; inChild; i++){ + fts5ExprSetEof(pNode->apChild[i]); + } +} + +static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + int i; + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + pPhrase->poslist.n = 0; + } + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprNodeZeroPoslist(pNode->apChild[i]); + } + } +} + + + +/* +** Compare the values currently indicated by the two nodes as follows: +** +** res = (*p1) - (*p2) +** +** Nodes that point to values that come later in the iteration order are +** considered to be larger. Nodes at EOF are the largest of all. +** +** This means that if the iteration order is ASC, then numerically larger +** rowids are considered larger. Or if it is the default DESC, numerically +** smaller rowids are larger. +*/ +static int fts5NodeCompare( + Fts5Expr *pExpr, + Fts5ExprNode *p1, + Fts5ExprNode *p2 +){ + if( p2->bEof ) return -1; + if( p1->bEof ) return +1; + return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); +} + +/* +** All individual term iterators in pNear are guaranteed to be valid when +** this function is called. This function checks if all term iterators +** point to the same rowid, and if not, advances them until they do. +** If an EOF is reached before this happens, *pbEof is set to true before +** returning. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if an iterator reaches +** EOF. +*/ +static int fts5ExprNodeTest_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; + int rc = SQLITE_OK; + i64 iLast; /* Lastest rowid any iterator points to */ + int i, j; /* Phrase and token index, respectively */ + int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; + + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + || pNear->apPhrase[0]->aTerm[0].bFirst + ); + + /* Initialize iLast, the "lastest" rowid any iterator points to. If the + ** iterator skips through rowids in the default ascending order, this means + ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it + ** means the minimum rowid. */ + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); + }else{ + iLast = pLeft->aTerm[0].pIter->iRowid; + } + + do { + bMatch = 1; + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bNomatch = 0; + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + if( pIter->iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } + } + } + } + }while( bMatch==0 ); + + pNode->iRowid = iLast; + pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); + assert( pNode->bEof==0 || pNode->bNomatch==0 ); + + return rc; +} + +/* +** Advance the first term iterator in the first phrase of pNear. Set output +** variable *pbEof to true if it reaches EOF or if an error occurs. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5ExprNodeNext_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ + int bFromValid, + i64 iFrom +){ + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; + int rc = SQLITE_OK; + + pNode->bNomatch = 0; + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); + + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 ii = p->pIter->iRowid; + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } + if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); + }else{ + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); + } + + if( pNode->bEof==0 ){ + assert( rc==SQLITE_OK ); + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + } + + return rc; +} + + +static int fts5ExprNodeTest_TERM( + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ +){ + /* As this "NEAR" object is actually a single phrase that consists + ** of a single term only, grab pointers into the poslist managed by the + ** fts5_index.c iterator object. This is much faster than synthesizing + ** a new poslist the way we have to for more complicated phrase or NEAR + ** expressions. */ + Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + + assert( pNode->eType==FTS5_TERM ); + assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); + + pPhrase->poslist.n = pIter->nData; + if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ + pPhrase->poslist.p = (u8*)pIter->pData; + } + pNode->iRowid = pIter->iRowid; + pNode->bNomatch = (pPhrase->poslist.n==0); + return SQLITE_OK; +} + +/* +** xNext() method for a node of type FTS5_TERM. +*/ +static int fts5ExprNodeNext_TERM( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc; + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + + assert( pNode->bEof==0 ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + }else{ + pNode->bEof = 1; + pNode->bNomatch = 0; + } + return rc; +} + +static void fts5ExprNodeTest_OR( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + Fts5ExprNode *pNext = pNode->apChild[0]; + int i; + + for(i=1; inChild; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + int cmp = fts5NodeCompare(pExpr, pNext, pChild); + if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ + pNext = pChild; + } + } + pNode->iRowid = pNext->iRowid; + pNode->bEof = pNext->bEof; + pNode->bNomatch = pNext->bNomatch; +} + +static int fts5ExprNodeNext_OR( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int i; + i64 iLast = pNode->iRowid; + + for(i=0; inChild; i++){ + Fts5ExprNode *p1 = pNode->apChild[i]; + assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); + if( p1->bEof==0 ){ + if( (p1->iRowid==iLast) + || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) + ){ + int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + return rc; + } + } + } + } + + fts5ExprNodeTest_OR(pExpr, pNode); + return SQLITE_OK; +} + +/* +** Argument pNode is an FTS5_AND node. +*/ +static int fts5ExprNodeTest_AND( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pAnd /* FTS5_AND node to advance */ +){ + int iChild; + i64 iLast = pAnd->iRowid; + int rc = SQLITE_OK; + int bMatch; + + assert( pAnd->bEof==0 ); + do { + pAnd->bNomatch = 0; + bMatch = 1; + for(iChild=0; iChildnChild; iChild++){ + Fts5ExprNode *pChild = pAnd->apChild[iChild]; + int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); + if( cmp>0 ){ + /* Advance pChild until it points to iLast or laster */ + rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); + if( rc!=SQLITE_OK ){ + pAnd->bNomatch = 0; + return rc; + } + } + + /* If the child node is now at EOF, so is the parent AND node. Otherwise, + ** the child node is guaranteed to have advanced at least as far as + ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the + ** new lastest rowid seen so far. */ + assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); + if( pChild->bEof ){ + fts5ExprSetEof(pAnd); + bMatch = 1; + break; + }else if( iLast!=pChild->iRowid ){ + bMatch = 0; + iLast = pChild->iRowid; + } + + if( pChild->bNomatch ){ + pAnd->bNomatch = 1; + } + } + }while( bMatch==0 ); + + if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ + fts5ExprNodeZeroPoslist(pAnd); + } + pAnd->iRowid = iLast; + return SQLITE_OK; +} + +static int fts5ExprNodeNext_AND( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_AND(pExpr, pNode); + }else{ + pNode->bNomatch = 0; + } + return rc; +} + +static int fts5ExprNodeTest_NOT( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode /* FTS5_NOT node to advance */ +){ + int rc = SQLITE_OK; + Fts5ExprNode *p1 = pNode->apChild[0]; + Fts5ExprNode *p2 = pNode->apChild[1]; + assert( pNode->nChild==2 ); + + while( rc==SQLITE_OK && p1->bEof==0 ){ + int cmp = fts5NodeCompare(pExpr, p1, p2); + if( cmp>0 ){ + rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid); + cmp = fts5NodeCompare(pExpr, p1, p2); + } + assert( rc!=SQLITE_OK || cmp<=0 ); + if( cmp || p2->bNomatch ) break; + rc = fts5ExprNodeNext(pExpr, p1, 0, 0); + } + pNode->bEof = p1->bEof; + pNode->bNomatch = p1->bNomatch; + pNode->iRowid = p1->iRowid; + if( p1->bEof ){ + fts5ExprNodeZeroPoslist(p2); + } + return rc; +} + +static int fts5ExprNodeNext_NOT( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + } + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + } + return rc; +} + +/* +** If pNode currently points to a match, this function returns SQLITE_OK +** without modifying it. Otherwise, pNode is advanced until it does point +** to a match or EOF is reached. +*/ +static int fts5ExprNodeTest( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + int rc = SQLITE_OK; + if( pNode->bEof==0 ){ + switch( pNode->eType ){ + + case FTS5_STRING: { + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + break; + } + + case FTS5_TERM: { + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + break; + } + + case FTS5_AND: { + rc = fts5ExprNodeTest_AND(pExpr, pNode); + break; + } + + case FTS5_OR: { + fts5ExprNodeTest_OR(pExpr, pNode); + break; + } + + default: assert( pNode->eType==FTS5_NOT ); { + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + break; + } + } + } + return rc; +} + + +/* +** Set node pNode, which is part of expression pExpr, to point to the first +** match. If there are no matches, set the Node.bEof flag to indicate EOF. +** +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +** It is not an error if there are no matches. +*/ +static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ + int rc = SQLITE_OK; + pNode->bEof = 0; + pNode->bNomatch = 0; + + if( Fts5NodeIsString(pNode) ){ + /* Initialize all term iterators in the NEAR object. */ + rc = fts5ExprNearInitAll(pExpr, pNode); + }else if( pNode->xNext==0 ){ + pNode->bEof = 1; + }else{ + int i; + int nEof = 0; + for(i=0; inChild && rc==SQLITE_OK; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]); + assert( pChild->bEof==0 || pChild->bEof==1 ); + nEof += pChild->bEof; + } + pNode->iRowid = pNode->apChild[0]->iRowid; + + switch( pNode->eType ){ + case FTS5_AND: + if( nEof>0 ) fts5ExprSetEof(pNode); + break; + + case FTS5_OR: + if( pNode->nChild==nEof ) fts5ExprSetEof(pNode); + break; + + default: + assert( pNode->eType==FTS5_NOT ); + pNode->bEof = pNode->apChild[0]->bEof; + break; + } + } + + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest(pExpr, pNode); + } + return rc; +} + + +/* +** Begin iterating through the set of documents in index pIdx matched by +** the MATCH expression passed as the first argument. If the "bDesc" +** parameter is passed a non-zero value, iteration is in descending rowid +** order. Or, if it is zero, in ascending order. +** +** If iterating in ascending rowid order (bDesc==0), the first document +** visited is that with the smallest rowid that is larger than or equal +** to parameter iFirst. Or, if iterating in ascending order (bDesc==1), +** then the first document visited must have a rowid smaller than or +** equal to iFirst. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It +** is not considered an error if the query does not match any documents. +*/ +int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ + Fts5ExprNode *pRoot = p->pRoot; + int rc; /* Return code */ + + p->pIndex = pIdx; + p->bDesc = bDesc; + rc = fts5ExprNodeFirst(p, pRoot); + + /* If not at EOF but the current rowid occurs earlier than iFirst in + ** the iteration order, move to document iFirst or later. */ + if( rc==SQLITE_OK + && 0==pRoot->bEof + && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 + ){ + rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); + } + + /* If the iterator is not at a real match, skip forward until it is. */ + while( pRoot->bNomatch && rc==SQLITE_OK ){ + assert( pRoot->bEof==0 ); + rc = fts5ExprNodeNext(p, pRoot, 0, 0); + } + return rc; +} + +/* +** Move to the next document +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It +** is not considered an error if the query does not match any documents. +*/ +int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ + int rc; + Fts5ExprNode *pRoot = p->pRoot; + assert( pRoot->bEof==0 && pRoot->bNomatch==0 ); + do { + rc = fts5ExprNodeNext(p, pRoot, 0, 0); + assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) ); + }while( pRoot->bNomatch ); + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } + return rc; +} + +int sqlite3Fts5ExprEof(Fts5Expr *p){ + return p->pRoot->bEof; +} + +i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ + return p->pRoot->iRowid; +} + +static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ + int rc = SQLITE_OK; + *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n); + return rc; +} + +/* +** Free the phrase object passed as the only argument. +*/ +static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ + if( pPhrase ){ + int i; + for(i=0; inTerm; i++){ + Fts5ExprTerm *pSyn; + Fts5ExprTerm *pNext; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + sqlite3_free(pTerm->pTerm); + sqlite3Fts5IterClose(pTerm->pIter); + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ + pNext = pSyn->pSynonym; + sqlite3Fts5IterClose(pSyn->pIter); + fts5BufferFree((Fts5Buffer*)&pSyn[1]); + sqlite3_free(pSyn); + } + } + if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); + sqlite3_free(pPhrase); + } +} + +/* +** Set the "bFirst" flag on the first token of the phrase passed as the +** only argument. +*/ +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){ + if( pPhrase && pPhrase->nTerm ){ + pPhrase->aTerm[0].bFirst = 1; + } +} + +/* +** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated +** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is +** appended to it and the results returned. +** +** If an OOM error occurs, both the pNear and pPhrase objects are freed and +** NULL returned. +*/ +Fts5ExprNearset *sqlite3Fts5ParseNearset( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNearset *pNear, /* Existing nearset, or NULL */ + Fts5ExprPhrase *pPhrase /* Recently parsed phrase */ +){ + const int SZALLOC = 8; + Fts5ExprNearset *pRet = 0; + + if( pParse->rc==SQLITE_OK ){ + if( pNear==0 ){ + sqlite3_int64 nByte; + nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + pParse->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, (size_t)nByte); + } + }else if( (pNear->nPhrase % SZALLOC)==0 ){ + int nNew = pNear->nPhrase + SZALLOC; + sqlite3_int64 nByte; + + nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); + if( pRet==0 ){ + pParse->rc = SQLITE_NOMEM; + } + }else{ + pRet = pNear; + } + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3Fts5ParseNearsetFree(pNear); + sqlite3Fts5ParsePhraseFree(pPhrase); + }else{ + if( pRet->nPhrase>0 ){ + Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; + assert( pParse!=0 ); + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>=2 ); + assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); + if( pPhrase->nTerm==0 ){ + fts5ExprPhraseFree(pPhrase); + pRet->nPhrase--; + pParse->nPhrase--; + pPhrase = pLast; + }else if( pLast->nTerm==0 ){ + fts5ExprPhraseFree(pLast); + pParse->apPhrase[pParse->nPhrase-2] = pPhrase; + pParse->nPhrase--; + pRet->nPhrase--; + } + } + pRet->apPhrase[pRet->nPhrase++] = pPhrase; + } + return pRet; +} + +typedef struct TokenCtx TokenCtx; +struct TokenCtx { + Fts5ExprPhrase *pPhrase; + Fts5Config *pConfig; + int rc; +}; + +/* +** Callback for tokenizing terms used by ParseTerm(). +*/ +static int fts5ParseTokenize( + void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + int rc = SQLITE_OK; + const int SZALLOC = 8; + TokenCtx *pCtx = (TokenCtx*)pContext; + Fts5ExprPhrase *pPhrase = pCtx->pPhrase; + + UNUSED_PARAM2(iUnused1, iUnused2); + + /* If an error has already occurred, this is a no-op */ + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + + if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ + Fts5ExprTerm *pSyn; + sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc64(nByte); + if( pSyn==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pSyn, 0, (size_t)nByte); + pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + pSyn->nFullTerm = pSyn->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata ){ + pSyn->nQueryTerm = (int)strlen(pSyn->pTerm); + } + memcpy(pSyn->pTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + } + }else{ + Fts5ExprTerm *pTerm; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ + Fts5ExprPhrase *pNew; + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + + pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + ); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } + } + + if( rc==SQLITE_OK ){ + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->nFullTerm = pTerm->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ + pTerm->nQueryTerm = (int)strlen(pTerm->pTerm); + } + } + } + + pCtx->rc = rc; + return rc; +} + + +/* +** Free the phrase object passed as the only argument. +*/ +void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){ + fts5ExprPhraseFree(pPhrase); +} + +/* +** Free the phrase object passed as the second argument. +*/ +void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){ + if( pNear ){ + int i; + for(i=0; inPhrase; i++){ + fts5ExprPhraseFree(pNear->apPhrase[i]); + } + sqlite3_free(pNear->pColset); + sqlite3_free(pNear); + } +} + +void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){ + assert( pParse->pExpr==0 ); + pParse->pExpr = p; +} + +static int parseGrowPhraseArray(Fts5Parse *pParse){ + if( (pParse->nPhrase % 8)==0 ){ + sqlite3_int64 nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); + Fts5ExprPhrase **apNew; + apNew = (Fts5ExprPhrase**)sqlite3_realloc64(pParse->apPhrase, nByte); + if( apNew==0 ){ + pParse->rc = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + pParse->apPhrase = apNew; + } + return SQLITE_OK; +} + +/* +** This function is called by the parser to process a string token. The +** string may or may not be quoted. In any case it is tokenized and a +** phrase object consisting of all tokens returned. +*/ +Fts5ExprPhrase *sqlite3Fts5ParseTerm( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprPhrase *pAppend, /* Phrase to append to */ + Fts5Token *pToken, /* String to tokenize */ + int bPrefix /* True if there is a trailing "*" */ +){ + Fts5Config *pConfig = pParse->pConfig; + TokenCtx sCtx; /* Context object passed to callback */ + int rc; /* Tokenize return code */ + char *z = 0; + + memset(&sCtx, 0, sizeof(TokenCtx)); + sCtx.pPhrase = pAppend; + sCtx.pConfig = pConfig; + + rc = fts5ParseStringFromToken(pToken, &z); + if( rc==SQLITE_OK ){ + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); + int n; + sqlite3Fts5Dequote(z); + n = (int)strlen(z); + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); + } + sqlite3_free(z); + if( rc || (rc = sCtx.rc) ){ + pParse->rc = rc; + fts5ExprPhraseFree(sCtx.pPhrase); + sCtx.pPhrase = 0; + }else{ + + if( pAppend==0 ){ + if( parseGrowPhraseArray(pParse) ){ + fts5ExprPhraseFree(sCtx.pPhrase); + return 0; + } + pParse->nPhrase++; + } + + if( sCtx.pPhrase==0 ){ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + }else if( sCtx.pPhrase->nTerm ){ + sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; + } + assert( pParse->apPhrase!=0 ); + pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; + } + + return sCtx.pPhrase; +} + +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +int sqlite3Fts5ExprClonePhrase( + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ + if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + rc = SQLITE_RANGE; + }else{ + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + } + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ + Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; + if( pColsetOrig ){ + sqlite3_int64 nByte; + Fts5Colset *pColset; + nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); + if( pColset ){ + memcpy(pColset, pColsetOrig, (size_t)nByte); + } + pNew->pRoot->pNear->pColset = pColset; + } + } + + if( rc==SQLITE_OK ){ + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } + } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); + } + } + + if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->pConfig = pExpr->pConfig; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 + && pOrig->aTerm[0].pSynonym==0 + && pOrig->aTerm[0].bFirst==0 + ){ + pNew->pRoot->eType = FTS5_TERM; + pNew->pRoot->xNext = fts5ExprNodeNext_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + pNew->pRoot->xNext = fts5ExprNodeNext_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + +/* +** Token pTok has appeared in a MATCH expression where the NEAR operator +** is expected. If token pTok does not contain "NEAR", store an error +** in the pParse object. +*/ +void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){ + if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){ + sqlite3Fts5ParseError( + pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p + ); + } +} + +void sqlite3Fts5ParseSetDistance( + Fts5Parse *pParse, + Fts5ExprNearset *pNear, + Fts5Token *p +){ + if( pNear ){ + int nNear = 0; + int i; + if( p->n ){ + for(i=0; in; i++){ + char c = (char)p->p[i]; + if( c<'0' || c>'9' ){ + sqlite3Fts5ParseError( + pParse, "expected integer, got \"%.*s\"", p->n, p->p + ); + return; + } + nNear = nNear * 10 + (p->p[i] - '0'); + } + }else{ + nNear = FTS5_DEFAULT_NEARDIST; + } + pNear->nNear = nNear; + } +} + +/* +** The second argument passed to this function may be NULL, or it may be +** an existing Fts5Colset object. This function returns a pointer to +** a new colset object containing the contents of (p) with new value column +** number iCol appended. +** +** If an OOM error occurs, store an error code in pParse and return NULL. +** The old colset object (if any) is not freed in this case. +*/ +static Fts5Colset *fts5ParseColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *p, /* Existing colset object */ + int iCol /* New column to add to colset object */ +){ + int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */ + Fts5Colset *pNew; /* New colset object to return */ + + assert( pParse->rc==SQLITE_OK ); + assert( iCol>=0 && iColpConfig->nCol ); + + pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + if( pNew==0 ){ + pParse->rc = SQLITE_NOMEM; + }else{ + int *aiCol = pNew->aiCol; + int i, j; + for(i=0; iiCol ) break; + } + for(j=nCol; j>i; j--){ + aiCol[j] = aiCol[j-1]; + } + aiCol[i] = iCol; + pNew->nCol = nCol+1; + +#ifndef NDEBUG + /* Check that the array is in order and contains no duplicate entries. */ + for(i=1; inCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] ); +#endif + } + + return pNew; +} + +/* +** Allocate and return an Fts5Colset object specifying the inverse of +** the colset passed as the second argument. Free the colset passed +** as the second argument before returning. +*/ +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ + Fts5Colset *pRet; + int nCol = pParse->pConfig->nCol; + + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, + sizeof(Fts5Colset) + sizeof(int)*nCol + ); + if( pRet ){ + int i; + int iOld = 0; + for(i=0; i=p->nCol || p->aiCol[iOld]!=i ){ + pRet->aiCol[pRet->nCol++] = i; + }else{ + iOld++; + } + } + } + + sqlite3_free(p); + return pRet; +} + +Fts5Colset *sqlite3Fts5ParseColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *pColset, /* Existing colset object */ + Fts5Token *p +){ + Fts5Colset *pRet = 0; + int iCol; + char *z; /* Dequoted copy of token p */ + + z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n); + if( pParse->rc==SQLITE_OK ){ + Fts5Config *pConfig = pParse->pConfig; + sqlite3Fts5Dequote(z); + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break; + } + if( iCol==pConfig->nCol ){ + sqlite3Fts5ParseError(pParse, "no such column: %s", z); + }else{ + pRet = fts5ParseColset(pParse, pColset, iCol); + } + sqlite3_free(z); + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3_free(pColset); + } + + return pRet; +} + +/* +** If argument pOrig is NULL, or if (*pRc) is set to anything other than +** SQLITE_OK when this function is called, NULL is returned. +** +** Otherwise, a copy of (*pOrig) is made into memory obtained from +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned. +*/ +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ + Fts5Colset *pRet; + if( pOrig ){ + sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); + if( pRet ){ + memcpy(pRet, pOrig, (size_t)nByte); + } + }else{ + pRet = 0; + } + return pRet; +} + +/* +** Remove from colset pColset any columns that are not also in colset pMerge. +*/ +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){ + int iIn = 0; /* Next input in pColset */ + int iMerge = 0; /* Next input in pMerge */ + int iOut = 0; /* Next output slot in pColset */ + + while( iInnCol && iMergenCol ){ + int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge]; + if( iDiff==0 ){ + pColset->aiCol[iOut++] = pMerge->aiCol[iMerge]; + iMerge++; + iIn++; + }else if( iDiff>0 ){ + iMerge++; + }else{ + iIn++; + } + } + pColset->nCol = iOut; +} + +/* +** Recursively apply colset pColset to expression node pNode and all of +** its decendents. If (*ppFree) is not NULL, it contains a spare copy +** of pColset. This function may use the spare copy and set (*ppFree) to +** zero, or it may create copies of pColset using fts5CloneColset(). +*/ +static void fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pNode, + Fts5Colset *pColset, + Fts5Colset **ppFree +){ + if( pParse->rc==SQLITE_OK ){ + assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING + || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR + || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF + ); + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->pColset ){ + fts5MergeColset(pNear->pColset, pColset); + if( pNear->pColset->nCol==0 ){ + pNode->eType = FTS5_EOF; + pNode->xNext = 0; + } + }else if( *ppFree ){ + pNear->pColset = pColset; + *ppFree = 0; + }else{ + pNear->pColset = fts5CloneColset(&pParse->rc, pColset); + } + }else{ + int i; + assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 ); + for(i=0; inChild; i++){ + fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree); + } + } + } +} + +/* +** Apply colset pColset to expression node pExpr and all of its descendents. +*/ +void sqlite3Fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pExpr, + Fts5Colset *pColset +){ + Fts5Colset *pFree = pColset; + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + sqlite3Fts5ParseError(pParse, + "fts5: column queries are not supported (detail=none)" + ); + }else{ + fts5ParseSetColset(pParse, pExpr, pColset, &pFree); + } + sqlite3_free(pFree); +} + +static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ + switch( pNode->eType ){ + case FTS5_STRING: { + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + && pNear->apPhrase[0]->aTerm[0].bFirst==0 + ){ + pNode->eType = FTS5_TERM; + pNode->xNext = fts5ExprNodeNext_TERM; + }else{ + pNode->xNext = fts5ExprNodeNext_STRING; + } + break; + }; + + case FTS5_OR: { + pNode->xNext = fts5ExprNodeNext_OR; + break; + }; + + case FTS5_AND: { + pNode->xNext = fts5ExprNodeNext_AND; + break; + }; + + default: assert( pNode->eType==FTS5_NOT ); { + pNode->xNext = fts5ExprNodeNext_NOT; + break; + }; + } +} + +/* +** Add pSub as a child of p. +*/ +static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + int ii = p->nChild; + if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ + int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; + memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); + p->nChild += pSub->nChild; + sqlite3_free(pSub); + }else{ + p->apChild[p->nChild++] = pSub; + } + for( ; iinChild; ii++){ + p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1); + } +} + +/* +** This function is used when parsing LIKE or GLOB patterns against +** trigram indexes that specify either detail=column or detail=none. +** It converts a phrase: +** +** abc + def + ghi +** +** into an AND tree: +** +** abc AND def AND ghi +*/ +static Fts5ExprNode *fts5ParsePhraseToAnd( + Fts5Parse *pParse, + Fts5ExprNearset *pNear +){ + int nTerm = pNear->apPhrase[0]->nTerm; + int ii; + int nByte; + Fts5ExprNode *pRet; + + assert( pNear->nPhrase==1 ); + assert( pParse->bPhraseToAnd ); + + nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + if( pRet ){ + pRet->eType = FTS5_AND; + pRet->nChild = nTerm; + pRet->iHeight = 1; + fts5ExprAssignXNext(pRet); + pParse->nPhrase--; + for(ii=0; iirc, sizeof(Fts5ExprPhrase) + ); + if( pPhrase ){ + if( parseGrowPhraseArray(pParse) ){ + fts5ExprPhraseFree(pPhrase); + }else{ + Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii]; + Fts5ExprTerm *pTo = &pPhrase->aTerm[0]; + pParse->apPhrase[pParse->nPhrase++] = pPhrase; + pPhrase->nTerm = 1; + pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm); + pTo->nQueryTerm = p->nQueryTerm; + pTo->nFullTerm = p->nFullTerm; + pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, + 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) + ); + } + } + } + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; + }else{ + sqlite3Fts5ParseNearsetFree(pNear); + } + } + + return pRet; +} + +/* +** Allocate and return a new expression object. If anything goes wrong (i.e. +** OOM error), leave an error code in pParse and return NULL. +*/ +Fts5ExprNode *sqlite3Fts5ParseNode( + Fts5Parse *pParse, /* Parse context */ + int eType, /* FTS5_STRING, AND, OR or NOT */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight, /* Right hand child expression */ + Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */ +){ + Fts5ExprNode *pRet = 0; + + if( pParse->rc==SQLITE_OK ){ + int nChild = 0; /* Number of children of returned node */ + sqlite3_int64 nByte; /* Bytes of space to allocate for this node */ + + assert( (eType!=FTS5_STRING && !pNear) + || (eType==FTS5_STRING && !pLeft && !pRight) + ); + if( eType==FTS5_STRING && pNear==0 ) return 0; + if( eType!=FTS5_STRING && pLeft==0 ) return pRight; + if( eType!=FTS5_STRING && pRight==0 ) return pLeft; + + if( eType==FTS5_STRING + && pParse->bPhraseToAnd + && pNear->apPhrase[0]->nTerm>1 + ){ + pRet = fts5ParsePhraseToAnd(pParse, pNear); + }else{ + if( eType==FTS5_NOT ){ + nChild = 2; + }else if( eType==FTS5_AND || eType==FTS5_OR ){ + nChild = 2; + if( pLeft->eType==eType ) nChild += pLeft->nChild-1; + if( pRight->eType==eType ) nChild += pRight->nChild-1; + } + + nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + + if( pRet ){ + pRet->eType = eType; + pRet->pNear = pNear; + fts5ExprAssignXNext(pRet); + if( eType==FTS5_STRING ){ + int iPhrase; + for(iPhrase=0; iPhrasenPhrase; iPhrase++){ + pNear->apPhrase[iPhrase]->pNode = pRet; + if( pNear->apPhrase[iPhrase]->nTerm==0 ){ + pRet->xNext = 0; + pRet->eType = FTS5_EOF; + } + } + + if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + if( pNear->nPhrase!=1 + || pPhrase->nTerm>1 + || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst) + ){ + sqlite3Fts5ParseError(pParse, + "fts5: %s queries are not supported (detail!=full)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; + pNear = 0; + assert( pLeft==0 && pRight==0 ); + } + } + }else{ + assert( pNear==0 ); + fts5ExprAddChildren(pRet, pLeft); + fts5ExprAddChildren(pRet, pRight); + pLeft = pRight = 0; + if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ + sqlite3Fts5ParseError(pParse, + "fts5 expression tree is too large (maximum depth %d)", + SQLITE_FTS5_MAX_EXPR_DEPTH + ); + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; + } + } + } + } + } + + if( pRet==0 ){ + assert( pParse->rc!=SQLITE_OK ); + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + sqlite3Fts5ParseNearsetFree(pNear); + } + return pRet; +} + +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight /* Right hand child expression */ +){ + Fts5ExprNode *pRet = 0; + Fts5ExprNode *pPrev; + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + }else{ + + assert( pLeft->eType==FTS5_STRING + || pLeft->eType==FTS5_TERM + || pLeft->eType==FTS5_EOF + || pLeft->eType==FTS5_AND + ); + assert( pRight->eType==FTS5_STRING + || pRight->eType==FTS5_TERM + || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) + ); + + if( pLeft->eType==FTS5_AND ){ + pPrev = pLeft->apChild[pLeft->nChild-1]; + }else{ + pPrev = pLeft; + } + assert( pPrev->eType==FTS5_STRING + || pPrev->eType==FTS5_TERM + || pPrev->eType==FTS5_EOF + ); + + if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>0 ); + assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); + sqlite3Fts5ParseNodeFree(pRight); + pRet = pLeft; + pParse->nPhrase--; + } + else if( pPrev->eType==FTS5_EOF ){ + Fts5ExprPhrase **ap; + + if( pPrev==pLeft ){ + pRet = pRight; + }else{ + pLeft->apChild[pLeft->nChild-1] = pRight; + pRet = pLeft; + } + + ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase]; + assert( ap[0]==pPrev->pNear->apPhrase[0] ); + memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase); + pParse->nPhrase--; + + sqlite3Fts5ParseNodeFree(pPrev); + } + else{ + pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0); + } + } + + return pRet; +} + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ + sqlite3_int64 nByte = 0; + Fts5ExprTerm *p; + char *zQuoted; + + /* Determine the maximum amount of space required. */ + for(p=pTerm; p; p=p->pSynonym){ + nByte += pTerm->nQueryTerm * 2 + 3 + 2; + } + zQuoted = sqlite3_malloc64(nByte); + + if( zQuoted ){ + int i = 0; + for(p=pTerm; p; p=p->pSynonym){ + char *zIn = p->pTerm; + char *zEnd = &zIn[p->nQueryTerm]; + zQuoted[i++] = '"'; + while( zInpSynonym ) zQuoted[i++] = '|'; + } + if( pTerm->bPrefix ){ + zQuoted[i++] = ' '; + zQuoted[i++] = '*'; + } + zQuoted[i++] = '\0'; + } + return zQuoted; +} + +static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){ + char *zNew; + va_list ap; + va_start(ap, zFmt); + zNew = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zApp && zNew ){ + char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew); + sqlite3_free(zNew); + zNew = zNew2; + } + sqlite3_free(zApp); + return zNew; +} + +/* +** Compose a tcl-readable representation of expression pExpr. Return a +** pointer to a buffer containing that representation. It is the +** responsibility of the caller to at some point free the buffer using +** sqlite3_free(). +*/ +static char *fts5ExprPrintTcl( + Fts5Config *pConfig, + const char *zNearsetCmd, + Fts5ExprNode *pExpr +){ + char *zRet = 0; + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pExpr->pNear; + int i; + int iTerm; + + zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd); + if( zRet==0 ) return 0; + if( pNear->pColset ){ + int *aiCol = pNear->pColset->aiCol; + int nCol = pNear->pColset->nCol; + if( nCol==1 ){ + zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]); + }else{ + zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]); + for(i=1; ipColset->nCol; i++){ + zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]); + } + zRet = fts5PrintfAppend(zRet, "} "); + } + if( zRet==0 ) return 0; + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear); + if( zRet==0 ) return 0; + } + + zRet = fts5PrintfAppend(zRet, "--"); + if( zRet==0 ) return 0; + + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + + zRet = fts5PrintfAppend(zRet, " {"); + for(iTerm=0; zRet && iTermnTerm; iTerm++){ + Fts5ExprTerm *p = &pPhrase->aTerm[iTerm]; + zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", + p->nQueryTerm, p->pTerm + ); + if( pPhrase->aTerm[iTerm].bPrefix ){ + zRet = fts5PrintfAppend(zRet, "*"); + } + } + + if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); + if( zRet==0 ) return 0; + } + + }else if( pExpr->eType==0 ){ + zRet = sqlite3_mprintf("{}"); + }else{ + char const *zOp = 0; + int i; + switch( pExpr->eType ){ + case FTS5_AND: zOp = "AND"; break; + case FTS5_NOT: zOp = "NOT"; break; + default: + assert( pExpr->eType==FTS5_OR ); + zOp = "OR"; + break; + } + + zRet = sqlite3_mprintf("%s", zOp); + for(i=0; zRet && inChild; i++){ + char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]); + if( !z ){ + sqlite3_free(zRet); + zRet = 0; + }else{ + zRet = fts5PrintfAppend(zRet, " [%z]", z); + } + } + } + + return zRet; +} + +static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ + char *zRet = 0; + if( pExpr->eType==0 ){ + return sqlite3_mprintf("\"\""); + }else + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pExpr->pNear; + int i; + int iTerm; + + if( pNear->pColset ){ + int ii; + Fts5Colset *pColset = pNear->pColset; + if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{"); + for(ii=0; iinCol; ii++){ + zRet = fts5PrintfAppend(zRet, "%s%s", + pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " " + ); + } + if( zRet ){ + zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : ""); + } + if( zRet==0 ) return 0; + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, "NEAR("); + if( zRet==0 ) return 0; + } + + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( i!=0 ){ + zRet = fts5PrintfAppend(zRet, " "); + if( zRet==0 ) return 0; + } + for(iTerm=0; iTermnTerm; iTerm++){ + char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]); + if( zTerm ){ + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm); + sqlite3_free(zTerm); + } + if( zTerm==0 || zRet==0 ){ + sqlite3_free(zRet); + return 0; + } + } + } + + if( pNear->nPhrase>1 ){ + zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear); + if( zRet==0 ) return 0; + } + + }else{ + char const *zOp = 0; + int i; + + switch( pExpr->eType ){ + case FTS5_AND: zOp = " AND "; break; + case FTS5_NOT: zOp = " NOT "; break; + default: + assert( pExpr->eType==FTS5_OR ); + zOp = " OR "; + break; + } + + for(i=0; inChild; i++){ + char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]); + if( z==0 ){ + sqlite3_free(zRet); + zRet = 0; + }else{ + int e = pExpr->apChild[i]->eType; + int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); + zRet = fts5PrintfAppend(zRet, "%s%s%z%s", + (i==0 ? "" : zOp), + (b?"(":""), z, (b?")":"") + ); + } + if( zRet==0 ) break; + } + } + + return zRet; +} + +/* +** The implementation of user-defined scalar functions fts5_expr() (bTcl==0) +** and fts5_expr_tcl() (bTcl!=0). +*/ +static void fts5ExprFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal, /* Function arguments */ + int bTcl +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zExpr = 0; + char *zErr = 0; + Fts5Expr *pExpr = 0; + int rc; + int i; + + const char **azConfig; /* Array of arguments for Fts5Config */ + const char *zNearsetCmd = "nearset"; + int nConfig; /* Size of azConfig[] */ + Fts5Config *pConfig = 0; + int iArg = 1; + + if( nArg<1 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function %s", + bTcl ? "fts5_expr_tcl" : "fts5_expr" + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + + if( bTcl && nArg>1 ){ + zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); + iArg = 2; + } + + nConfig = 3 + (nArg-iArg); + azConfig = (const char**)sqlite3_malloc64(sizeof(char*) * nConfig); + if( azConfig==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + azConfig[0] = 0; + azConfig[1] = "main"; + azConfig[2] = "tbl"; + for(i=3; iArgnCol, zExpr, &pExpr, &zErr); + } + if( rc==SQLITE_OK ){ + char *zText; + if( pExpr->pRoot->xNext==0 ){ + zText = sqlite3_mprintf(""); + }else if( bTcl ){ + zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); + }else{ + zText = fts5ExprPrint(pConfig, pExpr->pRoot); + } + if( zText==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + } + + if( rc!=SQLITE_OK ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free((void *)azConfig); + sqlite3Fts5ConfigFree(pConfig); + sqlite3Fts5ExprFree(pExpr); +} + +static void fts5ExprFunctionHr( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + fts5ExprFunction(pCtx, nArg, apVal, 0); +} +static void fts5ExprFunctionTcl( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + fts5ExprFunction(pCtx, nArg, apVal, 1); +} + +/* +** The implementation of an SQLite user-defined-function that accepts a +** single integer as an argument. If the integer is an alpha-numeric +** unicode code point, 1 is returned. Otherwise 0. +*/ +static void fts5ExprIsAlnum( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + int iCode; + u8 aArr[32]; + if( nArg!=1 ){ + sqlite3_result_error(pCtx, + "wrong number of arguments to function fts5_isalnum", -1 + ); + return; + } + memset(aArr, 0, sizeof(aArr)); + sqlite3Fts5UnicodeCatParse("L*", aArr); + sqlite3Fts5UnicodeCatParse("N*", aArr); + sqlite3Fts5UnicodeCatParse("Co", aArr); + iCode = sqlite3_value_int(apVal[0]); + sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]); +} + +static void fts5ExprFold( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apVal /* Function arguments */ +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(pCtx, + "wrong number of arguments to function fts5_fold", -1 + ); + }else{ + int iCode; + int bRemoveDiacritics = 0; + iCode = sqlite3_value_int(apVal[0]); + if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]); + sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); + } +} +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +/* +** This is called during initialization to register the fts5_expr() scalar +** UDF with the SQLite handle passed as the only argument. +*/ +int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) + struct Fts5ExprFunc { + const char *z; + void (*x)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "fts5_expr", fts5ExprFunctionHr }, + { "fts5_expr_tcl", fts5ExprFunctionTcl }, + { "fts5_isalnum", fts5ExprIsAlnum }, + { "fts5_fold", fts5ExprFold }, + }; + int i; + int rc = SQLITE_OK; + void *pCtx = (void*)pGlobal; + + for(i=0; rc==SQLITE_OK && iz, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); + } +#else + int rc = SQLITE_OK; + UNUSED_PARAM2(pGlobal,db); +#endif + + /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and + ** sqlite3Fts5ParserFallback() are unused */ +#ifndef NDEBUG + (void)sqlite3Fts5ParserTrace; +#endif + (void)sqlite3Fts5ParserFallback; + + return rc; +} + +/* +** Return the number of phrases in expression pExpr. +*/ +int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){ + return (pExpr ? pExpr->nPhrase : 0); +} + +/* +** Return the number of terms in the iPhrase'th phrase in pExpr. +*/ +int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){ + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0; + return pExpr->apExprPhrase[iPhrase]->nTerm; +} + +/* +** This function is used to access the current position list for phrase +** iPhrase. +*/ +int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ + int nRet; + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ + *pa = pPhrase->poslist.p; + nRet = pPhrase->poslist.n; + }else{ + *pa = 0; + nRet = 0; + } + return nRet; +} + +struct Fts5PoslistPopulator { + Fts5PoslistWriter writer; + int bOk; /* True if ok to populate */ + int bMiss; +}; + +/* +** Clear the position lists associated with all phrases in the expression +** passed as the first argument. Argument bLive is true if the expression +** might be pointing to a real entry, otherwise it has just been reset. +** +** At present this function is only used for detail=col and detail=none +** fts5 tables. This implies that all phrases must be at most 1 token +** in size, as phrase matches are not supported without detail=full. +*/ +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ + Fts5PoslistPopulator *pRet; + pRet = sqlite3_malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + if( pRet ){ + int i; + memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + for(i=0; inPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + assert( pExpr->apExprPhrase[i]->nTerm<=1 ); + if( bLive && + (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) + ){ + pRet[i].bMiss = 1; + }else{ + pBuf->n = 0; + } + } + } + return pRet; +} + +struct Fts5ExprCtx { + Fts5Expr *pExpr; + Fts5PoslistPopulator *aPopulator; + i64 iOff; +}; +typedef struct Fts5ExprCtx Fts5ExprCtx; + +/* +** TODO: Make this more efficient! +*/ +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +/* +** pToken is a buffer nToken bytes in size that may or may not contain +** an embedded 0x00 byte. If it does, return the number of bytes in +** the buffer before the 0x00. If it does not, return nToken. +*/ +static int fts5QueryTerm(const char *pToken, int nToken){ + int ii; + for(ii=0; iipExpr; + int i; + int nQuery = nToken; + i64 iRowid = pExpr->pRoot->iRowid; + + UNUSED_PARAM2(iUnused1, iUnused2); + + if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE; + if( pExpr->pConfig->bTokendata ){ + nQuery = fts5QueryTerm(pToken, nQuery); + } + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pT; + if( p->aPopulator[i].bOk==0 ) continue; + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + if( (pT->nQueryTerm==nQuery || (pT->nQueryTermbPrefix)) + && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0 + ){ + int rc = sqlite3Fts5PoslistWriterAppend( + &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff + ); + if( rc==SQLITE_OK && (pExpr->pConfig->bTokendata || pT->bPrefix) ){ + int iCol = p->iOff>>32; + int iTokOff = p->iOff & 0x7FFFFFFF; + rc = sqlite3Fts5IndexIterWriteTokendata( + pT->pIter, pToken, nToken, iRowid, iCol, iTokOff + ); + } + if( rc ) return rc; + break; + } + } + } + return SQLITE_OK; +} + +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config *pConfig, + Fts5Expr *pExpr, + Fts5PoslistPopulator *aPopulator, + int iCol, + const char *z, int n +){ + int i; + Fts5ExprCtx sCtx; + sCtx.pExpr = pExpr; + sCtx.aPopulator = aPopulator; + sCtx.iOff = (((i64)iCol) << 32) - 1; + + for(i=0; inPhrase; i++){ + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + Fts5Colset *pColset = pNode->pNear->pColset; + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) + || aPopulator[i].bMiss + ){ + aPopulator[i].bOk = 0; + }else{ + aPopulator[i].bOk = 1; + } + } + + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + ); +} + +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ + pNode->pNear->apPhrase[0]->poslist.n = 0; + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprClearPoslists(pNode->apChild[i]); + } + } +} + +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case 0: + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + break; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + break; + } + } + return 1; +} + +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); +} + +/* +** This function is only called for detail=columns tables. +*/ +int sqlite3Fts5ExprPhraseCollist( + Fts5Expr *pExpr, + int iPhrase, + const u8 **ppCollist, + int *pnCollist +){ + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + int rc = SQLITE_OK; + + assert( iPhrase>=0 && iPhrasenPhrase ); + assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + + if( pNode->bEof==0 + && pNode->iRowid==pExpr->pRoot->iRowid + && pPhrase->poslist.n>0 + ){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; + if( pTerm->pSynonym ){ + Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1]; + rc = fts5ExprSynonymList( + pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist + ); + }else{ + *ppCollist = pPhrase->aTerm[0].pIter->pData; + *pnCollist = pPhrase->aTerm[0].pIter->nData; + } + }else{ + *ppCollist = 0; + *pnCollist = 0; + } + + return rc; +} + +/* +** Does the work of the fts5_api.xQueryToken() API method. +*/ +int sqlite3Fts5ExprQueryToken( + Fts5Expr *pExpr, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + + *ppOut = pPhrase->aTerm[iToken].pTerm; + *pnOut = pPhrase->aTerm[iToken].nFullTerm; + return SQLITE_OK; +} + +/* +** Does the work of the fts5_api.xInstToken() API method. +*/ +int sqlite3Fts5ExprInstToken( + Fts5Expr *pExpr, + i64 iRowid, + int iPhrase, + int iCol, + int iOff, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + Fts5ExprTerm *pTerm = 0; + int rc = SQLITE_OK; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + pTerm = &pPhrase->aTerm[iToken]; + if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm, + iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; + } + return rc; +} + +/* +** Clear the token mappings for all Fts5IndexIter objects mannaged by +** the expression passed as the only argument. +*/ +void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ + int ii; + for(ii=0; iinPhrase; ii++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterClearTokendata(pT->pIter); + } + } +} diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c new file mode 100644 index 0000000000000000000000000000000000000000..5e0959aa8ef3ba324c173f4f189d1aaeacb15c53 --- /dev/null +++ b/ext/fts5/fts5_hash.c @@ -0,0 +1,590 @@ +/* +** 2014 August 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + + +#include "fts5Int.h" + +typedef struct Fts5HashEntry Fts5HashEntry; + +/* +** This file contains the implementation of an in-memory hash table used +** to accumuluate "term -> doclist" content before it is flused to a level-0 +** segment. +*/ + + +struct Fts5Hash { + int eDetail; /* Copy of Fts5Config.eDetail */ + int *pnByte; /* Pointer to bytes counter */ + int nEntry; /* Number of entries currently in hash */ + int nSlot; /* Size of aSlot[] array */ + Fts5HashEntry *pScan; /* Current ordered scan item */ + Fts5HashEntry **aSlot; /* Array of hash slots */ +}; + +/* +** Each entry in the hash table is represented by an object of the +** following type. Each object, its key, and its current data are stored +** in a single memory allocation. The key immediately follows the object +** in memory. The position list data immediately follows the key data +** in memory. +** +** The key is Fts5HashEntry.nKey bytes in size. It consists of a single +** byte identifying the index (either the main term index or a prefix-index), +** followed by the term data. For example: "0token". There is no +** nul-terminator - in this case nKey=6. +** +** The data that follows the key is in a similar, but not identical format +** to the doclist data stored in the database. It is: +** +** * Rowid, as a varint +** * Position list, without 0x00 terminator. +** * Size of previous position list and rowid, as a 4 byte +** big-endian integer. +** +** iRowidOff: +** Offset of last rowid written to data area. Relative to first byte of +** structure. +** +** nData: +** Bytes of data written since iRowidOff. +*/ +struct Fts5HashEntry { + Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ + Fts5HashEntry *pScanNext; /* Next entry in sorted order */ + + int nAlloc; /* Total size of allocation */ + int iSzPoslist; /* Offset of space for 4-byte poslist size */ + int nData; /* Total bytes of data (incl. structure) */ + int nKey; /* Length of key in bytes */ + u8 bDel; /* Set delete-flag @ iSzPoslist */ + u8 bContent; /* Set content-flag (detail=none mode) */ + i16 iCol; /* Column of last value written */ + int iPos; /* Position of last value written */ + i64 iRowid; /* Rowid of last value written */ +}; + +/* +** Eqivalent to: +** +** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } +*/ +#define fts5EntryKey(p) ( ((char *)(&(p)[1])) ) + + +/* +** Allocate a new hash table. +*/ +int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ + int rc = SQLITE_OK; + Fts5Hash *pNew; + + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_int64 nByte; + memset(pNew, 0, sizeof(Fts5Hash)); + pNew->pnByte = pnByte; + pNew->eDetail = pConfig->eDetail; + + pNew->nSlot = 1024; + nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; + pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc64(nByte); + if( pNew->aSlot==0 ){ + sqlite3_free(pNew); + *ppNew = 0; + rc = SQLITE_NOMEM; + }else{ + memset(pNew->aSlot, 0, (size_t)nByte); + } + } + return rc; +} + +/* +** Free a hash table object. +*/ +void sqlite3Fts5HashFree(Fts5Hash *pHash){ + if( pHash ){ + sqlite3Fts5HashClear(pHash); + sqlite3_free(pHash->aSlot); + sqlite3_free(pHash); + } +} + +/* +** Empty (but do not delete) a hash table. +*/ +void sqlite3Fts5HashClear(Fts5Hash *pHash){ + int i; + for(i=0; inSlot; i++){ + Fts5HashEntry *pNext; + Fts5HashEntry *pSlot; + for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){ + pNext = pSlot->pHashNext; + sqlite3_free(pSlot); + } + } + memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); + pHash->nEntry = 0; +} + +static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + return (h % nSlot); +} + +static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + h = (h << 3) ^ h ^ b; + return (h % nSlot); +} + +/* +** Resize the hash table by doubling the number of slots. +*/ +static int fts5HashResize(Fts5Hash *pHash){ + int nNew = pHash->nSlot*2; + int i; + Fts5HashEntry **apNew; + Fts5HashEntry **apOld = pHash->aSlot; + + apNew = (Fts5HashEntry**)sqlite3_malloc64(nNew*sizeof(Fts5HashEntry*)); + if( !apNew ) return SQLITE_NOMEM; + memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); + + for(i=0; inSlot; i++){ + while( apOld[i] ){ + unsigned int iHash; + Fts5HashEntry *p = apOld[i]; + apOld[i] = p->pHashNext; + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey); + p->pHashNext = apNew[iHash]; + apNew[iHash] = p; + } + } + + sqlite3_free(apOld); + pHash->nSlot = nNew; + pHash->aSlot = apNew; + return SQLITE_OK; +} + +static int fts5HashAddPoslistSize( + Fts5Hash *pHash, + Fts5HashEntry *p, + Fts5HashEntry *p2 +){ + int nRet = 0; + if( p->iSzPoslist ){ + u8 *pPtr = p2 ? (u8*)p2 : (u8*)p; + int nData = p->nData; + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + assert( nData==p->iSzPoslist ); + if( p->bDel ){ + pPtr[nData++] = 0x00; + if( p->bContent ){ + pPtr[nData++] = 0x00; + } + } + }else{ + int nSz = (nData - p->iSzPoslist - 1); /* Size in bytes */ + int nPos = nSz*2 + p->bDel; /* Value of nPos field */ + + assert( p->bDel==0 || p->bDel==1 ); + if( nPos<=127 ){ + pPtr[p->iSzPoslist] = (u8)nPos; + }else{ + int nByte = sqlite3Fts5GetVarintLen((u32)nPos); + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); + nData += (nByte-1); + } + } + + nRet = nData - p->nData; + if( p2==0 ){ + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; + p->nData = nData; + } + } + return nRet; +} + +/* +** Add an entry to the in-memory hash table. The key is the concatenation +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). +** +** (bByte || pToken) -> (iRowid,iCol,iPos) +** +** Or, if iCol is negative, then the value is a delete marker. +*/ +int sqlite3Fts5HashWrite( + Fts5Hash *pHash, + i64 iRowid, /* Rowid for this entry */ + int iCol, /* Column token appears in (-ve -> delete) */ + int iPos, /* Position of token within column */ + char bByte, /* First byte of token */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +){ + unsigned int iHash; + Fts5HashEntry *p; + u8 *pPtr; + int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ + int bNew; /* If non-delete entry should be written */ + + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); + + /* Attempt to locate an existing hash entry */ + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + char *zKey = fts5EntryKey(p); + if( zKey[0]==bByte + && p->nKey==nToken+1 + && memcmp(&zKey[1], pToken, nToken)==0 + ){ + break; + } + } + + /* If an existing hash entry cannot be found, create a new one. */ + if( p==0 ){ + /* Figure out how much space to allocate */ + char *zKey; + sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; + if( nByte<128 ) nByte = 128; + + /* Grow the Fts5Hash.aSlot[] array if necessary. */ + if( (pHash->nEntry*2)>=pHash->nSlot ){ + int rc = fts5HashResize(pHash); + if( rc!=SQLITE_OK ) return rc; + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); + } + + /* Allocate new Fts5HashEntry and add it to the hash table. */ + p = (Fts5HashEntry*)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, sizeof(Fts5HashEntry)); + p->nAlloc = (int)nByte; + zKey = fts5EntryKey(p); + zKey[0] = bByte; + memcpy(&zKey[1], pToken, nToken); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); + p->nKey = nToken+1; + zKey[nToken+1] = '\0'; + p->nData = nToken+1 + sizeof(Fts5HashEntry); + p->pHashNext = pHash->aSlot[iHash]; + pHash->aSlot[iHash] = p; + pHash->nEntry++; + + /* Add the first rowid field to the hash-entry */ + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); + p->iRowid = iRowid; + + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + } + + }else{ + + /* Appending to an existing hash-entry. Check that there is enough + ** space to append the largest possible new entry. Worst case scenario + ** is: + ** + ** + 9 bytes for a new rowid, + ** + 4 byte reserved for the "poslist size" varint. + ** + 1 byte for a "new column" byte, + ** + 3 bytes for a new column number (16-bit max) as a varint, + ** + 5 bytes for the new position offset (32-bit max). + */ + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ + sqlite3_int64 nNew = p->nAlloc * 2; + Fts5HashEntry *pNew; + Fts5HashEntry **pp; + pNew = (Fts5HashEntry*)sqlite3_realloc64(p, nNew); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->nAlloc = (int)nNew; + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); + *pp = pNew; + p = pNew; + } + nIncr -= p->nData; + } + assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); + + pPtr = (u8*)p; + + /* If this is a new rowid, append the 4-byte size field for the previous + ** entry, and the new rowid for this entry. */ + if( iRowid!=p->iRowid ){ + u64 iDiff = (u64)iRowid - (u64)p->iRowid; + fts5HashAddPoslistSize(pHash, p, 0); + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iDiff); + p->iRowid = iRowid; + bNew = 1; + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + p->iPos = 0; + } + } + + if( iCol>=0 ){ + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + p->bContent = 1; + }else{ + /* Append a new column value, if necessary */ + assert_nc( iCol>=p->iCol ); + if( iCol!=p->iCol ){ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ + pPtr[p->nData++] = 0x01; + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); + p->iCol = (i16)iCol; + p->iPos = 0; + }else{ + bNew = 1; + p->iCol = (i16)(iPos = iCol); + } + } + + /* Append the new position offset, if necessary */ + if( bNew ){ + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); + p->iPos = iPos; + } + } + }else{ + /* This is a delete. Set the delete flag. */ + p->bDel = 1; + } + + nIncr += p->nData; + *pHash->pnByte += nIncr; + return SQLITE_OK; +} + + +/* +** Arguments pLeft and pRight point to linked-lists of hash-entry objects, +** each sorted in key order. This function merges the two lists into a +** single list and returns a pointer to its first element. +*/ +static Fts5HashEntry *fts5HashEntryMerge( + Fts5HashEntry *pLeft, + Fts5HashEntry *pRight +){ + Fts5HashEntry *p1 = pLeft; + Fts5HashEntry *p2 = pRight; + Fts5HashEntry *pRet = 0; + Fts5HashEntry **ppOut = &pRet; + + while( p1 || p2 ){ + if( p1==0 ){ + *ppOut = p2; + p2 = 0; + }else if( p2==0 ){ + *ppOut = p1; + p1 = 0; + }else{ + char *zKey1 = fts5EntryKey(p1); + char *zKey2 = fts5EntryKey(p2); + int nMin = MIN(p1->nKey, p2->nKey); + + int cmp = memcmp(zKey1, zKey2, nMin); + if( cmp==0 ){ + cmp = p1->nKey - p2->nKey; + } + assert( cmp!=0 ); + + if( cmp>0 ){ + /* p2 is smaller */ + *ppOut = p2; + ppOut = &p2->pScanNext; + p2 = p2->pScanNext; + }else{ + /* p1 is smaller */ + *ppOut = p1; + ppOut = &p1->pScanNext; + p1 = p1->pScanNext; + } + *ppOut = 0; + } + } + + return pRet; +} + +/* +** Link all tokens from hash table iHash into a list in sorted order. The +** tokens are not removed from the hash table. +*/ +static int fts5HashEntrySort( + Fts5Hash *pHash, + const char *pTerm, int nTerm, /* Query prefix, if any */ + Fts5HashEntry **ppSorted +){ + const int nMergeSlot = 32; + Fts5HashEntry **ap; + Fts5HashEntry *pList; + int iSlot; + int i; + + *ppSorted = 0; + ap = sqlite3_malloc64(sizeof(Fts5HashEntry*) * nMergeSlot); + if( !ap ) return SQLITE_NOMEM; + memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot); + + for(iSlot=0; iSlotnSlot; iSlot++){ + Fts5HashEntry *pIter; + for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ + if( pTerm==0 + || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + ){ + Fts5HashEntry *pEntry = pIter; + pEntry->pScanNext = 0; + for(i=0; ap[i]; i++){ + pEntry = fts5HashEntryMerge(pEntry, ap[i]); + ap[i] = 0; + } + ap[i] = pEntry; + } + } + } + + pList = 0; + for(i=0; inSlot, (const u8*)pTerm, nTerm); + char *zKey = 0; + Fts5HashEntry *p; + + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + zKey = fts5EntryKey(p); + if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break; + } + + if( p ){ + int nHashPre = sizeof(Fts5HashEntry) + nTerm; + int nList = p->nData - nHashPre; + u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); + if( pRet ){ + Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre]; + memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList); + nList += fts5HashAddPoslistSize(pHash, p, pFaux); + *pnDoclist = nList; + }else{ + *pnDoclist = 0; + return SQLITE_NOMEM; + } + }else{ + *ppOut = 0; + *pnDoclist = 0; + } + + return SQLITE_OK; +} + +int sqlite3Fts5HashScanInit( + Fts5Hash *p, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +){ + return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); +} + +#ifdef SQLITE_DEBUG +static int fts5HashCount(Fts5Hash *pHash){ + int nEntry = 0; + int ii; + for(ii=0; iinSlot; ii++){ + Fts5HashEntry *p = 0; + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ + nEntry++; + } + } + return nEntry; +} +#endif + +/* +** Return true if the hash table is empty, false otherwise. +*/ +int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ + assert( pHash->nEntry==fts5HashCount(pHash) ); + return pHash->nEntry==0; +} + +void sqlite3Fts5HashScanNext(Fts5Hash *p){ + assert( !sqlite3Fts5HashScanEof(p) ); + p->pScan = p->pScan->pScanNext; +} + +int sqlite3Fts5HashScanEof(Fts5Hash *p){ + return (p->pScan==0); +} + +void sqlite3Fts5HashScanEntry( + Fts5Hash *pHash, + const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ + const u8 **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +){ + Fts5HashEntry *p; + if( (p = pHash->pScan) ){ + char *zKey = fts5EntryKey(p); + int nTerm = p->nKey; + fts5HashAddPoslistSize(pHash, p, 0); + *pzTerm = zKey; + *pnTerm = nTerm; + *ppDoclist = (const u8*)&zKey[nTerm]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm); + }else{ + *pzTerm = 0; + *pnTerm = 0; + *ppDoclist = 0; + *pnDoclist = 0; + } +} diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c new file mode 100644 index 0000000000000000000000000000000000000000..242258af705754a7a2f51d80115f0a661da0529f --- /dev/null +++ b/ext/fts5/fts5_index.c @@ -0,0 +1,9411 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Low level access to the FTS index stored in the database file. The +** routines in this file file implement all read and write access to the +** %_data table. Other parts of the system access this functionality via +** the interface defined in fts5Int.h. +*/ + + +#include "fts5Int.h" + +/* +** Overview: +** +** The %_data table contains all the FTS indexes for an FTS5 virtual table. +** As well as the main term index, there may be up to 31 prefix indexes. +** The format is similar to FTS3/4, except that: +** +** * all segment b-tree leaf data is stored in fixed size page records +** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is +** taken to ensure it is possible to iterate in either direction through +** the entries in a doclist, or to seek to a specific entry within a +** doclist, without loading it into memory. +** +** * large doclists that span many pages have associated "doclist index" +** records that contain a copy of the first rowid on each page spanned by +** the doclist. This is used to speed up seek operations, and merges of +** large doclists with very small doclists. +** +** * extra fields in the "structure record" record the state of ongoing +** incremental merge operations. +** +*/ + + +#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ +#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ + +#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ + +#define FTS5_MAIN_PREFIX '0' + +#if FTS5_MAX_PREFIX_INDEXES > 31 +# error "FTS5_MAX_PREFIX_INDEXES is too large" +#endif + +#define FTS5_MAX_LEVEL 64 + +/* +** There are two versions of the format used for the structure record: +** +** 1. the legacy format, that may be read by all fts5 versions, and +** +** 2. the V2 format, which is used by contentless_delete=1 databases. +** +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy +** format structure record contains a varint - the number of levels in +** the structure. Whereas a V2 structure record contains the constant +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a +** varint has to be at least 16256 to begin with "0xFF". And the default +** maximum number of levels is 64. +** +** See below for more on structure record formats. +*/ +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" + +/* +** Details: +** +** The %_data table managed by this module, +** +** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); +** +** , contains the following 6 types of records. See the comments surrounding +** the FTS5_*_ROWID macros below for a description of how %_data rowids are +** assigned to each fo them. +** +** 1. Structure Records: +** +** The set of segments that make up an index - the index structure - are +** recorded in a single record within the %_data table. The record consists +** of a single 32-bit configuration cookie value followed by a list of +** SQLite varints. +** +** If the structure record is a V2 record, the configuration cookie is +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. +** +** Next, the record continues with three varints: +** +** + number of levels, +** + total number of segments on all levels, +** + value of write counter. +** +** Then, for each level from 0 to nMax: +** +** + number of input segments in ongoing merge. +** + total number of segments in level. +** + for each segment from oldest to newest: +** + segment id (always > 0) +** + first leaf page number (often 1, always greater than 0) +** + final leaf page number +** +** Then, for V2 structures only: +** +** + lower origin counter value, +** + upper origin counter value, +** + the number of tombstone hash pages. +** +** 2. The Averages Record: +** +** A single record within the %_data table. The data is a list of varints. +** The first value is the number of rows in the index. Then, for each column +** from left to right, the total number of tokens in the column for all +** rows of the table. +** +** 3. Segment leaves: +** +** TERM/DOCLIST FORMAT: +** +** Most of each segment leaf is taken up by term/doclist data. The +** general format of term/doclist, starting with the first term +** on the leaf page, is: +** +** varint : size of first term +** blob: first term data +** doclist: first doclist +** zero-or-more { +** varint: number of bytes in common with previous term +** varint: number of bytes of new term data (nNew) +** blob: nNew bytes of new term data +** doclist: next doclist +** } +** +** doclist format: +** +** varint: first rowid +** poslist: first poslist +** zero-or-more { +** varint: rowid delta (always > 0) +** poslist: next poslist +** } +** +** poslist format: +** +** varint: size of poslist in bytes multiplied by 2, not including +** this field. Plus 1 if this entry carries the "delete" flag. +** collist: collist for column 0 +** zero-or-more { +** 0x01 byte +** varint: column number (I) +** collist: collist for column I +** } +** +** collist format: +** +** varint: first offset + 2 +** zero-or-more { +** varint: offset delta + 2 +** } +** +** PAGE FORMAT +** +** Each leaf page begins with a 4-byte header containing 2 16-bit +** unsigned integer fields in big-endian format. They are: +** +** * The byte offset of the first rowid on the page, if it exists +** and occurs before the first term (otherwise 0). +** +** * The byte offset of the start of the page footer. If the page +** footer is 0 bytes in size, then this field is the same as the +** size of the leaf page in bytes. +** +** The page footer consists of a single varint for each term located +** on the page. Each varint is the byte offset of the current term +** within the page, delta-compressed against the previous value. In +** other words, the first varint in the footer is the byte offset of +** the first term, the second is the byte offset of the second less that +** of the first, and so on. +** +** The term/doclist format described above is accurate if the entire +** term/doclist data fits on a single leaf page. If this is not the case, +** the format is changed in two ways: +** +** + if the first rowid on a page occurs before the first term, it +** is stored as a literal value: +** +** varint: first rowid +** +** + the first term on each page is stored in the same way as the +** very first term of the segment: +** +** varint : size of first term +** blob: first term data +** +** 5. Segment doclist indexes: +** +** Doclist indexes are themselves b-trees, however they usually consist of +** a single leaf record only. The format of each doclist index leaf page +** is: +** +** * Flags byte. Bits are: +** 0x01: Clear if leaf is also the root page, otherwise set. +** +** * Page number of fts index leaf page. As a varint. +** +** * First rowid on page indicated by previous field. As a varint. +** +** * A list of varints, one for each subsequent termless page. A +** positive delta if the termless page contains at least one rowid, +** or an 0x00 byte otherwise. +** +** Internal doclist index nodes are: +** +** * Flags byte. Bits are: +** 0x01: Clear for root page, otherwise set. +** +** * Page number of first child page. As a varint. +** +** * Copy of first rowid on page indicated by previous field. As a varint. +** +** * A list of delta-encoded varints - the first rowid on each subsequent +** child page. +** +** 6. Tombstone Hash Page +** +** These records are only ever present in contentless_delete=1 tables. +** There are zero or more of these associated with each segment. They +** are used to store the tombstone rowids for rows contained in the +** associated segments. +** +** The set of nHashPg tombstone hash pages associated with a single +** segment together form a single hash table containing tombstone rowids. +** To find the page of the hash on which a key might be stored: +** +** iPg = (rowid % nHashPg) +** +** Then, within page iPg, which has nSlot slots: +** +** iSlot = (rowid / nHashPg) % nSlot +** +** Each tombstone hash page begins with an 8 byte header: +** +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the +** first tombstone hash page for each segment (iPg=0). If set, +** the hash table contains rowid 0. If clear, it does not. +** Rowid 0 is handled specially. +** 2-bytes: unused. +** 4-bytes: Big-endian integer containing number of entries on page. +** +** Following this are nSlot 4 or 8 byte slots (depending on the key-size +** in the first byte of the page header). The number of slots may be +** determined based on the size of the page record and the key-size: +** +** nSlot = (nByte - 8) / key-size +*/ + +/* +** Rowids for the averages and structure records in the %_data table. +*/ +#define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ +#define FTS5_STRUCTURE_ROWID 10 /* The structure record */ + +/* +** Macros determining the rowids used by segment leaves and dlidx leaves +** and nodes. All nodes and leaves are stored in the %_data table with large +** positive rowids. +** +** Each segment has a unique non-zero 16-bit id. +** +** The rowid for each segment leaf is found by passing the segment id and +** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered +** sequentially starting from 1. +*/ +#define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ +#define FTS5_DATA_DLI_B 1 /* Doclist-index flag (1 bit) */ +#define FTS5_DATA_HEIGHT_B 5 /* Max dlidx tree height of 32 */ +#define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ + +#define fts5_dri(segid, dlidx, height, pgno) ( \ + ((i64)(segid) << (FTS5_DATA_PAGE_B+FTS5_DATA_HEIGHT_B+FTS5_DATA_DLI_B)) + \ + ((i64)(dlidx) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ + ((i64)(height) << (FTS5_DATA_PAGE_B)) + \ + ((i64)(pgno)) \ +) + +#define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) +#define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) + +#ifdef SQLITE_DEBUG +int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } +#endif + + +/* +** Each time a blob is read from the %_data table, it is padded with this +** many zero bytes. This makes it easier to decode the various record formats +** without overreading if the records are corrupt. +*/ +#define FTS5_DATA_ZERO_PADDING 8 +#define FTS5_DATA_PADDING 20 + +typedef struct Fts5Data Fts5Data; +typedef struct Fts5DlidxIter Fts5DlidxIter; +typedef struct Fts5DlidxLvl Fts5DlidxLvl; +typedef struct Fts5DlidxWriter Fts5DlidxWriter; +typedef struct Fts5Iter Fts5Iter; +typedef struct Fts5PageWriter Fts5PageWriter; +typedef struct Fts5SegIter Fts5SegIter; +typedef struct Fts5DoclistIter Fts5DoclistIter; +typedef struct Fts5SegWriter Fts5SegWriter; +typedef struct Fts5Structure Fts5Structure; +typedef struct Fts5StructureLevel Fts5StructureLevel; +typedef struct Fts5StructureSegment Fts5StructureSegment; +typedef struct Fts5TokenDataIter Fts5TokenDataIter; +typedef struct Fts5TokenDataMap Fts5TokenDataMap; +typedef struct Fts5TombstoneArray Fts5TombstoneArray; + +struct Fts5Data { + u8 *p; /* Pointer to buffer containing record */ + int nn; /* Size of record in bytes */ + int szLeaf; /* Size of leaf without page-index */ +}; + +/* +** One object per %_data table. +** +** nContentlessDelete: +** The number of contentless delete operations since the most recent +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked +** so that extra auto-merge work can be done by fts5IndexFlush() to +** account for the delete operations. +*/ +struct Fts5Index { + Fts5Config *pConfig; /* Virtual table configuration */ + char *zDataTbl; /* Name of %_data table */ + int nWorkUnit; /* Leaf pages in a "unit" of work */ + + /* + ** Variables related to the accumulation of tokens and doclists within the + ** in-memory hash tables before they are flushed to disk. + */ + Fts5Hash *pHash; /* Hash table for in-memory data */ + int nPendingData; /* Current bytes of pending data */ + i64 iWriteRowid; /* Rowid for current doc being written */ + int bDelete; /* Current write is a delete */ + int nContentlessDelete; /* Number of contentless delete ops */ + int nPendingRow; /* Number of INSERT in hash table */ + + /* Error state. */ + int rc; /* Current error code */ + int flushRc; + + /* State used by the fts5DataXXX() functions. */ + sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ + sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ + sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ + sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ + sqlite3_stmt *pIdxSelect; + sqlite3_stmt *pIdxNextSelect; + int nRead; /* Total number of blocks read */ + + sqlite3_stmt *pDeleteFromIdx; + + sqlite3_stmt *pDataVersion; + i64 iStructVersion; /* data_version when pStruct read */ + Fts5Structure *pStruct; /* Current db structure (or NULL) */ +}; + +struct Fts5DoclistIter { + u8 *aEof; /* Pointer to 1 byte past end of doclist */ + + /* Output variables. aPoslist==0 at EOF */ + i64 iRowid; + u8 *aPoslist; + int nPoslist; + int nSize; +}; + +/* +** The contents of the "structure" record for each index are represented +** using an Fts5Structure record in memory. Which uses instances of the +** other Fts5StructureXXX types as components. +** +** nOriginCntr: +** This value is set to non-zero for structure records created for +** contentlessdelete=1 tables only. In that case it represents the +** origin value to apply to the next top-level segment created. +*/ +struct Fts5StructureSegment { + int iSegid; /* Segment id */ + int pgnoFirst; /* First leaf page number in segment */ + int pgnoLast; /* Last leaf page number in segment */ + + /* contentlessdelete=1 tables only: */ + u64 iOrigin1; + u64 iOrigin2; + int nPgTombstone; /* Number of tombstone hash table pages */ + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ + u64 nEntry; /* Number of rows in this segment */ +}; +struct Fts5StructureLevel { + int nMerge; /* Number of segments in incr-merge */ + int nSeg; /* Total number of segments on level */ + Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ +}; +struct Fts5Structure { + int nRef; /* Object reference count */ + u64 nWriteCounter; /* Total leaves written to level 0 */ + u64 nOriginCntr; /* Origin value for next top-level segment */ + int nSegment; /* Total segments in this structure */ + int nLevel; /* Number of levels in this index */ + Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ +}; + +/* +** An object of type Fts5SegWriter is used to write to segments. +*/ +struct Fts5PageWriter { + int pgno; /* Page number for this page */ + int iPrevPgidx; /* Previous value written into pgidx */ + Fts5Buffer buf; /* Buffer containing leaf data */ + Fts5Buffer pgidx; /* Buffer containing page-index */ + Fts5Buffer term; /* Buffer containing previous term on page */ +}; +struct Fts5DlidxWriter { + int pgno; /* Page number for this page */ + int bPrevValid; /* True if iPrev is valid */ + i64 iPrev; /* Previous rowid value written to page */ + Fts5Buffer buf; /* Buffer containing page data */ +}; +struct Fts5SegWriter { + int iSegid; /* Segid to write to */ + Fts5PageWriter writer; /* PageWriter object */ + i64 iPrevRowid; /* Previous rowid written to current leaf */ + u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ + u8 bFirstRowidInPage; /* True if next rowid is first in page */ + /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */ + u8 bFirstTermInPage; /* True if next term will be first in leaf */ + int nLeafWritten; /* Number of leaf pages written */ + int nEmpty; /* Number of contiguous term-less nodes */ + + int nDlidx; /* Allocated size of aDlidx[] array */ + Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ + + /* Values to insert into the %_idx table */ + Fts5Buffer btterm; /* Next term to insert into %_idx table */ + int iBtPage; /* Page number corresponding to btterm */ +}; + +typedef struct Fts5CResult Fts5CResult; +struct Fts5CResult { + u16 iFirst; /* aSeg[] index of firstest iterator */ + u8 bTermEq; /* True if the terms are equal */ +}; + +/* +** Object for iterating through a single segment, visiting each term/rowid +** pair in the segment. +** +** pSeg: +** The segment to iterate through. +** +** iLeafPgno: +** Current leaf page number within segment. +** +** iLeafOffset: +** Byte offset within the current leaf that is the first byte of the +** position list data (one byte passed the position-list size field). +** +** pLeaf: +** Buffer containing current leaf page data. Set to NULL at EOF. +** +** iTermLeafPgno, iTermLeafOffset: +** Leaf page number containing the last term read from the segment. And +** the offset immediately following the term data. +** +** flags: +** Mask of FTS5_SEGITER_XXX values. Interpreted as follows: +** +** FTS5_SEGITER_ONETERM: +** If set, set the iterator to point to EOF after the current doclist +** has been exhausted. Do not proceed to the next term in the segment. +** +** FTS5_SEGITER_REVERSE: +** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If +** it is set, iterate through rowid in descending order instead of the +** default ascending order. +** +** iRowidOffset/nRowidOffset/aRowidOffset: +** These are used if the FTS5_SEGITER_REVERSE flag is set. +** +** For each rowid on the page corresponding to the current term, the +** corresponding aRowidOffset[] entry is set to the byte offset of the +** start of the "position-list-size" field within the page. +** +** iTermIdx: +** Index of current term on iTermLeafPgno. +** +** apTombstone/nTombstone: +** These are used for contentless_delete=1 tables only. When the cursor +** is first allocated, the apTombstone[] array is allocated so that it +** is large enough for all tombstones hash pages associated with the +** segment. The pages themselves are loaded lazily from the database as +** they are required. +*/ +struct Fts5SegIter { + Fts5StructureSegment *pSeg; /* Segment to iterate through */ + int flags; /* Mask of configuration flags */ + int iLeafPgno; /* Current leaf page number */ + Fts5Data *pLeaf; /* Current leaf data */ + Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ + i64 iLeafOffset; /* Byte offset within current leaf */ + Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */ + + /* Next method */ + void (*xNext)(Fts5Index*, Fts5SegIter*, int*); + + /* The page and offset from which the current term was read. The offset + ** is the offset of the first rowid in the current doclist. */ + int iTermLeafPgno; + int iTermLeafOffset; + + int iPgidxOff; /* Next offset in pgidx */ + int iEndofDoclist; + + /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ + int iRowidOffset; /* Current entry in aRowidOffset[] */ + int nRowidOffset; /* Allocated size of aRowidOffset[] array */ + int *aRowidOffset; /* Array of offset to rowid fields */ + + Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ + + /* Variables populated based on current entry. */ + Fts5Buffer term; /* Current term */ + i64 iRowid; /* Current rowid */ + int nPos; /* Number of bytes in current position list */ + u8 bDel; /* True if the delete flag is set */ +}; + +/* +** Array of tombstone pages. Reference counted. +*/ +struct Fts5TombstoneArray { + int nRef; /* Number of pointers to this object */ + int nTombstone; + Fts5Data *apTombstone[1]; /* Array of tombstone pages */ +}; + +/* +** Argument is a pointer to an Fts5Data structure that contains a +** leaf page. +*/ +#define ASSERT_SZLEAF_OK(x) assert( \ + (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ +) + +#define FTS5_SEGITER_ONETERM 0x01 +#define FTS5_SEGITER_REVERSE 0x02 + +/* +** Argument is a pointer to an Fts5Data structure that contains a leaf +** page. This macro evaluates to true if the leaf contains no terms, or +** false if it contains at least one term. +*/ +#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) + +#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) + +#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) + +/* +** Object for iterating through the merged results of one or more segments, +** visiting each term/rowid pair in the merged data. +** +** nSeg is always a power of two greater than or equal to the number of +** segments that this object is merging data from. Both the aSeg[] and +** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded +** with zeroed objects - these are handled as if they were iterators opened +** on empty segments. +** +** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an +** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the +** comparison in this context is the index of the iterator that currently +** points to the smaller term/rowid combination. Iterators at EOF are +** considered to be greater than all other iterators. +** +** aFirst[1] contains the index in aSeg[] of the iterator that points to +** the smallest key overall. aFirst[0] is unused. +** +** poslist: +** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. +** There is no way to tell if this is populated or not. +** +** pColset: +** If not NULL, points to an object containing a set of column indices. +** Only matches that occur in one of these columns will be returned. +** The Fts5Iter does not own the Fts5Colset object, and so it is not +** freed when the iterator is closed - it is owned by the upper layer. +*/ +struct Fts5Iter { + Fts5IndexIter base; /* Base class containing output vars */ + Fts5TokenDataIter *pTokenDataIter; + + Fts5Index *pIndex; /* Index that owns this iterator */ + Fts5Buffer poslist; /* Buffer containing current poslist */ + Fts5Colset *pColset; /* Restrict matches to these columns */ + + /* Invoked to set output variables. */ + void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); + + int nSeg; /* Size of aSeg[] array */ + int bRev; /* True to iterate in reverse order */ + u8 bSkipEmpty; /* True to skip deleted entries */ + + i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ + Fts5CResult *aFirst; /* Current merge state (see above) */ + Fts5SegIter aSeg[1]; /* Array of segment iterators */ +}; + +/* +** An instance of the following type is used to iterate through the contents +** of a doclist-index record. +** +** pData: +** Record containing the doclist-index data. +** +** bEof: +** Set to true once iterator has reached EOF. +** +** iOff: +** Set to the current offset within record pData. +*/ +struct Fts5DlidxLvl { + Fts5Data *pData; /* Data for current page of this level */ + int iOff; /* Current offset into pData */ + int bEof; /* At EOF already */ + int iFirstOff; /* Used by reverse iterators */ + + /* Output variables */ + int iLeafPgno; /* Page number of current leaf page */ + i64 iRowid; /* First rowid on leaf iLeafPgno */ +}; +struct Fts5DlidxIter { + int nLvl; + int iSegid; + Fts5DlidxLvl aLvl[1]; +}; + +static void fts5PutU16(u8 *aOut, u16 iVal){ + aOut[0] = (iVal>>8); + aOut[1] = (iVal&0xFF); +} + +static u16 fts5GetU16(const u8 *aIn){ + return ((u16)aIn[0] << 8) + aIn[1]; +} + +/* +** The only argument points to a buffer at least 8 bytes in size. This +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian +** unsigned integer and returns the result. +*/ +static u64 fts5GetU64(u8 *a){ + return ((u64)a[0] << 56) + + ((u64)a[1] << 48) + + ((u64)a[2] << 40) + + ((u64)a[3] << 32) + + ((u64)a[4] << 24) + + ((u64)a[5] << 16) + + ((u64)a[6] << 8) + + ((u64)a[7] << 0); +} + +/* +** The only argument points to a buffer at least 4 bytes in size. This +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian +** unsigned integer and returns the result. +*/ +static u32 fts5GetU32(const u8 *a){ + return ((u32)a[0] << 24) + + ((u32)a[1] << 16) + + ((u32)a[2] << 8) + + ((u32)a[3] << 0); +} + +/* +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU64(u8 *a, u64 iVal){ + a[0] = ((iVal >> 56) & 0xFF); + a[1] = ((iVal >> 48) & 0xFF); + a[2] = ((iVal >> 40) & 0xFF); + a[3] = ((iVal >> 32) & 0xFF); + a[4] = ((iVal >> 24) & 0xFF); + a[5] = ((iVal >> 16) & 0xFF); + a[6] = ((iVal >> 8) & 0xFF); + a[7] = ((iVal >> 0) & 0xFF); +} + +/* +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU32(u8 *a, u32 iVal){ + a[0] = ((iVal >> 24) & 0xFF); + a[1] = ((iVal >> 16) & 0xFF); + a[2] = ((iVal >> 8) & 0xFF); + a[3] = ((iVal >> 0) & 0xFF); +} + +/* +** Allocate and return a buffer at least nByte bytes in size. +** +** If an OOM error is encountered, return NULL and set the error code in +** the Fts5Index handle passed as the first argument. +*/ +static void *fts5IdxMalloc(Fts5Index *p, sqlite3_int64 nByte){ + return sqlite3Fts5MallocZero(&p->rc, nByte); +} + +/* +** Compare the contents of the pLeft buffer with the pRight/nRight blob. +** +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or +** +ve if pRight is smaller than pLeft. In other words: +** +** res = *pLeft - *pRight +*/ +#ifdef SQLITE_DEBUG +static int fts5BufferCompareBlob( + Fts5Buffer *pLeft, /* Left hand side of comparison */ + const u8 *pRight, int nRight /* Right hand side of comparison */ +){ + int nCmp = MIN(pLeft->n, nRight); + int res = memcmp(pLeft->p, pRight, nCmp); + return (res==0 ? (pLeft->n - nRight) : res); +} +#endif + +/* +** Compare the contents of the two buffers using memcmp(). If one buffer +** is a prefix of the other, it is considered the lesser. +** +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or +** +ve if pRight is smaller than pLeft. In other words: +** +** res = *pLeft - *pRight +*/ +static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ + int nCmp, res; + nCmp = MIN(pLeft->n, pRight->n); + assert( nCmp<=0 || pLeft->p!=0 ); + assert( nCmp<=0 || pRight->p!=0 ); + res = fts5Memcmp(pLeft->p, pRight->p, nCmp); + return (res==0 ? (pLeft->n - pRight->n) : res); +} + +static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ + int ret; + fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); + return ret; +} + +/* +** Close the read-only blob handle, if it is open. +*/ +void sqlite3Fts5IndexCloseReader(Fts5Index *p){ + if( p->pReader ){ + sqlite3_blob *pReader = p->pReader; + p->pReader = 0; + sqlite3_blob_close(pReader); + } +} + +/* +** Retrieve a record from the %_data table. +** +** If an error occurs, NULL is returned and an error left in the +** Fts5Index object. +*/ +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = 0; + if( p->rc==SQLITE_OK ){ + int rc = SQLITE_OK; + + if( p->pReader ){ + /* This call may return SQLITE_ABORT if there has been a savepoint + ** rollback since it was last used. In this case a new blob handle + ** is required. */ + sqlite3_blob *pBlob = p->pReader; + p->pReader = 0; + rc = sqlite3_blob_reopen(pBlob, iRowid); + assert( p->pReader==0 ); + p->pReader = pBlob; + if( rc!=SQLITE_OK ){ + sqlite3Fts5IndexCloseReader(p); + } + if( rc==SQLITE_ABORT ) rc = SQLITE_OK; + } + + /* If the blob handle is not open at this point, open it and seek + ** to the requested entry. */ + if( p->pReader==0 && rc==SQLITE_OK ){ + Fts5Config *pConfig = p->pConfig; + rc = sqlite3_blob_open(pConfig->db, + pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader + ); + } + + /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls + ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead. + ** All the reasons those functions might return SQLITE_ERROR - missing + ** table, missing row, non-blob/text in block column - indicate + ** backing store corruption. */ + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + + if( rc==SQLITE_OK ){ + u8 *aOut = 0; /* Read blob data into this buffer */ + int nByte = sqlite3_blob_bytes(p->pReader); + int szData = (sizeof(Fts5Data) + 7) & ~7; + sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); + if( pRet ){ + pRet->nn = nByte; + aOut = pRet->p = (u8*)pRet + szData; + }else{ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pRet); + pRet = 0; + }else{ + /* TODO1: Fix this */ + pRet->p[nByte] = 0x00; + pRet->p[nByte+1] = 0x00; + pRet->szLeaf = fts5GetU16(&pRet->p[2]); + } + } + p->rc = rc; + p->nRead++; + } + + assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) ); + return pRet; +} + + +/* +** Release a reference to data record returned by an earlier call to +** fts5DataRead(). +*/ +static void fts5DataRelease(Fts5Data *pData){ + sqlite3_free(pData); +} + +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = fts5DataRead(p, iRowid); + if( pRet ){ + if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ + p->rc = FTS5_CORRUPT; + fts5DataRelease(pRet); + pRet = 0; + } + } + return pRet; +} + +static int fts5IndexPrepareStmt( + Fts5Index *p, + sqlite3_stmt **ppStmt, + char *zSql +){ + if( p->rc==SQLITE_OK ){ + if( zSql ){ + p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB, + ppStmt, 0); + }else{ + p->rc = SQLITE_NOMEM; + } + } + sqlite3_free(zSql); + return p->rc; +} + + +/* +** INSERT OR REPLACE a record into the %_data table. +*/ +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ + if( p->rc!=SQLITE_OK ) return; + + if( p->pWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", + pConfig->zDb, pConfig->zName + )); + if( p->rc ) return; + } + + sqlite3_bind_int64(p->pWriter, 1, iRowid); + sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); + sqlite3_step(p->pWriter); + p->rc = sqlite3_reset(p->pWriter); + sqlite3_bind_null(p->pWriter, 2); +} + +/* +** Execute the following SQL: +** +** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast +*/ +static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ + if( p->rc!=SQLITE_OK ) return; + + if( p->pDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + char *zSql = sqlite3_mprintf( + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", + pConfig->zDb, pConfig->zName + ); + if( fts5IndexPrepareStmt(p, &p->pDeleter, zSql) ) return; + } + + sqlite3_bind_int64(p->pDeleter, 1, iFirst); + sqlite3_bind_int64(p->pDeleter, 2, iLast); + sqlite3_step(p->pDeleter); + p->rc = sqlite3_reset(p->pDeleter); +} + +/* +** Remove all records associated with segment iSegid. +*/ +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ + int iSegid = pSeg->iSegid; + i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); + i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; + fts5DataDelete(p, iFirst, iLast); + + if( pSeg->nPgTombstone ){ + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); + fts5DataDelete(p, iTomb1, iTomb2); + } + if( p->pIdxDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE segid=?", + pConfig->zDb, pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); + sqlite3_step(p->pIdxDeleter); + p->rc = sqlite3_reset(p->pIdxDeleter); + } +} + +/* +** Release a reference to an Fts5Structure object returned by an earlier +** call to fts5StructureRead() or fts5StructureDecode(). +*/ +static void fts5StructureRelease(Fts5Structure *pStruct){ + if( pStruct && 0>=(--pStruct->nRef) ){ + int i; + assert( pStruct->nRef==0 ); + for(i=0; inLevel; i++){ + sqlite3_free(pStruct->aLevel[i].aSeg); + } + sqlite3_free(pStruct); + } +} + +static void fts5StructureRef(Fts5Structure *pStruct){ + pStruct->nRef++; +} + +void *sqlite3Fts5StructureRef(Fts5Index *p){ + fts5StructureRef(p->pStruct); + return (void*)p->pStruct; +} +void sqlite3Fts5StructureRelease(void *p){ + if( p ){ + fts5StructureRelease((Fts5Structure*)p); + } +} +int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){ + if( p->pStruct!=(Fts5Structure*)pStruct ){ + return SQLITE_ABORT; + } + return SQLITE_OK; +} + +/* +** Ensure that structure object (*pp) is writable. +** +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. If +** an error occurs, (*pRc) is set to an SQLite error code before returning. +*/ +static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){ + Fts5Structure *p = *pp; + if( *pRc==SQLITE_OK && p->nRef>1 ){ + i64 nByte = sizeof(Fts5Structure)+(p->nLevel-1)*sizeof(Fts5StructureLevel); + Fts5Structure *pNew; + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte); + if( pNew ){ + int i; + memcpy(pNew, p, nByte); + for(i=0; inLevel; i++) pNew->aLevel[i].aSeg = 0; + for(i=0; inLevel; i++){ + Fts5StructureLevel *pLvl = &pNew->aLevel[i]; + nByte = sizeof(Fts5StructureSegment) * pNew->aLevel[i].nSeg; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(pRc, nByte); + if( pLvl->aSeg==0 ){ + for(i=0; inLevel; i++){ + sqlite3_free(pNew->aLevel[i].aSeg); + } + sqlite3_free(pNew); + return; + } + memcpy(pLvl->aSeg, p->aLevel[i].aSeg, nByte); + } + p->nRef--; + pNew->nRef = 1; + } + *pp = pNew; + } +} + +/* +** Deserialize and return the structure record currently stored in serialized +** form within buffer pData/nData. +** +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array +** are over-allocated by one slot. This allows the structure contents +** to be more easily edited. +** +** If an error occurs, *ppOut is set to NULL and an SQLite error code +** returned. Otherwise, *ppOut is set to point to the new object and +** SQLITE_OK returned. +*/ +static int fts5StructureDecode( + const u8 *pData, /* Buffer containing serialized structure */ + int nData, /* Size of buffer pData in bytes */ + int *piCookie, /* Configuration cookie value */ + Fts5Structure **ppOut /* OUT: Deserialized object */ +){ + int rc = SQLITE_OK; + int i = 0; + int iLvl; + int nLevel = 0; + int nSegment = 0; + sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ + Fts5Structure *pRet = 0; /* Structure object to return */ + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ + u64 nOriginCntr = 0; /* Largest origin value seen so far */ + + /* Grab the cookie value */ + if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); + i = 4; + + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ + i += 4; + bStructureV2 = 1; + } + + /* Read the total number of levels and segments from the start of the + ** structure record. */ + i += fts5GetVarint32(&pData[i], nLevel); + i += fts5GetVarint32(&pData[i], nSegment); + if( nLevel>FTS5_MAX_SEGMENT || nLevel<0 + || nSegment>FTS5_MAX_SEGMENT || nSegment<0 + ){ + return FTS5_CORRUPT; + } + nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ + ); + pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); + + if( pRet ){ + pRet->nRef = 1; + pRet->nLevel = nLevel; + pRet->nSegment = nSegment; + i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); + + for(iLvl=0; rc==SQLITE_OK && iLvlaLevel[iLvl]; + int nTotal = 0; + int iSeg; + + if( i>=nData ){ + rc = FTS5_CORRUPT; + }else{ + i += fts5GetVarint32(&pData[i], pLvl->nMerge); + i += fts5GetVarint32(&pData[i], nTotal); + if( nTotalnMerge ) rc = FTS5_CORRUPT; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, + nTotal * sizeof(Fts5StructureSegment) + ); + nSegment -= nTotal; + } + + if( rc==SQLITE_OK ){ + pLvl->nSeg = nTotal; + for(iSeg=0; iSegaSeg[iSeg]; + if( i>=nData ){ + rc = FTS5_CORRUPT; + break; + } + assert( pSeg!=0 ); + i += fts5GetVarint32(&pData[i], pSeg->iSegid); + i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); + i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( bStructureV2 ){ + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntry); + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); + } + if( pSeg->pgnoLastpgnoFirst ){ + rc = FTS5_CORRUPT; + break; + } + } + if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT; + if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT; + } + } + if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + if( bStructureV2 ){ + pRet->nOriginCntr = nOriginCntr+1; + } + + if( rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + *ppOut = pRet; + return rc; +} + +/* +** Add a level to the Fts5Structure.aLevel[] array of structure object +** (*ppStruct). +*/ +static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ + fts5StructureMakeWritable(pRc, ppStruct); + assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK ); + if( *pRc==SQLITE_OK ){ + Fts5Structure *pStruct = *ppStruct; + int nLevel = pStruct->nLevel; + sqlite3_int64 nByte = ( + sizeof(Fts5Structure) + /* Main structure */ + sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ + ); + + pStruct = sqlite3_realloc64(pStruct, nByte); + if( pStruct ){ + memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel)); + pStruct->nLevel++; + *ppStruct = pStruct; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Extend level iLvl so that there is room for at least nExtra more +** segments. +*/ +static void fts5StructureExtendLevel( + int *pRc, + Fts5Structure *pStruct, + int iLvl, + int nExtra, + int bInsert +){ + if( *pRc==SQLITE_OK ){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureSegment *aNew; + sqlite3_int64 nByte; + + nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment); + aNew = sqlite3_realloc64(pLvl->aSeg, nByte); + if( aNew ){ + if( bInsert==0 ){ + memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra); + }else{ + int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment); + memmove(&aNew[nExtra], aNew, nMove); + memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra); + } + pLvl->aSeg = aNew; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ + Fts5Structure *pRet = 0; + Fts5Config *pConfig = p->pConfig; + int iCookie; /* Configuration cookie */ + Fts5Data *pData; + + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc==SQLITE_OK ){ + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); + if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + fts5DataRelease(pData); + if( p->rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + return pRet; +} + +static i64 fts5IndexDataVersion(Fts5Index *p){ + i64 iVersion = 0; + + if( p->rc==SQLITE_OK ){ + if( p->pDataVersion==0 ){ + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, + sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) + ); + if( p->rc ) return 0; + } + + if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ + iVersion = sqlite3_column_int64(p->pDataVersion, 0); + } + p->rc = sqlite3_reset(p->pDataVersion); + } + + return iVersion; +} + +/* +** Read, deserialize and return the structure record. +** +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array +** are over-allocated as described for function fts5StructureDecode() +** above. +** +** If an error occurs, NULL is returned and an error code left in the +** Fts5Index handle. If an error has already occurred when this function +** is called, it is a no-op. +*/ +static Fts5Structure *fts5StructureRead(Fts5Index *p){ + + if( p->pStruct==0 ){ + p->iStructVersion = fts5IndexDataVersion(p); + if( p->rc==SQLITE_OK ){ + p->pStruct = fts5StructureReadUncached(p); + } + } + +#if 0 + else{ + Fts5Structure *pTest = fts5StructureReadUncached(p); + if( pTest ){ + int i, j; + assert_nc( p->pStruct->nSegment==pTest->nSegment ); + assert_nc( p->pStruct->nLevel==pTest->nLevel ); + for(i=0; inLevel; i++){ + assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); + assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); + for(j=0; jaLevel[i].nSeg; j++){ + Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; + Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; + assert_nc( p1->iSegid==p2->iSegid ); + assert_nc( p1->pgnoFirst==p2->pgnoFirst ); + assert_nc( p1->pgnoLast==p2->pgnoLast ); + } + } + fts5StructureRelease(pTest); + } + } +#endif + + if( p->rc!=SQLITE_OK ) return 0; + assert( p->iStructVersion!=0 ); + assert( p->pStruct!=0 ); + fts5StructureRef(p->pStruct); + return p->pStruct; +} + +static void fts5StructureInvalidate(Fts5Index *p){ + if( p->pStruct ){ + fts5StructureRelease(p->pStruct); + p->pStruct = 0; + } +} + +/* +** Return the total number of segments in index structure pStruct. This +** function is only ever used as part of assert() conditions. +*/ +#ifdef SQLITE_DEBUG +static int fts5StructureCountSegments(Fts5Structure *pStruct){ + int nSegment = 0; /* Total number of segments */ + if( pStruct ){ + int iLvl; /* Used to iterate through levels */ + for(iLvl=0; iLvlnLevel; iLvl++){ + nSegment += pStruct->aLevel[iLvl].nSeg; + } + } + + return nSegment; +} +#endif + +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ +} + + +/* +** Serialize and store the "structure" record. +** +** If an error occurs, leave an error code in the Fts5Index object. If an +** error has already occurred, this function is a no-op. +*/ +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ + if( p->rc==SQLITE_OK ){ + Fts5Buffer buf; /* Buffer to serialize record into */ + int iLvl; /* Used to iterate through levels */ + int iCookie; /* Cookie value to store */ + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); + + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + memset(&buf, 0, sizeof(Fts5Buffer)); + + /* Append the current configuration cookie */ + iCookie = p->pConfig->iCookie; + if( iCookie<0 ) iCookie = 0; + + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ + sqlite3Fts5Put32(buf.p, iCookie); + buf.n = 4; + if( pStruct->nOriginCntr>0 ){ + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); + } + fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); + fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); + fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); + } + + for(iLvl=0; iLvlnLevel; iLvl++){ + int iSeg; /* Used to iterate through segments */ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge); + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg); + assert( pLvl->nMerge<=pLvl->nSeg ); + + for(iSeg=0; iSegnSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); + if( pStruct->nOriginCntr>0 ){ + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); + } + } + } + + fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); + fts5BufferFree(&buf); + } +} + +#if 0 +static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*); +static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ + int rc = SQLITE_OK; + Fts5Buffer buf; + memset(&buf, 0, sizeof(buf)); + fts5DebugStructure(&rc, &buf, pStruct); + fprintf(stdout, "%s: %s\n", zCaption, buf.p); + fflush(stdout); + fts5BufferFree(&buf); +} +#else +# define fts5PrintStructure(x,y) +#endif + +static int fts5SegmentSize(Fts5StructureSegment *pSeg){ + return 1 + pSeg->pgnoLast - pSeg->pgnoFirst; +} + +/* +** Return a copy of index structure pStruct. Except, promote as many +** segments as possible to level iPromote. If an OOM occurs, NULL is +** returned. +*/ +static void fts5StructurePromoteTo( + Fts5Index *p, + int iPromote, + int szPromote, + Fts5Structure *pStruct +){ + int il, is; + Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote]; + + if( pOut->nMerge==0 ){ + for(il=iPromote+1; ilnLevel; il++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; + if( pLvl->nMerge ) return; + for(is=pLvl->nSeg-1; is>=0; is--){ + int sz = fts5SegmentSize(&pLvl->aSeg[is]); + if( sz>szPromote ) return; + fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1); + if( p->rc ) return; + memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment)); + pOut->nSeg++; + pLvl->nSeg--; + } + } + } +} + +/* +** A new segment has just been written to level iLvl of index structure +** pStruct. This function determines if any segments should be promoted +** as a result. Segments are promoted in two scenarios: +** +** a) If the segment just written is smaller than one or more segments +** within the previous populated level, it is promoted to the previous +** populated level. +** +** b) If the segment just written is larger than the newest segment on +** the next populated level, then that segment, and any other adjacent +** segments that are also smaller than the one just written, are +** promoted. +** +** If one or more segments are promoted, the structure object is updated +** to reflect this. +*/ +static void fts5StructurePromote( + Fts5Index *p, /* FTS5 backend object */ + int iLvl, /* Index level just updated */ + Fts5Structure *pStruct /* Index structure */ +){ + if( p->rc==SQLITE_OK ){ + int iTst; + int iPromote = -1; + int szPromote = 0; /* Promote anything this size or smaller */ + Fts5StructureSegment *pSeg; /* Segment just written */ + int szSeg; /* Size of segment just written */ + int nSeg = pStruct->aLevel[iLvl].nSeg; + + if( nSeg==0 ) return; + pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; + szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); + + /* Check for condition (a) */ + for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); + if( iTst>=0 ){ + int i; + int szMax = 0; + Fts5StructureLevel *pTst = &pStruct->aLevel[iTst]; + assert( pTst->nMerge==0 ); + for(i=0; inSeg; i++){ + int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; + if( sz>szMax ) szMax = sz; + } + if( szMax>=szSeg ){ + /* Condition (a) is true. Promote the newest segment on level + ** iLvl to level iTst. */ + iPromote = iTst; + szPromote = szMax; + } + } + + /* If condition (a) is not met, assume (b) is true. StructurePromoteTo() + ** is a no-op if it is not. */ + if( iPromote<0 ){ + iPromote = iLvl; + szPromote = szSeg; + } + fts5StructurePromoteTo(p, iPromote, szPromote, pStruct); + } +} + + +/* +** Advance the iterator passed as the only argument. If the end of the +** doclist-index page is reached, return non-zero. +*/ +static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ + Fts5Data *pData = pLvl->pData; + + if( pLvl->iOff==0 ){ + assert( pLvl->bEof==0 ); + pLvl->iOff = 1; + pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno); + pLvl->iOff += fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid); + pLvl->iFirstOff = pLvl->iOff; + }else{ + int iOff; + for(iOff=pLvl->iOff; iOffnn; iOff++){ + if( pData->p[iOff] ) break; + } + + if( iOffnn ){ + u64 iVal; + pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; + iOff += fts5GetVarint(&pData->p[iOff], &iVal); + pLvl->iRowid += iVal; + pLvl->iOff = iOff; + }else{ + pLvl->bEof = 1; + } + } + + return pLvl->bEof; +} + +/* +** Advance the iterator passed as the only argument. +*/ +static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; + + assert( iLvlnLvl ); + if( fts5DlidxLvlNext(pLvl) ){ + if( (iLvl+1) < pIter->nLvl ){ + fts5DlidxIterNextR(p, pIter, iLvl+1); + if( pLvl[1].bEof==0 ){ + fts5DataRelease(pLvl->pData); + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) + ); + if( pLvl->pData ) fts5DlidxLvlNext(pLvl); + } + } + } + + return pIter->aLvl[0].bEof; +} +static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){ + return fts5DlidxIterNextR(p, pIter, 0); +} + +/* +** The iterator passed as the first argument has the following fields set +** as follows. This function sets up the rest of the iterator so that it +** points to the first rowid in the doclist-index. +** +** pData: +** pointer to doclist-index record, +** +** When this function is called pIter->iLeafPgno is the page number the +** doclist is associated with (the one featuring the term). +*/ +static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){ + int i; + for(i=0; inLvl; i++){ + fts5DlidxLvlNext(&pIter->aLvl[i]); + } + return pIter->aLvl[0].bEof; +} + + +static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){ + return p->rc!=SQLITE_OK || pIter->aLvl[0].bEof; +} + +static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){ + int i; + + /* Advance each level to the last entry on the last page */ + for(i=pIter->nLvl-1; p->rc==SQLITE_OK && i>=0; i--){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[i]; + while( fts5DlidxLvlNext(pLvl)==0 ); + pLvl->bEof = 0; + + if( i>0 ){ + Fts5DlidxLvl *pChild = &pLvl[-1]; + fts5DataRelease(pChild->pData); + memset(pChild, 0, sizeof(Fts5DlidxLvl)); + pChild->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno) + ); + } + } +} + +/* +** Move the iterator passed as the only argument to the previous entry. +*/ +static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ + int iOff = pLvl->iOff; + + assert( pLvl->bEof==0 ); + if( iOff<=pLvl->iFirstOff ){ + pLvl->bEof = 1; + }else{ + u8 *a = pLvl->pData->p; + + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; + + while( a[ii]==0 ){ + nZero++; + ii++; + } + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; + } + } + + return pLvl->bEof; +} + +static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; + + assert( iLvlnLvl ); + if( fts5DlidxLvlPrev(pLvl) ){ + if( (iLvl+1) < pIter->nLvl ){ + fts5DlidxIterPrevR(p, pIter, iLvl+1); + if( pLvl[1].bEof==0 ){ + fts5DataRelease(pLvl->pData); + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) + ); + if( pLvl->pData ){ + while( fts5DlidxLvlNext(pLvl)==0 ); + pLvl->bEof = 0; + } + } + } + } + + return pIter->aLvl[0].bEof; +} +static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){ + return fts5DlidxIterPrevR(p, pIter, 0); +} + +/* +** Free a doclist-index iterator object allocated by fts5DlidxIterInit(). +*/ +static void fts5DlidxIterFree(Fts5DlidxIter *pIter){ + if( pIter ){ + int i; + for(i=0; inLvl; i++){ + fts5DataRelease(pIter->aLvl[i].pData); + } + sqlite3_free(pIter); + } +} + +static Fts5DlidxIter *fts5DlidxIterInit( + Fts5Index *p, /* Fts5 Backend to iterate within */ + int bRev, /* True for ORDER BY ASC */ + int iSegid, /* Segment id */ + int iLeafPg /* Leaf page number to load dlidx for */ +){ + Fts5DlidxIter *pIter = 0; + int i; + int bDone = 0; + + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ + sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + Fts5DlidxIter *pNew; + + pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg); + Fts5DlidxLvl *pLvl = &pNew->aLvl[i]; + pIter = pNew; + memset(pLvl, 0, sizeof(Fts5DlidxLvl)); + pLvl->pData = fts5DataRead(p, iRowid); + if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){ + bDone = 1; + } + pIter->nLvl = i+1; + } + } + + if( p->rc==SQLITE_OK ){ + pIter->iSegid = iSegid; + if( bRev==0 ){ + fts5DlidxIterFirst(pIter); + }else{ + fts5DlidxIterLast(p, pIter); + } + } + + if( p->rc!=SQLITE_OK ){ + fts5DlidxIterFree(pIter); + pIter = 0; + } + + return pIter; +} + +static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){ + return pIter->aLvl[0].iRowid; +} +static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){ + return pIter->aLvl[0].iLeafPgno; +} + +/* +** Load the next leaf page into the segment iterator. +*/ +static void fts5SegIterNextPage( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter /* Iterator to advance to next page */ +){ + Fts5Data *pLeaf; + Fts5StructureSegment *pSeg = pIter->pSeg; + fts5DataRelease(pIter->pLeaf); + pIter->iLeafPgno++; + if( pIter->pNextLeaf ){ + pIter->pLeaf = pIter->pNextLeaf; + pIter->pNextLeaf = 0; + }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ + pIter->pLeaf = fts5LeafRead(p, + FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) + ); + }else{ + pIter->pLeaf = 0; + } + pLeaf = pIter->pLeaf; + + if( pLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf; + if( fts5LeafIsTermless(pLeaf) ){ + pIter->iEndofDoclist = pLeaf->nn+1; + }else{ + pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff], + pIter->iEndofDoclist + ); + } + } +} + +/* +** Argument p points to a buffer containing a varint to be interpreted as a +** position list size field. Read the varint and return the number of bytes +** read. Before returning, set *pnSz to the number of bytes in the position +** list, and *pbDel to true if the delete flag is set, or false otherwise. +*/ +static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ + int nSz; + int n = 0; + fts5FastGetVarint32(p, n, nSz); + assert_nc( nSz>=0 ); + *pnSz = nSz/2; + *pbDel = nSz & 0x0001; + return n; +} + +/* +** Fts5SegIter.iLeafOffset currently points to the first byte of a +** position-list size field. Read the value of the field and store it +** in the following variables: +** +** Fts5SegIter.nPos +** Fts5SegIter.bDel +** +** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the +** position list content (if any). +*/ +static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ + if( p->rc==SQLITE_OK ){ + int iOff = pIter->iLeafOffset; /* Offset to read at */ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); + pIter->bDel = 0; + pIter->nPos = 1; + if( iOffpLeaf->p[iOff]==0 ){ + pIter->bDel = 1; + iOff++; + if( iOffpLeaf->p[iOff]==0 ){ + pIter->nPos = 1; + iOff++; + }else{ + pIter->nPos = 0; + } + } + }else{ + int nSz; + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } + pIter->iLeafOffset = iOff; + } +} + +static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ + i64 iOff = pIter->iLeafOffset; + + ASSERT_SZLEAF_OK(pIter->pLeaf); + while( iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + return; + } + iOff = 4; + a = pIter->pLeaf->p; + } + iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; +} + +/* +** Fts5SegIter.iLeafOffset currently points to the first byte of the +** "nSuffix" field of a term. Function parameter nKeep contains the value +** of the "nPrefix" field (if there was one - it is passed 0 if this is +** the first term in the segment). +** +** This function populates: +** +** Fts5SegIter.term +** Fts5SegIter.rowid +** +** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of +** the first position list. The position list belonging to document +** (Fts5SegIter.iRowid). +*/ +static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ + i64 iOff = pIter->iLeafOffset; /* Offset to read at */ + int nNew; /* Bytes of new data */ + + iOff += fts5GetVarint32(&a[iOff], nNew); + if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ + p->rc = FTS5_CORRUPT; + return; + } + pIter->term.n = nKeep; + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + assert( pIter->term.n<=pIter->term.nSpace ); + iOff += nNew; + pIter->iTermLeafOffset = iOff; + pIter->iTermLeafPgno = pIter->iLeafPgno; + pIter->iLeafOffset = iOff; + + if( pIter->iPgidxOff>=pIter->pLeaf->nn ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); + pIter->iEndofDoclist += nExtra; + } + + fts5SegIterLoadRowid(p, pIter); +} + +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); + +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + pIter->xNext = fts5SegIterNext_Reverse; + }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xNext = fts5SegIterNext_None; + }else{ + pIter->xNext = fts5SegIterNext; + } +} + +/* +** Allocate a tombstone hash page array object (pIter->pTombArray) for +** the iterator passed as the second argument. If an OOM error occurs, +** leave an error in the Fts5Index object. +*/ +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ + const int nTomb = pIter->pSeg->nPgTombstone; + if( nTomb>0 ){ + int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + Fts5TombstoneArray *pNew; + pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pNew ){ + pNew->nTombstone = nTomb; + pNew->nRef = 1; + pIter->pTombArray = pNew; + } + } +} + +/* +** Initialize the iterator object pIter to iterate through the entries in +** segment pSeg. The iterator is left pointing to the first entry when +** this function returns. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterInit( + Fts5Index *p, /* FTS index object */ + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + if( pSeg->pgnoFirst==0 ){ + /* This happens if the segment is being used as an input to an incremental + ** merge and all data has already been "trimmed". See function + ** fts5TrimSegments() for details. In this case leave the iterator empty. + ** The caller will see the (pIter->pLeaf==0) and assume the iterator is + ** at EOF already. */ + assert( pIter->pLeaf==0 ); + return; + } + + if( p->rc==SQLITE_OK ){ + memset(pIter, 0, sizeof(*pIter)); + fts5SegIterSetNext(p, pIter); + pIter->pSeg = pSeg; + pIter->iLeafPgno = pSeg->pgnoFirst-1; + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); + } + + if( p->rc==SQLITE_OK && pIter->pLeaf ){ + pIter->iLeafOffset = 4; + assert( pIter->pLeaf!=0 ); + assert_nc( pIter->pLeaf->nn>4 ); + assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); + pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + fts5SegIterAllocTombstone(p, pIter); + } +} + +/* +** This function is only ever called on iterators created by calls to +** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set. +** +** The iterator is in an unusual state when this function is called: the +** Fts5SegIter.iLeafOffset variable is set to the offset of the start of +** the position-list size field for the first relevant rowid on the page. +** Fts5SegIter.rowid is set, but nPos and bDel are not. +** +** This function advances the iterator so that it points to the last +** relevant rowid on the page and, if necessary, initializes the +** aRowidOffset[] and iRowidOffset variables. At this point the iterator +** is in its regular state - Fts5SegIter.iLeafOffset points to the first +** byte of the position list content associated with said rowid. +*/ +static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ + int eDetail = p->pConfig->eDetail; + int n = pIter->pLeaf->szLeaf; + int i = pIter->iLeafOffset; + u8 *a = pIter->pLeaf->p; + int iRowidOffset = 0; + + if( n>pIter->iEndofDoclist ){ + n = pIter->iEndofDoclist; + } + + ASSERT_SZLEAF_OK(pIter->pLeaf); + while( 1 ){ + u64 iDelta = 0; + + if( eDetail==FTS5_DETAIL_NONE ){ + /* todo */ + if( i=n ) break; + i += fts5GetVarint(&a[i], &iDelta); + pIter->iRowid += iDelta; + + /* If necessary, grow the pIter->aRowidOffset[] array. */ + if( iRowidOffset>=pIter->nRowidOffset ){ + int nNew = pIter->nRowidOffset + 8; + int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + pIter->aRowidOffset = aNew; + pIter->nRowidOffset = nNew; + } + + pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset; + pIter->iLeafOffset = i; + } + pIter->iRowidOffset = iRowidOffset; + fts5SegIterLoadNPos(p, pIter); +} + +/* +** +*/ +static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ + Fts5Data *pNew; + pIter->iLeafPgno--; + pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( + pIter->pSeg->iSegid, pIter->iLeafPgno + )); + if( pNew ){ + /* iTermLeafOffset may be equal to szLeaf if the term is the last + ** thing on the page - i.e. the first rowid is on the following page. + ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */ + if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ + assert( pIter->pLeaf==0 ); + if( pIter->iTermLeafOffsetszLeaf ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; + } + }else{ + int iRowidOff; + iRowidOff = fts5LeafFirstRowidOff(pNew); + if( iRowidOff ){ + if( iRowidOff>=pNew->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + pIter->pLeaf = pNew; + pIter->iLeafOffset = iRowidOff; + } + } + } + + if( pIter->pLeaf ){ + u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; + pIter->iLeafOffset += fts5GetVarint(a, (u64*)&pIter->iRowid); + break; + }else{ + fts5DataRelease(pNew); + } + } + } + + if( pIter->pLeaf ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + fts5SegIterReverseInitPage(p, pIter); + } +} + +/* +** Return true if the iterator passed as the second argument currently +** points to a delete marker. A delete marker is an entry with a 0 byte +** position-list. +*/ +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used by reverse iterators. +*/ +static void fts5SegIterNext_Reverse( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbUnused /* Unused */ +){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->pNextLeaf==0 ); + UNUSED_PARAM(pbUnused); + + if( pIter->iRowidOffset>0 ){ + u8 *a = pIter->pLeaf->p; + int iOff; + u64 iDelta; + + pIter->iRowidOffset--; + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; + fts5SegIterLoadNPos(p, pIter); + iOff = pIter->iLeafOffset; + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ + iOff += pIter->nPos; + } + fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid -= iDelta; + }else{ + fts5SegIterReverseNewPage(p, pIter); + } +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used if detail=none and the +** iterator is not a reverse direction iterator. +*/ +static void fts5SegIterNext_None( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + int iOff; + + assert( p->rc==SQLITE_OK ); + assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); + + ASSERT_SZLEAF_OK(pIter->pLeaf); + iOff = pIter->iLeafOffset; + + /* Next entry is on the next page */ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( p->rc || pIter->pLeaf==0 ) return; + pIter->iRowid = 0; + iOff = 4; + } + + if( iOffiEndofDoclist ){ + /* Next entry is on the current page */ + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); + pIter->iLeafOffset = iOff; + pIter->iRowid += iDelta; + }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ + if( pIter->pSeg ){ + int nKeep = 0; + if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ + iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); + } + pIter->iLeafOffset = iOff; + fts5SegIterLoadTerm(p, pIter, nKeep); + }else{ + const u8 *pList = 0; + const char *zTerm = 0; + int nTerm = 0; + int nList; + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); + if( pList==0 ) goto next_none_eof; + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList; + sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + } + + if( pbNewTerm ) *pbNewTerm = 1; + }else{ + goto next_none_eof; + } + + fts5SegIterLoadNPos(p, pIter); + + return; + next_none_eof: + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; +} + + +/* +** Advance iterator pIter to the next entry. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. It +** is not considered an error if the iterator reaches EOF. If an error has +** already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterNext( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + Fts5Data *pLeaf = pIter->pLeaf; + int iOff; + int bNewTerm = 0; + int nKeep = 0; + u8 *a; + int n; + + assert( pbNewTerm==0 || *pbNewTerm==0 ); + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + /* Search for the end of the position list within the current page. */ + a = pLeaf->p; + n = pLeaf->szLeaf; + + ASSERT_SZLEAF_OK(pLeaf); + iOff = pIter->iLeafOffset + pIter->nPos; + + if( iOffiEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ + bNewTerm = 1; + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); + } + }else{ + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); + } + pIter->iLeafOffset = iOff; + + }else if( pIter->pSeg==0 ){ + const u8 *pList = 0; + const char *zTerm = 0; + int nTerm = 0; + int nList = 0; + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); + } + if( pList==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + *pbNewTerm = 1; + } + }else{ + iOff = 0; + /* Next entry is not on the current page */ + while( iOff==0 ){ + fts5SegIterNextPage(p, pIter); + pLeaf = pIter->pLeaf; + if( pLeaf==0 ) break; + ASSERT_SZLEAF_OK(pLeaf); + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + ); + } + } + else if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff + ); + pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; + bNewTerm = 1; + } + assert_nc( iOffszLeaf ); + if( iOff>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } + } + } + + /* Check if the iterator is now at EOF. If so, return early. */ + if( pIter->pLeaf ){ + if( bNewTerm ){ + if( pIter->flags & FTS5_SEGITER_ONETERM ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + fts5SegIterLoadTerm(p, pIter, nKeep); + fts5SegIterLoadNPos(p, pIter); + if( pbNewTerm ) *pbNewTerm = 1; + } + }else{ + /* The following could be done by calling fts5SegIterLoadNPos(). But + ** this block is particularly performance critical, so equivalent + ** code is inlined. */ + int nSz; + assert_nc( pIter->iLeafOffset<=pIter->pLeaf->nn ); + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } + } +} + +#define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } + +#define fts5IndexSkipVarint(a, iOff) { \ + int iEnd = iOff+9; \ + while( (a[iOff++] & 0x80) && iOffpDlidx; + Fts5Data *pLast = 0; + int pgnoLast = 0; + + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ + int iSegid = pIter->pSeg->iSegid; + pgnoLast = fts5DlidxIterPgno(pDlidx); + pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); + }else{ + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ + + /* Currently, Fts5SegIter.iLeafOffset points to the first byte of + ** position-list content for the current rowid. Back it up so that it + ** points to the start of the position-list size field. */ + int iPoslist; + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + iPoslist = pIter->iTermLeafOffset; + }else{ + iPoslist = 4; + } + fts5IndexSkipVarint(pLeaf->p, iPoslist); + pIter->iLeafOffset = iPoslist; + + /* If this condition is true then the largest rowid for the current + ** term may not be stored on the current page. So search forward to + ** see where said rowid really is. */ + if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ + int pgno; + Fts5StructureSegment *pSeg = pIter->pSeg; + + /* The last rowid in the doclist may not be on the current page. Search + ** forward to find the page containing the last rowid. */ + for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + Fts5Data *pNew = fts5LeafRead(p, iAbs); + if( pNew ){ + int iRowid, bTermless; + iRowid = fts5LeafFirstRowidOff(pNew); + bTermless = fts5LeafIsTermless(pNew); + if( iRowid ){ + SWAPVAL(Fts5Data*, pNew, pLast); + pgnoLast = pgno; + } + fts5DataRelease(pNew); + if( bTermless==0 ) break; + } + } + } + } + + /* If pLast is NULL at this point, then the last rowid for this doclist + ** lies on the page currently indicated by the iterator. In this case + ** pIter->iLeafOffset is already set to point to the position-list size + ** field associated with the first relevant rowid on the page. + ** + ** Or, if pLast is non-NULL, then it is the page that contains the last + ** rowid. In this case configure the iterator so that it points to the + ** first rowid on this page. + */ + if( pLast ){ + int iOff; + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = pLast; + pIter->iLeafPgno = pgnoLast; + iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } + } + + fts5SegIterReverseInitPage(p, pIter); +} + +/* +** Iterator pIter currently points to the first rowid of a doclist. +** There is a doclist-index associated with the final term on the current +** page. If the current term is the last term on the page, load the +** doclist-index from disk and initialize an iterator at (pIter->pDlidx). +*/ +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ + int iSeg = pIter->pSeg->iSegid; + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ + + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + assert( pIter->pDlidx==0 ); + + /* Check if the current doclist ends on this page. If it does, return + ** early without loading the doclist-index (as it belongs to a different + ** term. */ + if( pIter->iTermLeafPgno==pIter->iLeafPgno + && pIter->iEndofDoclistszLeaf + ){ + return; + } + + pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); +} + +/* +** The iterator object passed as the second argument currently contains +** no valid values except for the Fts5SegIter.pLeaf member variable. This +** function searches the leaf page for a term matching (pTerm/nTerm). +** +** If the specified term is found on the page, then the iterator is left +** pointing to it. If argument bGe is zero and the term is not found, +** the iterator is left pointing at EOF. +** +** If bGe is non-zero and the specified term is not found, then the +** iterator is left pointing to the smallest term in the segment that +** is larger than the specified term, even if this term is not on the +** current page. +*/ +static void fts5LeafSeek( + Fts5Index *p, /* Leave any error code here */ + int bGe, /* True for a >= search */ + Fts5SegIter *pIter, /* Iterator to seek */ + const u8 *pTerm, int nTerm /* Term to search for */ +){ + u32 iOff; + const u8 *a = pIter->pLeaf->p; + u32 n = (u32)pIter->pLeaf->nn; + + u32 nMatch = 0; + u32 nKeep = 0; + u32 nNew = 0; + u32 iTermOff; + u32 iPgidx; /* Current offset in pgidx */ + int bEndOfPage = 0; + + assert( p->rc==SQLITE_OK ); + + iPgidx = (u32)pIter->pLeaf->szLeaf; + iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); + iOff = iTermOff; + if( iOff>n ){ + p->rc = FTS5_CORRUPT; + return; + } + + while( 1 ){ + + /* Figure out how many new bytes are in this term */ + fts5FastGetVarint32(a, iOff, nNew); + if( nKeep=nMatch ); + if( nKeep==nMatch ){ + u32 nCmp; + u32 i; + nCmp = (u32)MIN(nNew, nTerm-nMatch); + for(i=0; ipTerm[nMatch] ){ + goto search_failed; + } + } + + if( iPgidx>=n ){ + bEndOfPage = 1; + break; + } + + iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); + iTermOff += nKeep; + iOff = iTermOff; + + if( iOff>=n ){ + p->rc = FTS5_CORRUPT; + return; + } + + /* Read the nKeep field of the next term. */ + fts5FastGetVarint32(a, iOff, nKeep); + } + + search_failed: + if( bGe==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + return; + }else if( bEndOfPage ){ + do { + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) return; + a = pIter->pLeaf->p; + if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ + iPgidx = (u32)pIter->pLeaf->szLeaf; + iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); + if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + }else{ + nKeep = 0; + iTermOff = iOff; + n = (u32)pIter->pLeaf->nn; + iOff += fts5GetVarint32(&a[iOff], nNew); + break; + } + } + }while( 1 ); + } + + search_success: + if( (i64)iOff+nNew>n || nNew<1 ){ + p->rc = FTS5_CORRUPT; + return; + } + pIter->iLeafOffset = iOff + nNew; + pIter->iTermLeafOffset = pIter->iLeafOffset; + pIter->iTermLeafPgno = pIter->iLeafPgno; + + fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + + if( iPgidx>=n ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + iPgidx += fts5GetVarint32(&a[iPgidx], nExtra); + pIter->iEndofDoclist = iTermOff + nExtra; + } + pIter->iPgidxOff = iPgidx; + + fts5SegIterLoadRowid(p, pIter); + fts5SegIterLoadNPos(p, pIter); +} + +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + } + return p->pIdxSelect; +} + +/* +** Initialize the object pIter to point to term pTerm/nTerm within segment +** pSeg. If there is no such term in the index, the iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterSeekInit( + Fts5Index *p, /* FTS5 backend */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = 1; + int bGe = (flags & FTS5INDEX_QUERY_SCAN); + int bDlidx = 0; /* True if there is a doclist-index */ + sqlite3_stmt *pIdxSelect = 0; + + assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); + assert( pTerm && nTerm ); + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + + /* This block sets stack variable iPg to the leaf page number that may + ** contain term (pTerm/nTerm), if it is present in the segment. */ + pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc ) return; + sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ + i64 val = sqlite3_column_int(pIdxSelect, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); + + if( iPgpgnoFirst ){ + iPg = pSeg->pgnoFirst; + bDlidx = 0; + } + + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + + if( pIter->pLeaf ){ + fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); + } + + if( p->rc==SQLITE_OK && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM)) ){ + pIter->flags |= FTS5_SEGITER_ONETERM; + if( pIter->pLeaf ){ + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + } + if( bDlidx ){ + fts5SegIterLoadDlidx(p, pIter); + } + if( flags & FTS5INDEX_QUERY_DESC ){ + fts5SegIterReverse(p, pIter); + } + } + } + + fts5SegIterSetNext(p, pIter); + if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM) ){ + fts5SegIterAllocTombstone(p, pIter); + } + + /* Either: + ** + ** 1) an error has occurred, or + ** 2) the iterator points to EOF, or + ** 3) the iterator points to an entry with term (pTerm/nTerm), or + ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points + ** to an entry with a term greater than or equal to (pTerm/nTerm). + */ + assert_nc( p->rc!=SQLITE_OK /* 1 */ + || pIter->pLeaf==0 /* 2 */ + || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */ + || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */ + ); +} + + +/* +** SQL used by fts5SegIterNextInit() to find the page to open. +*/ +static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){ + if( p->pIdxNextSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term>? ORDER BY term ASC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + + } + return p->pIdxNextSelect; +} + +/* +** This is similar to fts5SegIterSeekInit(), except that it initializes +** the segment iterator to point to the first term following the page +** with pToken/nToken on it. +*/ +static void fts5SegIterNextInit( + Fts5Index *p, + const char *pTerm, int nTerm, + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = -1; /* Page of segment to open */ + int bDlidx = 0; + sqlite3_stmt *pSel = 0; /* SELECT to find iPg */ + + pSel = fts5IdxNextStmt(p); + if( pSel ){ + assert( p->rc==SQLITE_OK ); + sqlite3_bind_int(pSel, 1, pSeg->iSegid); + sqlite3_bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC); + + if( sqlite3_step(pSel)==SQLITE_ROW ){ + i64 val = sqlite3_column_int64(pSel, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pSel); + sqlite3_bind_null(pSel, 2); + if( p->rc ) return; + } + + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + pIter->flags |= FTS5_SEGITER_ONETERM; + if( iPg>=0 ){ + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + fts5SegIterSetNext(p, pIter); + } + if( pIter->pLeaf ){ + const u8 *a = pIter->pLeaf->p; + int iTermOff = 0; + + pIter->iPgidxOff = pIter->pLeaf->szLeaf; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff); + pIter->iLeafOffset = iTermOff; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + if( bDlidx ) fts5SegIterLoadDlidx(p, pIter); + + assert( p->rc!=SQLITE_OK || + fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0 + ); + } +} + +/* +** Initialize the object pIter to point to term pTerm/nTerm within the +** in-memory hash table. If there is no such term in the hash-table, the +** iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterHashInit( + Fts5Index *p, /* FTS5 backend */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5SegIter *pIter /* Object to populate */ +){ + int nList = 0; + const u8 *z = 0; + int n = 0; + Fts5Data *pLeaf = 0; + + assert( p->pHash ); + assert( p->rc==SQLITE_OK ); + + if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ + const u8 *pList = 0; + + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList); + if( pList ){ + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); + if( pLeaf ){ + pLeaf->p = (u8*)pList; + } + } + + /* The call to sqlite3Fts5HashScanInit() causes the hash table to + ** fill the size field of all existing position lists. This means they + ** can no longer be appended to. Since the only scenario in which they + ** can be appended to is if the previous operation on this table was + ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this + ** possibility altogether. */ + p->bDelete = 0; + }else{ + p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), + (const char*)pTerm, nTerm, (void**)&pLeaf, &nList + ); + if( pLeaf ){ + pLeaf->p = (u8*)&pLeaf[1]; + } + z = pTerm; + n = nTerm; + pIter->flags |= FTS5_SEGITER_ONETERM; + } + + if( pLeaf ){ + sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); + pLeaf->nn = pLeaf->szLeaf = nList; + pIter->pLeaf = pLeaf; + pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pLeaf->nn; + + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + } + + fts5SegIterSetNext(p, pIter); +} + +/* +** Array ap[] contains n elements. Release each of these elements using +** fts5DataRelease(). Then free the array itself using sqlite3_free(). +*/ +static void fts5IndexFreeArray(Fts5Data **ap, int n){ + if( ap ){ + int ii; + for(ii=0; iinRef--; + if( p->nRef<=0 ){ + int ii; + for(ii=0; iinTombstone; ii++){ + fts5DataRelease(p->apTombstone[ii]); + } + sqlite3_free(p); + } + } +} + +/* +** Zero the iterator passed as the only argument. +*/ +static void fts5SegIterClear(Fts5SegIter *pIter){ + fts5BufferFree(&pIter->term); + fts5DataRelease(pIter->pLeaf); + fts5DataRelease(pIter->pNextLeaf); + fts5TombstoneArrayDelete(pIter->pTombArray); + fts5DlidxIterFree(pIter->pDlidx); + sqlite3_free(pIter->aRowidOffset); + memset(pIter, 0, sizeof(Fts5SegIter)); +} + +#ifdef SQLITE_DEBUG + +/* +** This function is used as part of the big assert() procedure implemented by +** fts5AssertMultiIterSetup(). It ensures that the result currently stored +** in *pRes is the correct result of comparing the current positions of the +** two iterators. +*/ +static void fts5AssertComparisonResult( + Fts5Iter *pIter, + Fts5SegIter *p1, + Fts5SegIter *p2, + Fts5CResult *pRes +){ + int i1 = p1 - pIter->aSeg; + int i2 = p2 - pIter->aSeg; + + if( p1->pLeaf || p2->pLeaf ){ + if( p1->pLeaf==0 ){ + assert( pRes->iFirst==i2 ); + }else if( p2->pLeaf==0 ){ + assert( pRes->iFirst==i1 ); + }else{ + int nMin = MIN(p1->term.n, p2->term.n); + int res = fts5Memcmp(p1->term.p, p2->term.p, nMin); + if( res==0 ) res = p1->term.n - p2->term.n; + + if( res==0 ){ + assert( pRes->bTermEq==1 ); + assert( p1->iRowid!=p2->iRowid ); + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1; + }else{ + assert( pRes->bTermEq==0 ); + } + + if( res<0 ){ + assert( pRes->iFirst==i1 ); + }else{ + assert( pRes->iFirst==i2 ); + } + } + } +} + +/* +** This function is a no-op unless SQLITE_DEBUG is defined when this module +** is compiled. In that case, this function is essentially an assert() +** statement used to verify that the contents of the pIter->aFirst[] array +** are correct. +*/ +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int i; + + assert( (pFirst->pLeaf==0)==pIter->base.bEof ); + + /* Check that pIter->iSwitchRowid is set correctly. */ + for(i=0; inSeg; i++){ + Fts5SegIter *p1 = &pIter->aSeg[i]; + assert( p1==pFirst + || p1->pLeaf==0 + || fts5BufferCompare(&pFirst->term, &p1->term) + || p1->iRowid==pIter->iSwitchRowid + || (p1->iRowidiSwitchRowid)==pIter->bRev + ); + } + + for(i=0; inSeg; i+=2){ + Fts5SegIter *p1 = &pIter->aSeg[i]; + Fts5SegIter *p2 = &pIter->aSeg[i+1]; + Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2]; + fts5AssertComparisonResult(pIter, p1, p2, pRes); + } + + for(i=1; i<(pIter->nSeg / 2); i+=2){ + Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ]; + Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ]; + Fts5CResult *pRes = &pIter->aFirst[i]; + fts5AssertComparisonResult(pIter, p1, p2, pRes); + } + } +} +#else +# define fts5AssertMultiIterSetup(x,y) +#endif + +/* +** Do the comparison necessary to populate pIter->aFirst[iOut]. +** +** If the returned value is non-zero, then it is the index of an entry +** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing +** to a key that is a duplicate of another, higher priority, +** segment-iterator in the pSeg->aSeg[] array. +*/ +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ + int i1; /* Index of left-hand Fts5SegIter */ + int i2; /* Index of right-hand Fts5SegIter */ + int iRes; + Fts5SegIter *p1; /* Left-hand Fts5SegIter */ + Fts5SegIter *p2; /* Right-hand Fts5SegIter */ + Fts5CResult *pRes = &pIter->aFirst[iOut]; + + assert( iOutnSeg && iOut>0 ); + assert( pIter->bRev==0 || pIter->bRev==1 ); + + if( iOut>=(pIter->nSeg/2) ){ + i1 = (iOut - pIter->nSeg/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pIter->aFirst[iOut*2].iFirst; + i2 = pIter->aFirst[iOut*2+1].iFirst; + } + p1 = &pIter->aSeg[i1]; + p2 = &pIter->aSeg[i2]; + + pRes->bTermEq = 0; + if( p1->pLeaf==0 ){ /* If p1 is at EOF */ + iRes = i2; + }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */ + iRes = i1; + }else{ + int res = fts5BufferCompare(&p1->term, &p2->term); + if( res==0 ){ + assert_nc( i2>i1 ); + assert_nc( i2!=0 ); + pRes->bTermEq = 1; + if( p1->iRowid==p2->iRowid ){ + return i2; + } + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; + } + assert( res!=0 ); + if( res<0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pRes->iFirst = (u16)iRes; + return 0; +} + +/* +** Move the seg-iter so that it points to the first rowid on page iLeafPgno. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. +*/ +static void fts5SegIterGotoPage( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int iLeafPgno +){ + assert( iLeafPgno>pIter->iLeafPgno ); + + if( iLeafPgno>pIter->pSeg->pgnoLast ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5DataRelease(pIter->pNextLeaf); + pIter->pNextLeaf = 0; + pIter->iLeafPgno = iLeafPgno-1; + + while( p->rc==SQLITE_OK ){ + int iOff; + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; + iOff = fts5LeafFirstRowidOff(pIter->pLeaf); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; + } + } + } +} + +/* +** Advance the iterator passed as the second argument until it is at or +** past rowid iFrom. Regardless of the value of iFrom, the iterator is +** always advanced at least once. +*/ +static void fts5SegIterNextFrom( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + i64 iMatch /* Advance iterator at least this far */ +){ + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); + Fts5DlidxIter *pDlidx = pIter->pDlidx; + int iLeafPgno = pIter->iLeafPgno; + int bMove = 1; + + assert( pIter->flags & FTS5_SEGITER_ONETERM ); + assert( pIter->pDlidx ); + assert( pIter->pLeaf ); + + if( bRev==0 ){ + while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){ + iLeafPgno = fts5DlidxIterPgno(pDlidx); + fts5DlidxIterNext(p, pDlidx); + } + assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc ); + if( iLeafPgno>pIter->iLeafPgno ){ + fts5SegIterGotoPage(p, pIter, iLeafPgno); + bMove = 0; + } + }else{ + assert( pIter->pNextLeaf==0 ); + assert( iMatchiRowid ); + while( !fts5DlidxIterEof(p, pDlidx) && iMatchiLeafPgno ); + + if( iLeafPgnoiLeafPgno ){ + pIter->iLeafPgno = iLeafPgno+1; + fts5SegIterReverseNewPage(p, pIter); + bMove = 0; + } + } + + do{ + if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); + if( pIter->pLeaf==0 ) break; + if( bRev==0 && pIter->iRowid>=iMatch ) break; + if( bRev!=0 && pIter->iRowid<=iMatch ) break; + bMove = 1; + }while( p->rc==SQLITE_OK ); +} + +/* +** Free the iterator object passed as the second argument. +*/ +static void fts5MultiIterFree(Fts5Iter *pIter){ + if( pIter ){ + int i; + for(i=0; inSeg; i++){ + fts5SegIterClear(&pIter->aSeg[i]); + } + fts5BufferFree(&pIter->poslist); + sqlite3_free(pIter); + } +} + +static void fts5MultiIterAdvanced( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + int iMinset /* Minimum entry in aFirst[] to set */ +){ + int i; + for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, 0); + i = pIter->nSeg + iEq; + } + } +} + +/* +** Sub-iterator iChanged of iterator pIter has just been advanced. It still +** points to the same term though - just a different rowid. This function +** attempts to update the contents of the pIter->aFirst[] accordingly. +** If it does so successfully, 0 is returned. Otherwise 1. +** +** If non-zero is returned, the caller should call fts5MultiIterAdvanced() +** on the iterator instead. That function does the same as this one, except +** that it deals with more complicated cases as well. +*/ +static int fts5MultiIterAdvanceRowid( + Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ + int iChanged, /* Index of sub-iterator just advanced */ + Fts5SegIter **ppFirst +){ + Fts5SegIter *pNew = &pIter->aSeg[iChanged]; + + if( pNew->iRowid==pIter->iSwitchRowid + || (pNew->iRowidiSwitchRowid)==pIter->bRev + ){ + int i; + Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001]; + pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64 : LARGEST_INT64; + for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){ + Fts5CResult *pRes = &pIter->aFirst[i]; + + assert( pNew->pLeaf ); + assert( pRes->bTermEq==0 || pOther->pLeaf ); + + if( pRes->bTermEq ){ + if( pNew->iRowid==pOther->iRowid ){ + return 1; + }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){ + pIter->iSwitchRowid = pOther->iRowid; + pNew = pOther; + }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){ + pIter->iSwitchRowid = pOther->iRowid; + } + } + pRes->iFirst = (u16)(pNew - pIter->aSeg); + if( i==1 ) break; + + pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ]; + } + } + + *ppFirst = pNew; + return 0; +} + +/* +** Set the pIter->bEof variable based on the state of the sub-iterators. +*/ +static void fts5MultiIterSetEof(Fts5Iter *pIter){ + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + pIter->base.bEof = pSeg->pLeaf==0; + pIter->iSwitchRowid = pSeg->iRowid; +} + +/* +** The argument to this macro must be an Fts5Data structure containing a +** tombstone hash page. This macro returns the key-size of the hash-page. +*/ +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) + +#define TOMBSTONE_NSLOT(pPg) \ + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) + +/* +** Query a single tombstone hash table for rowid iRowid. Return true if +** it is found or false otherwise. The tombstone hash table is one of +** nHashTable tables. +*/ +static int fts5IndexTombstoneQuery( + Fts5Data *pHash, /* Hash table page to query */ + int nHashTable, /* Number of pages attached to segment */ + u64 iRowid /* Rowid to query hash for */ +){ + const int szKey = TOMBSTONE_KEYSIZE(pHash); + const int nSlot = TOMBSTONE_NSLOT(pHash); + int iSlot = (iRowid / nHashTable) % nSlot; + int nCollide = nSlot; + + if( iRowid==0 ){ + return pHash->p[1]; + }else if( szKey==4 ){ + u32 *aSlot = (u32*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + }else{ + u64 *aSlot = (u64*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + } + + return 0; +} + +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + Fts5TombstoneArray *pArray = pSeg->pTombArray; + + if( pSeg->pLeaf && pArray ){ + /* Figure out which page the rowid might be present on. */ + int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone; + assert( iPg>=0 ); + + /* If tombstone hash page iPg has not yet been loaded from the + ** database, load it now. */ + if( pArray->apTombstone[iPg]==0 ){ + pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) + ); + if( pArray->apTombstone[iPg]==0 ) return 0; + } + + return fts5IndexTombstoneQuery( + pArray->apTombstone[iPg], + pArray->nTombstone, + pSeg->iRowid + ); + } + + return 0; +} + +/* +** Move the iterator to the next entry. +** +** If an error occurs, an error code is left in Fts5Index.rc. It is not +** considered an error if the iterator reaches EOF, or if it is already at +** EOF when this function is called. +*/ +static void fts5MultiIterNext( + Fts5Index *p, + Fts5Iter *pIter, + int bFrom, /* True if argument iFrom is valid */ + i64 iFrom /* Advance at least as far as this */ +){ + int bUseFrom = bFrom; + assert( pIter->base.bEof==0 ); + while( p->rc==SQLITE_OK ){ + int iFirst = pIter->aFirst[1].iFirst; + int bNewTerm = 0; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + assert( p->rc==SQLITE_OK ); + if( bUseFrom && pSeg->pDlidx ){ + fts5SegIterNextFrom(p, pSeg, iFrom); + }else{ + pSeg->xNext(p, pSeg, &bNewTerm); + } + + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + if( pSeg->pLeaf==0 ) return; + } + + fts5AssertMultiIterSetup(p, pIter); + assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); + if( (pIter->bSkipEmpty==0 || pSeg->nPos) + && 0==fts5MultiIterIsDeleted(pIter) + ){ + pIter->xSetOutputs(pIter, pSeg); + return; + } + bUseFrom = 0; + } +} + +static void fts5MultiIterNext2( + Fts5Index *p, + Fts5Iter *pIter, + int *pbNewTerm /* OUT: True if *might* be new term */ +){ + assert( pIter->bSkipEmpty ); + if( p->rc==SQLITE_OK ){ + *pbNewTerm = 0; + do{ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + int bNewTerm = 0; + + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, &bNewTerm); + if( pSeg->pLeaf==0 || bNewTerm + || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) + ){ + fts5MultiIterAdvanced(p, pIter, iFirst, 1); + fts5MultiIterSetEof(pIter); + *pbNewTerm = 1; + } + fts5AssertMultiIterSetup(p, pIter); + + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) + && (p->rc==SQLITE_OK) + ); + } +} + +static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){ + UNUSED_PARAM2(pUnused1, pUnused2); +} + +static Fts5Iter *fts5MultiIterAlloc( + Fts5Index *p, /* FTS5 backend to iterate within */ + int nSeg +){ + Fts5Iter *pNew; + i64 nSlot; /* Power of two >= nSeg */ + + for(nSlot=2; nSlotaSeg[] */ + sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ + ); + if( pNew ){ + pNew->nSeg = nSlot; + pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; + pNew->pIndex = p; + pNew->xSetOutputs = fts5IterSetOutputs_Noop; + } + return pNew; +} + +static void fts5PoslistCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); + } +} + +typedef struct PoslistCallbackCtx PoslistCallbackCtx; +struct PoslistCallbackCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int eState; /* See above */ +}; + +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; + +/* +** TODO: Make this more efficient! +*/ +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static void fts5PoslistOffsetsCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( iiRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; + } + } + } +} + +static void fts5PoslistFilterCallback( + Fts5Index *pUnused, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; + UNUSED_PARAM(pUnused); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + /* Search through to find the first varint with value 1. This is the + ** start of the next columns hits. */ + int i = 0; + int iStart = 0; + + if( pCtx->eState==2 ){ + int iCol; + fts5FastGetVarint32(pChunk, i, iCol); + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ + pCtx->eState = 1; + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); + }else{ + pCtx->eState = 0; + } + } + + do { + while( ieState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + } + if( i=nChunk ){ + pCtx->eState = 2; + }else{ + fts5FastGetVarint32(pChunk, i, iCol); + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + iStart = i; + } + } + } + }while( inPos; /* Number of bytes still to come */ + Fts5Data *pData = 0; + u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); + int pgno = pSeg->iLeafPgno; + int pgnoSave = 0; + + /* This function does not work with detail=none databases. */ + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ + pgnoSave = pgno+1; + } + + while( 1 ){ + xChunk(p, pCtx, pChunk, nChunk); + nRem -= nChunk; + fts5DataRelease(pData); + if( nRem<=0 ){ + break; + }else if( pSeg->pSeg==0 ){ + p->rc = FTS5_CORRUPT; + return; + }else{ + pgno++; + pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); + if( pData==0 ) break; + pChunk = &pData->p[4]; + nChunk = MIN(nRem, pData->szLeaf - 4); + if( pgno==pgnoSave ){ + assert( pSeg->pNextLeaf==0 ); + pSeg->pNextLeaf = pData; + pData = 0; + } + } + } +} + +/* +** Iterator pIter currently points to a valid entry (not EOF). This +** function appends the position list data for the current entry to +** buffer pBuf. It does not make a copy of the position-list size +** field. +*/ +static void fts5SegiterPoslist( + Fts5Index *p, + Fts5SegIter *pSeg, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + assert( pBuf!=0 ); + assert( pSeg!=0 ); + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+FTS5_DATA_ZERO_PADDING) ){ + assert( pBuf->p!=0 ); + assert( pBuf->nSpace >= pBuf->n+pSeg->nPos+FTS5_DATA_ZERO_PADDING ); + memset(&pBuf->p[pBuf->n+pSeg->nPos], 0, FTS5_DATA_ZERO_PADDING); + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + } + } + } +} + +/* +** Parameter pPos points to a buffer containing a position list, size nPos. +** This function filters it according to pColset (which must be non-NULL) +** and sets pIter->base.pData/nData to point to the new position list. +** If memory is required for the new position list, use buffer pIter->poslist. +** Or, if the new position list is a contiguous subset of the input, set +** pIter->base.pData/nData to point directly to it. +** +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. If an OOM error is encountered, *pRc is set to SQLITE_NOMEM +** before returning. +*/ +static void fts5IndexExtractColset( + int *pRc, + Fts5Colset *pColset, /* Colset to filter on */ + const u8 *pPos, int nPos, /* Position list */ + Fts5Iter *pIter +){ + if( *pRc==SQLITE_OK ){ + const u8 *p = pPos; + const u8 *aCopy = p; + const u8 *pEnd = &p[nPos]; /* One byte past end of position list */ + int i = 0; + int iCurrent = 0; + + if( pColset->nCol>1 && sqlite3Fts5BufferSize(pRc, &pIter->poslist, nPos) ){ + return; + } + + while( 1 ){ + while( pColset->aiCol[i]nCol ){ + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + return; + } + } + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( paiCol[i]==iCurrent ){ + if( pColset->nCol==1 ){ + pIter->base.pData = aCopy; + pIter->base.nData = p-aCopy; + return; + } + fts5BufferSafeAppendBlob(&pIter->poslist, aCopy, p-aCopy); + } + if( p>=pEnd ){ + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + return; + } + aCopy = p++; + iCurrent = *p++; + if( iCurrent & 0x80 ){ + p--; + p += fts5GetVarint32(p, iCurrent); + } + } + } + +} + +/* +** xSetOutputs callback used by detail=none tables. +*/ +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; +} + +/* +** xSetOutputs callback used by detail=full and detail=col tables when no +** column filters are specified. +*/ +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; + + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); + assert( pIter->pColset==0 ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match +** against no columns at all). +*/ +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + UNUSED_PARAM(pSeg); + pIter->base.nData = 0; +} + +/* +** xSetOutputs callback used by detail=col when there is a column filter +** and there are 100 or more columns. Also called as a fallback from +** fts5IterSetOutputs_Col100 if the column-list spans more than one page. +*/ +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + +/* +** xSetOutputs callback used when: +** +** * detail=col, +** * there is a column filter, and +** * the table contains 100 or fewer columns. +** +** The last point is to ensure all column numbers are stored as +** single-byte varints. +*/ +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pIter->pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ + fts5IterSetOutputs_Col(pIter, pSeg); + }else{ + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + int iPrev = 0; + int *aiCol = pIter->pColset->aiCol; + int *aiColEnd = &aiCol[pIter->pColset->nCol]; + + u8 *aOut = pIter->poslist.p; + int iPrevOut = 0; + + pIter->base.iRowid = pSeg->iRowid; + + while( abase.pData = pIter->poslist.p; + pIter->base.nData = aOut - pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=full when there is a column filter. +*/ +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int *pRc = &pIter->pIndex->rc; + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, pIter); + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } +} + +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + assert( pIter!=0 || (*pRc)!=SQLITE_OK ); + if( *pRc==SQLITE_OK ){ + Fts5Config *pConfig = pIter->pIndex->pConfig; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xSetOutputs = fts5IterSetOutputs_None; + } + + else if( pIter->pColset==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; + } + + else if( pIter->pColset->nCol==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; + } + + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ + pIter->xSetOutputs = fts5IterSetOutputs_Full; + } + + else{ + assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pConfig->nCol<=100 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col100; + sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); + }else{ + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } + } + } +} + +/* +** All the component segment-iterators of pIter have been set up. This +** functions finishes setup for iterator pIter itself. +*/ +static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){ + int iIter; + for(iIter=pIter->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pIter, iEq, iIter); + } + } + fts5MultiIterSetEof(pIter); + fts5AssertMultiIterSetup(p, pIter); + + if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter)) + || fts5MultiIterIsDeleted(pIter) + ){ + fts5MultiIterNext(p, pIter, 0, 0); + }else if( pIter->base.bEof==0 ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + pIter->xSetOutputs(pIter, pSeg); + } +} + +/* +** Allocate a new Fts5Iter object. +** +** The new object will be used to iterate through data in structure pStruct. +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel +** is zero or greater, data from the first nSegment segments on level iLevel +** is merged. +** +** The iterator initially points to the first term/rowid entry in the +** iterated data. +*/ +static void fts5MultiIterNew( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Structure *pStruct, /* Structure of specific index */ + int flags, /* FTS5INDEX_QUERY_XXX flags */ + Fts5Colset *pColset, /* Colset to filter on (or NULL) */ + const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ + int iLevel, /* Level to iterate (-1 for all) */ + int nSegment, /* Number of segments to merge (iLevel>=0) */ + Fts5Iter **ppOut /* New object */ +){ + int nSeg = 0; /* Number of segment-iters in use */ + int iIter = 0; /* */ + int iSeg; /* Used to iterate through segments */ + Fts5StructureLevel *pLvl; + Fts5Iter *pNew; + + assert( (pTerm==0 && nTerm==0) || iLevel<0 ); + + /* Allocate space for the new multi-seg-iterator. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); + }else{ + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + } + } + *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); + if( pNew==0 ){ + assert( p->rc!=SQLITE_OK ); + goto fts5MultiIterNew_post_check; + } + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); + pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->pColset = pColset; + if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ + fts5IterSetOutputCb(&p->rc, pNew); + } + + /* Initialize each of the component segment iterators. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ + /* Add a segment iterator for the current contents of the hash table. */ + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); + } + for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + if( pTerm==0 ){ + fts5SegIterInit(p, pSeg, pIter); + }else{ + fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); + } + } + } + }else{ + pLvl = &pStruct->aLevel[iLevel]; + for(iSeg=nSeg-1; iSeg>=0; iSeg--){ + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); + } + } + assert( iIter==nSeg ); + } + + /* If the above was successful, each component iterator now points + ** to the first entry in its segment. In this case initialize the + ** aFirst[] array. Or, if an error has occurred, free the iterator + ** object and set the output variable to NULL. */ + if( p->rc==SQLITE_OK ){ + fts5MultiIterFinishSetup(p, pNew); + }else{ + fts5MultiIterFree(pNew); + *ppOut = 0; + } + +fts5MultiIterNew_post_check: + assert( (*ppOut)!=0 || p->rc!=SQLITE_OK ); + return; +} + +/* +** Create an Fts5Iter that iterates through the doclist provided +** as the second argument. +*/ +static void fts5MultiIterNew2( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Data *pData, /* Doclist to iterate through */ + int bDesc, /* True for descending rowid order */ + Fts5Iter **ppOut /* New object */ +){ + Fts5Iter *pNew; + pNew = fts5MultiIterAlloc(p, 2); + if( pNew ){ + Fts5SegIter *pIter = &pNew->aSeg[1]; + pIter->flags = FTS5_SEGITER_ONETERM; + if( pData->szLeaf>0 ){ + pIter->pLeaf = pData; + pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; + pNew->aFirst[1].iFirst = 1; + if( bDesc ){ + pNew->bRev = 1; + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + pData = 0; + }else{ + pNew->base.bEof = 1; + } + fts5SegIterSetNext(p, pIter); + + *ppOut = pNew; + } + + fts5DataRelease(pData); +} + +/* +** Return true if the iterator is at EOF or if an error has occurred. +** False otherwise. +*/ +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ + assert( pIter!=0 || p->rc!=SQLITE_OK ); + assert( p->rc!=SQLITE_OK + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof + ); + return (p->rc || pIter->base.bEof); +} + +/* +** Return the rowid of the entry that the iterator currently points +** to. If the iterator points to EOF when this function is called the +** results are undefined. +*/ +static i64 fts5MultiIterRowid(Fts5Iter *pIter){ + assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); + return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; +} + +/* +** Move the iterator to the next entry at or following iMatch. +*/ +static void fts5MultiIterNextFrom( + Fts5Index *p, + Fts5Iter *pIter, + i64 iMatch +){ + while( 1 ){ + i64 iRowid; + fts5MultiIterNext(p, pIter, 1, iMatch); + if( fts5MultiIterEof(p, pIter) ) break; + iRowid = fts5MultiIterRowid(pIter); + if( pIter->bRev==0 && iRowid>=iMatch ) break; + if( pIter->bRev!=0 && iRowid<=iMatch ) break; + } +} + +/* +** Return a pointer to a buffer containing the term associated with the +** entry that the iterator currently points to. +*/ +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ + Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + *pn = p->term.n; + return p->term.p; +} + +/* +** Allocate a new segment-id for the structure pStruct. The new segment +** id must be between 1 and 65335 inclusive, and must not be used by +** any currently existing segment. If a free segment id cannot be found, +** SQLITE_FULL is returned. +** +** If an error has already occurred, this function is a no-op. 0 is +** returned in this case. +*/ +static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ + int iSegid = 0; + + if( p->rc==SQLITE_OK ){ + if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ + p->rc = SQLITE_FULL; + }else{ + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT && iId>0 ){ + aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32); + } + } + } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + assert_nc( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert_nc( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); + + { + sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc==SQLITE_OK ){ + u8 aBlob[2] = {0xff, 0xff}; + sqlite3_bind_int(pIdxSelect, 1, iSegid); + sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); + assert_nc( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); + p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); + } + } +#endif + } + } + + return iSegid; +} + +/* +** Discard all data currently cached in the hash-tables. +*/ +static void fts5IndexDiscardData(Fts5Index *p){ + assert( p->pHash || p->nPendingData==0 ); + if( p->pHash ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + p->nPendingRow = 0; + p->flushRc = SQLITE_OK; + } + p->nContentlessDelete = 0; +} + +/* +** Return the size of the prefix, in bytes, that buffer +** (pNew/) shares with buffer (pOld/nOld). +** +** Buffer (pNew/) is guaranteed to be greater +** than buffer (pOld/nOld). +*/ +static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ + int i; + for(i=0; inDlidx>0 && pWriter->aDlidx[0].buf.n>0) ); + for(i=0; inDlidx; i++){ + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; + if( pDlidx->buf.n==0 ) break; + if( bFlush ){ + assert( pDlidx->pgno!=0 ); + fts5DataWrite(p, + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), + pDlidx->buf.p, pDlidx->buf.n + ); + } + sqlite3Fts5BufferZero(&pDlidx->buf); + pDlidx->bPrevValid = 0; + } +} + +/* +** Grow the pWriter->aDlidx[] array to at least nLvl elements in size. +** Any new array elements are zeroed before returning. +*/ +static int fts5WriteDlidxGrow( + Fts5Index *p, + Fts5SegWriter *pWriter, + int nLvl +){ + if( p->rc==SQLITE_OK && nLvl>=pWriter->nDlidx ){ + Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc64( + pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl + ); + if( aDlidx==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); + memset(&aDlidx[pWriter->nDlidx], 0, nByte); + pWriter->aDlidx = aDlidx; + pWriter->nDlidx = nLvl; + } + } + return p->rc; +} + +/* +** If the current doclist-index accumulating in pWriter->aDlidx[] is large +** enough, flush it to disk and return 1. Otherwise discard it and return +** zero. +*/ +static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag = 0; + + /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written + ** to the database, also write the doclist-index to disk. */ + if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ + bFlag = 1; + } + fts5WriteDlidxClear(p, pWriter, bFlag); + pWriter->nEmpty = 0; + return bFlag; +} + +/* +** This function is called whenever processing of the doclist for the +** last term on leaf page (pWriter->iBtPage) is completed. +** +** The doclist-index for that term is currently stored in-memory within the +** Fts5SegWriter.aDlidx[] array. If it is large enough, this function +** writes it out to disk. Or, if it is too small to bother with, discards +** it. +** +** Fts5SegWriter.btterm currently contains the first term on page iBtPage. +*/ +static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag; + + assert( pWriter->iBtPage || pWriter->nEmpty==0 ); + if( pWriter->iBtPage==0 ) return; + bFlag = fts5WriteFlushDlidx(p, pWriter); + + if( p->rc==SQLITE_OK ){ + const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); + /* The following was already done in fts5WriteInit(): */ + /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ + sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); + sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); + sqlite3_step(p->pIdxWriter); + p->rc = sqlite3_reset(p->pIdxWriter); + sqlite3_bind_null(p->pIdxWriter, 2); + } + pWriter->iBtPage = 0; +} + +/* +** This is called once for each leaf page except the first that contains +** at least one term. Argument (nTerm/pTerm) is the split-key - a term that +** is larger than all terms written to earlier leaves, and equal to or +** smaller than the first term on the new leaf. +** +** If an error occurs, an error code is left in Fts5Index.rc. If an error +** has already occurred when this function is called, it is a no-op. +*/ +static void fts5WriteBtreeTerm( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegWriter *pWriter, /* Writer object */ + int nTerm, const u8 *pTerm /* First term on new page */ +){ + fts5WriteFlushBtree(p, pWriter); + if( p->rc==SQLITE_OK ){ + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); + pWriter->iBtPage = pWriter->writer.pgno; + } +} + +/* +** This function is called when flushing a leaf page that contains no +** terms at all to disk. +*/ +static void fts5WriteBtreeNoTerm( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegWriter *pWriter /* Writer object */ +){ + /* If there were no rowids on the leaf page either and the doclist-index + ** has already been started, append an 0x00 byte to it. */ + if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){ + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0]; + assert( pDlidx->bPrevValid ); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0); + } + + /* Increment the "number of sequential leaves without a term" counter. */ + pWriter->nEmpty++; +} + +static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){ + i64 iRowid; + int iOff; + + iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid); + fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid); + return iRowid; +} + +/* +** Rowid iRowid has just been appended to the current leaf page. It is the +** first on the page. This function appends an appropriate entry to the current +** doclist-index. +*/ +static void fts5WriteDlidxAppend( + Fts5Index *p, + Fts5SegWriter *pWriter, + i64 iRowid +){ + int i; + int bDone = 0; + + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ + i64 iVal; + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; + + if( pDlidx->buf.n>=p->pConfig->pgsz ){ + /* The current doclist-index page is full. Write it to disk and push + ** a copy of iRowid (which will become the first rowid on the next + ** doclist-index leaf page) up into the next level of the b-tree + ** hierarchy. If the node being flushed is currently the root node, + ** also push its first rowid upwards. */ + pDlidx->buf.p[0] = 0x01; /* Not the root node */ + fts5DataWrite(p, + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), + pDlidx->buf.p, pDlidx->buf.n + ); + fts5WriteDlidxGrow(p, pWriter, i+2); + pDlidx = &pWriter->aDlidx[i]; + if( p->rc==SQLITE_OK && pDlidx[1].buf.n==0 ){ + i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf); + + /* This was the root node. Push its first rowid up to the new root. */ + pDlidx[1].pgno = pDlidx->pgno; + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst); + pDlidx[1].bPrevValid = 1; + pDlidx[1].iPrev = iFirst; + } + + sqlite3Fts5BufferZero(&pDlidx->buf); + pDlidx->bPrevValid = 0; + pDlidx->pgno++; + }else{ + bDone = 1; + } + + if( pDlidx->bPrevValid ){ + iVal = (u64)iRowid - (u64)pDlidx->iPrev; + }else{ + i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); + assert( pDlidx->buf.n==0 ); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); + iVal = iRowid; + } + + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal); + pDlidx->bPrevValid = 1; + pDlidx->iPrev = iRowid; + } +} + +static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ + static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; + Fts5PageWriter *pPage = &pWriter->writer; + i64 iRowid; + + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); + + /* Set the szLeaf header field. */ + assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); + + if( pWriter->bFirstTermInPage ){ + /* No term was written to this page. */ + assert( pPage->pgidx.n==0 ); + fts5WriteBtreeNoTerm(p, pWriter); + }else{ + /* Append the pgidx to the page buffer. Set the szLeaf header field. */ + fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p); + } + + /* Write the page out to disk */ + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno); + fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); + + /* Initialize the next page. */ + fts5BufferZero(&pPage->buf); + fts5BufferZero(&pPage->pgidx); + fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); + pPage->iPrevPgidx = 0; + pPage->pgno++; + + /* Increase the leaves written counter */ + pWriter->nLeafWritten++; + + /* The new leaf holds no terms or rowids */ + pWriter->bFirstTermInPage = 1; + pWriter->bFirstRowidInPage = 1; +} + +/* +** Append term pTerm/nTerm to the segment being written by the writer passed +** as the second argument. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5WriteAppendTerm( + Fts5Index *p, + Fts5SegWriter *pWriter, + int nTerm, const u8 *pTerm +){ + int nPrefix; /* Bytes of prefix compression for term */ + Fts5PageWriter *pPage = &pWriter->writer; + Fts5Buffer *pPgidx = &pWriter->writer.pgidx; + int nMin = MIN(pPage->term.n, nTerm); + + assert( p->rc==SQLITE_OK ); + assert( pPage->buf.n>=4 ); + assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); + + /* If the current leaf page is full, flush it to disk. */ + if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ + if( pPage->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + if( p->rc!=SQLITE_OK ) return; + } + fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); + } + + /* TODO1: Updating pgidx here. */ + pPgidx->n += sqlite3Fts5PutVarint( + &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx + ); + pPage->iPrevPgidx = pPage->buf.n; +#if 0 + fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); + pPgidx->n += 2; +#endif + + if( pWriter->bFirstTermInPage ){ + nPrefix = 0; + if( pPage->pgno!=1 ){ + /* This is the first term on a leaf that is not the leftmost leaf in + ** the segment b-tree. In this case it is necessary to add a term to + ** the b-tree hierarchy that is (a) larger than the largest term + ** already written to the segment and (b) smaller than or equal to + ** this term. In other words, a prefix of (pTerm/nTerm) that is one + ** byte longer than the longest prefix (pTerm/nTerm) shares with the + ** previous term. + ** + ** Usually, the previous term is available in pPage->term. The exception + ** is if this is the first term written in an incremental-merge step. + ** In this case the previous term is not available, so just write a + ** copy of (pTerm/nTerm) into the parent node. This is slightly + ** inefficient, but still correct. */ + int n = nTerm; + if( pPage->term.n ){ + n = 1 + fts5PrefixCompress(nMin, pPage->term.p, pTerm); + } + fts5WriteBtreeTerm(p, pWriter, n, pTerm); + if( p->rc!=SQLITE_OK ) return; + pPage = &pWriter->writer; + } + }else{ + nPrefix = fts5PrefixCompress(nMin, pPage->term.p, pTerm); + fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); + } + + /* Append the number of bytes of new data, then the term data itself + ** to the page. */ + fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix); + fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]); + + /* Update the Fts5PageWriter.term field. */ + fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm); + pWriter->bFirstTermInPage = 0; + + pWriter->bFirstRowidInPage = 0; + pWriter->bFirstRowidInDoclist = 1; + + assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) ); + pWriter->aDlidx[0].pgno = pPage->pgno; +} + +/* +** Append a rowid and position-list size field to the writers output. +*/ +static void fts5WriteAppendRowid( + Fts5Index *p, + Fts5SegWriter *pWriter, + i64 iRowid +){ + if( p->rc==SQLITE_OK ){ + Fts5PageWriter *pPage = &pWriter->writer; + + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ + fts5WriteFlushLeaf(p, pWriter); + } + + /* If this is to be the first rowid written to the page, set the + ** rowid-pointer in the page-header. Also append a value to the dlidx + ** buffer, in case a doclist-index is required. */ + if( pWriter->bFirstRowidInPage ){ + fts5PutU16(pPage->buf.p, (u16)pPage->buf.n); + fts5WriteDlidxAppend(p, pWriter, iRowid); + } + + /* Write the rowid. */ + if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); + }else{ + assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); + fts5BufferAppendVarint(&p->rc, &pPage->buf, + (u64)iRowid - (u64)pWriter->iPrevRowid + ); + } + pWriter->iPrevRowid = iRowid; + pWriter->bFirstRowidInDoclist = 0; + pWriter->bFirstRowidInPage = 0; + } +} + +static void fts5WriteAppendPoslistData( + Fts5Index *p, + Fts5SegWriter *pWriter, + const u8 *aData, + int nData +){ + Fts5PageWriter *pPage = &pWriter->writer; + const u8 *a = aData; + int n = nData; + + assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK ); + while( p->rc==SQLITE_OK + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz + ){ + int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; + int nCopy = 0; + while( nCopyrc, &pPage->buf, nCopy, a); + a += nCopy; + n -= nCopy; + fts5WriteFlushLeaf(p, pWriter); + } + if( n>0 ){ + fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); + } +} + +/* +** Flush any data cached by the writer object to the database. Free any +** allocations associated with the writer. +*/ +static void fts5WriteFinish( + Fts5Index *p, + Fts5SegWriter *pWriter, /* Writer object */ + int *pnLeaf /* OUT: Number of leaf pages in b-tree */ +){ + int i; + Fts5PageWriter *pLeaf = &pWriter->writer; + if( p->rc==SQLITE_OK ){ + assert( pLeaf->pgno>=1 ); + if( pLeaf->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + } + *pnLeaf = pLeaf->pgno-1; + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } + } + fts5BufferFree(&pLeaf->term); + fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pLeaf->pgidx); + fts5BufferFree(&pWriter->btterm); + + for(i=0; inDlidx; i++){ + sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); + } + sqlite3_free(pWriter->aDlidx); +} + +static void fts5WriteInit( + Fts5Index *p, + Fts5SegWriter *pWriter, + int iSegid +){ + const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; + + memset(pWriter, 0, sizeof(Fts5SegWriter)); + pWriter->iSegid = iSegid; + + fts5WriteDlidxGrow(p, pWriter, 1); + pWriter->writer.pgno = 1; + pWriter->bFirstTermInPage = 1; + pWriter->iBtPage = 1; + + assert( pWriter->writer.buf.n==0 ); + assert( pWriter->writer.pgidx.n==0 ); + + /* Grow the two buffers to pgsz + padding bytes in size. */ + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer); + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer); + + if( p->pIdxWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", + pConfig->zDb, pConfig->zName + )); + } + + if( p->rc==SQLITE_OK ){ + /* Initialize the 4-byte leaf-page header to 0x00. */ + memset(pWriter->writer.buf.p, 0, 4); + pWriter->writer.buf.n = 4; + + /* Bind the current output segment id to the index-writer. This is an + ** optimization over binding the same value over and over as rows are + ** inserted into %_idx by the current writer. */ + sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); + } +} + +/* +** Iterator pIter was used to iterate through the input segments of on an +** incremental merge operation. This function is called if the incremental +** merge step has finished but the input has not been completely exhausted. +*/ +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ + int i; + Fts5Buffer buf; + memset(&buf, 0, sizeof(Fts5Buffer)); + for(i=0; inSeg && p->rc==SQLITE_OK; i++){ + Fts5SegIter *pSeg = &pIter->aSeg[i]; + if( pSeg->pSeg==0 ){ + /* no-op */ + }else if( pSeg->pLeaf==0 ){ + /* All keys from this input segment have been transfered to the output. + ** Set both the first and last page-numbers to 0 to indicate that the + ** segment is now empty. */ + pSeg->pSeg->pgnoLast = 0; + pSeg->pSeg->pgnoFirst = 0; + }else{ + int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */ + i64 iLeafRowid; + Fts5Data *pData; + int iId = pSeg->pSeg->iSegid; + u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; + + iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); + pData = fts5LeafRead(p, iLeafRowid); + if( pData ){ + if( iOff>pData->szLeaf ){ + /* This can occur if the pages that the segments occupy overlap - if + ** a single page has been assigned to more than one segment. In + ** this case a prior iteration of this loop may have corrupted the + ** segment currently being trimmed. */ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferZero(&buf); + fts5BufferGrow(&p->rc, &buf, pData->nn); + fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); + fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); + if( p->rc==SQLITE_OK ){ + /* Set the szLeaf field */ + fts5PutU16(&buf.p[2], (u16)buf.n); + } + + /* Set up the new page-index array */ + fts5BufferAppendVarint(&p->rc, &buf, 4); + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno + && pSeg->iEndofDoclistszLeaf + && pSeg->iPgidxOff<=pData->nn + ){ + int nDiff = pData->szLeaf - pSeg->iEndofDoclist; + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); + fts5BufferAppendBlob(&p->rc, &buf, + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] + ); + } + + pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); + fts5DataWrite(p, iLeafRowid, buf.p, buf.n); + } + fts5DataRelease(pData); + } + } + } + fts5BufferFree(&buf); +} + +static void fts5MergeChunkCallback( + Fts5Index *p, + void *pCtx, + const u8 *pChunk, int nChunk +){ + Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; + fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk); +} + +/* +** +*/ +static void fts5IndexMergeLevel( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ + int iLvl, /* Level to read input from */ + int *pnRem /* Write up to this many output leaves */ +){ + Fts5Structure *pStruct = *ppStruct; + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + Fts5StructureLevel *pLvlOut; + Fts5Iter *pIter = 0; /* Iterator to read input data */ + int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ + int nInput; /* Number of input segments */ + Fts5SegWriter writer; /* Writer object */ + Fts5StructureSegment *pSeg; /* Output segment */ + Fts5Buffer term; + int bOldest; /* True if the output segment is the oldest */ + int eDetail = p->pConfig->eDetail; + const int flags = FTS5INDEX_QUERY_NOOUTPUT; + int bTermWritten = 0; /* True if current term already output */ + + assert( iLvlnLevel ); + assert( pLvl->nMerge<=pLvl->nSeg ); + + memset(&writer, 0, sizeof(Fts5SegWriter)); + memset(&term, 0, sizeof(Fts5Buffer)); + if( pLvl->nMerge ){ + pLvlOut = &pStruct->aLevel[iLvl+1]; + assert( pLvlOut->nSeg>0 ); + nInput = pLvl->nMerge; + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; + + fts5WriteInit(p, &writer, pSeg->iSegid); + writer.writer.pgno = pSeg->pgnoLast+1; + writer.iBtPage = 0; + }else{ + int iSegid = fts5AllocateSegid(p, pStruct); + + /* Extend the Fts5Structure object as required to ensure the output + ** segment exists. */ + if( iLvl==pStruct->nLevel-1 ){ + fts5StructureAddLevel(&p->rc, ppStruct); + pStruct = *ppStruct; + } + fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); + if( p->rc ) return; + pLvl = &pStruct->aLevel[iLvl]; + pLvlOut = &pStruct->aLevel[iLvl+1]; + + fts5WriteInit(p, &writer, iSegid); + + /* Add the new segment to the output level */ + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg]; + pLvlOut->nSeg++; + pSeg->pgnoFirst = 1; + pSeg->iSegid = iSegid; + pStruct->nSegment++; + + /* Read input from all segments in the input level */ + nInput = pLvl->nSeg; + + /* Set the range of origins that will go into the output segment. */ + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; + } + } + bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); + + assert( iLvl>=0 ); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int nPos; /* position-list size field value */ + int nTerm; + const u8 *pTerm; + + pTerm = fts5MultiIterTerm(pIter, &nTerm); + if( nTerm!=term.n || fts5Memcmp(pTerm, term.p, nTerm) ){ + if( pnRem && writer.nLeafWritten>nRem ){ + break; + } + fts5BufferSet(&p->rc, &term, nTerm, pTerm); + bTermWritten =0; + } + + /* Check for key annihilation. */ + if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + /* This is a new term. Append a term to the output segment. */ + fts5WriteAppendTerm(p, &writer, nTerm, pTerm); + bTermWritten = 1; + } + + /* Append the rowid to the output */ + /* WRITEPOSLISTSIZE */ + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); + + if( eDetail==FTS5_DETAIL_NONE ){ + if( pSegIter->bDel ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + if( pSegIter->nPos>0 ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + } + } + }else{ + /* Append the position-list data to the output */ + nPos = pSegIter->nPos*2 + pSegIter->bDel; + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + } + } + + /* Flush the last leaf page to disk. Set the output segment b-tree height + ** and last leaf page number at the same time. */ + fts5WriteFinish(p, &writer, &pSeg->pgnoLast); + + assert( pIter!=0 || p->rc!=SQLITE_OK ); + if( fts5MultiIterEof(p, pIter) ){ + int i; + + /* Remove the redundant segments from the %_data table */ + assert( pSeg->nEntry==0 ); + for(i=0; iaSeg[i]; + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); + fts5DataRemoveSegment(p, pOld); + } + + /* Remove the redundant segments from the input level */ + if( pLvl->nSeg!=nInput ){ + int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); + memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove); + } + pStruct->nSegment -= nInput; + pLvl->nSeg -= nInput; + pLvl->nMerge = 0; + if( pSeg->pgnoLast==0 ){ + pLvlOut->nSeg--; + pStruct->nSegment--; + } + }else{ + assert( pSeg->pgnoLast>0 ); + fts5TrimSegments(p, pIter); + pLvl->nMerge = nInput; + } + + fts5MultiIterFree(pIter); + fts5BufferFree(&term); + if( pnRem ) *pnRem -= writer.nLeafWritten; +} + +/* +** If this is not a contentless_delete=1 table, or if the 'deletemerge' +** configuration option is set to 0, then this function always returns -1. +** Otherwise, it searches the structure object passed as the second argument +** for a level suitable for merging due to having a large number of +** tombstones in the tombstone hash. If one is found, its index is returned. +** Otherwise, if there is no suitable level, -1. +*/ +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ + Fts5Config *pConfig = p->pConfig; + int iRet = -1; + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ + int ii; + int nBest = 0; + + for(ii=0; iinLevel; ii++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; + i64 nEntry = 0; + i64 nTomb = 0; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + nEntry += pLvl->aSeg[iSeg].nEntry; + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; + } + assert_nc( nEntry>0 || pLvl->nSeg==0 ); + if( nEntry>0 ){ + int nPercent = (nTomb * 100) / nEntry; + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ + iRet = ii; + nBest = nPercent; + } + } + + /* If pLvl is already the input level to an ongoing merge, look no + ** further for a merge candidate. The caller should be allowed to + ** continue merging from pLvl first. */ + if( pLvl->nMerge ) break; + } + } + return iRet; +} + +/* +** Do up to nPg pages of automerge work on the index. +** +** Return true if any changes were actually made, or false otherwise. +*/ +static int fts5IndexMerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ + int nPg, /* Pages of work to do */ + int nMin /* Minimum number of segments to merge */ +){ + int nRem = nPg; + int bRet = 0; + Fts5Structure *pStruct = *ppStruct; + while( nRem>0 && p->rc==SQLITE_OK ){ + int iLvl; /* To iterate through levels */ + int iBestLvl = 0; /* Level offering the most input segments */ + int nBest = 0; /* Number of input segments on best level */ + + /* Set iBestLvl to the level to read input segments from. Or to -1 if + ** there is no level suitable to merge segments from. */ + assert( pStruct->nLevel>0 ); + for(iLvl=0; iLvlnLevel; iLvl++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; + if( pLvl->nMerge ){ + if( pLvl->nMerge>nBest ){ + iBestLvl = iLvl; + nBest = nMin; + } + break; + } + if( pLvl->nSeg>nBest ){ + nBest = pLvl->nSeg; + iBestLvl = iLvl; + } + } + if( nBestrc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ + fts5StructurePromote(p, iBestLvl+1, pStruct); + } + + if( nMin==1 ) nMin = 2; + } + *ppStruct = pStruct; + return bRet; +} + +/* +** A total of nLeaf leaf pages of data has just been flushed to a level-0 +** segment. This function updates the write-counter accordingly and, if +** necessary, performs incremental merge work. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5IndexAutomerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ + int nLeaf /* Number of output leaves just written */ +){ + if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 && ALWAYS((*ppStruct)!=0) ){ + Fts5Structure *pStruct = *ppStruct; + u64 nWrite; /* Initial value of write-counter */ + int nWork; /* Number of work-quanta to perform */ + int nRem; /* Number of leaf pages left to write */ + + /* Update the write-counter. While doing so, set nWork. */ + nWrite = pStruct->nWriteCounter; + nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit)); + pStruct->nWriteCounter += nLeaf; + nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); + + fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); + } +} + +static void fts5IndexCrisismerge( + Fts5Index *p, /* FTS5 backend object */ + Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ +){ + const int nCrisis = p->pConfig->nCrisisMerge; + Fts5Structure *pStruct = *ppStruct; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; + } +} + +static int fts5IndexReturn(Fts5Index *p){ + int rc = p->rc; + p->rc = SQLITE_OK; + return rc; +} + +typedef struct Fts5FlushCtx Fts5FlushCtx; +struct Fts5FlushCtx { + Fts5Index *pIdx; + Fts5SegWriter writer; +}; + +/* +** Buffer aBuf[] contains a list of varints, all small enough to fit +** in a 32-bit integer. Return the size of the largest prefix of this +** list nMax bytes or less in size. +*/ +static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ + int ret; + u32 dummy; + ret = fts5GetVarint32(aBuf, dummy); + if( ret nMax ) break; + ret += i; + } + } + return ret; +} + +/* +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else if( bDetailNone ){ + break; + }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + i1 += fts5GetVarint32(&aPg[i1], iFirst); + if( iFirstrc = FTS5_CORRUPT; + break; + } + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** The offset of the first byte of the rowid or delta-rowid + ** value for the doclist entry being removed. + ** + ** iDelta: + ** The value of the rowid or delta-rowid value for the doclist + ** entry being removed. + ** + ** iNextOff: + ** The offset of the next entry following the position list + ** for the one being removed. If the position list for this + ** entry overflows onto the next leaf page, this value will be + ** greater than pLeaf->szLeaf. + */ + { + int iSOP; /* Start-Of-Position-list */ + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOPiLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + + /* If the position-list for the entry being removed flows over past + ** the end of this page, delete the portion of the position-list on the + ** next page and beyond. + ** + ** Set variable bLastInDoclist to true if this entry happens + ** to be the last rowid in the doclist for its term. */ + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + } + + if( pSeg->bDel==0 ){ + if( iNextOff!=iPgIdx ){ + /* Loop through the page-footer. If iNextOff (offset of the + ** entry following the one we are removing) is equal to the + ** offset of a key on this page, then the entry is the last + ** in its doclist. */ + int iKeyOff = 0; + for(iIdx=0; iIdxbDel ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta); + aPg[iOff++] = 0x01; + }else if( bLastInDoclist==0 ){ + if( iNextOff!=iPgIdx ){ + u64 iNextDelta = 0; + iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta); + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta); + } + }else if( + pSeg->iLeafPgno==pSeg->iTermLeafPgno + && iStart==pSeg->iTermLeafOffset + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + int iKeyOff = 0; + + /* Set iKeyOff to the offset of the term that will be removed - the + ** last offset in the footer that is not greater than iStart. */ + for(iIdx=0; iIdx(u32)iStart ) break; + iKeyOff += iVal; + } + assert_nc( iKey>=1 ); + + /* Set iDelKeyOff to the value of the footer entry to remove from + ** the page. */ + iDelKeyOff = iOff = iKeyOff; + + if( iNextOff!=iPgIdx ){ + /* This is the only position-list associated with the term, and there + ** is another term following it on this page. So the subsequent term + ** needs to be moved to replace the term associated with the entry + ** being removed. */ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>pSeg->term.n ){ + p->rc = FTS5_CORRUPT; + }else if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + /* Assuming no error has occurred, this block does final edits to the + ** leaf page before writing it back to disk. Input variables are: + ** + ** nPg: Total initial size of leaf page. + ** iPgIdx: Initial offset of page footer. + ** + ** iOff: Offset to move data to + ** iNextOff: Offset to move data from + */ + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; /* Number of bytes to move */ + int nShift = iNextOff - iOff; /* Distance to move them */ + + int iPrevKeyOut = 0; + int iKeyIn = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= nShift; + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + for(iIdx=0; iIdxiOff ? nShift : 0)); + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut); + iPrevKeyOut = iKeyOut; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + } + sqlite3_free(aIdx); +} + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + int nTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThisrc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + fts5DoSecureDelete(p, pSeg); + } + } + + fts5MultiIterFree(pIter); +} + + +/* +** Flush the contents of in-memory hash table iHash to a new level-0 +** segment on disk. Also update the corresponding structure record. +** +** If an error occurs, set the Fts5Index.rc error code. If an error has +** already occurred, this function is a no-op. +*/ +static void fts5FlushOneHash(Fts5Index *p){ + Fts5Hash *pHash = p->pHash; + Fts5Structure *pStruct; + int iSegid; + int pgnoLast = 0; /* Last leaf page number in segment */ + + /* Obtain a reference to the index structure and allocate a new segment-id + ** for the new level-0 segment. */ + pStruct = fts5StructureRead(p); + fts5StructureInvalidate(p); + + if( sqlite3Fts5HashIsEmpty(pHash)==0 ){ + iSegid = fts5AllocateSegid(p, pStruct); + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ + + /* Get the term and doclist for this entry. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } + + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + int bTermWritten = !bSecureDelete; + i64 iRowid = 0; + i64 iPrev = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } + } + } + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } + + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + u64 iRowidDelta = (u64)iRowid - (u64)iPrev; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta); + } + if( p->rc!=SQLITE_OK ) break; + assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffp[pBuf->n++] = 0; + iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + }else{ + int bDel = 0; + int nPos = 0; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel); + if( bDel && bSecureDelete ){ + fts5BufferAppendVarint(&p->rc, pBuf, nPos*2); + iOff += nCopy; + nCopy = nPos; + }else{ + nCopy += nPos; + } + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; + } + } + } + + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); + } + fts5WriteFinish(p, &writer, &pgnoLast); + + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pStruct->nOriginCntr; + pSeg->iOrigin2 = pStruct->nOriginCntr; + pSeg->nEntry = p->nPendingRow; + pStruct->nOriginCntr++; + } + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); + } + } + } + + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); + fts5IndexCrisismerge(p, &pStruct); + fts5StructureWrite(p, pStruct); + fts5StructureRelease(pStruct); +} + +/* +** Flush any data stored in the in-memory hash tables to the database. +*/ +static void fts5IndexFlush(Fts5Index *p){ + /* Unless it is empty, flush the hash table to disk */ + if( p->flushRc ){ + p->rc = p->flushRc; + return; + } + if( p->nPendingData || p->nContentlessDelete ){ + assert( p->pHash ); + fts5FlushOneHash(p); + if( p->rc==SQLITE_OK ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + p->nPendingRow = 0; + p->nContentlessDelete = 0; + }else if( p->nPendingData || p->nContentlessDelete ){ + p->flushRc = p->rc; + } + } +} + +static Fts5Structure *fts5IndexOptimizeStruct( + Fts5Index *p, + Fts5Structure *pStruct +){ + Fts5Structure *pNew = 0; + sqlite3_int64 nByte = sizeof(Fts5Structure); + int nSeg = pStruct->nSegment; + int i; + + /* Figure out if this structure requires optimization. A structure does + ** not require optimization if either: + ** + ** 1. it consists of fewer than two segments, or + ** 2. all segments are on the same level, or + ** 3. all segments except one are currently inputs to a merge operation. + ** + ** In the first case, if there are no tombstone hash pages, return NULL. In + ** the second, increment the ref-count on *pStruct and return a copy of the + ** pointer to it. + */ + if( nSeg==0 ) return 0; + for(i=0; inLevel; i++){ + int nThis = pStruct->aLevel[i].nSeg; + int nMerge = pStruct->aLevel[i].nMerge; + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ + return 0; + } + fts5StructureRef(pStruct); + return pStruct; + } + assert( pStruct->aLevel[i].nMerge<=nThis ); + } + + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); + + if( pNew ){ + Fts5StructureLevel *pLvl; + nByte = nSeg * sizeof(Fts5StructureSegment); + pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); + pNew->nRef = 1; + pNew->nWriteCounter = pStruct->nWriteCounter; + pNew->nOriginCntr = pStruct->nOriginCntr; + pLvl = &pNew->aLevel[pNew->nLevel-1]; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pLvl->aSeg ){ + int iLvl, iSeg; + int iSegOut = 0; + /* Iterate through all segments, from oldest to newest. Add them to + ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest + ** segment in the data structure. */ + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; + iSegOut++; + } + } + pNew->nSegment = pLvl->nSeg = nSeg; + }else{ + sqlite3_free(pNew); + pNew = 0; + } + } + + return pNew; +} + +int sqlite3Fts5IndexOptimize(Fts5Index *p){ + Fts5Structure *pStruct; + Fts5Structure *pNew = 0; + + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 ); + pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || pStruct!=0 ); + fts5StructureInvalidate(p); + + if( pStruct ){ + pNew = fts5IndexOptimizeStruct(p, pStruct); + } + fts5StructureRelease(pStruct); + + assert( pNew==0 || pNew->nSegment>0 ); + if( pNew ){ + int iLvl; + for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){} + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ + int nRem = FTS5_OPT_WORK_UNIT; + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); + } + + fts5StructureWrite(p, pNew); + fts5StructureRelease(pNew); + } + + return fts5IndexReturn(p); +} + +/* +** This is called to implement the special "VALUES('merge', $nMerge)" +** INSERT command. +*/ +int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ + Fts5Structure *pStruct = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + if( pStruct ){ + int nMin = p->pConfig->nUsermerge; + fts5StructureInvalidate(p); + if( nMerge<0 ){ + Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); + fts5StructureRelease(pStruct); + pStruct = pNew; + nMin = 1; + nMerge = nMerge*-1; + } + if( pStruct && pStruct->nLevel ){ + if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ + fts5StructureWrite(p, pStruct); + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +static void fts5AppendRowid( + Fts5Index *p, + u64 iDelta, + Fts5Iter *pUnused, + Fts5Buffer *pBuf +){ + UNUSED_PARAM(pUnused); + fts5BufferAppendVarint(&p->rc, pBuf, iDelta); +} + +static void fts5AppendPoslist( + Fts5Index *p, + u64 iDelta, + Fts5Iter *pMulti, + Fts5Buffer *pBuf +){ + int nData = pMulti->base.nData; + int nByte = nData + 9 + 9 + FTS5_DATA_ZERO_PADDING; + assert( nData>0 ); + if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nByte) ){ + fts5BufferSafeAppendVarint(pBuf, iDelta); + fts5BufferSafeAppendVarint(pBuf, nData*2); + fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); + memset(&pBuf->p[pBuf->n], 0, FTS5_DATA_ZERO_PADDING); + } +} + + +static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ + u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; + + assert( pIter->aPoslist || (p==0 && pIter->aPoslist==0) ); + if( p>=pIter->aEof ){ + pIter->aPoslist = 0; + }else{ + i64 iDelta; + + p += fts5GetVarint(p, (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* Read position list size */ + if( p[0] & 0x80 ){ + int nPos; + pIter->nSize = fts5GetVarint32(p, nPos); + pIter->nPoslist = (nPos>>1); + }else{ + pIter->nPoslist = ((int)(p[0])) >> 1; + pIter->nSize = 1; + } + + pIter->aPoslist = p; + if( &pIter->aPoslist[pIter->nPoslist]>pIter->aEof ){ + pIter->aPoslist = 0; + } + } +} + +static void fts5DoclistIterInit( + Fts5Buffer *pBuf, + Fts5DoclistIter *pIter +){ + memset(pIter, 0, sizeof(*pIter)); + if( pBuf->n>0 ){ + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; + fts5DoclistIterNext(pIter); + } +} + +#if 0 +/* +** Append a doclist to buffer pBuf. +** +** This function assumes that space within the buffer has already been +** allocated. +*/ +static void fts5MergeAppendDocid( + Fts5Buffer *pBuf, /* Buffer to write to */ + i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ + i64 iRowid /* Rowid to append */ +){ + assert( pBuf->n!=0 || (*piLastRowid)==0 ); + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); + *piLastRowid = iRowid; +} +#endif + +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \ + (iLastRowid) = (iRowid); \ +} + +/* +** Swap the contents of buffer *p1 with that of *p2. +*/ +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ + Fts5Buffer tmp = *p1; + *p1 = *p2; + *p2 = tmp; +} + +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ + int i = *piOff; + if( i>=pBuf->n ){ + *piOff = -1; + }else{ + u64 iVal; + *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); + *piRowid += iVal; + } +} + +/* +** This is the equivalent of fts5MergePrefixLists() for detail=none mode. +** In this case the buffers consist of a delta-encoded list of rowids only. +*/ +static void fts5MergeRowidLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + int nBuf, /* Number of entries in apBuf[] */ + Fts5Buffer *aBuf /* Array of other lists to merge into p1 */ +){ + int i1 = 0; + int i2 = 0; + i64 iRowid1 = 0; + i64 iRowid2 = 0; + i64 iOut = 0; + Fts5Buffer *p2 = &aBuf[0]; + Fts5Buffer out; + + (void)nBuf; + memset(&out, 0, sizeof(out)); + assert( nBuf==1 ); + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); + if( p->rc ) return; + + fts5NextRowid(p1, &i1, &iRowid1); + fts5NextRowid(p2, &i2, &iRowid2); + while( i1>=0 || i2>=0 ){ + if( i1>=0 && (i2<0 || iRowid1iOut ); + fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); + iOut = iRowid1; + fts5NextRowid(p1, &i1, &iRowid1); + }else{ + assert( iOut==0 || iRowid2>iOut ); + fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); + iOut = iRowid2; + if( i1>=0 && iRowid1==iRowid2 ){ + fts5NextRowid(p1, &i1, &iRowid1); + } + fts5NextRowid(p2, &i2, &iRowid2); + } + } + + fts5BufferSwap(&out, p1); + fts5BufferFree(&out); +} + +typedef struct PrefixMerger PrefixMerger; +struct PrefixMerger { + Fts5DoclistIter iter; /* Doclist iterator */ + i64 iPos; /* For iterating through a position list */ + int iOff; + u8 *aPos; + PrefixMerger *pNext; /* Next in docid/poslist order */ +}; + +static void fts5PrefixMergerInsertByRowid( + PrefixMerger **ppHead, + PrefixMerger *p +){ + if( p->iter.aPoslist ){ + PrefixMerger **pp = ppHead; + while( *pp && p->iter.iRowid>(*pp)->iter.iRowid ){ + pp = &(*pp)->pNext; + } + p->pNext = *pp; + *pp = p; + } +} + +static void fts5PrefixMergerInsertByPosition( + PrefixMerger **ppHead, + PrefixMerger *p +){ + if( p->iPos>=0 ){ + PrefixMerger **pp = ppHead; + while( *pp && p->iPos>(*pp)->iPos ){ + pp = &(*pp)->pNext; + } + p->pNext = *pp; + *pp = p; + } +} + + +/* +** Array aBuf[] contains nBuf doclists. These are all merged in with the +** doclist in buffer p1. +*/ +static void fts5MergePrefixLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + int nBuf, /* Number of buffers in array aBuf[] */ + Fts5Buffer *aBuf /* Other lists to merge in */ +){ +#define fts5PrefixMergerNextPosition(p) \ + sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos) +#define FTS5_MERGE_NLIST 16 + PrefixMerger aMerger[FTS5_MERGE_NLIST]; + PrefixMerger *pHead = 0; + int i; + int nOut = 0; + Fts5Buffer out = {0, 0, 0}; + Fts5Buffer tmp = {0, 0, 0}; + i64 iLastRowid = 0; + + /* Initialize a doclist-iterator for each input buffer. Arrange them in + ** a linked-list starting at pHead in ascending order of rowid. Avoid + ** linking any iterators already at EOF into the linked list at all. */ + assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) ); + memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1)); + pHead = &aMerger[nBuf]; + fts5DoclistIterInit(p1, &pHead->iter); + for(i=0; in + 9 + 10*nBuf; + + /* The maximum size of the output is equal to the sum of the + ** input sizes + 1 varint (9 bytes). The extra varint is because if the + ** first rowid in one input is a large negative number, and the first in + ** the other a non-negative number, the delta for the non-negative + ** number will be larger on disk than the literal integer value + ** was. + ** + ** Or, if the input position-lists are corrupt, then the output might + ** include up to (nBuf+1) extra 10-byte positions created by interpreting -1 + ** (the value PoslistNext64() uses for EOF) as a position and appending + ** it to the output. This can happen at most once for each input + ** position-list, hence (nBuf+1) 10 byte paddings. */ + if( sqlite3Fts5BufferSize(&p->rc, &out, nOut) ) return; + + while( pHead ){ + fts5MergeAppendDocid(&out, iLastRowid, pHead->iter.iRowid); + + if( pHead->pNext && iLastRowid==pHead->pNext->iter.iRowid ){ + /* Merge data from two or more poslists */ + i64 iPrev = 0; + int nTmp = FTS5_DATA_ZERO_PADDING; + int nMerge = 0; + PrefixMerger *pSave = pHead; + PrefixMerger *pThis = 0; + int nTail = 0; + + pHead = 0; + while( pSave && pSave->iter.iRowid==iLastRowid ){ + PrefixMerger *pNext = pSave->pNext; + pSave->iOff = 0; + pSave->iPos = 0; + pSave->aPos = &pSave->iter.aPoslist[pSave->iter.nSize]; + fts5PrefixMergerNextPosition(pSave); + nTmp += pSave->iter.nPoslist + 10; + nMerge++; + fts5PrefixMergerInsertByPosition(&pHead, pSave); + pSave = pNext; + } + + if( pHead==0 || pHead->pNext==0 ){ + p->rc = FTS5_CORRUPT; + break; + } + + /* See the earlier comment in this function for an explanation of why + ** corrupt input position lists might cause the output to consume + ** at most nMerge*10 bytes of unexpected space. */ + if( sqlite3Fts5BufferSize(&p->rc, &tmp, nTmp+nMerge*10) ){ + break; + } + fts5BufferZero(&tmp); + + pThis = pHead; + pHead = pThis->pNext; + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos); + fts5PrefixMergerNextPosition(pThis); + fts5PrefixMergerInsertByPosition(&pHead, pThis); + + while( pHead->pNext ){ + pThis = pHead; + if( pThis->iPos!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos); + } + fts5PrefixMergerNextPosition(pThis); + pHead = pThis->pNext; + fts5PrefixMergerInsertByPosition(&pHead, pThis); + } + + if( pHead->iPos!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pHead->iPos); + } + nTail = pHead->iter.nPoslist - pHead->iOff; + + /* WRITEPOSLISTSIZE */ + assert_nc( tmp.n+nTail<=nTmp ); + assert( tmp.n+nTail<=nTmp+nMerge*10 ); + if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + break; + } + fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); + if( nTail>0 ){ + fts5BufferSafeAppendBlob(&out, &pHead->aPos[pHead->iOff], nTail); + } + + pHead = pSave; + for(i=0; iiter.aPoslist && pX->iter.iRowid==iLastRowid ){ + fts5DoclistIterNext(&pX->iter); + fts5PrefixMergerInsertByRowid(&pHead, pX); + } + } + + }else{ + /* Copy poslist from pHead to output */ + PrefixMerger *pThis = pHead; + Fts5DoclistIter *pI = &pThis->iter; + fts5BufferSafeAppendBlob(&out, pI->aPoslist, pI->nPoslist+pI->nSize); + fts5DoclistIterNext(pI); + pHead = pThis->pNext; + fts5PrefixMergerInsertByRowid(&pHead, pThis); + } + } + + fts5BufferFree(p1); + fts5BufferFree(&tmp); + memset(&out.p[out.n], 0, FTS5_DATA_ZERO_PADDING); + *p1 = out; +} + + +/* +** Iterate through a range of entries in the FTS index, invoking the xVisit +** callback for each of them. +** +** Parameter pToken points to an nToken buffer containing an FTS index term +** (i.e. a document term with the preceding 1 byte index identifier - +** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits +** all entries for terms that have pToken/nToken as a prefix. If bPrefix +** is false, then only entries with pToken/nToken as the entire key are +** visited. +** +** If the current table is a tokendata=1 table, then if bPrefix is true then +** each index term is treated separately. However, if bPrefix is false, then +** all index terms corresponding to pToken/nToken are collapsed into a single +** term before the callback is invoked. +** +** The callback invoked for each entry visited is specified by paramter xVisit. +** Each time it is invoked, it is passed a pointer to the Fts5Index object, +** a copy of the 7th paramter to this function (pCtx) and a pointer to the +** iterator that indicates the current entry. If the current entry is the +** first with a new term (i.e. different from that of the previous entry, +** including the very first term), then the final two parameters are passed +** a pointer to the term and its size in bytes, respectively. If the current +** entry is not the first associated with its term, these two parameters +** are passed 0. +** +** If parameter pColset is not NULL, then it is used to filter entries before +** the callback is invoked. +*/ +static int fts5VisitEntries( + Fts5Index *p, /* Fts5 index object */ + Fts5Colset *pColset, /* Columns filter to apply, or NULL */ + u8 *pToken, /* Buffer containing token */ + int nToken, /* Size of buffer pToken in bytes */ + int bPrefix, /* True for a prefix scan */ + void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int), + void *pCtx /* Passed as second argument to xVisit() */ +){ + const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN : 0) + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ + int bNewTerm = 1; + Fts5Structure *pStruct = fts5StructureRead(p); + + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nNew = 0; + const u8 *pNew = 0; + + p1->xSetOutputs(p1, pSeg); + if( p->rc ) break; + + if( bNewTerm ){ + nNew = pSeg->term.n; + pNew = pSeg->term.p; + if( nNewrc; +} + + +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits (so all iRowid fields are the same). +** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an +** array of these for the entire query (in which case iRowid fields may take +** a variety of values). +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +** +** iRowid: +** Rowid for the current entry. +** +** iPos: +** Position of current entry within row. In the usual ((iCol<<32)+iOff) +** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()). +** +** iIter: +** If the Fts5TokenDataIter iterator that the entry is part of is +** actually an iterator (i.e. with nIter>0, not just a container for +** Fts5TokenDataMap structures), then this variable is an index into +** the apIter[] array. The corresponding term is that which the iterator +** at apIter[iIter] currently points to. +** +** Or, if the Fts5TokenDataIter iterator is just a container object +** (nIter==0), then iIter is an index into the term.p[] buffer where +** the term is stored. +** +** nByte: +** In the case where iIter is an index into term.p[], this variable +** is the size of the term in bytes. If iIter is an index into apIter[], +** this variable is unused. +*/ +struct Fts5TokenDataMap { + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ + int nByte; /* Length of token in bytes (or 0) */ +}; + +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +** +** This object serves two purposes. The first is as a container for an array +** of Fts5TokenDataMap structures, which are used to find the token required +** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and +** aMap[] variables. +*/ +struct Fts5TokenDataIter { + int nMapAlloc; /* Allocated size of aMap[] in entries */ + int nMap; /* Number of valid entries in aMap[] */ + Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ + + /* The following are used for prefix-queries only. */ + Fts5Buffer terms; + + /* The following are used for other full-token tokendata queries only. */ + int nIter; + int nIterAlloc; + Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; + Fts5Iter *apIter[1]; +}; + +/* +** The two input arrays - a1[] and a2[] - are in sorted order. This function +** merges the two arrays together and writes the result to output array +** aOut[]. aOut[] is guaranteed to be large enough to hold the result. +** +** Duplicate entries are copied into the output. So the size of the output +** array is always (n1+n2) entries. +*/ +static void fts5TokendataMerge( + Fts5TokenDataMap *a1, int n1, /* Input array 1 */ + Fts5TokenDataMap *a2, int n2, /* Input array 2 */ + Fts5TokenDataMap *aOut /* Output array */ +){ + int i1 = 0; + int i2 = 0; + + assert( n1>=0 && n2>=0 ); + while( i1=n2 || (i1rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nAlloc = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->aMap[pT->nMap].nByte = nByte; + pT->nMap++; + } +} + +/* +** Sort the contents of the pT->aMap[] array. +** +** The sorting algorithm requries a malloc(). If this fails, an error code +** is left in Fts5Index.rc before returning. +*/ +static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ + Fts5TokenDataMap *aTmp = 0; + int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + + aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( aTmp ){ + Fts5TokenDataMap *a1 = pT->aMap; + Fts5TokenDataMap *a2 = aTmp; + i64 nHalf; + + for(nHalf=1; nHalfnMap; nHalf=nHalf*2){ + int i1; + for(i1=0; i1nMap; i1+=(nHalf*2)){ + int n1 = MIN(nHalf, pT->nMap-i1); + int n2 = MIN(nHalf, pT->nMap-i1-n1); + fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]); + } + SWAPVAL(Fts5TokenDataMap*, a1, a2); + } + + if( a1!=pT->aMap ){ + memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap)); + } + sqlite3_free(aTmp); + +#ifdef SQLITE_DEBUG + { + int ii; + for(ii=1; iinMap; ii++){ + Fts5TokenDataMap *p1 = &pT->aMap[ii-1]; + Fts5TokenDataMap *p2 = &pT->aMap[ii]; + assert( p1->iRowidiRowid + || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos) + ); + } + } +#endif + } +} + +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + fts5BufferFree(&pSet->terms); + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + + +/* +** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata() +** to pass data to prefixIterSetupTokendataCb(). +*/ +typedef struct TokendataSetupCtx TokendataSetupCtx; +struct TokendataSetupCtx { + Fts5TokenDataIter *pT; /* Object being populated with mappings */ + int iTermOff; /* Offset of current term in terms.p[] */ + int nTermByte; /* Size of current term in bytes */ +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This +** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each +** position in the current position-list. It doesn't matter that some of +** these may be out of order - they will be sorted later. +*/ +static void prefixIterSetupTokendataCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx; + int iPosOff = 0; + i64 iPos = 0; + + if( pNew ){ + pSetup->nTermByte = nNew-1; + pSetup->iTermOff = pSetup->pT->terms.n; + fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1); + } + + while( 0==sqlite3Fts5PoslistNext64( + p1->base.pData, p1->base.nData, &iPosOff, &iPos + ) ){ + fts5TokendataIterAppendMap(p, + pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos + ); + } +} + + +/* +** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries(). +*/ +typedef struct PrefixSetupCtx PrefixSetupCtx; +struct PrefixSetupCtx { + void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); + i64 iLastRowid; + int nMerge; + Fts5Buffer *aBuf; + int nBuf; + Fts5Buffer doclist; + TokendataSetupCtx *pTokendata; +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIter() +*/ +static void prefixIterSetupCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx; + const int nMerge = pSetup->nMerge; + + if( p1->base.nData>0 ){ + if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){ + int i; + for(i=0; p->rc==SQLITE_OK && pSetup->doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=pSetup->nBuf ); + for(iStore=i1; iStoreaBuf[iStore].n==0 ){ + fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]); + fts5BufferZero(&pSetup->doclist); + break; + } + } + if( iStore==i1+nMerge ){ + pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]); + for(iStore=i1; iStoreaBuf[iStore]); + } + } + } + pSetup->iLastRowid = 0; + } + + pSetup->xAppend( + p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist + ); + pSetup->iLastRowid = p1->base.iRowid; + } + + if( pSetup->pTokendata ){ + prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew); + } +} + +static void fts5SetupPrefixIter( + Fts5Index *p, /* Index to read from */ + int bDesc, /* True for "ORDER BY rowid DESC" */ + int iIdx, /* Index to scan for data */ + u8 *pToken, /* Buffer containing prefix to match */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset, /* Restrict matches to these columns */ + Fts5Iter **ppIter /* OUT: New iterator */ +){ + Fts5Structure *pStruct; + PrefixSetupCtx s; + TokendataSetupCtx s2; + + memset(&s, 0, sizeof(s)); + memset(&s2, 0, sizeof(s2)); + + s.nMerge = 1; + s.iLastRowid = 0; + s.nBuf = 32; + if( iIdx==0 + && p->pConfig->eDetail==FTS5_DETAIL_FULL + && p->pConfig->bPrefixInsttoken + ){ + s.pTokendata = &s2; + s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, sizeof(*s2.pT)); + } + + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + s.xMerge = fts5MergeRowidLists; + s.xAppend = fts5AppendRowid; + }else{ + s.nMerge = FTS5_MERGE_NLIST-1; + s.nBuf = s.nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ + s.xMerge = fts5MergePrefixLists; + s.xAppend = fts5AppendPoslist; + } + + s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf); + pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) ); + + if( p->rc==SQLITE_OK ){ + void *pCtx = (void*)&s; + int i; + Fts5Data *pData; + + /* If iIdx is non-zero, then it is the number of a prefix-index for + ** prefixes 1 character longer than the prefix being queried for. That + ** index contains all the doclists required, except for the one + ** corresponding to the prefix itself. That one is extracted from the + ** main term index here. */ + if( iIdx!=0 ){ + pToken[0] = FTS5_MAIN_PREFIX; + fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx); + } + + pToken[0] = FTS5_MAIN_PREFIX + iIdx; + fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx); + + assert( (s.nBuf%s.nMerge)==0 ); + for(i=0; irc==SQLITE_OK ){ + s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]); + } + for(iFree=i; iFreerc!=SQLITE_OK ); + if( pData ){ + pData->p = (u8*)&pData[1]; + pData->nn = pData->szLeaf = s.doclist.n; + if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n); + fts5MultiIterNew2(p, pData, bDesc, ppIter); + } + + assert( (*ppIter)!=0 || p->rc!=SQLITE_OK ); + if( p->rc==SQLITE_OK && s.pTokendata ){ + fts5TokendataIterSortMap(p, s2.pT); + (*ppIter)->pTokenDataIter = s2.pT; + s2.pT = 0; + } + } + + fts5TokendataIterDelete(s2.pT); + fts5BufferFree(&s.doclist); + fts5StructureRelease(pStruct); + sqlite3_free(s.aBuf); +} + + +/* +** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain +** to the document with rowid iRowid. +*/ +int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ + assert( p->rc==SQLITE_OK ); + + /* Allocate the hash table if it has not already been allocated */ + if( p->pHash==0 ){ + p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); + } + + /* Flush the hash table to disk if required */ + if( iRowidiWriteRowid + || (iRowid==p->iWriteRowid && p->bDelete==0) + || (p->nPendingData > p->pConfig->nHashSize) + ){ + fts5IndexFlush(p); + } + + p->iWriteRowid = iRowid; + p->bDelete = bDelete; + if( bDelete==0 ){ + p->nPendingRow++; + } + return fts5IndexReturn(p); +} + +/* +** Commit data to disk. +*/ +int sqlite3Fts5IndexSync(Fts5Index *p){ + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + sqlite3Fts5IndexCloseReader(p); + return fts5IndexReturn(p); +} + +/* +** Discard any data stored in the in-memory hash tables. Do not write it +** to the database. Additionally, assume that the contents of the %_data +** table may have changed on disk. So any in-memory caches of %_data +** records must be invalidated. +*/ +int sqlite3Fts5IndexRollback(Fts5Index *p){ + sqlite3Fts5IndexCloseReader(p); + fts5IndexDiscardData(p); + fts5StructureInvalidate(p); + /* assert( p->rc==SQLITE_OK ); */ + return SQLITE_OK; +} + +/* +** The %_data table is completely empty when this function is called. This +** function populates it with the initial structure objects for each index, +** and the initial version of the "averages" record (a zero-byte blob). +*/ +int sqlite3Fts5IndexReinit(Fts5Index *p){ + Fts5Structure s; + fts5StructureInvalidate(p); + fts5IndexDiscardData(p); + memset(&s, 0, sizeof(Fts5Structure)); + if( p->pConfig->bContentlessDelete ){ + s.nOriginCntr = 1; + } + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); + fts5StructureWrite(p, &s); + return fts5IndexReturn(p); +} + +/* +** Open a new Fts5Index handle. If the bCreate argument is true, create +** and initialize the underlying %_data table. +** +** If successful, set *pp to point to the new object and return SQLITE_OK. +** Otherwise, set *pp to NULL and return an SQLite error code. +*/ +int sqlite3Fts5IndexOpen( + Fts5Config *pConfig, + int bCreate, + Fts5Index **pp, + char **pzErr +){ + int rc = SQLITE_OK; + Fts5Index *p; /* New object */ + + *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); + if( rc==SQLITE_OK ){ + p->pConfig = pConfig; + p->nWorkUnit = FTS5_WORK_UNIT; + p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); + if( p->zDataTbl && bCreate ){ + rc = sqlite3Fts5CreateTable( + pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable(pConfig, "idx", + "segid, term, pgno, PRIMARY KEY(segid, term)", + 1, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p); + } + } + } + + assert( rc!=SQLITE_OK || p->rc==SQLITE_OK ); + if( rc ){ + sqlite3Fts5IndexClose(p); + *pp = 0; + } + return rc; +} + +/* +** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). +*/ +int sqlite3Fts5IndexClose(Fts5Index *p){ + int rc = SQLITE_OK; + if( p ){ + assert( p->pReader==0 ); + fts5StructureInvalidate(p); + sqlite3_finalize(p->pWriter); + sqlite3_finalize(p->pDeleter); + sqlite3_finalize(p->pIdxWriter); + sqlite3_finalize(p->pIdxDeleter); + sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pIdxNextSelect); + sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); + sqlite3Fts5HashFree(p->pHash); + sqlite3_free(p->zDataTbl); + sqlite3_free(p); + } + return rc; +} + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +){ + int n = 0; + int i; + for(i=0; i=nByte ) return 0; /* Input contains fewer than nChar chars */ + if( (unsigned char)p[n++]>=0xc0 ){ + if( n>=nByte ) return 0; + while( (p[n] & 0xc0)==0x80 ){ + n++; + if( n>=nByte ){ + if( i+1==nChar ) break; + return 0; + } + } + } + } + return n; +} + +/* +** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of +** unicode characters in the string. +*/ +static int fts5IndexCharlen(const char *pIn, int nIn){ + int nChar = 0; + int i = 0; + while( i=0xc0 ){ + while( i delete) */ + int iPos, /* Position of token within column */ + const char *pToken, int nToken /* Token to add or remove to or from index */ +){ + int i; /* Used to iterate through indexes */ + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pConfig = p->pConfig; + + assert( p->rc==SQLITE_OK ); + assert( (iCol<0)==p->bDelete ); + + /* Add the entry to the main terms index. */ + rc = sqlite3Fts5HashWrite( + p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken + ); + + for(i=0; inPrefix && rc==SQLITE_OK; i++){ + const int nChar = pConfig->aPrefix[i]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5HashWrite(p->pHash, + p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, + nByte + ); + } + } + + return rc; +} + +/* +** pToken points to a buffer of size nToken bytes containing a search +** term, including the index number at the start, used on a tokendata=1 +** table. This function returns true if the term in buffer pBuf matches +** token pToken/nToken. +*/ +static int fts5IsTokendataPrefix( + Fts5Buffer *pBuf, + const u8 *pToken, + int nToken +){ + return ( + pBuf->n>=nToken + && 0==memcmp(pBuf->p, pToken, nToken) + && (pBuf->n==nToken || pBuf->p[nToken]==0x00) + ); +} + +/* +** Ensure the segment-iterator passed as the only argument points to EOF. +*/ +static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; +} + +/* +** This function appends iterator pAppend to Fts5TokenDataIter pIn and +** returns the result. +*/ +static Fts5TokenDataIter *fts5AppendTokendataIter( + Fts5Index *p, /* Index object (for error code) */ + Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */ + Fts5Iter *pAppend /* Append this iterator */ +){ + Fts5TokenDataIter *pRet = pIn; + + if( p->rc==SQLITE_OK ){ + if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ + int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); + Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + if( pIn==0 ) memset(pNew, 0, nByte); + pRet = pNew; + pNew->nIterAlloc = nAlloc; + } + } + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pAppend); + }else{ + pRet->apIter[pRet->nIter++] = pAppend; + } + assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc ); + + return pRet; +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function sets the iterator output +** variables (pIter->base.*) according to the contents of the current +** row. +*/ +static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ + int ii; + int nHit = 0; + i64 iRowid = SMALLEST_INT64; + int iMin = 0; + + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + + pIter->base.nData = 0; + pIter->base.pData = 0; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 ){ + if( nHit==0 || p->base.iRowidbase.iRowid; + nHit = 1; + pIter->base.pData = p->base.pData; + pIter->base.nData = p->base.nData; + iMin = ii; + }else if( p->base.iRowid==iRowid ){ + nHit++; + } + } + } + + if( nHit==0 ){ + pIter->base.bEof = 1; + }else{ + int eDetail = pIter->pIndex->pConfig->eDetail; + pIter->base.bEof = 0; + pIter->base.iRowid = iRowid; + + if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1); + }else + if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ + int nReader = 0; + int nByte = 0; + i64 iPrev = 0; + + /* Allocate array of iterators if they are not already allocated. */ + if( pT->aPoslistReader==0 ){ + pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero( + &pIter->pIndex->rc, + pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)) + ); + if( pT->aPoslistReader==0 ) return; + pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter]; + } + + /* Populate an iterator for each poslist that will be merged */ + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( iRowid==p->base.iRowid ){ + pT->aPoslistToIter[nReader] = ii; + sqlite3Fts5PoslistReaderInit( + p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++] + ); + nByte += p->base.nData; + } + } + + /* Ensure the output buffer is large enough */ + if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10) ){ + return; + } + + /* Ensure the token-mapping is large enough */ + if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ + int nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + pT->aMap, nNew*sizeof(Fts5TokenDataMap) + ); + if( aNew==0 ){ + pIter->pIndex->rc = SQLITE_NOMEM; + return; + } + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pIter->poslist.n = 0; + + while( 1 ){ + i64 iMinPos = LARGEST_INT64; + + /* Find smallest position */ + iMin = 0; + for(ii=0; iiaPoslistReader[ii]; + if( pReader->bEof==0 ){ + if( pReader->iPosiPos; + iMin = ii; + } + } + } + + /* If all readers were at EOF, break out of the loop. */ + if( iMinPos==LARGEST_INT64 ) break; + + sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos); + sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]); + + if( eDetail==FTS5_DETAIL_FULL ){ + pT->aMap[pT->nMap].iPos = iMinPos; + pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin]; + pT->aMap[pT->nMap].iRowid = iRowid; + pT->nMap++; + } + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + } +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function advances the iterator. If +** argument bFrom is false, then the iterator is advanced to the next +** entry. Or, if bFrom is true, it is advanced to the first entry with +** a rowid of iFrom or greater. +*/ +static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ + int ii; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 + && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidbase.bEof==0 + && p->base.iRowidrc==SQLITE_OK + ){ + fts5MultiIterNext(pIndex, p, 0, 0); + } + } + } + + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } +} + +/* +** If the segment-iterator passed as the first argument is at EOF, then +** set pIter->term to a copy of buffer pTerm. +*/ +static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ + if( pIter && pIter->aSeg[0].pLeaf==0 ){ + fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p); + } +} + +/* +** This function sets up an iterator to use for a non-prefix query on a +** tokendata=1 table. +*/ +static Fts5Iter *fts5SetupTokendataIter( + Fts5Index *p, /* FTS index to query */ + const u8 *pToken, /* Buffer containing query term */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset /* Colset to filter on */ +){ + Fts5Iter *pRet = 0; + Fts5TokenDataIter *pSet = 0; + Fts5Structure *pStruct = 0; + const int flags = FTS5INDEX_QUERY_SCANONETERM | FTS5INDEX_QUERY_SCAN; + + Fts5Buffer bSeek = {0, 0, 0}; + Fts5Buffer *pSmall = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + + while( p->rc==SQLITE_OK ){ + Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; + Fts5Iter *pNew = 0; + Fts5SegIter *pNewIter = 0; + Fts5SegIter *pPrevIter = 0; + + int iLvl, iSeg, ii; + + pNew = fts5MultiIterAlloc(p, pStruct->nSegment); + if( pSmall ){ + fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); + fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); + }else{ + fts5BufferSet(&p->rc, &bSeek, nToken, pToken); + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + pNewIter = &pNew->aSeg[0]; + pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + int bDone = 0; + + if( pPrevIter ){ + if( fts5BufferCompare(pSmall, &pPrevIter->term) ){ + memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter)); + memset(pPrevIter, 0, sizeof(Fts5SegIter)); + bDone = 1; + }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){ + fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter); + bDone = 1; + } + } + + if( bDone==0 ){ + fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + } + + if( pPrevIter ){ + if( pPrevIter->pTombArray ){ + pNewIter->pTombArray = pPrevIter->pTombArray; + pNewIter->pTombArray->nRef++; + } + }else{ + fts5SegIterAllocTombstone(p, pNewIter); + } + + pNewIter++; + if( pPrevIter ) pPrevIter++; + if( p->rc ) break; + } + } + fts5TokendataSetTermIfEof(pPrev, pSmall); + + pNew->bSkipEmpty = 1; + pNew->pColset = pColset; + fts5IterSetOutputCb(&p->rc, pNew); + + /* Loop through all segments in the new iterator. Find the smallest + ** term that any segment-iterator points to. Iterator pNew will be + ** used for this term. Also, set any iterator that points to a term that + ** does not match pToken/nToken to point to EOF */ + pSmall = 0; + for(ii=0; iinSeg; ii++){ + Fts5SegIter *pII = &pNew->aSeg[ii]; + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ + fts5SegIterSetEOF(pII); + } + if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ + pSmall = &pII->term; + } + } + + /* If pSmall is still NULL at this point, then the new iterator does + ** not point to any terms that match the query. So delete it and break + ** out of the loop - all required iterators have been collected. */ + if( pSmall==0 ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + /* Append this iterator to the set and continue. */ + pSet = fts5AppendTokendataIter(p, pSet, pNew); + } + + if( p->rc==SQLITE_OK && pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Iter *pIter = pSet->apIter[ii]; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM; + } + fts5MultiIterFinishSetup(p, pIter); + } + } + + if( p->rc==SQLITE_OK ){ + pRet = fts5MultiIterAlloc(p, 0); + } + if( pRet ){ + pRet->nSeg = 0; + pRet->pTokenDataIter = pSet; + if( pSet ){ + fts5IterSetOutputsTokendata(pRet); + }else{ + pRet->base.bEof = 1; + } + }else{ + fts5TokendataIterDelete(pSet); + } + + fts5StructureRelease(pStruct); + fts5BufferFree(&bSeek); + return pRet; +} + +/* +** Open a new iterator to iterate though all rowid that match the +** specified token or token prefix. +*/ +int sqlite3Fts5IndexQuery( + Fts5Index *p, /* FTS index to query */ + const char *pToken, int nToken, /* Token (or prefix) to query for */ + int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ +){ + Fts5Config *pConfig = p->pConfig; + Fts5Iter *pRet = 0; + Fts5Buffer buf = {0, 0, 0}; + + /* If the QUERY_SCAN flag is set, all other flags must be clear. */ + assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); + + if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ + int iIdx = 0; /* Index to search */ + int iPrefixIdx = 0; /* +1 prefix index */ + int bTokendata = pConfig->bTokendata; + assert( buf.p!=0 ); + if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + + /* The NOTOKENDATA flag is set when each token in a tokendata=1 table + ** should be treated individually, instead of merging all those with + ** a common prefix into a single entry. This is used, for example, by + ** queries performed as part of an integrity-check, or by the fts5vocab + ** module. */ + if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ + bTokendata = 0; + } + + /* Figure out which index to search and set iIdx accordingly. If this + ** is a prefix query for which there is no prefix index, set iIdx to + ** greater than pConfig->nPrefix to indicate that the query will be + ** satisfied by scanning multiple terms in the main index. + ** + ** If the QUERY_TEST_NOIDX flag was specified, then this must be a + ** prefix-query. Instead of using a prefix-index (if one exists), + ** evaluate the prefix query using the main FTS index. This is used + ** for internal sanity checking by the integrity-check in debug + ** mode only. */ +#ifdef SQLITE_DEBUG + if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ + assert( flags & FTS5INDEX_QUERY_PREFIX ); + iIdx = 1+pConfig->nPrefix; + }else +#endif + if( flags & FTS5INDEX_QUERY_PREFIX ){ + int nChar = fts5IndexCharlen(pToken, nToken); + for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ + int nIdxChar = pConfig->aPrefix[iIdx-1]; + if( nIdxChar==nChar ) break; + if( nIdxChar==nChar+1 ) iPrefixIdx = iIdx; + } + } + + if( bTokendata && iIdx==0 ){ + buf.p[0] = FTS5_MAIN_PREFIX; + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); + }else if( iIdx<=pConfig->nPrefix ){ + /* Straight index lookup */ + Fts5Structure *pStruct = fts5StructureRead(p); + buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); + if( pStruct ){ + fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, + pColset, buf.p, nToken+1, -1, 0, &pRet + ); + fts5StructureRelease(pStruct); + } + }else{ + /* Scan multiple terms in the main index for a prefix query. */ + int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; + fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); + if( pRet==0 ){ + assert( p->rc!=SQLITE_OK ); + }else{ + assert( pRet->pColset==0 ); + fts5IterSetOutputCb(&p->rc, pRet); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } + } + } + + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pRet); + pRet = 0; + sqlite3Fts5IndexCloseReader(p); + } + + *ppIter = (Fts5IndexIter*)pRet; + sqlite3Fts5BufferFree(&buf); + } + return fts5IndexReturn(p); +} + +/* +** Return true if the iterator passed as the only argument is at EOF. +*/ +/* +** Move to the next matching rowid. +*/ +int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + assert( pIter->pIndex->rc==SQLITE_OK ); + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); + fts5TokendataIterNext(pIter, 0, 0); + }else{ + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + } + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Move to the next matching term/rowid. Used by the fts5vocab module. +*/ +int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *p = pIter->pIndex; + + assert( pIter->pIndex->rc==SQLITE_OK ); + + fts5MultiIterNext(p, pIter, 0, 0); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; + pIter->base.bEof = 1; + } + } + + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Move to the next matching rowid that occurs at or after iMatch. The +** definition of "at or after" depends on whether this iterator iterates +** in ascending or descending rowid order. +*/ +int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); + fts5TokendataIterNext(pIter, 1, iMatch); + }else{ + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + } + return fts5IndexReturn(pIter->pIndex); +} + +/* +** Return the current term. +*/ +const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ + int n; + const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); + assert_nc( z || n<=1 ); + *pn = n-1; + return (z ? &z[1] : 0); +} + +/* +** pIter is a prefix query. This function populates pIter->pTokenDataIter +** with an Fts5TokenDataIter object containing mappings for all rows +** matched by the query. +*/ +static int fts5SetupPrefixIterTokendata( + Fts5Iter *pIter, + const char *pToken, /* Token prefix to search for */ + int nToken /* Size of pToken in bytes */ +){ + Fts5Index *p = pIter->pIndex; + Fts5Buffer token = {0, 0, 0}; + TokendataSetupCtx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + fts5BufferGrow(&p->rc, &token, nToken+1); + assert( token.p!=0 || p->rc!=SQLITE_OK ); + ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*ctx.pT)); + + if( p->rc==SQLITE_OK ){ + + /* Fill in the token prefix to search for */ + token.p[0] = FTS5_MAIN_PREFIX; + memcpy(&token.p[1], pToken, nToken); + token.n = nToken+1; + + fts5VisitEntries( + p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx + ); + + fts5TokendataIterSortMap(p, ctx.pT); + } + + if( p->rc==SQLITE_OK ){ + pIter->pTokenDataIter = ctx.pT; + }else{ + fts5TokendataIterDelete(ctx.pT); + } + fts5BufferFree(&token); + + return fts5IndexReturn(p); +} + +/* +** This is used by xInstToken() to access the token at offset iOff, column +** iCol of row iRowid. The token is returned via output variables *ppOut +** and *pnOut. The iterator passed as the first argument must be a tokendata=1 +** iterator (pIter->pTokenDataIter!=0). +** +** pToken/nToken: +*/ +int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + i64 iPos = (((i64)iCol)<<32) + iOff; + Fts5TokenDataMap *aMap = 0; + int i1 = 0; + int i2 = 0; + int iTest = 0; + + assert( pT || (pToken && pIter->nSeg>0) ); + if( pT==0 ){ + int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken); + if( rc!=SQLITE_OK ) return rc; + pT = pIter->pTokenDataIter; + } + + i2 = pT->nMap; + aMap = pT->aMap; + + while( i2>i1 ){ + iTest = (i1 + i2) / 2; + + if( aMap[iTest].iRowidiRowid ){ + i2 = iTest; + }else{ + if( aMap[iTest].iPosiPos ){ + i2 = iTest; + }else{ + break; + } + } + } + + if( i2>i1 ){ + if( pIter->nSeg==0 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + }else{ + Fts5TokenDataMap *p = &aMap[iTest]; + *ppOut = (const char*)&pT->terms.p[p->iIter]; + *pnOut = aMap[iTest].nByte; + } + } + + return SQLITE_OK; +} + +/* +** Clear any existing entries from the token-map associated with the +** iterator passed as the only argument. +*/ +void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter && pIter->pTokenDataIter + && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL) + ){ + pIter->pTokenDataIter->nMap = 0; + } +} + +/* +** Set a token-mapping for the iterator passed as the first argument. This +** is used in detail=column or detail=none mode when a token is requested +** using the xInstToken() API. In this case the caller tokenizers the +** current row and configures the token-mapping via multiple calls to this +** function. +*/ +int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, int iCol, int iOff +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *p = pIter->pIndex; + i64 iPos = (((i64)iCol)<<32) + iOff; + + assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); + assert( pIter->pTokenDataIter || pIter->nSeg>0 ); + if( pIter->nSeg>0 ){ + /* This is a prefix term iterator. */ + if( pT==0 ){ + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); + pIter->pTokenDataIter = pT; + } + if( pT ){ + fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos); + fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken); + } + }else{ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos); + } + } + return fts5IndexReturn(p); +} + +/* +** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). +*/ +void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ + if( pIndexIter ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *pIndex = pIter->pIndex; + fts5TokendataIterDelete(pIter->pTokenDataIter); + fts5MultiIterFree(pIter); + sqlite3Fts5IndexCloseReader(pIndex); + } +} + +/* +** Read and decode the "averages" record from the database. +** +** Parameter anSize must point to an array of size nCol, where nCol is +** the number of user defined columns in the FTS table. +*/ +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ + int nCol = p->pConfig->nCol; + Fts5Data *pData; + + *pnRow = 0; + memset(anSize, 0, sizeof(i64) * nCol); + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); + if( p->rc==SQLITE_OK && pData->nn ){ + int i = 0; + int iCol; + i += fts5GetVarint(&pData->p[i], (u64*)pnRow); + for(iCol=0; inn && iColp[i], (u64*)&anSize[iCol]); + } + } + + fts5DataRelease(pData); + return fts5IndexReturn(p); +} + +/* +** Replace the current "averages" record with the contents of the buffer +** supplied as the second argument. +*/ +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ + assert( p->rc==SQLITE_OK ); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); + return fts5IndexReturn(p); +} + +/* +** Return the total number of blocks this module has read from the %_data +** table since it was created. +*/ +int sqlite3Fts5IndexReads(Fts5Index *p){ + return p->nRead; +} + +/* +** Set the 32-bit cookie value stored at the start of all structure +** records to the value passed as the second argument. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ + int rc; /* Return code */ + Fts5Config *pConfig = p->pConfig; /* Configuration object */ + u8 aCookie[4]; /* Binary representation of iNew */ + sqlite3_blob *pBlob = 0; + + assert( p->rc==SQLITE_OK ); + sqlite3Fts5Put32(aCookie, iNew); + + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, + "block", FTS5_STRUCTURE_ROWID, 1, &pBlob + ); + if( rc==SQLITE_OK ){ + sqlite3_blob_write(pBlob, aCookie, 4, 0); + rc = sqlite3_blob_close(pBlob); + } + + return rc; +} + +int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + fts5StructureRelease(pStruct); + return fts5IndexReturn(p); +} + +/* +** Retrieve the origin value that will be used for the segment currently +** being accumulated in the in-memory hash table when it is flushed to +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to +** the queried value. Or, if an error occurs, an error code is returned +** and the final value of (*piOrigin) is undefined. +*/ +int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + *piOrigin = pStruct->nOriginCntr; + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/* +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages +** associated with the same segment. This function adds rowid iRowid to +** the hash table. The caller is required to guarantee that there is at +** least one free slot on the page. +** +** If parameter bForce is false and the hash table is deemed to be full +** (more than half of the slots are occupied), then non-zero is returned +** and iRowid not inserted. Or, if bForce is true or if the hash table page +** is not full, iRowid is inserted and zero returned. +*/ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ + const int szKey = TOMBSTONE_KEYSIZE(pPg); + const int nSlot = TOMBSTONE_NSLOT(pPg); + const int nElem = fts5GetU32(&pPg->p[4]); + int iSlot = (iRowid / nPg) % nSlot; + int nCollide = nSlot; + + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; + if( iRowid==0 ){ + pPg->p[1] = 0x01; + return 0; + } + + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + + fts5PutU32(&pPg->p[4], nElem+1); + if( szKey==4 ){ + u32 *aSlot = (u32*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); + }else{ + u64 *aSlot = (u64*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU64((u8*)&aSlot[iSlot], iRowid); + } + + return 0; +} + +/* +** This function attempts to build a new hash containing all the keys +** currently in the tombstone hash table for segment pSeg. The new +** hash will be stored in the nOut buffers passed in array apOut[]. +** All pages of the new hash use key-size szKey (4 or 8). +** +** Return 0 if the hash is successfully rebuilt into the nOut pages. +** Or non-zero if it is not (because one page became overfull). In this +** case the caller should retry with a larger nOut parameter. +** +** Parameter pData1 is page iPg1 of the hash table being rebuilt. +*/ +static int fts5IndexTombstoneRehash( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int nOut, /* Number of output pages */ + Fts5Data **apOut /* Array of output hash pages */ +){ + int ii; + int res = 0; + + /* Initialize the headers of all the output pages */ + for(ii=0; iip[0] = szKey; + fts5PutU32(&apOut[ii]->p[4], 0); + } + + /* Loop through the current pages of the hash table. */ + for(ii=0; res==0 && iinPgTombstone; ii++){ + Fts5Data *pData = 0; /* Page ii of the current hash table */ + Fts5Data *pFree = 0; /* Free this at the end of the loop */ + + if( iPg1==ii ){ + pData = pData1; + }else{ + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); + } + + if( pData ){ + int szKeyIn = TOMBSTONE_KEYSIZE(pData); + int nSlotIn = (pData->nn - 8) / szKeyIn; + int iIn; + for(iIn=0; iInp[8]; + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); + }else{ + u64 *aSlot = (u64*)&pData->p[8]; + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); + } + + /* If iVal is not 0 at this point, insert it into the new hash table */ + if( iVal ){ + Fts5Data *pPg = apOut[(iVal % nOut)]; + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); + if( res ) break; + } + } + + /* If this is page 0 of the old hash, copy the rowid-0-flag from the + ** old hash to the new. */ + if( ii==0 ){ + apOut[0]->p[1] = pData->p[1]; + } + } + fts5DataRelease(pFree); + } + + return res; +} + +/* +** This is called to rebuild the hash table belonging to segment pSeg. +** If parameter pData1 is not NULL, then one page of the existing hash table +** has already been loaded - pData1, which is page iPg1. The key-size for +** the new hash table is szKey (4 or 8). +** +** If successful, the new hash table is not written to disk. Instead, +** output parameter (*pnOut) is set to the number of pages in the new +** hash table, and (*papOut) to point to an array of buffers containing +** the new page data. +** +** If an error occurs, an error code is left in the Fts5Index object and +** both output parameters set to 0 before returning. +*/ +static void fts5IndexTombstoneRebuild( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int *pnOut, /* OUT: Number of output pages */ + Fts5Data ***papOut /* OUT: Output hash pages */ +){ + const int MINSLOT = 32; + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); + int nSlot = 0; /* Number of slots in each output page */ + int nOut = 0; + + /* Figure out how many output pages (nOut) and how many slots per + ** page (nSlot). There are three possibilities: + ** + ** 1. The hash table does not yet exist. In this case the new hash + ** table will consist of a single page with MINSLOT slots. + ** + ** 2. The hash table exists but is currently a single page. In this + ** case an attempt is made to grow the page to accommodate the new + ** entry. The page is allowed to grow up to nSlotPerPage (see above) + ** slots. + ** + ** 3. The hash table already consists of more than one page, or of + ** a single page already so large that it cannot be grown. In this + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage + ** slots each, where nPg is the current number of pages in the + ** hash table. + */ + if( pSeg->nPgTombstone==0 ){ + /* Case 1. */ + nOut = 1; + nSlot = MINSLOT; + }else if( pSeg->nPgTombstone==1 ){ + /* Case 2. */ + int nElem = (int)fts5GetU32(&pData1->p[4]); + assert( pData1 && iPg1==0 ); + nOut = 1; + nSlot = MAX(nElem*4, MINSLOT); + if( nSlot>nSlotPerPage ) nOut = 0; + } + if( nOut==0 ){ + /* Case 3. */ + nOut = (pSeg->nPgTombstone * 2 + 1); + nSlot = nSlotPerPage; + } + + /* Allocate the required array and output pages */ + while( 1 ){ + int res = 0; + int ii = 0; + int szPage = 0; + Fts5Data **apOut = 0; + + /* Allocate space for the new hash table */ + assert( nSlot>=MINSLOT ); + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); + szPage = 8 + nSlot*szKey; + for(ii=0; iirc, + sizeof(Fts5Data)+szPage + ); + if( pNew ){ + pNew->nn = szPage; + pNew->p = (u8*)&pNew[1]; + apOut[ii] = pNew; + } + } + + /* Rebuild the hash table. */ + if( p->rc==SQLITE_OK ){ + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); + } + if( res==0 ){ + if( p->rc ){ + fts5IndexFreeArray(apOut, nOut); + apOut = 0; + nOut = 0; + } + *pnOut = nOut; + *papOut = apOut; + break; + } + + /* If control flows to here, it was not possible to rebuild the hash + ** table. Free all buffers and then try again with more pages. */ + assert( p->rc==SQLITE_OK ); + fts5IndexFreeArray(apOut, nOut); + nSlot = nSlotPerPage; + nOut = nOut*2 + 1; + } +} + + +/* +** Add a tombstone for rowid iRowid to segment pSeg. +*/ +static void fts5IndexTombstoneAdd( + Fts5Index *p, + Fts5StructureSegment *pSeg, + u64 iRowid +){ + Fts5Data *pPg = 0; + int iPg = -1; + int szKey = 0; + int nHash = 0; + Fts5Data **apHash = 0; + + p->nContentlessDelete++; + + if( pSeg->nPgTombstone>0 ){ + iPg = iRowid % pSeg->nPgTombstone; + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); + if( pPg==0 ){ + assert( p->rc!=SQLITE_OK ); + return; + } + + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); + fts5DataRelease(pPg); + return; + } + } + + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; + if( iRowid>0xFFFFFFFF ) szKey = 8; + + /* Rebuild the hash table */ + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); + + /* If all has succeeded, write the new rowid into one of the new hash + ** table pages, then write them all out to disk. */ + if( nHash ){ + int ii = 0; + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); + for(ii=0; iiiSegid, ii); + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); + } + pSeg->nPgTombstone = nHash; + fts5StructureWrite(p, p->pStruct); + } + + fts5DataRelease(pPg); + fts5IndexFreeArray(apHash, nHash); +} + +/* +** Add iRowid to the tombstone list of the segment or segments that contain +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite +** error code otherwise. +*/ +int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ + int iLvl; + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + int iSeg; + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ + if( bFound==0 ){ + pSeg->nEntryTombstone++; + bFound = 1; + } + fts5IndexTombstoneAdd(p, pSeg, iRowid); + } + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/************************************************************************* +************************************************************************** +** Below this point is the implementation of the integrity-check +** functionality. +*/ + +/* +** Return a simple checksum value based on the arguments. +*/ +u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +){ + int i; + u64 ret = iRowid; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); + for(i=0; iiLeaf ); + cksum1 += iRowid + ((i64)pgno<<32); + } + fts5DlidxIterFree(pDlidx); + pDlidx = 0; + + for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterPrev(p, pDlidx) + ){ + i64 iRowid = fts5DlidxIterRowid(pDlidx); + int pgno = fts5DlidxIterPgno(pDlidx); + assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); + cksum2 += iRowid + ((i64)pgno<<32); + } + fts5DlidxIterFree(pDlidx); + pDlidx = 0; + + if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; +} + +static int fts5QueryCksum( + Fts5Index *p, /* Fts5 index object */ + int iIdx, + const char *z, /* Index key to query for */ + int n, /* Size of index key in bytes */ + int flags, /* Flags for Fts5IndexQuery */ + u64 *pCksum /* IN/OUT: Checksum value */ +){ + int eDetail = p->pConfig->eDetail; + u64 cksum = *pCksum; + Fts5IndexIter *pIter = 0; + int rc = sqlite3Fts5IndexQuery( + p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA), 0, &pIter + ); + + while( rc==SQLITE_OK && ALWAYS(pIter!=0) && 0==sqlite3Fts5IterEof(pIter) ){ + i64 rowid = pIter->iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); + }else{ + Fts5PoslistReader sReader; + for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader); + sReader.bEof==0; + sqlite3Fts5PoslistReaderNext(&sReader) + ){ + int iCol = FTS5_POS2COLUMN(sReader.iPos); + int iOff = FTS5_POS2OFFSET(sReader.iPos); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IterNext(pIter); + } + } + sqlite3Fts5IterClose(pIter); + + *pCksum = cksum; + return rc; +} + +/* +** Check if buffer z[], size n bytes, contains as series of valid utf-8 +** encoded codepoints. If so, return 0. Otherwise, if the buffer does not +** contain valid utf-8, return non-zero. +*/ +static int fts5TestUtf8(const char *z, int n){ + int i = 0; + assert_nc( n>0 ); + while( i=n || (z[i+1] & 0xC0)!=0x80 ) return 1; + i += 2; + }else + if( (z[i] & 0xF0)==0xE0 ){ + if( i+2>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; + i += 3; + }else + if( (z[i] & 0xF8)==0xF0 ){ + if( i+3>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; + if( (z[i+2] & 0xC0)!=0x80 ) return 1; + i += 3; + }else{ + return 1; + } + } + + return 0; +} + +/* +** This function is also purely an internal test. It does not contribute to +** FTS functionality, or even the integrity-check, in any way. +*/ +static void fts5TestTerm( + Fts5Index *p, + Fts5Buffer *pPrev, /* Previous term */ + const char *z, int n, /* Possibly new term to test */ + u64 expected, + u64 *pCksum +){ + int rc = p->rc; + if( pPrev->n==0 ){ + fts5BufferSet(&rc, pPrev, n, (const u8*)z); + }else + if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + u64 cksum3 = *pCksum; + const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ + int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ + int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX); + int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); + u64 ck1 = 0; + u64 ck2 = 0; + + /* Check that the results returned for ASC and DESC queries are + ** the same. If not, call this corruption. */ + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); + if( rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_DESC; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + } + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + + /* If this is a prefix query, check that the results returned if the + ** the index is disabled are the same. In both ASC and DESC order. + ** + ** This check may only be performed if the hash table is empty. This + ** is because the hash table only supports a single scan query at + ** a time, and the multi-iter loop from which this function is called + ** is already performing such a scan. + ** + ** Also only do this if buffer zTerm contains nTerm bytes of valid + ** utf-8. Otherwise, the last part of the buffer contents might contain + ** a non-utf-8 sequence that happens to be a prefix of a valid utf-8 + ** character stored in the main fts index, which will cause the + ** test to fail. */ + if( p->nPendingData==0 && 0==fts5TestUtf8(zTerm, nTerm) ){ + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + } + + cksum3 ^= ck1; + fts5BufferSet(&rc, pPrev, n, (const u8*)z); + + if( rc==SQLITE_OK && cksum3!=expected ){ + rc = FTS5_CORRUPT; + } + *pCksum = cksum3; + } + p->rc = rc; +} + +#else +# define fts5TestDlidxReverse(x,y,z) +# define fts5TestTerm(u,v,w,x,y,z) +#endif + +/* +** Check that: +** +** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and +** contain zero terms. +** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and +** contain zero rowids. +*/ +static void fts5IndexIntegrityCheckEmpty( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ + int iFirst, + int iNoRowid, + int iLast +){ + int i; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + if( pLeaf ){ + if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + } +} + +static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ + i64 iTermOff = 0; + int ii; + + Fts5Buffer buf1 = {0,0,0}; + Fts5Buffer buf2 = {0,0,0}; + + ii = pLeaf->szLeaf; + while( iinn && p->rc==SQLITE_OK ){ + int res; + i64 iOff; + int nIncr; + + ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); + iTermOff += nIncr; + iOff = iTermOff; + + if( iOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else if( iTermOff==nIncr ){ + int nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + }else{ + int nKeep, nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + buf1.n = nKeep; + fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + + if( p->rc==SQLITE_OK ){ + res = fts5BufferCompare(&buf1, &buf2); + if( res<=0 ) p->rc = FTS5_CORRUPT; + } + } + fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); + } + + fts5BufferFree(&buf1); + fts5BufferFree(&buf2); +} + +static void fts5IndexIntegrityCheckSegment( + Fts5Index *p, /* FTS5 backend object */ + Fts5StructureSegment *pSeg /* Segment to check internal consistency */ +){ + Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); + sqlite3_stmt *pStmt = 0; + int rc2; + int iIdxPrevLeaf = pSeg->pgnoFirst-1; + int iDlidxPrevLeaf = pSeg->pgnoLast; + + if( pSeg->pgnoFirst==0 ) return; + + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d " + "ORDER BY 1, 2", + pConfig->zDb, pConfig->zName, pSeg->iSegid + )); + + /* Iterate through the b-tree hierarchy. */ + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iRow; /* Rowid for this leaf */ + Fts5Data *pLeaf; /* Data for this leaf */ + + const char *zIdxTerm = (const char*)sqlite3_column_blob(pStmt, 1); + int nIdxTerm = sqlite3_column_bytes(pStmt, 1); + int iIdxLeaf = sqlite3_column_int(pStmt, 2); + int bIdxDlidx = sqlite3_column_int(pStmt, 3); + + /* If the leaf in question has already been trimmed from the segment, + ** ignore this b-tree entry. Otherwise, load it into memory. */ + if( iIdxLeafpgnoFirst ) continue; + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); + pLeaf = fts5LeafRead(p, iRow); + if( pLeaf==0 ) break; + + /* Check that the leaf contains at least one term, and that it is equal + ** to or larger than the split-key in zIdxTerm. Also check that if there + ** is also a rowid pointer within the leaf page header, it points to a + ** location before the term. */ + if( pLeaf->nn<=pLeaf->szLeaf ){ + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + + }else{ + int iOff; /* Offset of first term on leaf */ + int iRowidOff; /* Offset of first rowid on leaf */ + int nTerm; /* Size of term on leaf in bytes */ + int res; /* Comparison of term and split-key */ + + iOff = fts5LeafFirstTermOff(pLeaf); + iRowidOff = fts5LeafFirstRowidOff(pLeaf); + if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); + res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); + if( res==0 ) res = nTerm - nIdxTerm; + if( res<0 ) p->rc = FTS5_CORRUPT; + } + + fts5IntegrityCheckPgidx(p, pLeaf); + } + fts5DataRelease(pLeaf); + if( p->rc ) break; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + fts5IndexIntegrityCheckEmpty( + p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 + ); + if( p->rc ) break; + + /* If there is a doclist-index, check that it looks right. */ + if( bIdxDlidx ){ + Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ + int iPrevLeaf = iIdxLeaf; + int iSegid = pSeg->iSegid; + int iPg = 0; + i64 iKey; + + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); + fts5DlidxIterEof(p, pDlidx)==0; + fts5DlidxIterNext(p, pDlidx) + ){ + + /* Check any rowid-less pages that occur before the current leaf. */ + for(iPg=iPrevLeaf+1; iPgrc = FTS5_CORRUPT; + fts5DataRelease(pLeaf); + } + } + iPrevLeaf = fts5DlidxIterPgno(pDlidx); + + /* Check that the leaf page indicated by the iterator really does + ** contain the rowid suggested by the same. */ + iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf); + pLeaf = fts5DataRead(p, iKey); + if( pLeaf ){ + i64 iRowid; + int iRowidOff = fts5LeafFirstRowidOff(pLeaf); + ASSERT_SZLEAF_OK(pLeaf); + if( iRowidOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); + fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); + if( iRowidrc = FTS5_CORRUPT; + } + } + fts5DataRelease(pLeaf); + } + } + + iDlidxPrevLeaf = iPg; + fts5DlidxIterFree(pDlidx); + fts5TestDlidxReverse(p, iSegid, iIdxLeaf); + }else{ + iDlidxPrevLeaf = pSeg->pgnoLast; + /* TODO: Check there is no doclist index */ + } + + iIdxPrevLeaf = iIdxLeaf; + } + + rc2 = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ +#if 0 + if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ + p->rc = FTS5_CORRUPT; + } +#endif +} + + +/* +** Run internal checks to ensure that the FTS index (a) is internally +** consistent and (b) contains entries for which the XOR of the checksums +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. +** +** Return SQLITE_CORRUPT if any of the internal checks fail, or if the +** checksum does not match. Return SQLITE_OK if all checks pass without +** error, or some other SQLite error code if another error (e.g. OOM) +** occurs. +*/ +int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ + int eDetail = p->pConfig->eDetail; + u64 cksum2 = 0; /* Checksum based on contents of indexes */ + Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ + Fts5Iter *pIter; /* Used to iterate through entire index */ + Fts5Structure *pStruct; /* Index structure */ + int iLvl, iSeg; + +#ifdef SQLITE_DEBUG + /* Used by extra internal tests only run if NDEBUG is not defined */ + u64 cksum3 = 0; /* Checksum based on contents of indexes */ + Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ +#endif + const int flags = FTS5INDEX_QUERY_NOOUTPUT; + + /* Load the FTS index structure */ + pStruct = fts5StructureRead(p); + if( pStruct==0 ){ + assert( p->rc!=SQLITE_OK ); + return fts5IndexReturn(p); + } + + /* Check that the internal nodes of each segment match the leaves */ + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + fts5IndexIntegrityCheckSegment(p, pSeg); + } + } + + /* The cksum argument passed to this function is a checksum calculated + ** based on all expected entries in the FTS index (including prefix index + ** entries). This block checks that a checksum calculated based on the + ** actual contents of FTS index is identical. + ** + ** Two versions of the same checksum are calculated. The first (stack + ** variable cksum2) based on entries extracted from the full-text index + ** while doing a linear scan of each individual index in turn. + ** + ** As each term visited by the linear scans, a separate query for the + ** same term is performed. cksum3 is calculated based on the entries + ** extracted by these queries. + */ + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + int n; /* Size of term in bytes */ + i64 iPos = 0; /* Position read from poslist */ + int iOff = 0; /* Offset within poslist */ + i64 iRowid = fts5MultiIterRowid(pIter); + char *z = (char*)fts5MultiIterTerm(pIter, &n); + + /* If this is a new term, query for it. Update cksum3 with the results. */ + fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + if( p->rc ) break; + + if( eDetail==FTS5_DETAIL_NONE ){ + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + } + }else{ + poslist.n = 0; + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); + fts5BufferAppendBlob(&p->rc, &poslist, 4, (const u8*)"\0\0\0\0"); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + } + } + } + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + + fts5MultiIterFree(pIter); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; + + fts5StructureRelease(pStruct); +#ifdef SQLITE_DEBUG + fts5BufferFree(&term); +#endif + fts5BufferFree(&poslist); + return fts5IndexReturn(p); +} + +/************************************************************************* +************************************************************************** +** Below this point is the implementation of the fts5_decode() scalar +** function only. +*/ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +/* +** Decode a segment-data rowid from the %_data table. This function is +** the opposite of macro FTS5_SEGMENT_ROWID(). +*/ +static void fts5DecodeRowid( + i64 iRowid, /* Rowid from %_data table */ + int *pbTombstone, /* OUT: Tombstone hash flag */ + int *piSegid, /* OUT: Segment id */ + int *pbDlidx, /* OUT: Dlidx flag */ + int *piHeight, /* OUT: Height */ + int *piPgno /* OUT: Page number */ +){ + *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1)); + iRowid >>= FTS5_DATA_PAGE_B; + + *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1)); + iRowid >>= FTS5_DATA_HEIGHT_B; + + *pbDlidx = (int)(iRowid & 0x0001); + iRowid >>= FTS5_DATA_DLI_B; + + *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); + iRowid >>= FTS5_DATA_ID_B; + + *pbTombstone = (int)(iRowid & 0x0001); +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); + + if( iSegid==0 ){ + if( iKey==FTS5_AVERAGES_ROWID ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); + }else{ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); + } + } + else{ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", + bTomb ? "tombstone " : "", + iSegid, iHeight, iPgno + ); + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static void fts5DebugStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + Fts5Structure *p +){ + int iLvl, iSeg; /* Iterate through levels, segments */ + + for(iLvl=0; iLvlnLevel; iLvl++){ + Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg + ); + for(iSeg=0; iSegnSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", + pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast + ); + if( pSeg->iOrigin1>0 ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", + pSeg->iOrigin1, pSeg->iOrigin2 + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This +** function appends a human-readable representation of the same object +** to the buffer passed as the second argument. +*/ +static void fts5DecodeStructure( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int rc; /* Return code */ + Fts5Structure *p = 0; /* Decoded structure object */ + + rc = fts5StructureDecode(pBlob, nBlob, 0, &p); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + + fts5DebugStructure(pRc, pBuf, p); + fts5StructureRelease(p); +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain an "averages" record. This function +** appends a human-readable representation of record to the buffer passed +** as the second argument. +*/ +static void fts5DecodeAverages( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int i = 0; + const char *zSpace = ""; + + while( i0 ){ + iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); + } + while( iOffn*2 + 1); + if( *pRc==SQLITE_OK ){ + for(ii=0; iin; ii++){ + if( pTerm->p[ii]==0x00 ){ + pBuf->p[pBuf->n++] = '\\'; + pBuf->p[pBuf->n++] = '0'; + }else{ + pBuf->p[pBuf->n++] = pTerm->p[ii]; + } + } + pBuf->p[pBuf->n] = 0x00; + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +/* +** The implementation of user-defined scalar function fts5_decode(). +*/ +static void fts5DecodeFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args (always 2) */ + sqlite3_value **apVal /* Function arguments */ +){ + i64 iRowid; /* Rowid for record being decoded */ + int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + int bTomb; + const u8 *aBlob; int n; /* Record to decode */ + u8 *a = 0; + Fts5Buffer s; /* Build up text to return here */ + int rc = SQLITE_OK; /* Return code */ + sqlite3_int64 nSpace = 0; + int eDetailNone = (sqlite3_user_data(pCtx)!=0); + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + memset(&s, 0, sizeof(Fts5Buffer)); + iRowid = sqlite3_value_int64(apVal[0]); + + /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[] + ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents + ** buffer overreads even if the record is corrupt. */ + n = sqlite3_value_bytes(apVal[1]); + aBlob = sqlite3_value_blob(apVal[1]); + nSpace = n + FTS5_DATA_ZERO_PADDING; + a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); + if( a==0 ) goto decode_out; + if( n>0 ) memcpy(a, aBlob, n); + + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); + + fts5DebugRowid(&rc, &s, iRowid); + if( bDlidx ){ + Fts5Data dlidx; + Fts5DlidxLvl lvl; + + dlidx.p = a; + dlidx.nn = n; + + memset(&lvl, 0, sizeof(Fts5DlidxLvl)); + lvl.pData = &dlidx; + lvl.iLeafPgno = iPgno; + + for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, + " %d(%lld)", lvl.iLeafPgno, lvl.iRowid + ); + } + }else if( bTomb ){ + u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; iin ){ + rc = FTS5_CORRUPT; + goto decode_out; + } + } + + /* Decode the position list tail at the start of the page */ + if( iRowidOff!=0 ){ + iOff = iRowidOff; + }else if( iTermOff!=0 ){ + iOff = iTermOff; + }else{ + iOff = szLeaf; + } + if( iOff>n ){ + rc = FTS5_CORRUPT; + goto decode_out; + } + fts5DecodePoslist(&rc, &s, &a[4], iOff-4); + + /* Decode any more doclist data that appears on the page before the + ** first term. */ + nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff; + if( nDoclist+iOff>n ){ + rc = FTS5_CORRUPT; + goto decode_out; + } + fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist); + + while( iPgidxOffszLeaf ){ + rc = FTS5_CORRUPT; + break; + } + + if( bFirst==0 ){ + iOff += fts5GetVarint32(&a[iOff], nByte); + if( nByte>term.n ){ + rc = FTS5_CORRUPT; + break; + } + term.n = nByte; + } + iOff += fts5GetVarint32(&a[iOff], nByte); + if( iOff+nByte>n ){ + rc = FTS5_CORRUPT; + break; + } + fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]); + iOff += nByte; + + sqlite3Fts5BufferAppendPrintf(&rc, &s, " term="); + fts5BufferAppendTerm(&rc, &s, &term); + iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff); + } + + fts5BufferFree(&term); + } + + decode_out: + sqlite3_free(a); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + fts5BufferFree(&s); +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +/* +** The implementation of user-defined scalar function fts5_rowid(). +*/ +static void fts5RowidFunction( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args (always 2) */ + sqlite3_value **apVal /* Function arguments */ +){ + const char *zArg; + if( nArg==0 ){ + sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1); + }else{ + zArg = (const char*)sqlite3_value_text(apVal[0]); + if( 0==sqlite3_stricmp(zArg, "segment") ){ + i64 iRowid; + int segid, pgno; + if( nArg!=3 ){ + sqlite3_result_error(pCtx, + "should be: fts5_rowid('segment', segid, pgno))", -1 + ); + }else{ + segid = sqlite3_value_int(apVal[1]); + pgno = sqlite3_value_int(apVal[2]); + iRowid = FTS5_SEGMENT_ROWID(segid, pgno); + sqlite3_result_int64(pCtx, iRowid); + } + }else{ + sqlite3_result_error(pCtx, + "first arg to fts5_rowid() must be 'segment'" , -1 + ); + } + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) + +typedef struct Fts5StructVtab Fts5StructVtab; +struct Fts5StructVtab { + sqlite3_vtab base; +}; + +typedef struct Fts5StructVcsr Fts5StructVcsr; +struct Fts5StructVcsr { + sqlite3_vtab_cursor base; + Fts5Structure *pStruct; + int iLevel; + int iSeg; + int iRowid; +}; + +/* +** Create a new fts5_structure() table-valued function. +*/ +static int fts5structConnectMethod( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + Fts5StructVtab *pNew = 0; + int rc = SQLITE_OK; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE xyz(" + "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, " + "npgtombstone, nentrytombstone, nentry, struct HIDDEN);" + ); + if( rc==SQLITE_OK ){ + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + } + + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** We must have a single struct=? constraint that will be passed through +** into the xFilter method. If there is no valid struct=? constraint, +** then return an SQLITE_CONSTRAINT error. +*/ +static int fts5structBestIndexMethod( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int rc = SQLITE_CONSTRAINT; + struct sqlite3_index_constraint *p; + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + break; + } + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + int rc = SQLITE_OK; + Fts5StructVcsr *pNew = 0; + + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + *ppCsr = (sqlite3_vtab_cursor*)pNew; + + return SQLITE_OK; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + fts5StructureRelease(pCsr->pStruct); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + + assert( pCsr->pStruct ); + pCsr->iSeg++; + pCsr->iRowid++; + while( pCsr->iLevelnLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){ + pCsr->iLevel++; + pCsr->iSeg = 0; + } + if( pCsr->iLevel>=p->nLevel ){ + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + return pCsr->pStruct==0; +} + +static int fts5structRowidMethod( + sqlite3_vtab_cursor *cur, + sqlite_int64 *piRowid +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + *piRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int fts5structColumnMethod( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; + + switch( i ){ + case 0: /* level */ + sqlite3_result_int(ctx, pCsr->iLevel); + break; + case 1: /* segment */ + sqlite3_result_int(ctx, pCsr->iSeg); + break; + case 2: /* merge */ + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); + break; + case 3: /* segid */ + sqlite3_result_int(ctx, pSeg->iSegid); + break; + case 4: /* leaf1 */ + sqlite3_result_int(ctx, pSeg->pgnoFirst); + break; + case 5: /* leaf2 */ + sqlite3_result_int(ctx, pSeg->pgnoLast); + break; + case 6: /* origin1 */ + sqlite3_result_int64(ctx, pSeg->iOrigin1); + break; + case 7: /* origin2 */ + sqlite3_result_int64(ctx, pSeg->iOrigin2); + break; + case 8: /* npgtombstone */ + sqlite3_result_int(ctx, pSeg->nPgTombstone); + break; + case 9: /* nentrytombstone */ + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); + break; + case 10: /* nentry */ + sqlite3_result_int64(ctx, pSeg->nEntry); + break; + } + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int fts5structFilterMethod( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; + int rc = SQLITE_OK; + + const u8 *aBlob = 0; + int nBlob = 0; + + assert( argc==1 ); + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + + nBlob = sqlite3_value_bytes(argv[0]); + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); + if( rc==SQLITE_OK ){ + pCsr->iLevel = 0; + pCsr->iRowid = 0; + pCsr->iSeg = -1; + rc = fts5structNextMethod(pVtabCursor); + } + + return rc; +} + +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +/* +** This is called as part of registering the FTS5 module with database +** connection db. It registers several user-defined scalar functions useful +** with FTS5. +** +** If successful, SQLITE_OK is returned. If an error occurs, some other +** SQLite error code is returned instead. +*/ +int sqlite3Fts5IndexInit(sqlite3 *db){ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) + int rc = sqlite3_create_function( + db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 + ); + + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_decode_none", 2, + SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0 + ); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 + ); + } + + if( rc==SQLITE_OK ){ + static const sqlite3_module fts5structure_module = { + 0, /* iVersion */ + 0, /* xCreate */ + fts5structConnectMethod, /* xConnect */ + fts5structBestIndexMethod, /* xBestIndex */ + fts5structDisconnectMethod, /* xDisconnect */ + 0, /* xDestroy */ + fts5structOpenMethod, /* xOpen */ + fts5structCloseMethod, /* xClose */ + fts5structFilterMethod, /* xFilter */ + fts5structNextMethod, /* xNext */ + fts5structEofMethod, /* xEof */ + fts5structColumnMethod, /* xColumn */ + fts5structRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); + } + return rc; +#else + return SQLITE_OK; + UNUSED_PARAM(db); +#endif +} + + +int sqlite3Fts5IndexReset(Fts5Index *p){ + assert( p->pStruct==0 || p->iStructVersion!=0 ); + if( fts5IndexDataVersion(p)!=p->iStructVersion ){ + fts5StructureInvalidate(p); + } + return fts5IndexReturn(p); +} diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c new file mode 100644 index 0000000000000000000000000000000000000000..876420f24d8f3af6bffb8526c9e2aac2c6747367 --- /dev/null +++ b/ext/fts5/fts5_main.c @@ -0,0 +1,3869 @@ +/* +** 2014 Jun 09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + + +#include "fts5Int.h" + +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts5 code are activated - conditions that are +** only true if it is guaranteed that the fts5 database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +int sqlite3_fts5_may_be_corrupt = 1; +#endif + + +typedef struct Fts5Auxdata Fts5Auxdata; +typedef struct Fts5Auxiliary Fts5Auxiliary; +typedef struct Fts5Cursor Fts5Cursor; +typedef struct Fts5FullTable Fts5FullTable; +typedef struct Fts5Sorter Fts5Sorter; +typedef struct Fts5TokenizerModule Fts5TokenizerModule; + +/* +** NOTES ON TRANSACTIONS: +** +** SQLite invokes the following virtual table methods as transactions are +** opened and closed by the user: +** +** xBegin(): Start of a new transaction. +** xSync(): Initial part of two-phase commit. +** xCommit(): Final part of two-phase commit. +** xRollback(): Rollback the transaction. +** +** Anything that is required as part of a commit that may fail is performed +** in the xSync() callback. Current versions of SQLite ignore any errors +** returned by xCommit(). +** +** And as sub-transactions are opened/closed: +** +** xSavepoint(int S): Open savepoint S. +** xRelease(int S): Commit and close savepoint S. +** xRollbackTo(int S): Rollback to start of savepoint S. +** +** During a write-transaction the fts5_index.c module may cache some data +** in-memory. It is flushed to disk whenever xSync(), xRelease() or +** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo() +** is called. +** +** Additionally, if SQLITE_DEBUG is defined, an instance of the following +** structure is used to record the current transaction state. This information +** is not required, but it is used in the assert() statements executed by +** function fts5CheckTransactionState() (see below). +*/ +struct Fts5TransactionState { + int eState; /* 0==closed, 1==open, 2==synced */ + int iSavepoint; /* Number of open savepoints (0 -> none) */ +}; + +/* +** A single object of this type is allocated when the FTS5 module is +** registered with a database handle. It is used to store pointers to +** all registered FTS5 extensions - tokenizers and auxiliary functions. +*/ +struct Fts5Global { + fts5_api api; /* User visible part of object (see fts5.h) */ + sqlite3 *db; /* Associated database connection */ + i64 iNextId; /* Used to allocate unique cursor ids */ + Fts5Auxiliary *pAux; /* First in list of all aux. functions */ + Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ + Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ + Fts5Cursor *pCsr; /* First in list of all open cursors */ + u32 aLocaleHdr[4]; +}; + +/* +** Size of header on fts5_locale() values. And macro to access a buffer +** containing a copy of the header from an Fts5Config pointer. +*/ +#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) +#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) + +#define FTS5_INSTTOKEN_SUBTYPE 73 + +/* +** Each auxiliary function registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pAux list. +*/ +struct Fts5Auxiliary { + Fts5Global *pGlobal; /* Global context for this function */ + char *zFunc; /* Function name (nul-terminated) */ + void *pUserData; /* User-data pointer */ + fts5_extension_function xFunc; /* Callback function */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5Auxiliary *pNext; /* Next registered auxiliary function */ +}; + +/* +** Each tokenizer module registered with the FTS5 module is represented +** by an object of the following type. All such objects are stored as part +** of the Fts5Global.pTok list. +** +** bV2Native: +** True if the tokenizer was registered using xCreateTokenizer_v2(), false +** for xCreateTokenizer(). If this variable is true, then x2 is populated +** with the routines as supplied by the caller and x1 contains synthesized +** wrapper routines. In this case the user-data pointer passed to +** x1.xCreate should be a pointer to the Fts5TokenizerModule structure, +** not a copy of pUserData. +** +** Of course, if bV2Native is false, then x1 contains the real routines and +** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule +** object should be passed to x2.xCreate. +** +** The synthesized wrapper routines are necessary for xFindTokenizer(_v2) +** calls. +*/ +struct Fts5TokenizerModule { + char *zName; /* Name of tokenizer */ + void *pUserData; /* User pointer passed to xCreate() */ + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ + void (*xDestroy)(void*); /* Destructor function */ + Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ +}; + +struct Fts5FullTable { + Fts5Table p; /* Public class members from fts5Int.h */ + Fts5Storage *pStorage; /* Document store */ + Fts5Global *pGlobal; /* Global (connection wide) data */ + Fts5Cursor *pSortCsr; /* Sort data from this cursor */ + int iSavepoint; /* Successful xSavepoint()+1 */ + +#ifdef SQLITE_DEBUG + struct Fts5TransactionState ts; +#endif +}; + +struct Fts5MatchPhrase { + Fts5Buffer *pPoslist; /* Pointer to current poslist */ + int nTerm; /* Size of phrase in terms */ +}; + +/* +** pStmt: +** SELECT rowid, FROM ORDER BY +rank; +** +** aIdx[]: +** There is one entry in the aIdx[] array for each phrase in the query, +** the value of which is the offset within aPoslist[] following the last +** byte of the position list for the corresponding phrase. +*/ +struct Fts5Sorter { + sqlite3_stmt *pStmt; + i64 iRowid; /* Current rowid */ + const u8 *aPoslist; /* Position lists for current row */ + int nIdx; /* Number of entries in aIdx[] */ + int aIdx[1]; /* Offsets into aPoslist for current row */ +}; + + +/* +** Virtual-table cursor object. +** +** iSpecial: +** If this is a 'special' query (refer to function fts5SpecialMatch()), +** then this variable contains the result of the query. +** +** iFirstRowid, iLastRowid: +** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the +** cursor iterates in ascending order of rowids, iFirstRowid is the lower +** limit of rowids to return, and iLastRowid the upper. In other words, the +** WHERE clause in the user's query might have been: +** +** MATCH AND rowid BETWEEN $iFirstRowid AND $iLastRowid +** +** If the cursor iterates in descending order of rowid, iFirstRowid +** is the upper limit (i.e. the "first" rowid visited) and iLastRowid +** the lower. +*/ +struct Fts5Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ + int *aColumnSize; /* Values for xColumnSize() */ + i64 iCsrId; /* Cursor id */ + + /* Zero from this point onwards on cursor reset */ + int ePlan; /* FTS5_PLAN_XXX value */ + int bDesc; /* True for "ORDER BY rowid DESC" queries */ + i64 iFirstRowid; /* Return no rowids earlier than this */ + i64 iLastRowid; /* Return no rowids later than this */ + sqlite3_stmt *pStmt; /* Statement used to read %_content */ + Fts5Expr *pExpr; /* Expression for MATCH queries */ + Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ + int csrflags; /* Mask of cursor flags (see below) */ + i64 iSpecial; /* Result of special query */ + + /* "rank" function. Populated on demand from vtab.xColumn(). */ + char *zRank; /* Custom rank function */ + char *zRankArgs; /* Custom rank function args */ + Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ + int nRankArg; /* Number of trailing arguments for rank() */ + sqlite3_value **apRankArg; /* Array of trailing arguments */ + sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ + + /* Auxiliary data storage */ + Fts5Auxiliary *pAux; /* Currently executing extension function */ + Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ + + /* Cache used by auxiliary API functions xInst() and xInstCount() */ + Fts5PoslistReader *aInstIter; /* One for each phrase */ + int nInstAlloc; /* Size of aInst[] array (entries / 3) */ + int nInstCount; /* Number of phrase instances */ + int *aInst; /* 3 integers per phrase instance */ +}; + +/* +** Bits that make up the "idxNum" parameter passed indirectly by +** xBestIndex() to xFilter(). +*/ +#define FTS5_BI_MATCH 0x0001 /* MATCH ? */ +#define FTS5_BI_RANK 0x0002 /* rank MATCH ? */ +#define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */ +#define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */ +#define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */ + +#define FTS5_BI_ORDER_RANK 0x0020 +#define FTS5_BI_ORDER_ROWID 0x0040 +#define FTS5_BI_ORDER_DESC 0x0080 + +/* +** Values for Fts5Cursor.csrflags +*/ +#define FTS5CSR_EOF 0x01 +#define FTS5CSR_REQUIRE_CONTENT 0x02 +#define FTS5CSR_REQUIRE_DOCSIZE 0x04 +#define FTS5CSR_REQUIRE_INST 0x08 +#define FTS5CSR_FREE_ZRANK 0x10 +#define FTS5CSR_REQUIRE_RESEEK 0x20 +#define FTS5CSR_REQUIRE_POSLIST 0x40 + +#define BitFlagAllTest(x,y) (((x) & (y))==(y)) +#define BitFlagTest(x,y) (((x) & (y))!=0) + + +/* +** Macros to Set(), Clear() and Test() cursor flags. +*/ +#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag)) +#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag)) +#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag)) + +struct Fts5Auxdata { + Fts5Auxiliary *pAux; /* Extension to which this belongs */ + void *pPtr; /* Pointer value */ + void(*xDelete)(void*); /* Destructor */ + Fts5Auxdata *pNext; /* Next object in linked list */ +}; + +#ifdef SQLITE_DEBUG +#define FTS5_BEGIN 1 +#define FTS5_SYNC 2 +#define FTS5_COMMIT 3 +#define FTS5_ROLLBACK 4 +#define FTS5_SAVEPOINT 5 +#define FTS5_RELEASE 6 +#define FTS5_ROLLBACKTO 7 +static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ + switch( op ){ + case FTS5_BEGIN: + assert( p->ts.eState==0 ); + p->ts.eState = 1; + p->ts.iSavepoint = -1; + break; + + case FTS5_SYNC: + assert( p->ts.eState==1 || p->ts.eState==2 ); + p->ts.eState = 2; + break; + + case FTS5_COMMIT: + assert( p->ts.eState==2 ); + p->ts.eState = 0; + break; + + case FTS5_ROLLBACK: + assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 ); + p->ts.eState = 0; + break; + + case FTS5_SAVEPOINT: + assert( p->ts.eState>=1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint>=p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint; + break; + + case FTS5_RELEASE: + assert( p->ts.eState>=1 ); + assert( iSavepoint>=0 ); + assert( iSavepoint<=p->ts.iSavepoint ); + p->ts.iSavepoint = iSavepoint-1; + break; + + case FTS5_ROLLBACKTO: + assert( p->ts.eState>=1 ); + assert( iSavepoint>=-1 ); + /* The following assert() can fail if another vtab strikes an error + ** within an xSavepoint() call then SQLite calls xRollbackTo() - without + ** having called xSavepoint() on this vtab. */ + /* assert( iSavepoint<=p->ts.iSavepoint ); */ + p->ts.iSavepoint = iSavepoint; + break; + } +} +#else +# define fts5CheckTransactionState(x,y,z) +#endif + +/* +** Return true if pTab is a contentless table. If parameter bIncludeUnindexed +** is true, this includes contentless tables that store UNINDEXED columns +** only. +*/ +static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){ + int eContent = pTab->p.pConfig->eContent; + return ( + eContent==FTS5_CONTENT_NONE + || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED) + ); +} + +/* +** Delete a virtual table handle allocated by fts5InitVtab(). +*/ +static void fts5FreeVtab(Fts5FullTable *pTab){ + if( pTab ){ + sqlite3Fts5IndexClose(pTab->p.pIndex); + sqlite3Fts5StorageClose(pTab->pStorage); + sqlite3Fts5ConfigFree(pTab->p.pConfig); + sqlite3_free(pTab); + } +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5DisconnectMethod(sqlite3_vtab *pVtab){ + fts5FreeVtab((Fts5FullTable*)pVtab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5DestroyMethod(sqlite3_vtab *pVtab){ + Fts5Table *pTab = (Fts5Table*)pVtab; + int rc = sqlite3Fts5DropAll(pTab->pConfig); + if( rc==SQLITE_OK ){ + fts5FreeVtab((Fts5FullTable*)pVtab); + } + return rc; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts5InitVtab( + int bCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts5Global *pGlobal = (Fts5Global*)pAux; + const char **azConfig = (const char**)argv; + int rc = SQLITE_OK; /* Return code */ + Fts5Config *pConfig = 0; /* Results of parsing argc/argv */ + Fts5FullTable *pTab = 0; /* New virtual table object */ + + /* Allocate the new vtab object and parse the configuration */ + pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable)); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr); + assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); + } + if( rc==SQLITE_OK ){ + pConfig->pzErrmsg = pzErr; + pTab->p.pConfig = pConfig; + pTab->pGlobal = pGlobal; + if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } + } + + /* Open the index sub-system */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr); + } + + /* Open the storage sub-system */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageOpen( + pConfig, pTab->p.pIndex, bCreate, &pTab->pStorage, pzErr + ); + } + + /* Call sqlite3_declare_vtab() */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigDeclareVtab(pConfig); + } + + /* Load the initial configuration */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1); + } + + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + + if( pConfig ) pConfig->pzErrmsg = 0; + if( rc!=SQLITE_OK ){ + fts5FreeVtab(pTab); + pTab = 0; + }else if( bCreate ){ + fts5CheckTransactionState(pTab, FTS5_BEGIN, 0); + } + *ppVTab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5InitVtab(). +*/ +static int fts5ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** The different query plans. +*/ +#define FTS5_PLAN_MATCH 1 /* ( MATCH ?) */ +#define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */ +#define FTS5_PLAN_SPECIAL 3 /* An internal query */ +#define FTS5_PLAN_SORTED_MATCH 4 /* ( MATCH ? ORDER BY rank) */ +#define FTS5_PLAN_SCAN 5 /* No usable constraint */ +#define FTS5_PLAN_ROWID 6 /* (rowid = ?) */ + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008012 ) +#endif + { + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +static int fts5UsePatternMatch( + Fts5Config *pConfig, + struct sqlite3_index_constraint *p +){ + assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB ); + assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE ); + if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ + return 1; + } + if( pConfig->t.ePattern==FTS5_PATTERN_LIKE + && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB) + ){ + return 1; + } + return 0; +} + +/* +** Implementation of the xBestIndex method for FTS5 tables. Within the +** WHERE constraint, it searches for the following: +** +** 1. A MATCH constraint against the table column. +** 2. A MATCH constraint against the "rank" column. +** 3. A MATCH constraint against some other column. +** 4. An == constraint against the rowid column. +** 5. A < or <= constraint against the rowid column. +** 6. A > or >= constraint against the rowid column. +** +** Within the ORDER BY, the following are supported: +** +** 5. ORDER BY rank [ASC|DESC] +** 6. ORDER BY rowid [ASC|DESC] +** +** Information for the xFilter call is passed via both the idxNum and +** idxStr variables. Specifically, idxNum is a bitmask of the following +** flags used to encode the ORDER BY clause: +** +** FTS5_BI_ORDER_RANK +** FTS5_BI_ORDER_ROWID +** FTS5_BI_ORDER_DESC +** +** idxStr is used to encode data from the WHERE clause. For each argument +** passed to the xFilter method, the following is appended to idxStr: +** +** Match against table column: "m" +** Match against rank column: "r" +** Match against other column: "M" +** LIKE against other column: "L" +** GLOB against other column: "G" +** Equality constraint against the rowid: "=" +** A < or <= against the rowid: "<" +** A > or >= against the rowid: ">" +** +** This function ensures that there is at most one "r" or "=". And that if +** there exists an "=" then there is no "<" or ">". +** +** If an unusable MATCH operator is present in the WHERE clause, then +** SQLITE_CONSTRAINT is returned. +** +** Costs are assigned as follows: +** +** a) If a MATCH operator is present, the cost depends on the other +** constraints also present. As follows: +** +** * No other constraints: cost=1000.0 +** * One rowid range constraint: cost=750.0 +** * Both rowid range constraints: cost=500.0 +** * An == rowid constraint: cost=100.0 +** +** b) Otherwise, if there is no MATCH: +** +** * No other constraints: cost=1000000.0 +** * One rowid range constraint: cost=750000.0 +** * Both rowid range constraints: cost=250000.0 +** * An == rowid constraint: cost=10.0 +** +** Costs are not modified by the ORDER BY clause. +*/ +static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts5Table *pTab = (Fts5Table*)pVTab; + Fts5Config *pConfig = pTab->pConfig; + const int nCol = pConfig->nCol; + int idxFlags = 0; /* Parameter passed through to xFilter() */ + int i; + + char *idxStr; + int iIdxStr = 0; + int iCons = 0; + + int bSeenEq = 0; + int bSeenGt = 0; + int bSeenLt = 0; + int nSeenMatch = 0; + int bSeenRank = 0; + + + assert( SQLITE_INDEX_CONSTRAINT_EQbLock ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "recursively defined fts5 content table" + ); + return SQLITE_ERROR; + } + + idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + if( idxStr==0 ) return SQLITE_NOMEM; + pInfo->idxStr = idxStr; + pInfo->needToFreeIdxStr = 1; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + int iCol = p->iColumn; + if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH + || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol) + ){ + /* A MATCH operator or equivalent */ + if( p->usable==0 || iCol<0 ){ + /* As there exists an unusable MATCH constraint this is an + ** unusable plan. Return SQLITE_CONSTRAINT. */ + idxStr[iIdxStr] = 0; + return SQLITE_CONSTRAINT; + }else{ + if( iCol==nCol+1 ){ + if( bSeenRank ) continue; + idxStr[iIdxStr++] = 'r'; + bSeenRank = 1; + }else{ + nSeenMatch++; + idxStr[iIdxStr++] = 'M'; + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); + idxStr += strlen(&idxStr[iIdxStr]); + assert( idxStr[iIdxStr]=='\0' ); + } + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; + } + }else if( p->usable ){ + if( iCol>=0 && iColop==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB ); + idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE ? 'L' : 'G'; + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); + idxStr += strlen(&idxStr[iIdxStr]); + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + assert( idxStr[iIdxStr]=='\0' ); + nSeenMatch++; + }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){ + idxStr[iIdxStr++] = '='; + bSeenEq = 1; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + } + } + } + + if( bSeenEq==0 ){ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->iColumn<0 && p->usable ){ + int op = p->op; + if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){ + if( bSeenLt ) continue; + idxStr[iIdxStr++] = '<'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenLt = 1; + }else + if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){ + if( bSeenGt ) continue; + idxStr[iIdxStr++] = '>'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenGt = 1; + } + } + } + } + idxStr[iIdxStr] = '\0'; + + /* Set idxFlags flags for the ORDER BY clause + ** + ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC". + */ + if( pInfo->nOrderBy==1 ){ + int iSort = pInfo->aOrderBy[0].iColumn; + if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){ + idxFlags |= FTS5_BI_ORDER_RANK; + }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ + idxFlags |= FTS5_BI_ORDER_ROWID; + } + if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ + pInfo->orderByConsumed = 1; + if( pInfo->aOrderBy[0].desc ){ + idxFlags |= FTS5_BI_ORDER_DESC; + } + } + } + + /* Calculate the estimated cost based on the flags set in idxFlags. */ + if( bSeenEq ){ + pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0; + if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo); + }else if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0; + }else{ + pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0; + } + for(i=1; iestimatedCost *= 0.4; + } + + pInfo->idxNum = idxFlags; + return SQLITE_OK; +} + +static int fts5NewTransaction(Fts5FullTable *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; + } + return sqlite3Fts5StorageReset(pTab->pStorage); +} + +/* +** Implementation of xOpen method. +*/ +static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts5FullTable *pTab = (Fts5FullTable*)pVTab; + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Cursor *pCsr = 0; /* New cursor object */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + int rc; /* Return code */ + + rc = fts5NewTransaction(pTab); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); + pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte); + if( pCsr ){ + Fts5Global *pGlobal = pTab->pGlobal; + memset(pCsr, 0, (size_t)nByte); + pCsr->aColumnSize = (int*)&pCsr[1]; + pCsr->pNext = pGlobal->pCsr; + pGlobal->pCsr = pCsr; + pCsr->iCsrId = ++pGlobal->iNextId; + }else{ + rc = SQLITE_NOMEM; + } + } + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static int fts5StmtType(Fts5Cursor *pCsr){ + if( pCsr->ePlan==FTS5_PLAN_SCAN ){ + return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; + } + return FTS5_STMT_LOOKUP; +} + +/* +** This function is called after the cursor passed as the only argument +** is moved to point at a different row. It clears all cached data +** specific to the previous row stored by the cursor object. +*/ +static void fts5CsrNewrow(Fts5Cursor *pCsr){ + CsrFlagSet(pCsr, + FTS5CSR_REQUIRE_CONTENT + | FTS5CSR_REQUIRE_DOCSIZE + | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST + ); +} + +static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Auxdata *pData; + Fts5Auxdata *pNext; + + sqlite3_free(pCsr->aInstIter); + sqlite3_free(pCsr->aInst); + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + if( pCsr->pSorter ){ + Fts5Sorter *pSorter = pCsr->pSorter; + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + } + + if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ + sqlite3Fts5ExprFree(pCsr->pExpr); + } + + for(pData=pCsr->pAuxdata; pData; pData=pNext){ + pNext = pData->pNext; + if( pData->xDelete ) pData->xDelete(pData->pPtr); + sqlite3_free(pData); + } + + sqlite3_finalize(pCsr->pRankArgStmt); + sqlite3_free(pCsr->apRankArg); + + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ + sqlite3_free(pCsr->zRank); + sqlite3_free(pCsr->zRankArgs); + } + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr)); +} + + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ + if( pCursor ){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + Fts5Cursor **pp; + + fts5FreeCursorComponents(pCsr); + /* Remove the cursor from the Fts5Global.pCsr list */ + for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); + *pp = pCsr->pNext; + + sqlite3_free(pCsr); + } + return SQLITE_OK; +} + +static int fts5SorterNext(Fts5Cursor *pCsr){ + Fts5Sorter *pSorter = pCsr->pSorter; + int rc; + + rc = sqlite3_step(pSorter->pStmt); + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT); + }else if( rc==SQLITE_ROW ){ + const u8 *a; + const u8 *aBlob; + int nBlob; + int i; + int iOff = 0; + rc = SQLITE_OK; + + pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); + nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); + aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); + + /* nBlob==0 in detail=none mode. */ + if( nBlob>0 ){ + for(i=0; i<(pSorter->nIdx-1); i++){ + int iVal; + a += fts5GetVarint32(a, iVal); + iOff += iVal; + pSorter->aIdx[i] = iOff; + } + pSorter->aIdx[i] = &aBlob[nBlob] - a; + pSorter->aPoslist = a; + } + + fts5CsrNewrow(pCsr); + } + + return rc; +} + + +/* +** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors +** open on table pTab. +*/ +static void fts5TripCursors(Fts5FullTable *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && pCsr->base.pVtab==(sqlite3_vtab*)pTab + ){ + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK); + } + } +} + +/* +** If the REQUIRE_RESEEK flag is set on the cursor passed as the first +** argument, close and reopen all Fts5IndexIter iterators that the cursor +** is using. Then attempt to move the cursor to a rowid equal to or laster +** (in the cursors sort order - ASC or DESC) than the current rowid. +** +** If the new rowid is not equal to the old, set output parameter *pbSkip +** to 1 before returning. Otherwise, leave it unchanged. +** +** Return SQLITE_OK if successful or if no reseek was required, or an +** error code if an error occurred. +*/ +static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ + int rc = SQLITE_OK; + assert( *pbSkip==0 ); + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + int bDesc = pCsr->bDesc; + i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); + + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); + if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ + *pbSkip = 1; + } + + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); + fts5CsrNewrow(pCsr); + if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + *pbSkip = 1; + } + } + return rc; +} + + +/* +** Advance the cursor to the next row in the table that matches the +** search criteria. +** +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned +** even if we reach end-of-file. The fts5EofMethod() will be called +** subsequently to determine whether or not an EOF was hit. +*/ +static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc; + + assert( (pCsr->ePlan<3)== + (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) + ); + assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + + /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table, + ** clear any token mappings accumulated at the fts5_index.c level. In + ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH, + ** we need to retain the mappings for the entire query. */ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata + ){ + sqlite3Fts5ExprClearTokens(pCsr->pExpr); + } + + if( pCsr->ePlan<3 ){ + int bSkip = 0; + if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; + rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); + CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr)); + fts5CsrNewrow(pCsr); + }else{ + switch( pCsr->ePlan ){ + case FTS5_PLAN_SPECIAL: { + CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = SQLITE_OK; + break; + } + + case FTS5_PLAN_SORTED_MATCH: { + rc = fts5SorterNext(pCsr); + break; + } + + default: { + Fts5Config *pConfig = ((Fts5Table*)pCursor->pVtab)->pConfig; + pConfig->bLock++; + rc = sqlite3_step(pCsr->pStmt); + pConfig->bLock--; + if( rc!=SQLITE_ROW ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = sqlite3_reset(pCsr->pStmt); + if( rc!=SQLITE_OK ){ + pCursor->pVtab->zErrMsg = sqlite3_mprintf( + "%s", sqlite3_errmsg(pConfig->db) + ); + } + }else{ + rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE); + } + break; + } + } + } + + return rc; +} + + +static int fts5PrepareStatement( + sqlite3_stmt **ppStmt, + Fts5Config *pConfig, + const char *zFmt, + ... +){ + sqlite3_stmt *pRet = 0; + int rc; + char *zSql; + va_list ap; + + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT, &pRet, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db)); + } + sqlite3_free(zSql); + } + + va_end(ap); + *ppStmt = pRet; + return rc; +} + +static int fts5CursorFirstSorted( + Fts5FullTable *pTab, + Fts5Cursor *pCsr, + int bDesc +){ + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Sorter *pSorter; + int nPhrase; + sqlite3_int64 nByte; + int rc; + const char *zRank = pCsr->zRank; + const char *zRankArgs = pCsr->zRankArgs; + + nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); + if( pSorter==0 ) return SQLITE_NOMEM; + memset(pSorter, 0, (size_t)nByte); + pSorter->nIdx = nPhrase; + + /* TODO: It would be better to have some system for reusing statement + ** handles here, rather than preparing a new one for each query. But that + ** is not possible as SQLite reference counts the virtual table objects. + ** And since the statement required here reads from this very virtual + ** table, saving it creates a circular reference. + ** + ** If SQLite a built-in statement cache, this wouldn't be a problem. */ + rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, + "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s", + pConfig->zDb, pConfig->zName, zRank, pConfig->zName, + (zRankArgs ? ", " : ""), + (zRankArgs ? zRankArgs : ""), + bDesc ? "DESC" : "ASC" + ); + + pCsr->pSorter = pSorter; + if( rc==SQLITE_OK ){ + assert( pTab->pSortCsr==0 ); + pTab->pSortCsr = pCsr; + rc = fts5SorterNext(pCsr); + pTab->pSortCsr = 0; + } + + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + pCsr->pSorter = 0; + } + + return rc; +} + +static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ + int rc; + Fts5Expr *pExpr = pCsr->pExpr; + rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); + if( sqlite3Fts5ExprEof(pExpr) ){ + CsrFlagSet(pCsr, FTS5CSR_EOF); + } + fts5CsrNewrow(pCsr); + return rc; +} + +/* +** Process a "special" query. A special query is identified as one with a +** MATCH expression that begins with a '*' character. The remainder of +** the text passed to the MATCH operator are used as the special query +** parameters. +*/ +static int fts5SpecialMatch( + Fts5FullTable *pTab, + Fts5Cursor *pCsr, + const char *zQuery +){ + int rc = SQLITE_OK; /* Return code */ + const char *z = zQuery; /* Special query text */ + int n; /* Number of bytes in text at z */ + + while( z[0]==' ' ) z++; + for(n=0; z[n] && z[n]!=' '; n++); + + assert( pTab->p.base.zErrMsg==0 ); + pCsr->ePlan = FTS5_PLAN_SPECIAL; + + if( n==5 && 0==sqlite3_strnicmp("reads", z, n) ){ + pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex); + } + else if( n==2 && 0==sqlite3_strnicmp("id", z, n) ){ + pCsr->iSpecial = pCsr->iCsrId; + } + else{ + /* An unrecognized directive. Return an error message. */ + pTab->p.base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Search for an auxiliary function named zName that can be used with table +** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary +** structure. Otherwise, if no such function exists, return NULL. +*/ +static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){ + Fts5Auxiliary *pAux; + + for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){ + if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux; + } + + /* No function of the specified name was found. Return 0. */ + return 0; +} + + +static int fts5FindRankFunction(Fts5Cursor *pCsr){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + int rc = SQLITE_OK; + Fts5Auxiliary *pAux = 0; + const char *zRank = pCsr->zRank; + const char *zRankArgs = pCsr->zRankArgs; + + if( zRankArgs ){ + char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT, &pStmt, 0); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 ); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_int64 nByte; + pCsr->nRankArg = sqlite3_column_count(pStmt); + nByte = sizeof(sqlite3_value*)*pCsr->nRankArg; + pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte); + if( rc==SQLITE_OK ){ + int i; + for(i=0; inRankArg; i++){ + pCsr->apRankArg[i] = sqlite3_column_value(pStmt, i); + } + } + pCsr->pRankArgStmt = pStmt; + }else{ + rc = sqlite3_finalize(pStmt); + assert( rc!=SQLITE_OK ); + } + } + } + } + + if( rc==SQLITE_OK ){ + pAux = fts5FindAuxiliary(pTab, zRank); + if( pAux==0 ){ + assert( pTab->p.base.zErrMsg==0 ); + pTab->p.base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank); + rc = SQLITE_ERROR; + } + } + + pCsr->pRank = pAux; + return rc; +} + + +static int fts5CursorParseRank( + Fts5Config *pConfig, + Fts5Cursor *pCsr, + sqlite3_value *pRank +){ + int rc = SQLITE_OK; + if( pRank ){ + const char *z = (const char*)sqlite3_value_text(pRank); + char *zRank = 0; + char *zRankArgs = 0; + + if( z==0 ){ + if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + } + if( rc==SQLITE_OK ){ + pCsr->zRank = zRank; + pCsr->zRankArgs = zRankArgs; + CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK); + }else if( rc==SQLITE_ERROR ){ + pCsr->base.pVtab->zErrMsg = sqlite3_mprintf( + "parse error in rank function: %s", z + ); + } + }else{ + if( pConfig->zRank ){ + pCsr->zRank = (char*)pConfig->zRank; + pCsr->zRankArgs = (char*)pConfig->zRankArgs; + }else{ + pCsr->zRank = (char*)FTS5_DEFAULT_RANK; + pCsr->zRankArgs = 0; + } + } + return rc; +} + +static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** Set the error message on the virtual table passed as the first argument. +*/ +static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + sqlite3_free(p->p.base.zErrMsg); + p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale +** specified by pLocale/nLocale. The buffer indicated by pLocale must remain +** valid until after the final call to sqlite3Fts5Tokenize() that will use +** the locale. +*/ +static void sqlite3Fts5SetLocale( + Fts5Config *pConfig, + const char *zLocale, + int nLocale +){ + Fts5TokenizerConfig *pT = &pConfig->t; + pT->pLocale = zLocale; + pT->nLocale = nLocale; +} + +/* +** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale(). +*/ +void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ + sqlite3Fts5SetLocale(pConfig, 0, 0); +} + +/* +** Return true if the value passed as the only argument is an +** fts5_locale() value. +*/ +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ + int ret = 0; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case. + ** If the blob was created using zeroblob(), then sqlite3_value_blob() + ** may call malloc(). If this malloc() fails, then the values returned + ** by both value_blob() and value_bytes() will be 0. If value_bytes() were + ** called first, then the NULL pointer returned by value_blob() might + ** be dereferenced. */ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( nBlob>FTS5_LOCALE_HDR_SIZE + && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) + ){ + ret = 1; + } + } + return ret; +} + +/* +** Value pVal is guaranteed to be an fts5_locale() value, according to +** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale +** from the value and returns them separately. +** +** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set +** to point to buffers containing the text and locale, as utf-8, +** respectively. In this case output parameters (*pnText) and (*pnLoc) are +** set to the sizes in bytes of these two buffers. +** +** Or, if an error occurs, then an SQLite error code is returned. The final +** value of the four output parameters is undefined in this case. +*/ +int sqlite3Fts5DecodeLocaleValue( + sqlite3_value *pVal, + const char **ppText, + int *pnText, + const char **ppLoc, + int *pnLoc +){ + const char *p = sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + int nLoc = 0; + + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + assert( n>FTS5_LOCALE_HDR_SIZE ); + + for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){ + if( nLoc==(n-1) ){ + return SQLITE_MISMATCH; + } + } + *ppLoc = &p[FTS5_LOCALE_HDR_SIZE]; + *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE; + + *ppText = &p[nLoc+1]; + *pnText = n - nLoc - 1; + return SQLITE_OK; +} + +/* +** Argument pVal is the text of a full-text search expression. It may or +** may not have been wrapped by fts5_locale(). This function extracts +** the text of the expression, and sets output variable (*pzText) to +** point to a nul-terminated buffer containing the expression. +** +** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called +** to set the tokenizer to use the specified locale. +** +** If output variable (*pbFreeAndReset) is set to true, then the caller +** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer +** locale, and (b) call sqlite3_free() to free (*pzText). +*/ +static int fts5ExtractExprText( + Fts5Config *pConfig, /* Fts5 configuration */ + sqlite3_value *pVal, /* Value to extract expression text from */ + char **pzText, /* OUT: nul-terminated buffer of text */ + int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ +){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText); + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + } + *pbFreeAndReset = 1; + }else{ + *pzText = (char*)sqlite3_value_text(pVal); + *pbFreeAndReset = 0; + } + + return rc; +} + + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** There are three possible query strategies: +** +** 1. Full-text search using a MATCH operator. +** 2. A by-rowid lookup. +** 3. A full-table scan. +*/ +static int fts5FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; /* Error code */ + int bDesc; /* True if ORDER BY [rank|rowid] DESC */ + int bOrderByRank; /* True if ORDER BY rank */ + sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ + sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ + sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ + sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ + int iCol; /* Column on LHS of MATCH operator */ + char **pzErrmsg = pConfig->pzErrmsg; + int bPrefixInsttoken = pConfig->bPrefixInsttoken; + int i; + int iIdxStr = 0; + Fts5Expr *pExpr = 0; + + assert( pConfig->bLock==0 ); + if( pCsr->ePlan ){ + fts5FreeCursorComponents(pCsr); + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); + } + + assert( pCsr->pStmt==0 ); + assert( pCsr->pExpr==0 ); + assert( pCsr->csrflags==0 ); + assert( pCsr->pRank==0 ); + assert( pCsr->zRank==0 ); + assert( pCsr->zRankArgs==0 ); + assert( pTab->pSortCsr==0 || nVal==0 ); + + assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg ); + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + + /* Decode the arguments passed through to this function. */ + for(i=0; ibPrefixInsttoken = 1; + } + + iCol = 0; + do{ + iCol = iCol*10 + (idxStr[iIdxStr]-'0'); + iIdxStr++; + }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ); + + if( zText[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); + bInternal = 1; + }else{ + char **pzErr = &pTab->p.base.zErrMsg; + rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + } + + if( bFreeAndReset ){ + sqlite3_free(zText); + sqlite3Fts5ClearLocale(pConfig); + } + + if( bInternal || rc!=SQLITE_OK ) goto filter_out; + + break; + } + case 'L': + case 'G': { + int bGlob = (idxStr[iIdxStr-1]=='G'); + const char *zText = (const char*)sqlite3_value_text(apVal[i]); + iCol = 0; + do{ + iCol = iCol*10 + (idxStr[iIdxStr]-'0'); + iIdxStr++; + }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ); + if( zText ){ + rc = sqlite3Fts5ExprPattern(pConfig, bGlob, iCol, zText, &pExpr); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + if( rc!=SQLITE_OK ) goto filter_out; + break; + } + case '=': + pRowidEq = apVal[i]; + break; + case '<': + pRowidLe = apVal[i]; + break; + default: assert( idxStr[iIdxStr-1]=='>' ); + pRowidGe = apVal[i]; + break; + } + } + bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); + pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); + + /* Set the cursor upper and lower rowid limits. Only some strategies + ** actually use them. This is ok, as the xBestIndex() method leaves the + ** sqlite3_index_constraint.omit flag clear for range constraints + ** on the rowid field. */ + if( pRowidEq ){ + pRowidLe = pRowidGe = pRowidEq; + } + if( bDesc ){ + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + }else{ + pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + } + + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) goto filter_out; + + if( pTab->pSortCsr ){ + /* If pSortCsr is non-NULL, then this call is being made as part of + ** processing for a "... MATCH ORDER BY rank" query (ePlan is + ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will + ** return results to the user for this query. The current cursor + ** (pCursor) is used to execute the query issued by function + ** fts5CursorFirstSorted() above. */ + assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); + assert( nVal==0 && bOrderByRank==0 && bDesc==0 ); + assert( pCsr->iLastRowid==LARGEST_INT64 ); + assert( pCsr->iFirstRowid==SMALLEST_INT64 ); + if( pTab->pSortCsr->bDesc ){ + pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid; + }else{ + pCsr->iLastRowid = pTab->pSortCsr->iLastRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid; + } + pCsr->ePlan = FTS5_PLAN_SOURCE; + pCsr->pExpr = pTab->pSortCsr->pExpr; + rc = fts5CursorFirst(pTab, pCsr, bDesc); + }else if( pCsr->pExpr ){ + assert( rc==SQLITE_OK ); + rc = fts5CursorParseRank(pConfig, pCsr, pRank); + if( rc==SQLITE_OK ){ + if( bOrderByRank ){ + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); + }else{ + pCsr->ePlan = FTS5_PLAN_MATCH; + rc = fts5CursorFirst(pTab, pCsr, bDesc); + } + } + }else if( pConfig->zContent==0 ){ + fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName); + rc = SQLITE_ERROR; + }else{ + /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup + ** by rowid (ePlan==FTS5_PLAN_ROWID). */ + pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg + ); + if( rc==SQLITE_OK ){ + if( pRowidEq!=0 ){ + assert( pCsr->ePlan==FTS5_PLAN_ROWID ); + sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq); + }else{ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); + sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); + } + rc = fts5NextMethod(pCursor); + } + } + + filter_out: + sqlite3Fts5ExprFree(pExpr); + pConfig->pzErrmsg = pzErrmsg; + pConfig->bPrefixInsttoken = bPrefixInsttoken; + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0); +} + +/* +** Return the rowid that the cursor currently points to. +*/ +static i64 fts5CursorRowid(Fts5Cursor *pCsr){ + assert( pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + || pCsr->ePlan==FTS5_PLAN_SOURCE + || pCsr->ePlan==FTS5_PLAN_SCAN + || pCsr->ePlan==FTS5_PLAN_ROWID + ); + if( pCsr->pSorter ){ + return pCsr->pSorter->iRowid; + }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){ + return sqlite3_column_int64(pCsr->pStmt, 0); + }else{ + return sqlite3Fts5ExprRowid(pCsr->pExpr); + } +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts5 +** exposes %_content.rowid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int ePlan = pCsr->ePlan; + + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + if( ePlan==FTS5_PLAN_SPECIAL ){ + *pRowid = 0; + }else{ + *pRowid = fts5CursorRowid(pCsr); + } + + return SQLITE_OK; +} + + +/* +** If the cursor requires seeking (bSeekRequired flag is set), seek it. +** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. +** +** If argument bErrormsg is true and an error occurs, an error message may +** be left in sqlite3_vtab.zErrMsg. +*/ +static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ + int rc = SQLITE_OK; + + /* If the cursor does not yet have a statement handle, obtain one now. */ + if( pCsr->pStmt==0 ){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + int eStmt = fts5StmtType(pCsr); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->p.base.zErrMsg:0) + ); + assert( rc!=SQLITE_OK || pTab->p.base.zErrMsg==0 ); + assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); + } + + if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + assert( pCsr->pExpr ); + sqlite3_reset(pCsr->pStmt); + sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); + pTab->pConfig->bLock++; + rc = sqlite3_step(pCsr->pStmt); + pTab->pConfig->bLock--; + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT); + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + fts5SetVtabError((Fts5FullTable*)pTab, + "fts5: missing row %lld from content table %s", + fts5CursorRowid(pCsr), + pTab->pConfig->zContent + ); + }else if( pTab->pConfig->pzErrmsg ){ + fts5SetVtabError((Fts5FullTable*)pTab, + "%s", sqlite3_errmsg(pTab->pConfig->db) + ); + } + } + } + return rc; +} + +/* +** This function is called to handle an FTS INSERT command. In other words, +** an INSERT statement of the form: +** +** INSERT INTO fts(fts) VALUES($pCmd) +** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) +** +** Argument pVal is the value assigned to column "fts" by the INSERT +** statement. This function returns SQLITE_OK if successful, or an SQLite +** error code if an error occurs. +** +** The commands implemented by this function are documented in the "Special +** INSERT Directives" section of the documentation. It should be updated if +** more commands are added to this function. +*/ +static int fts5SpecialInsert( + Fts5FullTable *pTab, /* Fts5 table object */ + const char *zCmd, /* Text inserted into table-name column */ + sqlite3_value *pVal /* Value inserted into rank column */ +){ + Fts5Config *pConfig = pTab->p.pConfig; + int rc = SQLITE_OK; + int bError = 0; + int bLoadConfig = 0; + + if( 0==sqlite3_stricmp("delete-all", zCmd) ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5SetVtabError(pTab, + "'delete-all' may only be used with a " + "contentless or external content fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); + } + bLoadConfig = 1; + }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ + if( fts5IsContentless(pTab, 1) ){ + fts5SetVtabError(pTab, + "'rebuild' may not be used with a contentless fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageRebuild(pTab->pStorage); + } + bLoadConfig = 1; + }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ + rc = sqlite3Fts5StorageOptimize(pTab->pStorage); + }else if( 0==sqlite3_stricmp("merge", zCmd) ){ + int nMerge = sqlite3_value_int(pVal); + rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); + }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ + int iArg = sqlite3_value_int(pVal); + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg); +#ifdef SQLITE_DEBUG + }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ + pConfig->bPrefixIndex = sqlite3_value_int(pVal); +#endif + }else if( 0==sqlite3_stricmp("flush", zCmd) ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + }else{ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError); + } + if( rc==SQLITE_OK ){ + if( bError ){ + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0); + } + } + } + + if( rc==SQLITE_OK && bLoadConfig ){ + pTab->p.pConfig->iCookie--; + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + + return rc; +} + +static int fts5SpecialDelete( + Fts5FullTable *pTab, + sqlite3_value **apVal +){ + int rc = SQLITE_OK; + int eType1 = sqlite3_value_type(apVal[1]); + if( eType1==SQLITE_INTEGER ){ + sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0); + } + return rc; +} + +static void fts5StorageInsert( + int *pRc, + Fts5FullTable *pTab, + sqlite3_value **apVal, + i64 *piRowid +){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); + } + *pRc = rc; +} + +/* +** +** This function is called when the user attempts an UPDATE on a contentless +** table. Parameter bRowidModified is true if the UPDATE statement modifies +** the rowid value. Parameter apVal[] contains the new values for each user +** defined column of the fts5 table. pConfig is the configuration object of the +** table being updated (guaranteed to be contentless). The contentless_delete=1 +** and contentless_unindexed=1 options may or may not be set. +** +** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite +** error code if it cannot. In this case an error message is also loaded into +** pConfig. Output parameter (*pbContent) is set to true if the caller should +** update the %_content table only - not the FTS index or any other shadow +** table. This occurs when an UPDATE modifies only UNINDEXED columns of the +** table. +** +** An UPDATE may proceed if: +** +** * The only columns modified are UNINDEXED columns, or +** +** * The contentless_delete=1 option was specified and all of the indexed +** columns (not a subset) have been modified. +*/ +static int fts5ContentlessUpdate( + Fts5Config *pConfig, + sqlite3_value **apVal, + int bRowidModified, + int *pbContent +){ + int ii; + int bSeenIndex = 0; /* Have seen modified indexed column */ + int bSeenIndexNC = 0; /* Have seen unmodified indexed column */ + int rc = SQLITE_OK; + + for(ii=0; iinCol; ii++){ + if( pConfig->abUnindexed[ii]==0 ){ + if( sqlite3_value_nochange(apVal[ii]) ){ + bSeenIndexNC++; + }else{ + bSeenIndex++; + } + } + } + + if( bSeenIndex==0 && bRowidModified==0 ){ + *pbContent = 1; + }else{ + if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){ + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, + (pConfig->bContentlessDelete ? + "%s a subset of columns on fts5 contentless-delete table: %s" : + "%s contentless fts5 table: %s") + , "cannot UPDATE", pConfig->zName + ); + } + } + + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +** +** A delete specifies a single argument - the rowid of the row to remove. +** +** Update and insert operations pass: +** +** 1. The "old" rowid, or NULL. +** 2. The "new" rowid. +** 3. Values for each of the nCol matchable columns. +** 4. Values for the two hidden columns ( and "rank"). +*/ +static int fts5UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + Fts5Config *pConfig = pTab->p.pConfig; + int eType0; /* value_type() of apVal[0] */ + int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; + + /* A transaction must be open when this is called. */ + assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); + + assert( pVtab->zErrMsg==0 ); + assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER + || sqlite3_value_type(apVal[0])==SQLITE_NULL + ); + assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + + /* Put any active cursors into REQUIRE_SEEK state. */ + fts5TripCursors(pTab); + + eType0 = sqlite3_value_type(apVal[0]); + if( eType0==SQLITE_NULL + && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL + ){ + /* A "special" INSERT op. These are handled separately. */ + const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && 0==sqlite3_stricmp("delete", z) + ){ + if( pConfig->bContentlessDelete ){ + fts5SetVtabError(pTab, + "'delete' may not be used with a contentless_delete=1 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; + } + }else{ + rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); + } + }else{ + /* A regular INSERT, UPDATE or DELETE statement. The trick here is that + ** any conflict on the rowid value must be detected before any + ** modifications are made to the database file. There are 4 cases: + ** + ** 1) DELETE + ** 2) UPDATE (rowid not modified) + ** 3) UPDATE (rowid modified) + ** 4) INSERT + ** + ** Cases 3 and 4 may violate the rowid constraint. + */ + int eConflict = SQLITE_ABORT; + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ + eConflict = sqlite3_vtab_on_conflict(pConfig->db); + } + + assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); + assert( nArg!=1 || eType0==SQLITE_INTEGER ); + + /* DELETE */ + if( nArg==1 ){ + /* It is only possible to DELETE from a contentless table if the + ** contentless_delete=1 flag is set. */ + if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){ + fts5SetVtabError(pTab, + "cannot DELETE from contentless fts5 table: %s", pConfig->zName + ); + rc = SQLITE_ERROR; + }else{ + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0); + bUpdateOrDelete = 1; + } + } + + /* INSERT or UPDATE */ + else{ + int eType1 = sqlite3_value_numeric_type(apVal[1]); + + /* It is an error to write an fts5_locale() value to a table without + ** the locale=1 option. */ + if( pConfig->bLocale==0 ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + rc = SQLITE_MISMATCH; + goto update_out; + } + } + } + + if( eType0!=SQLITE_INTEGER ){ + /* An INSERT statement. If the conflict-mode is REPLACE, first remove + ** the current entry (if any). */ + if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ + i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); + bUpdateOrDelete = 1; + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + + /* UPDATE */ + else{ + Fts5Storage *pStorage = pTab->pStorage; + i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ + i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ + int bContent = 0; /* Content only update */ + + /* If this is a contentless table (including contentless_unindexed=1 + ** tables), check if the UPDATE may proceed. */ + if( fts5IsContentless(pTab, 1) ){ + rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent); + if( rc!=SQLITE_OK ) goto update_out; + } + + if( eType1!=SQLITE_INTEGER ){ + rc = SQLITE_MISMATCH; + }else if( iOld!=iNew ){ + assert( bContent==0 ); + if( eConflict==SQLITE_REPLACE ){ + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + }else{ + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid); + } + } + }else if( bContent ){ + /* This occurs when an UPDATE on a contentless table affects *only* + ** UNINDEXED columns. This is a no-op for contentless_unindexed=0 + ** tables, or a write to the %_content table only for =1 tables. */ + assert( fts5IsContentless(pTab, 1) ); + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid); + } + }else{ + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + bUpdateOrDelete = 1; + sqlite3Fts5StorageReleaseDeleteRow(pStorage); + } + + } + } + + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + + update_out: + pTab->p.pConfig->pzErrmsg = 0; + return rc; +} + +/* +** Implementation of xSync() method. +*/ +static int fts5SyncMethod(sqlite3_vtab *pVtab){ + int rc; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + fts5CheckTransactionState(pTab, FTS5_SYNC, 0); + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = sqlite3Fts5FlushToDisk(&pTab->p); + pTab->p.pConfig->pzErrmsg = 0; + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts5BeginMethod(sqlite3_vtab *pVtab){ + int rc = fts5NewTransaction((Fts5FullTable*)pVtab); + if( rc==SQLITE_OK ){ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); + } + return rc; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts5SyncMethod(). +*/ +static int fts5CommitMethod(sqlite3_vtab *pVtab){ + UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_COMMIT, 0); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts5RollbackMethod(sqlite3_vtab *pVtab){ + int rc; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + return rc; +} + +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); + +static void *fts5ApiUserData(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return pCsr->pAux->pUserData; +} + +static int fts5ApiColumnCount(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; +} + +static int fts5ApiColumnTotalSize( + Fts5Context *pCtx, + int iCol, + sqlite3_int64 *pnToken +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken); +} + +static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); +} + +/* +** Implementation of xTokenize_v2() API. +*/ +static int fts5ApiTokenize_v2( + Fts5Context *pCtx, + const char *pText, int nText, + const char *pLoc, int nLoc, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + int rc = SQLITE_OK; + + sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pTab->pConfig, + FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); + sqlite3Fts5SetLocale(pTab->pConfig, 0, 0); + + return rc; +} + +/* +** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0 +** passed as the locale. +*/ +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken); +} + +static int fts5ApiPhraseCount(Fts5Context *pCtx){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprPhraseCount(pCsr->pExpr); +} + +static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); +} + +/* +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This +** function extracts the text value of column iCol of the current row. +** Additionally, if there is an associated locale, it invokes +** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller +** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point +** after this function returns. +** +** If successful, (*ppText) is set to point to a buffer containing the text +** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that +** buffer in bytes. It is not guaranteed to be nul-terminated. If an error +** occurs, an SQLite error code is returned. The final values of the two +** output parameters are undefined in this case. +*/ +static int fts5TextFromStmt( + Fts5Config *pConfig, + sqlite3_stmt *pStmt, + int iCol, + const char **ppText, + int *pnText +){ + sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1); + const char *pLoc = 0; + int nLoc = 0; + int rc = SQLITE_OK; + + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc); + }else{ + *ppText = (const char*)sqlite3_value_text(pVal); + *pnText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol); + nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol); + } + } + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + return rc; +} + +static int fts5ApiColumnText( + Fts5Context *pCtx, + int iCol, + const char **pz, + int *pn +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pTab->pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){ + *pz = 0; + *pn = 0; + }else{ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn); + sqlite3Fts5ClearLocale(pTab->pConfig); + } + } + return rc; +} + +/* +** This is called by various API functions - xInst, xPhraseFirst, +** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase +** of the current row. This function works for both detail=full tables (in +** which case the position-list was read from the fts index) or for other +** detail= modes if the row content is available. +*/ +static int fts5CsrPoslist( + Fts5Cursor *pCsr, /* Fts5 cursor object */ + int iPhrase, /* Phrase to find position list for */ + const u8 **pa, /* OUT: Pointer to position list buffer */ + int *pn /* OUT: Size of (*pa) in bytes */ +){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + int rc = SQLITE_OK; + int bLive = (pCsr->pSorter==0); + + if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ + rc = SQLITE_RANGE; + }else if( pConfig->eDetail!=FTS5_DETAIL_FULL + && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + ){ + *pa = 0; + *pn = 0; + return SQLITE_OK; + }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5PoslistPopulator *aPopulator; + int i; + + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); + if( aPopulator==0 ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = fts5SeekCursor(pCsr, 0); + } + for(i=0; inCol && rc==SQLITE_OK; i++){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aPopulator, i, z, n + ); + } + sqlite3Fts5ClearLocale(pConfig); + } + sqlite3_free(aPopulator); + + if( pCsr->pSorter ){ + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); + } + + if( rc==SQLITE_OK ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } + }else{ + *pa = 0; + *pn = 0; + } + + return rc; +} + +/* +** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated +** correctly for the current view. Return SQLITE_OK if successful, or an +** SQLite error code otherwise. +*/ +static int fts5CacheInstArray(Fts5Cursor *pCsr){ + int rc = SQLITE_OK; + Fts5PoslistReader *aIter; /* One iterator for each phrase */ + int nIter; /* Number of iterators/phrases */ + int nCol = ((Fts5Table*)pCsr->base.pVtab)->pConfig->nCol; + + nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + if( pCsr->aInstIter==0 ){ + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nIter; + pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); + } + aIter = pCsr->aInstIter; + + if( aIter ){ + int nInst = 0; /* Number instances seen so far */ + int i; + + /* Initialize all iterators */ + for(i=0; i=pCsr->nInstAlloc ){ + int nNewSize = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc64( + pCsr->aInst, nNewSize*sizeof(int)*3 + ); + if( aInst ){ + pCsr->aInst = aInst; + pCsr->nInstAlloc = nNewSize; + }else{ + nInst--; + rc = SQLITE_NOMEM; + break; + } + } + + aInst = &pCsr->aInst[3 * (nInst-1)]; + aInst[0] = iBest; + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + assert( aInst[1]>=0 ); + if( aInst[1]>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + } + } + + pCsr->nInstCount = nInst; + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); + } + return rc; +} + +static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ + *pnInst = pCsr->nInstCount; + } + return rc; +} + +static int fts5ApiInst( + Fts5Context *pCtx, + int iIdx, + int *piPhrase, + int *piCol, + int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; + }else{ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 1]; + *piOff = pCsr->aInst[iIdx*3 + 2]; + } + } + return rc; +} + +static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ + return fts5CursorRowid((Fts5Cursor*)pCtx); +} + +static int fts5ColumnSizeCb( + void *pContext, /* Pointer to int */ + int tflags, + const char *pUnused, /* Buffer containing token */ + int nUnused, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + int *pCnt = (int*)pContext; + UNUSED_PARAM2(pUnused, nUnused); + UNUSED_PARAM2(iUnused1, iUnused2); + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + (*pCnt)++; + } + return SQLITE_OK; +} + +static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + int rc = SQLITE_OK; + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ + if( pConfig->bColumnsize ){ + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); + }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ + int i; + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + pCsr->aColumnSize[i] = -1; + } + } + }else{ + int i; + rc = fts5SeekCursor(pCsr, 0); + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + const char *z = 0; + int n = 0; + pCsr->aColumnSize[i] = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, + z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb + ); + } + sqlite3Fts5ClearLocale(pConfig); + } + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); + } + if( iCol<0 ){ + int i; + *pnToken = 0; + for(i=0; inCol; i++){ + *pnToken += pCsr->aColumnSize[i]; + } + }else if( iColnCol ){ + *pnToken = pCsr->aColumnSize[iCol]; + }else{ + *pnToken = 0; + rc = SQLITE_RANGE; + } + return rc; +} + +/* +** Implementation of the xSetAuxdata() method. +*/ +static int fts5ApiSetAuxdata( + Fts5Context *pCtx, /* Fts5 context */ + void *pPtr, /* Pointer to save as auxdata */ + void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */ +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Auxdata *pData; + + /* Search through the cursors list of Fts5Auxdata objects for one that + ** corresponds to the currently executing auxiliary function. */ + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ + if( pData->pAux==pCsr->pAux ) break; + } + + if( pData ){ + if( pData->xDelete ){ + pData->xDelete(pData->pPtr); + } + }else{ + int rc = SQLITE_OK; + pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata)); + if( pData==0 ){ + if( xDelete ) xDelete(pPtr); + return rc; + } + pData->pAux = pCsr->pAux; + pData->pNext = pCsr->pAuxdata; + pCsr->pAuxdata = pData; + } + + pData->xDelete = xDelete; + pData->pPtr = pPtr; + return SQLITE_OK; +} + +static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Auxdata *pData; + void *pRet = 0; + + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ + if( pData->pAux==pCsr->pAux ) break; + } + + if( pData ){ + pRet = pData->pPtr; + if( bClear ){ + pData->pPtr = 0; + pData->xDelete = 0; + } + } + + return pRet; +} + +static void fts5ApiPhraseNext( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + *piOff = -1; + }else{ + int iVal; + pIter->a += fts5GetVarint32(pIter->a, iVal); + if( iVal==1 ){ + /* Avoid returning a (*piCol) value that is too large for the table, + ** even if the position-list is corrupt. The caller might not be + ** expecting it. */ + int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol; + pIter->a += fts5GetVarint32(pIter->a, iVal); + *piCol = (iVal>=nCol ? nCol-1 : iVal); + *piOff = 0; + pIter->a += fts5GetVarint32(pIter->a, iVal); + } + *piOff += (iVal-2); + } +} + +static int fts5ApiPhraseFirst( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol, int *piOff +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int n; + int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); + *piCol = 0; + *piOff = 0; + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + } + return rc; +} + +static void fts5ApiPhraseNextColumn( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + }else{ + int iIncr; + pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); + *piCol += (iIncr-2); + } + }else{ + while( 1 ){ + int dummy; + if( pIter->a>=pIter->b ){ + *piCol = -1; + return; + } + if( pIter->a[0]==0x01 ) break; + pIter->a += fts5GetVarint32(pIter->a, dummy); + } + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + } +} + +static int fts5ApiPhraseFirstColumn( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int n; + if( pSorter ){ + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + n = pSorter->aIdx[iPhrase] - i1; + pIter->a = &pSorter->aPoslist[i1]; + }else{ + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + } + if( rc==SQLITE_OK ){ + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); + *piCol = 0; + fts5ApiPhraseNextColumn(pCtx, pIter, piCol); + } + }else{ + int n; + rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); + if( n<=0 ){ + *piCol = -1; + }else if( pIter->a[0]==0x01 ){ + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + }else{ + *piCol = 0; + } + } + } + + return rc; +} + +/* +** xQueryToken() API implemenetation. +*/ +static int fts5ApiQueryToken( + Fts5Context* pCtx, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); +} + +/* +** xInstToken() API implemenetation. +*/ +static int fts5ApiInstToken( + Fts5Context *pCtx, + int iIdx, + int iToken, + const char **ppOut, int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; + }else{ + int iPhrase = pCsr->aInst[iIdx*3]; + int iCol = pCsr->aInst[iIdx*3 + 1]; + int iOff = pCsr->aInst[iIdx*3 + 2]; + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5ExprInstToken( + pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut + ); + } + } + return rc; +} + + +static int fts5ApiQueryPhrase(Fts5Context*, int, void*, + int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) +); + +/* +** The xColumnLocale() API. +*/ +static int fts5ApiColumnLocale( + Fts5Context *pCtx, + int iCol, + const char **pzLocale, + int *pnLocale +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + *pzLocale = 0; + *pnLocale = 0; + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( + pConfig->abUnindexed[iCol]==0 + && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + && pConfig->bLocale + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + const char *zDummy = 0; + int nDummy = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy); + if( rc==SQLITE_OK ){ + *pzLocale = pConfig->t.pLocale; + *pnLocale = pConfig->t.nLocale; + } + sqlite3Fts5ClearLocale(pConfig); + } + } + + return rc; +} + +static const Fts5ExtensionApi sFts5Api = { + 4, /* iVersion */ + fts5ApiUserData, + fts5ApiColumnCount, + fts5ApiRowCount, + fts5ApiColumnTotalSize, + fts5ApiTokenize, + fts5ApiPhraseCount, + fts5ApiPhraseSize, + fts5ApiInstCount, + fts5ApiInst, + fts5ApiRowid, + fts5ApiColumnText, + fts5ApiColumnSize, + fts5ApiQueryPhrase, + fts5ApiSetAuxdata, + fts5ApiGetAuxdata, + fts5ApiPhraseFirst, + fts5ApiPhraseNext, + fts5ApiPhraseFirstColumn, + fts5ApiPhraseNextColumn, + fts5ApiQueryToken, + fts5ApiInstToken, + fts5ApiColumnLocale, + fts5ApiTokenize_v2 +}; + +/* +** Implementation of API function xQueryPhrase(). +*/ +static int fts5ApiQueryPhrase( + Fts5Context *pCtx, + int iPhrase, + void *pUserData, + int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*) +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + int rc; + Fts5Cursor *pNew = 0; + + rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); + if( rc==SQLITE_OK ){ + pNew->ePlan = FTS5_PLAN_MATCH; + pNew->iFirstRowid = SMALLEST_INT64; + pNew->iLastRowid = LARGEST_INT64; + pNew->base.pVtab = (sqlite3_vtab*)pTab; + rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr); + } + + if( rc==SQLITE_OK ){ + for(rc = fts5CursorFirst(pTab, pNew, 0); + rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; + rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) + ){ + rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + break; + } + } + } + + fts5CloseMethod((sqlite3_vtab_cursor*)pNew); + return rc; +} + +static void fts5ApiInvoke( + Fts5Auxiliary *pAux, + Fts5Cursor *pCsr, + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( pCsr->pAux==0 ); + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + pCsr->pAux = pAux; + pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); + pCsr->pAux = 0; +} + +static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ + Fts5Cursor *pCsr; + for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->iCsrId==iCsrId ) break; + } + return pCsr; +} + +/* +** Parameter zFmt is a printf() style formatting string. This function +** formats it using the trailing arguments and returns the result as +** an error message to the context passed as the first argument. +*/ +static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + va_end(ap); +} + +static void fts5ApiCallback( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + + Fts5Auxiliary *pAux; + Fts5Cursor *pCsr; + i64 iCsrId; + + assert( argc>=1 ); + pAux = (Fts5Auxiliary*)sqlite3_user_data(context); + iCsrId = sqlite3_value_int64(argv[0]); + + pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); + if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){ + fts5ResultError(context, "no such cursor: %lld", iCsrId); + }else{ + sqlite3_vtab *pTab = pCsr->base.pVtab; + fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + sqlite3_free(pTab->zErrMsg); + pTab->zErrMsg = 0; + } +} + + +/* +** Given cursor id iId, return a pointer to the corresponding Fts5Table +** object. Or NULL If the cursor id does not exist. +*/ +Fts5Table *sqlite3Fts5TableFromCsrid( + Fts5Global *pGlobal, /* FTS5 global context for db handle */ + i64 iCsrId /* Id of cursor to find */ +){ + Fts5Cursor *pCsr; + pCsr = fts5CursorFromCsrid(pGlobal, iCsrId); + if( pCsr ){ + return (Fts5Table*)pCsr->base.pVtab; + } + return 0; +} + +/* +** Return a "position-list blob" corresponding to the current position of +** cursor pCsr via sqlite3_result_blob(). A position-list blob contains +** the current position-list for each phrase in the query associated with +** cursor pCsr. +** +** A position-list blob begins with (nPhrase-1) varints, where nPhrase is +** the number of phrases in the query. Following the varints are the +** concatenated position lists for each phrase, in order. +** +** The first varint (if it exists) contains the size of the position list +** for phrase 0. The second (same disclaimer) contains the size of position +** list 1. And so on. There is no size field for the final position list, +** as it can be derived from the total size of the blob. +*/ +static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ + int i; + int rc = SQLITE_OK; + int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); + Fts5Buffer val; + + memset(&val, 0, sizeof(Fts5Buffer)); + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + + /* Append the varints */ + for(i=0; i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; ipExpr, i, &pPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + case FTS5_DETAIL_COLUMNS: + + /* Append the varints */ + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; rc==SQLITE_OK && ipExpr, i, &pPoslist, &nPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + default: + break; + } + + sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); + return rc; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +*/ +static int fts5ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->p.pConfig; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; + + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + + if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){ + if( iCol==pConfig->nCol ){ + sqlite3_result_int64(pCtx, pCsr->iSpecial); + } + }else + + if( iCol==pConfig->nCol ){ + /* User is requesting the value of the special column with the same name + ** as the table. Return the cursor integer id number. This value is only + ** useful in that it may be passed as the first argument to an FTS5 + ** auxiliary function. */ + sqlite3_result_int64(pCtx, pCsr->iCsrId); + }else if( iCol==pConfig->nCol+1 ){ + /* The value of the "rank" column. */ + + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ + fts5PoslistBlob(pCtx, pCsr); + }else if( + pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + ){ + if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){ + fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); + } + } + }else{ + if( !sqlite3_vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT); + } + sqlite3Fts5ClearLocale(pConfig); + }else{ + sqlite3_result_value(pCtx, pVal); + } + } + + pConfig->pzErrmsg = 0; + } + } + + return rc; +} + + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts5FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nUnused, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* OUT: User data for *pxFunc */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + Fts5Auxiliary *pAux; + + UNUSED_PARAM(nUnused); + pAux = fts5FindAuxiliary(pTab, zName); + if( pAux ){ + *pxFunc = fts5ApiCallback; + *ppArg = (void*)pAux; + return 1; + } + + /* No function of the specified name was found. Return 0. */ + return 0; +} + +/* +** Implementation of FTS5 xRename method. Rename an fts5 table. +*/ +static int fts5RenameMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + const char *zName /* New name of table */ +){ + int rc; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); + return rc; +} + +int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ + fts5TripCursors((Fts5FullTable*)pTab); + return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage); +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); + rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); + if( (iSavepoint+1)iSavepoint ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint; + } + } + return rc; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); + fts5TripCursors(pTab); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + pTab->p.pConfig->pgsz = 0; + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + } + return rc; +} + +/* +** Register a new auxiliary function with global context pGlobal. +*/ +static int fts5CreateAux( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_extension_function xFunc, /* Aux. function implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = sqlite3_overload_function(pGlobal->db, zName, -1); + if( rc==SQLITE_OK ){ + Fts5Auxiliary *pAux; + sqlite3_int64 nName; /* Size of zName in bytes, including \0 */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5Auxiliary) + nName; + pAux = (Fts5Auxiliary*)sqlite3_malloc64(nByte); + if( pAux ){ + memset(pAux, 0, (size_t)nByte); + pAux->zFunc = (char*)&pAux[1]; + memcpy(pAux->zFunc, zName, nName); + pAux->pGlobal = pGlobal; + pAux->pUserData = pUserData; + pAux->xFunc = xFunc; + pAux->xDestroy = xDestroy; + pAux->pNext = pGlobal->pAux; + pGlobal->pAux = pAux; + }else{ + rc = SQLITE_NOMEM; + } + } + + return rc; +} + +/* +** This function is used by xCreateTokenizer_v2() and xCreateTokenizer(). +** It allocates and partially populates a new Fts5TokenizerModule object. +** The new object is already linked into the Fts5Global context before +** returning. +** +** If successful, SQLITE_OK is returned and a pointer to the new +** Fts5TokenizerModule object returned via output parameter (*ppNew). All +** that is required is for the caller to fill in the methods in +** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native +** as appropriate. +** +** If an error occurs, an SQLite error code is returned and the final value +** of (*ppNew) undefined. +*/ +static int fts5NewTokenizerModule( + Fts5Global *pGlobal, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + void(*xDestroy)(void*), /* Destructor for pUserData */ + Fts5TokenizerModule **ppNew +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; + *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); + if( pNew ){ + pNew->zName = (char*)&pNew[1]; + memcpy(pNew->zName, zName, nName); + pNew->pUserData = pUserData; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + if( pNew->pNext==0 ){ + pGlobal->pDfltTok = pNew; + } + } + + return rc; +} + +/* +** An instance of this type is used as the Fts5Tokenizer object for +** wrapper tokenizers - those that provide access to a v1 tokenizer via +** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer +** via the fts5_tokenizer API. +*/ +typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; +struct Fts5VtoVTokenizer { + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ + Fts5Tokenizer *pReal; +}; + +/* +** Create a wrapper tokenizer. The context argument pCtx points to the +** Fts5TokenizerModule object. +*/ +static int fts5VtoVCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx; + Fts5VtoVTokenizer *pNew = 0; + int rc = SQLITE_OK; + + pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + if( rc==SQLITE_OK ){ + pNew->x1 = pMod->x1; + pNew->x2 = pMod->x2; + pNew->bV2Native = pMod->bV2Native; + if( pMod->bV2Native ){ + rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + }else{ + rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + } + } + + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Delete an Fts5VtoVTokenizer wrapper tokenizer. +*/ +static void fts5VtoVDelete(Fts5Tokenizer *pTok){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + if( p ){ + if( p->bV2Native ){ + p->x2.xDelete(p->pReal); + }else{ + p->x1.xDelete(p->pReal); + } + sqlite3_free(p); + } +} + + +/* +** xTokenizer method for a wrapper tokenizer that offers the v1 interface +** (no support for locales). +*/ +static int fts5V1toV2Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native ); + return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** xTokenizer method for a wrapper tokenizer that offers the v2 interface +** (with locale support). +*/ +static int fts5V2toV1Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native==0 ); + UNUSED_PARAM2(pLocale,nLocale); + return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer_v2() method. +*/ +static int fts5CreateTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + + if( pTokenizer->iVersion>2 ){ + rc = SQLITE_ERROR; + }else{ + Fts5TokenizerModule *pNew = 0; + rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew); + if( pNew ){ + pNew->x2 = *pTokenizer; + pNew->bV2Native = 1; + pNew->x1.xCreate = fts5VtoVCreate; + pNew->x1.xTokenize = fts5V1toV2Tokenize; + pNew->x1.xDelete = fts5VtoVDelete; + } + } + + return rc; +} + +/* +** The fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5TokenizerModule *pNew = 0; + int rc = SQLITE_OK; + + rc = fts5NewTokenizerModule( + (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew + ); + if( pNew ){ + pNew->x1 = *pTokenizer; + pNew->x2.xCreate = fts5VtoVCreate; + pNew->x2.xTokenize = fts5V2toV1Tokenize; + pNew->x2.xDelete = fts5VtoVDelete; + } + return rc; +} + +/* +** Search the global context passed as the first argument for a tokenizer +** module named zName. If found, return a pointer to the Fts5TokenizerModule +** object. Otherwise, return NULL. +*/ +static Fts5TokenizerModule *fts5LocateTokenizer( + Fts5Global *pGlobal, /* Global (one per db handle) object */ + const char *zName /* Name of tokenizer module to find */ +){ + Fts5TokenizerModule *pMod = 0; + + if( zName==0 ){ + pMod = pGlobal->pDfltTok; + }else{ + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ + if( sqlite3_stricmp(zName, pMod->zName)==0 ) break; + } + } + + return pMod; +} + +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer_v2() method. +*/ +static int fts5FindTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of tokenizer */ + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *ppTokenizer = &pMod->x2; + }else{ + *ppTokenizer = 0; + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer() method. +*/ +static int fts5FindTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void **ppUserData, + fts5_tokenizer *pTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native==0 ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *pTokenizer = pMod->x1; + }else{ + memset(pTokenizer, 0, sizeof(*pTokenizer)); + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Attempt to instantiate the tokenizer. +*/ +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ + const char **azArg = pConfig->t.azArg; + const int nArg = pConfig->t.nArg; + Fts5TokenizerModule *pMod = 0; + int rc = SQLITE_OK; + + pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]); + if( pMod==0 ){ + assert( nArg>0 ); + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]); + }else{ + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; + if( pMod->bV2Native ){ + xCreate = pMod->x2.xCreate; + pConfig->t.pApi2 = &pMod->x2; + }else{ + pConfig->t.pApi1 = &pMod->x1; + xCreate = pMod->x1.xCreate; + } + + rc = xCreate(pMod->pUserData, + (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok + ); + + if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_NOMEM ){ + sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor"); + } + }else if( pMod->bV2Native==0 ){ + pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( + pMod->x1.xCreate, pConfig->t.pTok + ); + } + } + + if( rc!=SQLITE_OK ){ + pConfig->t.pApi1 = 0; + pConfig->t.pApi2 = 0; + pConfig->t.pTok = 0; + } + + return rc; +} + + +/* +** xDestroy callback passed to sqlite3_create_module(). This is invoked +** when the db handle is being closed. Free memory associated with +** tokenizers and aux functions registered with this db handle. +*/ +static void fts5ModuleDestroy(void *pCtx){ + Fts5TokenizerModule *pTok, *pNextTok; + Fts5Auxiliary *pAux, *pNextAux; + Fts5Global *pGlobal = (Fts5Global*)pCtx; + + for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){ + pNextAux = pAux->pNext; + if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData); + sqlite3_free(pAux); + } + + for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){ + pNextTok = pTok->pNext; + if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData); + sqlite3_free(pTok); + } + + sqlite3_free(pGlobal); +} + +/* +** Implementation of the fts5() function used by clients to obtain the +** API pointer. +*/ +static void fts5Fts5Func( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); + fts5_api **ppApi; + UNUSED_PARAM(nArg); + assert( nArg==1 ); + ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr"); + if( ppApi ) *ppApi = &pGlobal->api; +} + +/* +** Implementation of fts5_source_id() function. +*/ +static void fts5SourceIdFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apUnused /* Function arguments */ +){ + assert( nArg==0 ); + UNUSED_PARAM2(nArg, apUnused); + sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT); +} + +/* +** Implementation of fts5_locale(LOCALE, TEXT) function. +** +** If parameter LOCALE is NULL, or a zero-length string, then a copy of +** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as +** text, and the value returned is a blob consisting of: +** +** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER). +** * The LOCALE, as utf-8 text, followed by +** * 0x00, followed by +** * The TEXT, as utf-8 text. +** +** There is no final nul-terminator following the TEXT value. +*/ +static void fts5LocaleFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + const char *zLocale = 0; + int nLocale = 0; + const char *zText = 0; + int nText = 0; + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + + zLocale = (const char*)sqlite3_value_text(apArg[0]); + nLocale = sqlite3_value_bytes(apArg[0]); + + zText = (const char*)sqlite3_value_text(apArg[1]); + nText = sqlite3_value_bytes(apArg[1]); + + if( zLocale==0 || zLocale[0]=='\0' ){ + sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT); + }else{ + Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); + u8 *pBlob = 0; + u8 *pCsr = 0; + int nBlob = 0; + + nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; + pBlob = (u8*)sqlite3_malloc(nBlob); + if( pBlob==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + + pCsr = pBlob; + memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE); + pCsr += FTS5_LOCALE_HDR_SIZE; + memcpy(pCsr, zLocale, nLocale); + pCsr += nLocale; + (*pCsr++) = 0x00; + if( zText ) memcpy(pCsr, zText, nText); + assert( &pCsr[nText]==&pBlob[nBlob] ); + + sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); + } +} + +/* +** Implementation of fts5_insttoken() function. +*/ +static void fts5InsttokenFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + assert( nArg==1 ); + (void)nArg; + sqlite3_result_value(pCtx, apArg[0]); + sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE); +} + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts5ShadowName(const char *zName){ + static const char *azName[] = { + "config", "content", "data", "docsize", "idx" + }; + unsigned int i; + for(i=0; ip.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } + } + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; + + return rc; +} + +static int fts5Init(sqlite3 *db){ + static const sqlite3_module fts5Mod = { + /* iVersion */ 4, + /* xCreate */ fts5CreateMethod, + /* xConnect */ fts5ConnectMethod, + /* xBestIndex */ fts5BestIndexMethod, + /* xDisconnect */ fts5DisconnectMethod, + /* xDestroy */ fts5DestroyMethod, + /* xOpen */ fts5OpenMethod, + /* xClose */ fts5CloseMethod, + /* xFilter */ fts5FilterMethod, + /* xNext */ fts5NextMethod, + /* xEof */ fts5EofMethod, + /* xColumn */ fts5ColumnMethod, + /* xRowid */ fts5RowidMethod, + /* xUpdate */ fts5UpdateMethod, + /* xBegin */ fts5BeginMethod, + /* xSync */ fts5SyncMethod, + /* xCommit */ fts5CommitMethod, + /* xRollback */ fts5RollbackMethod, + /* xFindFunction */ fts5FindFunctionMethod, + /* xRename */ fts5RenameMethod, + /* xSavepoint */ fts5SavepointMethod, + /* xRelease */ fts5ReleaseMethod, + /* xRollbackTo */ fts5RollbackToMethod, + /* xShadowName */ fts5ShadowName, + /* xIntegrity */ fts5IntegrityMethod + }; + + int rc; + Fts5Global *pGlobal = 0; + + pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); + if( pGlobal==0 ){ + rc = SQLITE_NOMEM; + }else{ + void *p = (void*)pGlobal; + memset(pGlobal, 0, sizeof(Fts5Global)); + pGlobal->db = db; + pGlobal->api.iVersion = 3; + pGlobal->api.xCreateFunction = fts5CreateAux; + pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; + pGlobal->api.xFindTokenizer = fts5FindTokenizer; + pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; + pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; + + /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector. + ** The constants below were generated randomly. */ + sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr); + pGlobal->aLocaleHdr[0] ^= 0xF924976D; + pGlobal->aLocaleHdr[1] ^= 0x16596E13; + pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA; + pGlobal->aLocaleHdr[3] ^= 0x9B03A67F; + assert( sizeof(pGlobal->aLocaleHdr)==16 ); + + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); + if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); + if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_source_id", 0, + SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS, + p, fts5SourceIdFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_locale", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE, + p, fts5LocaleFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_insttoken", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5InsttokenFunc, 0, 0 + ); + } + } + + /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file + ** fts5_test_mi.c is compiled and linked into the executable. And call + ** its entry point to enable the matchinfo() demo. */ +#ifdef SQLITE_FTS5_ENABLE_TEST_MI + if( rc==SQLITE_OK ){ + extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); + rc = sqlite3Fts5TestRegisterMatchinfo(db); + } +#endif + + return rc; +} + +/* +** The following functions are used to register the module with SQLite. If +** this module is being built as part of the SQLite core (SQLITE_CORE is +** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. +** +** Or, if this module is being built as a loadable extension, +** sqlite3Fts5Init() is omitted and the two standard entry points +** sqlite3_fts_init() and sqlite3_fts5_init() defined instead. +*/ +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts5_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} +#else +int sqlite3Fts5Init(sqlite3 *db){ + return fts5Init(db); +} +#endif diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c new file mode 100644 index 0000000000000000000000000000000000000000..31f5fc5dc3b19d7662ee73e6725c33e9e054b95b --- /dev/null +++ b/ext/fts5/fts5_storage.c @@ -0,0 +1,1512 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + + +#include "fts5Int.h" + +/* +** pSavedRow: +** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it +** does a by-rowid lookup to retrieve a single row from the %_content +** table or equivalent external-content table/view. +** +** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original +** values for a row being UPDATEd. In that case, the SQL statement is +** not reset and pSavedRow is set to point at it. This is so that the +** insert operation that follows the delete may access the original +** row values for any new values for which sqlite3_value_nochange() returns +** true. i.e. if the user executes: +** +** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1); +** ... +** UPDATE fts SET a=?, b=? WHERE rowid=?; +** +** then the value passed to the xUpdate() method of this table as the +** new.c value is an sqlite3_value_nochange() value. So in this case it +** must be read from the saved row stored in Fts5Storage.pSavedRow. +** +** This is necessary - using sqlite3_value_nochange() instead of just having +** SQLite pass the original value back via xUpdate() - so as not to discard +** any locale information associated with such values. +** +*/ +struct Fts5Storage { + Fts5Config *pConfig; + Fts5Index *pIndex; + int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ + i64 nTotalRow; /* Total number of rows in FTS table */ + i64 *aTotalSize; /* Total sizes of each column */ + sqlite3_stmt *pSavedRow; + sqlite3_stmt *aStmt[12]; +}; + + +#if FTS5_STMT_SCAN_ASC!=0 +# error "FTS5_STMT_SCAN_ASC mismatch" +#endif +#if FTS5_STMT_SCAN_DESC!=1 +# error "FTS5_STMT_SCAN_DESC mismatch" +#endif +#if FTS5_STMT_LOOKUP!=2 +# error "FTS5_STMT_LOOKUP mismatch" +#endif + +#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_INSERT_CONTENT 4 +#define FTS5_STMT_REPLACE_CONTENT 5 +#define FTS5_STMT_DELETE_CONTENT 6 +#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_DELETE_DOCSIZE 8 +#define FTS5_STMT_LOOKUP_DOCSIZE 9 +#define FTS5_STMT_REPLACE_CONFIG 10 +#define FTS5_STMT_SCAN 11 + +/* +** Prepare the two insert statements - Fts5Storage.pInsertContent and +** Fts5Storage.pInsertDocsize - if they have not already been prepared. +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageGetStmt( + Fts5Storage *p, /* Storage handle */ + int eStmt, /* FTS5_STMT_XXX constant */ + sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ + char **pzErrMsg /* OUT: Error message (if any) */ +){ + int rc = SQLITE_OK; + + /* If there is no %_docsize table, there should be no requests for + ** statements to operate on it. */ + assert( p->pConfig->bColumnsize || ( + eStmt!=FTS5_STMT_REPLACE_DOCSIZE + && eStmt!=FTS5_STMT_DELETE_DOCSIZE + && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE + )); + + assert( eStmt>=0 && eStmtaStmt) ); + if( p->aStmt[eStmt]==0 ){ + const char *azStmt[] = { + "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", + "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */ + + "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ + "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ + "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ + "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ + + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + + "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ + "SELECT %s FROM %s AS T", /* SCAN */ + }; + Fts5Config *pC = p->pConfig; + char *zSql = 0; + + assert( ArraySize(azStmt)==ArraySize(p->aStmt) ); + + switch( eStmt ){ + case FTS5_STMT_SCAN: + zSql = sqlite3_mprintf(azStmt[eStmt], + pC->zContentExprlist, pC->zContent + ); + break; + + case FTS5_STMT_SCAN_ASC: + case FTS5_STMT_SCAN_DESC: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, + pC->zContent, pC->zContentRowid, pC->zContentRowid, + pC->zContentRowid + ); + break; + + case FTS5_STMT_LOOKUP: + case FTS5_STMT_LOOKUP2: + zSql = sqlite3_mprintf(azStmt[eStmt], + pC->zContentExprlist, pC->zContent, pC->zContentRowid + ); + break; + + case FTS5_STMT_INSERT_CONTENT: + case FTS5_STMT_REPLACE_CONTENT: { + char *zBind = 0; + int i; + + assert( pC->eContent==FTS5_CONTENT_NORMAL + || pC->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Add bindings for the "c*" columns - those that store the actual + ** table content. If eContent==NORMAL, then there is one binding + ** for each column. Or, if eContent==UNINDEXED, then there are only + ** bindings for the UNINDEXED columns. */ + for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){ + if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1); + } + } + + /* Add bindings for any "l*" columns. Only non-UNINDEXED columns + ** require these. */ + if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){ + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pC->abUnindexed[i]==0 ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2); + } + } + } + + zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind); + sqlite3_free(zBind); + break; + } + + case FTS5_STMT_REPLACE_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, + (pC->bContentlessDelete ? ",?" : "") + ); + break; + + case FTS5_STMT_LOOKUP_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], + (pC->bContentlessDelete ? ",origin" : ""), + pC->zDb, pC->zName + ); + break; + + default: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); + break; + } + + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int f = SQLITE_PREPARE_PERSISTENT; + if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB; + p->pConfig->bLock++; + rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); + p->pConfig->bLock--; + sqlite3_free(zSql); + if( rc!=SQLITE_OK && pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); + } + } + } + + *ppStmt = p->aStmt[eStmt]; + sqlite3_reset(*ppStmt); + return rc; +} + + +static int fts5ExecPrintf( + sqlite3 *db, + char **pzErr, + const char *zFormat, + ... +){ + int rc; + va_list ap; /* ... printf arguments */ + char *zSql; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, 0, 0, pzErr); + sqlite3_free(zSql); + } + + va_end(ap); + return rc; +} + +/* +** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error +** code otherwise. +*/ +int sqlite3Fts5DropAll(Fts5Config *pConfig){ + int rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_data';" + "DROP TABLE IF EXISTS %Q.'%q_idx';" + "DROP TABLE IF EXISTS %Q.'%q_config';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_docsize';", + pConfig->zDb, pConfig->zName + ); + } + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_content';", + pConfig->zDb, pConfig->zName + ); + } + return rc; +} + +static void fts5StorageRenameOne( + Fts5Config *pConfig, /* Current FTS5 configuration */ + int *pRc, /* IN/OUT: Error code */ + const char *zTail, /* Tail of table name e.g. "data", "config" */ + const char *zName /* New name of FTS5 table */ +){ + if( *pRc==SQLITE_OK ){ + *pRc = fts5ExecPrintf(pConfig->db, 0, + "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';", + pConfig->zDb, pConfig->zName, zTail, zName, zTail + ); + } +} + +int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ + Fts5Config *pConfig = pStorage->pConfig; + int rc = sqlite3Fts5StorageSync(pStorage); + + fts5StorageRenameOne(pConfig, &rc, "data", zName); + fts5StorageRenameOne(pConfig, &rc, "idx", zName); + fts5StorageRenameOne(pConfig, &rc, "config", zName); + if( pConfig->bColumnsize ){ + fts5StorageRenameOne(pConfig, &rc, "docsize", zName); + } + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5StorageRenameOne(pConfig, &rc, "content", zName); + } + return rc; +} + +/* +** Create the shadow table named zPost, with definition zDefn. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +int sqlite3Fts5CreateTable( + Fts5Config *pConfig, /* FTS5 configuration */ + const char *zPost, /* Shadow table to create (e.g. "content") */ + const char *zDefn, /* Columns etc. for shadow table */ + int bWithout, /* True for without rowid */ + char **pzErr /* OUT: Error message */ +){ + int rc; + char *zErr = 0; + + rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", + pConfig->zDb, pConfig->zName, zPost, zDefn, +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID + bWithout?" WITHOUT ROWID": +#endif + "" + ); + if( zErr ){ + *pzErr = sqlite3_mprintf( + "fts5: error creating shadow table %q_%s: %s", + pConfig->zName, zPost, zErr + ); + sqlite3_free(zErr); + } + + return rc; +} + +/* +** Open a new Fts5Index handle. If the bCreate argument is true, create +** and initialize the underlying tables +** +** If successful, set *pp to point to the new object and return SQLITE_OK. +** Otherwise, set *pp to NULL and return an SQLite error code. +*/ +int sqlite3Fts5StorageOpen( + Fts5Config *pConfig, + Fts5Index *pIndex, + int bCreate, + Fts5Storage **pp, + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + Fts5Storage *p; /* New object */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nByte = sizeof(Fts5Storage) /* Fts5Storage object */ + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ + *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + + memset(p, 0, (size_t)nByte); + p->aTotalSize = (i64*)&p[1]; + p->pConfig = pConfig; + p->pIndex = pIndex; + + if( bCreate ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ + int nDefn = 32 + pConfig->nCol*10; + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20); + if( zDefn==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + int iOff; + sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); + iOff = (int)strlen(zDefn); + for(i=0; inCol; i++){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->abUnindexed[i] + ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + if( pConfig->bLocale ){ + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + } + rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); + } + sqlite3_free(zDefn); + } + + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; + if( pConfig->bContentlessDelete ){ + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; + } + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable( + pConfig, "config", "k PRIMARY KEY, v", 1, pzErr + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } + } + + if( rc ){ + sqlite3Fts5StorageClose(p); + *pp = 0; + } + return rc; +} + +/* +** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen(). +*/ +int sqlite3Fts5StorageClose(Fts5Storage *p){ + int rc = SQLITE_OK; + if( p ){ + int i; + + /* Finalize all SQL statements */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + + sqlite3_free(p); + } + return rc; +} + +typedef struct Fts5InsertCtx Fts5InsertCtx; +struct Fts5InsertCtx { + Fts5Storage *pStorage; + int iCol; + int szCol; /* Size of column value in tokens */ +}; + +/* +** Tokenization callback used when inserting tokens into the FTS index. +*/ +static int fts5StorageInsertCallback( + void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; + Fts5Index *pIdx = pCtx->pStorage->pIndex; + UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); +} + +/* +** This function is used as part of an UPDATE statement that modifies the +** rowid of a row. In that case, this function is called first to set +** Fts5Storage.pSavedRow to point to a statement that may be used to +** access the original values of the row being deleted - iDel. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if row iDel does not exist. In this case +** pSavedRow is not set and SQLITE_OK returned. +*/ +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ + int rc = SQLITE_OK; + sqlite3_stmt *pSeek = 0; + + assert( p->pSavedRow==0 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + rc = sqlite3_reset(pSeek); + }else{ + p->pSavedRow = pSeek; + } + } + + return rc; +} + +/* +** If a row with rowid iDel is present in the %_content table, add the +** delete-markers to the FTS index necessary to delete it. Do not actually +** remove the %_content row at this time though. +** +** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left +** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access +** the original values of the row being deleted. This is used by UPDATE +** statements. +*/ +static int fts5StorageDeleteFromIndex( + Fts5Storage *p, + i64 iDel, + sqlite3_value **apVal, + int bSaveRow /* True to set pSavedRow */ +){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ + int rc = SQLITE_OK; /* Return code */ + int rc2; /* sqlite3_reset() return code */ + int iCol; + Fts5InsertCtx ctx; + + assert( bSaveRow==0 || apVal==0 ); + assert( bSaveRow==0 || bSaveRow==1 ); + assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 ); + + if( apVal==0 ){ + if( p->pSavedRow && bSaveRow ){ + pSeek = p->pSavedRow; + p->pSavedRow = 0; + }else{ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } + } + } + + ctx.pStorage = p; + ctx.iCol = -1; + for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ + if( pConfig->abUnindexed[iCol-1]==0 ){ + sqlite3_value *pVal = 0; + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + + assert( pSeek==0 || apVal==0 ); + assert( pSeek!=0 || apVal!=0 ); + if( pSeek ){ + pVal = sqlite3_column_value(pSeek, iCol); + }else{ + pVal = apVal[iCol-1]; + } + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + pText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ + rc = FTS5_CORRUPT; + } + sqlite3Fts5ClearLocale(pConfig); + } + } + } + if( rc==SQLITE_OK && p->nTotalRow<1 ){ + rc = FTS5_CORRUPT; + }else{ + p->nTotalRow--; + } + + if( rc==SQLITE_OK && bSaveRow ){ + assert( p->pSavedRow==0 ); + p->pSavedRow = pSeek; + }else{ + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + } + return rc; +} + +/* +** Reset any saved statement pSavedRow. Zero pSavedRow as well. This +** should be called by the xUpdate() method of the fts5 table before +** returning from any operation that may have set Fts5Storage.pSavedRow. +*/ +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + assert( pStorage->pSavedRow==0 + || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] + ); + sqlite3_reset(pStorage->pSavedRow); + pStorage->pSavedRow = 0; +} + +/* +** This function is called to process a DELETE on a contentless_delete=1 +** table. It adds the tombstone required to delete the entry with rowid +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, +** an SQLite error code. +*/ +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ + i64 iOrigin = 0; + sqlite3_stmt *pLookup = 0; + int rc = SQLITE_OK; + + assert( p->pConfig->bContentlessDelete ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE + || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Look up the origin of the document in the %_docsize table. Store + ** this in stack variable iOrigin. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLookup, 1, iDel); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + iOrigin = sqlite3_column_int64(pLookup, 1); + } + rc = sqlite3_reset(pLookup); + } + + if( rc==SQLITE_OK && iOrigin!=0 ){ + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + } + + return rc; +} + +/* +** Insert a record into the %_docsize table. Specifically, do: +** +** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf); +** +** If there is no %_docsize table (as happens if the columnsize=0 option +** is specified when the FTS5 table is created), this function is a no-op. +*/ +static int fts5StorageInsertDocsize( + Fts5Storage *p, /* Storage module to write to */ + i64 iRowid, /* id value */ + Fts5Buffer *pBuf /* sz value */ +){ + int rc = SQLITE_OK; + if( p->pConfig->bColumnsize ){ + sqlite3_stmt *pReplace = 0; + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pReplace, 1, iRowid); + if( p->pConfig->bContentlessDelete ){ + i64 iOrigin = 0; + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); + sqlite3_bind_int64(pReplace, 3, iOrigin); + } + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } + } + return rc; +} + +/* +** Load the contents of the "averages" record from disk into the +** p->nTotalRow and p->aTotalSize[] variables. If successful, and if +** argument bCache is true, set the p->bTotalsValid flag to indicate +** that the contents of aTotalSize[] and nTotalRow are valid until +** further notice. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ + int rc = SQLITE_OK; + if( p->bTotalsValid==0 ){ + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); + p->bTotalsValid = bCache; + } + return rc; +} + +/* +** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** variables in the "averages" record on disk. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageSaveTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + int i; + Fts5Buffer buf; + int rc = SQLITE_OK; + memset(&buf, 0, sizeof(buf)); + + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); + for(i=0; iaTotalSize[i]); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); + } + sqlite3_free(buf.p); + + return rc; +} + +/* +** Remove a row from the FTS table. +*/ +int sqlite3Fts5StorageDelete( + Fts5Storage *p, /* Storage object */ + i64 iDel, /* Rowid to delete from table */ + sqlite3_value **apVal, /* Optional - values to remove from index */ + int bSaveRow /* If true, set pSavedRow for deleted row */ +){ + Fts5Config *pConfig = p->pConfig; + int rc; + sqlite3_stmt *pDel = 0; + + assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); + rc = fts5StorageLoadTotals(p, 1); + + /* Delete the index records */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + } + + if( rc==SQLITE_OK ){ + if( p->pConfig->bContentlessDelete ){ + rc = fts5StorageContentlessDelete(p, iDel); + if( rc==SQLITE_OK + && bSaveRow + && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ + rc = sqlite3Fts5StorageFindDeleteRow(p, iDel); + } + }else{ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow); + } + } + + /* Delete the %_docsize record */ + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + /* Delete the %_content record */ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + return rc; +} + +/* +** Delete all entries in the FTS5 index. +*/ +int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ + Fts5Config *pConfig = p->pConfig; + int rc; + + p->bTotalsValid = 0; + + /* Delete the contents of the %_data and %_docsize tables. */ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_idx';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName + ); + } + + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName + ); + } + + /* Reinitialize the %_data table. This call creates the initial structure + ** and averages records. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p->pIndex); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } + return rc; +} + +int sqlite3Fts5StorageRebuild(Fts5Storage *p){ + Fts5Buffer buf = {0,0,0}; + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pScan = 0; + Fts5InsertCtx ctx; + int rc, rc2; + + memset(&ctx, 0, sizeof(Fts5InsertCtx)); + ctx.pStorage = p; + rc = sqlite3Fts5StorageDeleteAll(p); + if( rc==SQLITE_OK ){ + rc = fts5StorageLoadTotals(p, 1); + } + + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ + i64 iRowid = sqlite3_column_int64(pScan, 0); + + sqlite3Fts5BufferZero(&buf); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ + ctx.szCol = 0; + if( pConfig->abUnindexed[ctx.iCol]==0 ){ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pLoc in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } + } + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + } + sqlite3_free(buf.p); + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + return rc; +} + +int sqlite3Fts5StorageOptimize(Fts5Storage *p){ + return sqlite3Fts5IndexOptimize(p->pIndex); +} + +int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ + return sqlite3Fts5IndexMerge(p->pIndex, nMerge); +} + +int sqlite3Fts5StorageReset(Fts5Storage *p){ + return sqlite3Fts5IndexReset(p->pIndex); +} + +/* +** Allocate a new rowid. This is used for "external content" tables when +** a NULL value is inserted into the rowid column. The new rowid is allocated +** by inserting a dummy row into the %_docsize table. The dummy will be +** overwritten later. +** +** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In +** this case the user is required to provide a rowid explicitly. +*/ +static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ + int rc = SQLITE_MISMATCH; + if( p->pConfig->bColumnsize ){ + sqlite3_stmt *pReplace = 0; + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pReplace, 1); + sqlite3_bind_null(pReplace, 2); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + if( rc==SQLITE_OK ){ + *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); + } + } + return rc; +} + +/* +** Insert a new row into the FTS content table. +*/ +int sqlite3Fts5StorageContentInsert( + Fts5Storage *p, + int bReplace, /* True to use REPLACE instead of INSERT */ + sqlite3_value **apVal, + i64 *piRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; + + /* Insert the new row into the %_content table. */ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && pConfig->eContent!=FTS5_CONTENT_UNINDEXED + ){ + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + *piRowid = sqlite3_value_int64(apVal[1]); + }else{ + rc = fts5StorageNewRowid(p, piRowid); + } + }else{ + sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ + int i; /* Counter variable */ + + assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT ); + assert( bReplace==0 || bReplace==1 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT+bReplace, &pInsert, 0); + if( pInsert ) sqlite3_clear_bindings(pInsert); + + /* Bind the rowid value */ + sqlite3_bind_value(pInsert, 1, apVal[1]); + + /* Loop through values for user-defined columns. i=2 is the leftmost + ** user-defined column. As is column 1 of pSavedRow. */ + for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + int bUnindexed = pConfig->abUnindexed[i-2]; + if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){ + sqlite3_value *pVal = apVal[i]; + + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + /* This is an UPDATE statement, and user-defined column (i-2) was not + ** modified. Retrieve the value from Fts5Storage.pSavedRow. */ + pVal = sqlite3_column_value(p->pSavedRow, i-1); + if( pConfig->bLocale && bUnindexed==0 ){ + sqlite3_bind_value(pInsert, pConfig->nCol + i, + sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1) + ); + } + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + const char *pLoc = 0; + int nText = 0; + int nLoc = 0; + assert( pConfig->bLocale ); + + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( bUnindexed==0 ){ + int iLoc = pConfig->nCol + i; + sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT); + } + } + + continue; + } + + rc = sqlite3_bind_value(pInsert, i, pVal); + } + } + if( rc==SQLITE_OK ){ + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + } + *piRowid = sqlite3_last_insert_rowid(pConfig->db); + } + + return rc; +} + +/* +** Insert new entries into the FTS index and %_docsize table. +*/ +int sqlite3Fts5StorageIndexInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 iRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; /* Return code */ + Fts5InsertCtx ctx; /* Tokenization callback context object */ + Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ + + memset(&buf, 0, sizeof(Fts5Buffer)); + ctx.pStorage = p; + rc = fts5StorageLoadTotals(p, 1); + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); + } + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ + ctx.szCol = 0; + if( pConfig->abUnindexed[ctx.iCol]==0 ){ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pText in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = apVal[ctx.iCol+2]; + if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ + pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol); + nLoc = sqlite3_column_bytes(p->pSavedRow, iCol); + } + }else{ + pVal = apVal[ctx.iCol+2]; + } + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } + } + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + /* Write the %_docsize record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + sqlite3_free(buf.p); + + return rc; +} + +static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){ + Fts5Config *pConfig = p->pConfig; + char *zSql; + int rc; + + zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", + pConfig->zDb, pConfig->zName, zSuffix + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pCnt = 0; + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pCnt) ){ + *pnRow = sqlite3_column_int64(pCnt, 0); + } + rc = sqlite3_finalize(pCnt); + } + } + + sqlite3_free(zSql); + return rc; +} + +/* +** Context object used by sqlite3Fts5StorageIntegrity(). +*/ +typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; +struct Fts5IntegrityCtx { + i64 iRowid; + int iCol; + int szCol; + u64 cksum; + Fts5Termset *pTermset; + Fts5Config *pConfig; +}; + + +/* +** Tokenization callback used by integrity check. +*/ +static int fts5StorageIntegrityCallback( + void *pContext, /* Pointer to Fts5IntegrityCtx object */ + int tflags, + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ +){ + Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; + Fts5Termset *pTermset = pCtx->pTermset; + int bPresent; + int ii; + int rc = SQLITE_OK; + int iPos; + int iCol; + + UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + + switch( pCtx->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + iPos = pCtx->szCol-1; + iCol = pCtx->iCol; + break; + + case FTS5_DETAIL_COLUMNS: + iPos = pCtx->iCol; + iCol = 0; + break; + + default: + assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); + iPos = 0; + iCol = 0; + break; + } + + rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); + if( rc==SQLITE_OK && bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, 0, pToken, nToken + ); + } + + for(ii=0; rc==SQLITE_OK && iipConfig->nPrefix; ii++){ + const int nChar = pCtx->pConfig->aPrefix[ii]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); + if( bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte + ); + } + } + } + + return rc; +} + +/* +** Check that the contents of the FTS index match that of the %_content +** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return +** some other SQLite error code if an error occurs while attempting to +** determine this. +*/ +int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; /* Return code */ + int *aColSize; /* Array of size pConfig->nCol */ + i64 *aTotalSize; /* Array of size pConfig->nCol */ + Fts5IntegrityCtx ctx; + sqlite3_stmt *pScan; + int bUseCksum; + + memset(&ctx, 0, sizeof(Fts5IntegrityCtx)); + ctx.pConfig = p->pConfig; + aTotalSize = (i64*)sqlite3_malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64))); + if( !aTotalSize ) return SQLITE_NOMEM; + aColSize = (int*)&aTotalSize[pConfig->nCol]; + memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); + + bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL + || (pConfig->eContent==FTS5_CONTENT_EXTERNAL && iArg) + ); + if( bUseCksum ){ + /* Generate the expected index checksum based on the contents of the + ** %_content table. This block stores the checksum in ctx.cksum. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + if( rc==SQLITE_OK ){ + int rc2; + while( SQLITE_ROW==sqlite3_step(pScan) ){ + int i; + ctx.iRowid = sqlite3_column_int64(pScan, 0); + ctx.szCol = 0; + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + sqlite3_value *pVal = sqlite3_column_value(pScan, i+1); + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue( + pVal, &pText, &nText, &pLoc, &nLoc + ); + }else{ + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = i + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + ctx.iCol = i; + ctx.szCol = 0; + + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } + + /* If this is not a columnsize=0 database, check that the number + ** of tokens in the value matches the aColSize[] value read from + ** the %_docsize table. */ + if( rc==SQLITE_OK + && pConfig->bColumnsize + && ctx.szCol!=aColSize[i] + ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } + } + } + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + + if( rc!=SQLITE_OK ) break; + } + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* Test that the "totals" (sometimes called "averages") record looks Ok */ + if( rc==SQLITE_OK ){ + int i; + rc = fts5StorageLoadTotals(p, 0); + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT; + } + } + + /* Check that the %_docsize and %_content tables contain the expected + ** number of rows. */ + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "content", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "docsize", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + } + + /* Pass the expected checksum down to the FTS index module. It will + ** verify, amongst other things, that it matches the checksum generated by + ** inspecting the index itself. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum); + } + + sqlite3_free(aTotalSize); + return rc; +} + +/* +** Obtain an SQLite statement handle that may be used to read data from the +** %_content table. +*/ +int sqlite3Fts5StorageStmt( + Fts5Storage *p, + int eStmt, + sqlite3_stmt **pp, + char **pzErrMsg +){ + int rc; + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg); + if( rc==SQLITE_OK ){ + assert( p->aStmt[eStmt]==*pp ); + p->aStmt[eStmt] = 0; + } + return rc; +} + +/* +** Release an SQLite statement handle obtained via an earlier call to +** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function +** must match that passed to the sqlite3Fts5StorageStmt() call. +*/ +void sqlite3Fts5StorageStmtRelease( + Fts5Storage *p, + int eStmt, + sqlite3_stmt *pStmt +){ + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + if( p->aStmt[eStmt]==0 ){ + sqlite3_reset(pStmt); + p->aStmt[eStmt] = pStmt; + }else{ + sqlite3_finalize(pStmt); + } +} + +static int fts5StorageDecodeSizeArray( + int *aCol, int nCol, /* Array to populate */ + const u8 *aBlob, int nBlob /* Record to read varints from */ +){ + int i; + int iOff = 0; + for(i=0; i=nBlob ) return 1; + iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]); + } + return (iOff!=nBlob); +} + +/* +** Argument aCol points to an array of integers containing one entry for +** each table column. This function reads the %_docsize record for the +** specified rowid and populates aCol[] with the results. +** +** An SQLite error code is returned if an error occurs, or SQLITE_OK +** otherwise. +*/ +int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ + int nCol = p->pConfig->nCol; /* Number of user columns in table */ + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ + int rc; /* Return Code */ + + assert( p->pConfig->bColumnsize ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( pLookup ){ + int bCorrupt = 1; + assert( rc==SQLITE_OK ); + sqlite3_bind_int64(pLookup, 1, iRowid); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + const u8 *aBlob = sqlite3_column_blob(pLookup, 0); + int nBlob = sqlite3_column_bytes(pLookup, 0); + if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){ + bCorrupt = 0; + } + } + rc = sqlite3_reset(pLookup); + if( bCorrupt && rc==SQLITE_OK ){ + rc = FTS5_CORRUPT; + } + }else{ + assert( rc!=SQLITE_OK ); + } + + return rc; +} + +int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ + int rc = fts5StorageLoadTotals(p, 0); + if( rc==SQLITE_OK ){ + *pnToken = 0; + if( iCol<0 ){ + int i; + for(i=0; ipConfig->nCol; i++){ + *pnToken += p->aTotalSize[i]; + } + }else if( iColpConfig->nCol ){ + *pnToken = p->aTotalSize[iCol]; + }else{ + rc = SQLITE_RANGE; + } + } + return rc; +} + +int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ + int rc = fts5StorageLoadTotals(p, 0); + if( rc==SQLITE_OK ){ + /* nTotalRow being zero does not necessarily indicate a corrupt + ** database - it might be that the FTS5 table really does contain zero + ** rows. However this function is only called from the xRowCount() API, + ** and there is no way for that API to be invoked if the table contains + ** no rows. Hence the FTS5_CORRUPT return. */ + *pnRow = p->nTotalRow; + if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT; + } + return rc; +} + +/* +** Flush any data currently held in-memory to disk. +*/ +int sqlite3Fts5StorageSync(Fts5Storage *p){ + int rc = SQLITE_OK; + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); + if( p->bTotalsValid ){ + rc = fts5StorageSaveTotals(p); + if( rc==SQLITE_OK ){ + p->bTotalsValid = 0; + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSync(p->pIndex); + } + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); + return rc; +} + +int sqlite3Fts5StorageRollback(Fts5Storage *p){ + p->bTotalsValid = 0; + return sqlite3Fts5IndexRollback(p->pIndex); +} + +int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, + const char *z, + sqlite3_value *pVal, + int iVal +){ + sqlite3_stmt *pReplace = 0; + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC); + if( pVal ){ + sqlite3_bind_value(pReplace, 2, pVal); + }else{ + sqlite3_bind_int(pReplace, 2, iVal); + } + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 1); + } + if( rc==SQLITE_OK && pVal ){ + int iNew = p->pConfig->iCookie + 1; + rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); + if( rc==SQLITE_OK ){ + p->pConfig->iCookie = iNew; + } + } + return rc; +} diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c new file mode 100644 index 0000000000000000000000000000000000000000..25cd5c063345727ab1d67e32116763c36110c61f --- /dev/null +++ b/ext/fts5/fts5_tcl.c @@ -0,0 +1,1741 @@ +/* +** 2014 Dec 01 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + +#ifdef SQLITE_TEST +#include "tclsqlite.h" + +#ifdef SQLITE_ENABLE_FTS5 + +#include "fts5.h" +#include +#include +#include + +#ifdef SQLITE_DEBUG +extern int sqlite3_fts5_may_be_corrupt; +#endif +extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); +extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*); + +/************************************************************************* +** This is a copy of the first part of the SqliteDb structure in +** tclsqlite.c. We need it here so that the get_sqlite_pointer routine +** can extract the sqlite3* pointer from an existing Tcl SQLite +** connection. +*/ + +extern const char *sqlite3ErrName(int); + +struct SqliteDb { + sqlite3 *db; +}; + +/* +** Decode a pointer to an sqlite3 object. +*/ +static int f5tDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb){ + struct SqliteDb *p; + Tcl_CmdInfo cmdInfo; + char *z = Tcl_GetString(pObj); + if( Tcl_GetCommandInfo(interp, z, &cmdInfo) ){ + p = (struct SqliteDb*)cmdInfo.objClientData; + *ppDb = p->db; + return TCL_OK; + } + return TCL_ERROR; +} + +/* End of code that accesses the SqliteDb struct. +**************************************************************************/ + +static int f5tResultToErrorCode(const char *zRes){ + struct ErrorCode { + int rc; + const char *zError; + } aErr[] = { + { SQLITE_DONE, "SQLITE_DONE" }, + { SQLITE_ERROR, "SQLITE_ERROR" }, + { SQLITE_OK, "SQLITE_OK" }, + { SQLITE_OK, "" }, + }; + int i; + + for(i=0; ipScript); + int rc; + + Tcl_IncrRefCount(pEval); + Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken)); + Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart)); + Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd)); + + rc = Tcl_EvalObjEx(p->interp, pEval, 0); + Tcl_DecrRefCount(pEval); + if( rc==TCL_OK ){ + rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp)); + } + + return rc; +} + +static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); + +static int xQueryPhraseCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pCtx +){ + F5tFunction *p = (F5tFunction*)pCtx; + static sqlite3_int64 iCmd = 0; + Tcl_Obj *pEval; + int rc; + + char zCmd[64]; + F5tApi sApi; + + sApi.pApi = pApi; + sApi.pFts = pFts; + sprintf(zCmd, "f5t_2_%lld", iCmd++); + Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0); + + pEval = Tcl_DuplicateObj(p->pScript); + Tcl_IncrRefCount(pEval); + Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1)); + rc = Tcl_EvalObjEx(p->interp, pEval, 0); + Tcl_DecrRefCount(pEval); + Tcl_DeleteCommand(p->interp, zCmd); + + if( rc==TCL_OK ){ + rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp)); + } + + return rc; +} + +static void xSetAuxdataDestructor(void *p){ + F5tAuxData *pData = (F5tAuxData*)p; + Tcl_DecrRefCount(pData->pObj); + sqlite3_free(pData); +} + +/* +** api sub-command... +** +** Description... +*/ +static int SQLITE_TCLAPI xF5tApi( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct Sub { + const char *zName; + int nArg; + const char *zMsg; + } aSub[] = { + { "xColumnCount", 0, "" }, /* 0 */ + { "xRowCount", 0, "" }, /* 1 */ + { "xColumnTotalSize", 1, "COL" }, /* 2 */ + { "xTokenize", 2, "TEXT SCRIPT" }, /* 3 */ + { "xPhraseCount", 0, "" }, /* 4 */ + { "xPhraseSize", 1, "PHRASE" }, /* 5 */ + { "xInstCount", 0, "" }, /* 6 */ + { "xInst", 1, "IDX" }, /* 7 */ + { "xRowid", 0, "" }, /* 8 */ + { "xColumnText", 1, "COL" }, /* 9 */ + { "xColumnSize", 1, "COL" }, /* 10 */ + { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ + { "xSetAuxdata", 1, "VALUE" }, /* 12 */ + { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ + { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ + { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ + { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ + { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ + + { "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */ + { "xInstToken", 2, "IDX ITERM" }, /* 19 */ + { "xColumnLocale", 1, "COL" }, /* 20 */ + { 0, 0, 0} + }; + + int rc; + int iSub = 0; + F5tApi *p = (F5tApi*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND"); + return TCL_ERROR; + } + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aSub, sizeof(aSub[0]), "SUB-COMMAND", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( aSub[iSub].nArg!=objc-2 ){ + Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + +#define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 ); + switch( iSub ){ + CASE(0, "xColumnCount") { + int nCol; + nCol = p->pApi->xColumnCount(p->pFts); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol)); + } + break; + } + CASE(1, "xRowCount") { + sqlite3_int64 nRow; + rc = p->pApi->xRowCount(p->pFts, &nRow); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow)); + } + break; + } + CASE(2, "xColumnTotalSize") { + int iCol; + sqlite3_int64 nSize; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR; + rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize)); + } + break; + } + CASE(3, "xTokenize") { + Tcl_Size nText; + char *zText = Tcl_GetStringFromObj(objv[2], &nText); + F5tFunction ctx; + ctx.interp = interp; + ctx.pScript = objv[3]; + rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb); + if( rc==SQLITE_OK ){ + Tcl_ResetResult(interp); + } + return rc; + } + CASE(4, "xPhraseCount") { + int nPhrase; + nPhrase = p->pApi->xPhraseCount(p->pFts); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase)); + } + break; + } + CASE(5, "xPhraseSize") { + int iPhrase; + int sz; + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){ + return TCL_ERROR; + } + sz = p->pApi->xPhraseSize(p->pFts, iPhrase); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(sz)); + } + break; + } + CASE(6, "xInstCount") { + int nInst; + rc = p->pApi->xInstCount(p->pFts, &nInst); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst)); + } + break; + } + CASE(7, "xInst") { + int iIdx, ip, ic, io; + if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){ + return TCL_ERROR; + } + rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io); + if( rc==SQLITE_OK ){ + Tcl_Obj *pList = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip)); + Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic)); + Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io)); + Tcl_SetObjResult(interp, pList); + } + break; + } + CASE(8, "xRowid") { + sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid)); + break; + } + CASE(9, "xColumnText") { + const char *z = 0; + int n = 0; + int iCol; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ + return TCL_ERROR; + } + rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); + } + break; + } + CASE(10, "xColumnSize") { + int n = 0; + int iCol; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ + return TCL_ERROR; + } + rc = p->pApi->xColumnSize(p->pFts, iCol, &n); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(n)); + } + break; + } + CASE(11, "xQueryPhrase") { + int iPhrase; + F5tFunction ctx; + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){ + return TCL_ERROR; + } + ctx.interp = interp; + ctx.pScript = objv[3]; + rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb); + if( rc==SQLITE_OK ){ + Tcl_ResetResult(interp); + } + break; + } + CASE(12, "xSetAuxdata") { + F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); + if( pData==0 ){ + Tcl_AppendResult(interp, "out of memory", (char*)0); + return TCL_ERROR; + } + pData->pObj = objv[2]; + Tcl_IncrRefCount(pData->pObj); + rc = p->pApi->xSetAuxdata(p->pFts, pData, xSetAuxdataDestructor); + break; + } + CASE(13, "xGetAuxdata") { + F5tAuxData *pData; + int bClear; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ){ + return TCL_ERROR; + } + pData = (F5tAuxData*)p->pApi->xGetAuxdata(p->pFts, bClear); + if( pData==0 ){ + Tcl_ResetResult(interp); + }else{ + Tcl_SetObjResult(interp, pData->pObj); + if( bClear ){ + xSetAuxdataDestructor((void*)pData); + } + } + break; + } + + /* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the + ** xSetAuxdata and xGetAuxdata methods implemented above. The difference + ** is that they may only save an integer value as auxiliary data, and + ** do not specify a destructor function. */ + CASE(14, "xSetAuxdataInt") { + int iVal; + if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR; + rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0); + break; + } + CASE(15, "xGetAuxdataInt") { + int iVal; + int bClear; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; + iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); + Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); + break; + } + + CASE(16, "xPhraseForeach") { + int iPhrase; + int iCol; + int iOff; + const char *zColvar; + const char *zOffvar; + Tcl_Obj *pScript = objv[5]; + Fts5PhraseIter iter; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + zColvar = Tcl_GetString(objv[3]); + zOffvar = Tcl_GetString(objv[4]); + + rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); + return TCL_ERROR; + } + for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); + Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc==TCL_CONTINUE ) rc = TCL_OK; + if( rc!=TCL_OK ){ + if( rc==TCL_BREAK ) rc = TCL_OK; + break; + } + } + + break; + } + + CASE(17, "xPhraseColumnForeach") { + int iPhrase; + int iCol; + const char *zColvar; + Tcl_Obj *pScript = objv[4]; + Fts5PhraseIter iter; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + zColvar = Tcl_GetString(objv[3]); + + rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){ + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc==TCL_CONTINUE ) rc = TCL_OK; + if( rc!=TCL_OK ){ + if( rc==TCL_BREAK ) rc = TCL_OK; + break; + } + } + + break; + } + + CASE(18, "xQueryToken") { + const char *pTerm = 0; + int nTerm = 0; + int iPhrase = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + + CASE(19, "xInstToken") { + const char *pTerm = 0; + int nTerm = 0; + int iIdx = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + + CASE(20, "xColumnLocale") { + const char *z = 0; + int n = 0; + int iCol; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ + return TCL_ERROR; + } + rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n); + if( rc==SQLITE_OK && z ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); + } + break; + } + + default: + assert( 0 ); + break; + } +#undef CASE + + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + + return TCL_OK; +} + +static void xF5tFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + F5tFunction *p = (F5tFunction*)pApi->xUserData(pFts); + Tcl_Obj *pEval; /* Script to evaluate */ + int i; + int rc; + + static sqlite3_int64 iCmd = 0; + char zCmd[64]; + F5tApi sApi; + sApi.pApi = pApi; + sApi.pFts = pFts; + + sprintf(zCmd, "f5t_%lld", iCmd++); + Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0); + pEval = Tcl_DuplicateObj(p->pScript); + Tcl_IncrRefCount(pEval); + Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1)); + + for(i=0; iinterp, pEval, pObj); + } + + rc = Tcl_EvalObjEx(p->interp, pEval, TCL_GLOBAL_ONLY); + Tcl_DecrRefCount(pEval); + Tcl_DeleteCommand(p->interp, zCmd); + + if( rc!=TCL_OK ){ + sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1); + }else{ + Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); + const char *zType = (pVar->typePtr ? pVar->typePtr->name : ""); + char c = zType[0]; + if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ + /* Only return a BLOB type if the Tcl variable is a bytearray and + ** has no string representation. */ + Tcl_Size nn; + unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn); + sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT); + }else if( c=='b' && strcmp(zType,"boolean")==0 ){ + int n; + Tcl_GetIntFromObj(0, pVar, &n); + sqlite3_result_int(pCtx, n); + }else if( c=='d' && strcmp(zType,"double")==0 ){ + double r; + Tcl_GetDoubleFromObj(0, pVar, &r); + sqlite3_result_double(pCtx, r); + }else if( (c=='w' && strcmp(zType,"wideInt")==0) || + (c=='i' && strcmp(zType,"int")==0) ){ + Tcl_WideInt v; + Tcl_GetWideIntFromObj(0, pVar, &v); + sqlite3_result_int64(pCtx, v); + }else{ + Tcl_Size nn; + unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn); + sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT); + } + } +} + +static void xF5tDestroy(void *pCtx){ + F5tFunction *p = (F5tFunction*)pCtx; + Tcl_DecrRefCount(p->pScript); + ckfree((char *)p); +} + +/* +** sqlite3_fts5_create_function DB NAME SCRIPT +** +** Description... +*/ +static int SQLITE_TCLAPI f5tCreateFunction( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char *zName; + Tcl_Obj *pScript; + sqlite3 *db = 0; + fts5_api *pApi = 0; + F5tFunction *pCtx = 0; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT"); + return TCL_ERROR; + } + if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR; + + zName = Tcl_GetString(objv[2]); + pScript = objv[3]; + pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction)); + pCtx->interp = interp; + pCtx->pScript = pScript; + Tcl_IncrRefCount(pScript); + + rc = pApi->xCreateFunction( + pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy + ); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); + return TCL_ERROR; + } + + return TCL_OK; +} + +typedef struct F5tTokenizeCtx F5tTokenizeCtx; +struct F5tTokenizeCtx { + Tcl_Obj *pRet; + int bSubst; + const char *zInput; +}; + +static int xTokenizeCb2( + void *pCtx, + int tflags, + const char *zToken, int nToken, + int iStart, int iEnd +){ + F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx; + if( p->bSubst ){ + Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken)); + Tcl_ListObjAppendElement( + 0, p->pRet, Tcl_NewStringObj(&p->zInput[iStart], iEnd-iStart) + ); + }else{ + Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken)); + Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iStart)); + Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iEnd)); + } + return SQLITE_OK; +} + + +/* +** sqlite3_fts5_tokenize DB TOKENIZER TEXT +** +** Description... +*/ +static int SQLITE_TCLAPI f5tTokenize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char *pCopy = 0; + char *zText = 0; + Tcl_Size nText = 0; + sqlite3 *db = 0; + fts5_api *pApi = 0; + Fts5Tokenizer *pTok = 0; + fts5_tokenizer tokenizer; + Tcl_Obj *pRet = 0; + void *pUserdata; + int rc; + + Tcl_Size nArg; + const char **azArg; + F5tTokenizeCtx ctx; + + if( objc!=4 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-subst? DB NAME TEXT"); + return TCL_ERROR; + } + if( objc==5 ){ + char *zOpt = Tcl_GetString(objv[1]); + if( strcmp("-subst", zOpt) ){ + Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0); + return TCL_ERROR; + } + } + if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ) return TCL_ERROR; + if( Tcl_SplitList(interp, Tcl_GetString(objv[objc-2]), &nArg, &azArg) ){ + return TCL_ERROR; + } + if( nArg==0 ){ + Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0); + Tcl_Free((void*)azArg); + return TCL_ERROR; + } + zText = Tcl_GetStringFromObj(objv[objc-1], &nText); + + rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0); + return TCL_ERROR; + } + + rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0); + return TCL_ERROR; + } + + if( nText>0 ){ + pCopy = sqlite3_malloc(nText); + if( pCopy==0 ){ + tokenizer.xDelete(pTok); + Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); + return TCL_ERROR; + }else{ + memcpy(pCopy, zText, nText); + } + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + ctx.bSubst = (objc==5); + ctx.pRet = pRet; + ctx.zInput = pCopy; + rc = tokenizer.xTokenize( + pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, pCopy,(int)nText, xTokenizeCb2 + ); + tokenizer.xDelete(pTok); + sqlite3_free(pCopy); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0); + Tcl_DecrRefCount(pRet); + return TCL_ERROR; + } + + Tcl_Free((void*)azArg); + Tcl_SetObjResult(interp, pRet); + Tcl_DecrRefCount(pRet); + return TCL_OK; +} + +/************************************************************************* +** Start of tokenizer wrapper. +*/ + +typedef struct F5tTokenizerContext F5tTokenizerContext; +typedef struct F5tTokenizerCb F5tTokenizerCb; +typedef struct F5tTokenizerModule F5tTokenizerModule; +typedef struct F5tTokenizerInstance F5tTokenizerInstance; + +struct F5tTokenizerContext { + void *pCtx; + int (*xToken)(void*, int, const char*, int, int, int); + F5tTokenizerInstance *pInst; +}; + +struct F5tTokenizerModule { + Tcl_Interp *interp; + Tcl_Obj *pScript; + void *pParentCtx; + fts5_tokenizer_v2 parent_v2; + fts5_tokenizer parent; + F5tTokenizerContext *pContext; +}; + +/* +** zLocale: +** Within a call to xTokenize_v2(), pLocale/nLocale store the locale +** passed to the call by fts5. This can be retrieved by a Tcl tokenize +** script using [sqlite3_fts5_locale]. +*/ +struct F5tTokenizerInstance { + Tcl_Interp *interp; + Tcl_Obj *pScript; + F5tTokenizerModule *pModule; + Fts5Tokenizer *pParent; + F5tTokenizerContext *pContext; + const char *pLocale; + int nLocale; +}; + +static int f5tTokenizerCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5Tokenizer *pParent = 0; + F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; + Tcl_Obj *pEval; + int rc = TCL_OK; + int i; + + assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 ); + if( pMod->parent_v2.xCreate ){ + rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + if( pMod->parent.xCreate ){ + rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + + pEval = Tcl_DuplicateObj(pMod->pScript); + Tcl_IncrRefCount(pEval); + for(i=0; rc==TCL_OK && iinterp, pEval, pObj); + } + + if( rc==TCL_OK ){ + rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY); + } + Tcl_DecrRefCount(pEval); + + if( rc==TCL_OK ){ + F5tTokenizerInstance *pInst; + pInst = (F5tTokenizerInstance*)ckalloc(sizeof(F5tTokenizerInstance)); + memset(pInst, 0, sizeof(F5tTokenizerInstance)); + pInst->interp = pMod->interp; + pInst->pScript = Tcl_GetObjResult(pMod->interp); + pInst->pContext = pMod->pContext; + pInst->pParent = pParent; + pInst->pModule = pMod; + Tcl_IncrRefCount(pInst->pScript); + *ppOut = (Fts5Tokenizer*)pInst; + } + + return rc; +} + + +static void f5tTokenizerDelete(Fts5Tokenizer *p){ + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + if( pInst ){ + if( pInst->pParent ){ + if( pInst->pModule->parent_v2.xDelete ){ + pInst->pModule->parent_v2.xDelete(pInst->pParent); + }else{ + pInst->pModule->parent.xDelete(pInst->pParent); + } + } + Tcl_DecrRefCount(pInst->pScript); + ckfree((char *)pInst); + } +} + + +static int f5tTokenizerReallyTokenize( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + F5tTokenizerInstance *pOldInst = 0; + void *pOldCtx; + int (*xOldToken)(void*, int, const char*, int, int, int); + Tcl_Obj *pEval; + int rc; + const char *zFlags; + + pOldCtx = pInst->pContext->pCtx; + xOldToken = pInst->pContext->xToken; + pOldInst = pInst->pContext->pInst; + + pInst->pContext->pCtx = pCtx; + pInst->pContext->xToken = xToken; + pInst->pContext->pInst = pInst; + + assert( + flags==FTS5_TOKENIZE_DOCUMENT + || flags==FTS5_TOKENIZE_AUX + || flags==FTS5_TOKENIZE_QUERY + || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) + ); + pEval = Tcl_DuplicateObj(pInst->pScript); + Tcl_IncrRefCount(pEval); + switch( flags ){ + case FTS5_TOKENIZE_DOCUMENT: + zFlags = "document"; + break; + case FTS5_TOKENIZE_AUX: + zFlags = "aux"; + break; + case FTS5_TOKENIZE_QUERY: + zFlags = "query"; + break; + case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY): + zFlags = "prefixquery"; + break; + default: + assert( 0 ); + zFlags = "invalid"; + break; + } + + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1)); + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText)); + rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); + Tcl_DecrRefCount(pEval); + + pInst->pContext->pCtx = pOldCtx; + pInst->pContext->xToken = xOldToken; + pInst->pContext->pInst = pOldInst; + return rc; +} + +typedef struct CallbackCtx CallbackCtx; +struct CallbackCtx { + Fts5Tokenizer *p; + void *pCtx; + int flags; + int (*xToken)(void*, int, const char*, int, int, int); +}; + +static int f5tTokenizeCallback( + void *pCtx, + int tflags, + const char *z, int n, + int iStart, int iEnd +){ + CallbackCtx *p = (CallbackCtx*)pCtx; + return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken); +} + +static int f5tTokenizerTokenize_v2( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + const char *pLoc, int nLoc, + int (*xToken)(void*, int, const char*, int, int, int) +){ + int rc = SQLITE_OK; + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + + pInst->pLocale = pLoc; + pInst->nLocale = nLoc; + + if( pInst->pParent ){ + CallbackCtx ctx; + ctx.p = p; + ctx.pCtx = pCtx; + ctx.flags = flags; + ctx.xToken = xToken; + if( pInst->pModule->parent_v2.xTokenize ){ + rc = pInst->pModule->parent_v2.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, + pLoc, nLoc, f5tTokenizeCallback + ); + }else{ + rc = pInst->pModule->parent.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback + ); + } + }else{ + rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); + } + + pInst->pLocale = 0; + pInst->nLocale = 0; + return rc; +} +static int f5tTokenizerTokenize( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** sqlite3_fts5_locale +*/ +static int SQLITE_TCLAPI f5tTokenizerLocale( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + if( p->xToken==0 ){ + Tcl_AppendResult(interp, + "sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0 + ); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, + Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale) + ); + return TCL_OK; +} + +/* +** sqlite3_fts5_token ?-colocated? TEXT START END +*/ +static int SQLITE_TCLAPI f5tTokenizerReturn( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; + int iStart; + int iEnd; + Tcl_Size nToken; + int tflags = 0; + char *zToken; + int rc; + + if( objc==5 ){ + Tcl_Size nArg; + char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); + if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ + tflags |= FTS5_TOKEN_COLOCATED; + }else{ + goto usage; + } + }else if( objc!=4 ){ + goto usage; + } + + zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken); + if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart) + || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd) + ){ + return TCL_ERROR; + } + + if( p->xToken==0 ){ + Tcl_AppendResult(interp, + "sqlite3_fts5_token may only be used by tokenizer callback", (char*)0 + ); + return TCL_ERROR; + } + + rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd); + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return rc==SQLITE_OK ? TCL_OK : TCL_ERROR; + + usage: + Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END"); + return TCL_ERROR; +} + +static void f5tDelTokenizer(void *pCtx){ + F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; + Tcl_DecrRefCount(pMod->pScript); + ckfree((char *)pMod); +} + +/* +** sqlite3_fts5_create_tokenizer DB NAME SCRIPT +** +** Register a tokenizer named NAME implemented by script SCRIPT. When +** a tokenizer instance is created (fts5_tokenizer.xCreate), any tokenizer +** arguments are appended to SCRIPT and the result executed. +** +** The value returned by (SCRIPT + args) is itself a tcl script. This +** script - call it SCRIPT2 - is executed to tokenize text using the +** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize +** text SCRIPT2 is invoked with a single argument appended to it - the +** text to tokenize. +** +** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each +** token within the tokenized text. +*/ +static int SQLITE_TCLAPI f5tCreateTokenizer( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData; + sqlite3 *db; + fts5_api *pApi; + char *zName; + Tcl_Obj *pScript; + F5tTokenizerModule *pMod; + int rc = SQLITE_OK; + int bV2 = 0; /* True to use _v2 API */ + int iVersion = 2; /* Value for _v2.iVersion */ + const char *zParent = 0; /* Name of parent tokenizer, if any */ + int ii = 0; + + if( objc<4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT"); + return TCL_ERROR; + } + + /* Parse any options. Set stack variables bV2 and zParent. */ + for(ii=1; iiinterp = interp; + pMod->pScript = pScript; + Tcl_IncrRefCount(pScript); + pMod->pContext = pContext; + if( zParent ){ + if( bV2 ){ + fts5_tokenizer_v2 *pParent = 0; + rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent); + if( rc==SQLITE_OK ){ + memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2)); + pMod->parent_v2.xDelete(0); + } + }else{ + rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent); + if( rc==SQLITE_OK ){ + pMod->parent.xDelete(0); + } + } + } + + if( rc==SQLITE_OK ){ + void *pModCtx = (void*)pMod; + if( bV2==0 ){ + fts5_tokenizer t; + t.xCreate = f5tTokenizerCreate; + t.xTokenize = f5tTokenizerTokenize; + t.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer); + }else{ + fts5_tokenizer_v2 t2; + memset(&t2, 0, sizeof(t2)); + t2.iVersion = iVersion; + t2.xCreate = f5tTokenizerCreate; + t2.xTokenize = f5tTokenizerTokenize_v2; + t2.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); + } + } + + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, ( + bV2 ? "error in fts5_api.xCreateTokenizer_v2()" + : "error in fts5_api.xCreateTokenizer()" + ), (char*)0); + return TCL_ERROR; + } + + return TCL_OK; +} + +static void SQLITE_TCLAPI xF5tFree(ClientData clientData){ + ckfree(clientData); +} + +/* +** sqlite3_fts5_may_be_corrupt BOOLEAN +** +** Set or clear the global "may-be-corrupt" flag. Return the old value. +*/ +static int SQLITE_TCLAPI f5tMayBeCorrupt( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_DEBUG + int bOld = sqlite3_fts5_may_be_corrupt; + + if( objc!=2 && objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?"); + return TCL_ERROR; + } + if( objc==2 ){ + int bNew; + if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR; + sqlite3_fts5_may_be_corrupt = bNew; + } + + Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld)); +#endif + return TCL_OK; +} + + +static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + return (h % nSlot); +} + +static int SQLITE_TCLAPI f5tTokenHash( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char *z; + Tcl_Size n; + unsigned int iVal; + int nSlot; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NSLOT TOKEN"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &nSlot) ){ + return TCL_ERROR; + } + z = Tcl_GetStringFromObj(objv[2], &n); + + iVal = f5t_fts5HashKey(nSlot, z, (int)n); + Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); + return TCL_OK; +} + +static int SQLITE_TCLAPI f5tRegisterMatchinfo( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + + rc = sqlite3Fts5TestRegisterMatchinfo(db); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} + +static int SQLITE_TCLAPI f5tRegisterTok( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db = 0; + fts5_api *pApi = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ + return TCL_ERROR; + } + + rc = sqlite3Fts5TestRegisterTok(db, pApi); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} + +typedef struct OriginTextCtx OriginTextCtx; +struct OriginTextCtx { + sqlite3 *db; + fts5_api *pApi; +}; + +typedef struct OriginTextTokenizer OriginTextTokenizer; +struct OriginTextTokenizer { + Fts5Tokenizer *pTok; /* Underlying tokenizer object */ + fts5_tokenizer tokapi; /* API implementation for pTok */ +}; + +/* +** Delete the OriginTextCtx object indicated by the only argument. +*/ +static void f5tOrigintextTokenizerDelete(void *pCtx){ + OriginTextCtx *p = (OriginTextCtx*)pCtx; + ckfree((char*)p); +} + +static int f5tOrigintextCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + OriginTextCtx *p = (OriginTextCtx*)pCtx; + OriginTextTokenizer *pTok = 0; + void *pTokCtx = 0; + int rc = SQLITE_OK; + + pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); + if( pTok==0 ){ + rc = SQLITE_NOMEM; + }else if( nArg<1 ){ + rc = SQLITE_ERROR; + }else{ + /* Locate the underlying tokenizer */ + rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi); + } + + /* Create the new tokenizer instance */ + if( rc==SQLITE_OK ){ + rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pTok); + pTok = 0; + } + *ppOut = (Fts5Tokenizer*)pTok; + return rc; +} + +static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){ + OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; + if( p->pTok ){ + p->tokapi.xDelete(p->pTok); + } + sqlite3_free(p); +} + +typedef struct OriginTextCb OriginTextCb; +struct OriginTextCb { + void *pCtx; + const char *pText; + int nText; + int (*xToken)(void *, int, const char *, int, int, int); + + char *aBuf; /* Buffer to use */ + int nBuf; /* Allocated size of aBuf[] */ +}; + +static int xOriginToken( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input */ +){ + OriginTextCb *p = (OriginTextCb*)pCtx; + int ret = 0; + + if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){ + /* Token exactly matches document text. Pass it through as is. */ + ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); + }else{ + int nReq = nToken + 1 + (iEnd-iStart); + if( nReq>p->nBuf ){ + sqlite3_free(p->aBuf); + p->aBuf = sqlite3_malloc(nReq*2); + if( p->aBuf==0 ) return SQLITE_NOMEM; + p->nBuf = nReq*2; + } + + memcpy(p->aBuf, pToken, nToken); + p->aBuf[nToken] = '\0'; + memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart); + ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd); + } + + return ret; +} + + +static int f5tOrigintextTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, int) +){ + OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; + OriginTextCb cb; + int ret; + + memset(&cb, 0, sizeof(cb)); + cb.pCtx = pCtx; + cb.pText = pText; + cb.nText = nText; + cb.xToken = xToken; + + ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken); + sqlite3_free(cb.aBuf); + return ret; +} + +/* +** sqlite3_fts5_register_origintext DB +** +** Description... +*/ +static int SQLITE_TCLAPI f5tRegisterOriginText( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + fts5_api *pApi = 0; + int rc; + fts5_tokenizer tok = {0, 0, 0}; + OriginTextCtx *pCtx = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR; + + pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx)); + pCtx->db = db; + pCtx->pApi = pApi; + + tok.xCreate = f5tOrigintextCreate; + tok.xDelete = f5tOrigintextDelete; + tok.xTokenize = f5tOrigintextTokenize; + rc = pApi->xCreateTokenizer( + pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete + ); + + Tcl_ResetResult(interp); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** This function is used to DROP an fts5 table. It works even if the data +** structures fts5 stores within the database are corrupt, which sometimes +** prevents a straight "DROP TABLE" command from succeeding. +** +** The first parameter is the database handle to use for the DROP TABLE +** operation. The second is the name of the database to drop the fts5 table +** from (i.e. "main", "temp" or the name of an attached database). The +** third parameter is the name of the fts5 table to drop. +** +** SQLITE_OK is returned if the table is successfully dropped. Or, if an +** error occurs, an SQLite error code. +*/ +static int sqlite3_fts5_drop_corrupt_table( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zTab /* Name of fts5 table to drop */ +){ + int rc = SQLITE_OK; + int bDef = 0; + + rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef); + if( rc==SQLITE_OK ){ + char *zScript = sqlite3_mprintf( + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_config';" + "INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');" + "INSERT INTO %Q.'%q_config' VALUES('version', 4);" + "DROP TABLE %Q.'%q';", + zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab + ); + + if( zScript==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + rc = sqlite3_exec(db, zScript, 0, 0, 0); + if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0); + sqlite3_free(zScript); + } + } + + return rc; +} + +/* +** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE +** +** Description... +*/ +static int SQLITE_TCLAPI f5tDropCorruptTable( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + const char *zTab = 0; + int rc = SQLITE_OK; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + zTab = Tcl_GetString(objv[3]); + + rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); + return TCL_ERROR; + } + + return TCL_OK; +} + +/* +** Free a buffer returned to SQLite by the str() function. +*/ +void f5tFree(void *p){ + char *x = (char *)p; + ckfree(&x[-8]); +} + +/* +** Implementation of str(). +*/ +void f5tStrFunc(sqlite3_context *pCtx, int nArg, sqlite3_value **apArg){ + const char *zText = 0; + assert( nArg==1 ); + + zText = (const char*)sqlite3_value_text(apArg[0]); + if( zText ){ + sqlite3_int64 nText = strlen(zText); + char *zCopy = (char*)ckalloc(nText+8); + if( zCopy==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + zCopy += 8; + memcpy(zCopy, zText, nText); + sqlite3_result_text64(pCtx, zCopy, nText, f5tFree, SQLITE_UTF8); + } + } +} + +/* +** sqlite3_fts5_register_str DB +** +** Register the str() function with database handle DB. str() interprets +** its only argument as text and returns a copy of the value in a +** non-nul-terminated buffer. +*/ +static int SQLITE_TCLAPI f5tRegisterStr( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + + sqlite3_create_function(db, "str", 1, SQLITE_UTF8, 0, f5tStrFunc, 0, 0); + + return TCL_OK; +} + +/* +** Entry point. +*/ +int Fts5tcl_Init(Tcl_Interp *interp){ + static struct Cmd { + char *zName; + Tcl_ObjCmdProc *xProc; + int bTokenizeCtx; + } aCmd[] = { + { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, + { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, + { "sqlite3_fts5_locale", f5tTokenizerLocale, 1 }, + { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, + { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, + { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, + { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, + { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, + { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }, + { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }, + { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }, + { "sqlite3_fts5_register_str", f5tRegisterStr, 0 } + }; + int i; + F5tTokenizerContext *pContext; + + pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext)); + memset(pContext, 0, sizeof(*pContext)); + + for(i=0; ibTokenizeCtx ) pCtx = (void*)pContext; + Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree)); + } + + return TCL_OK; +} +#else /* SQLITE_ENABLE_FTS5 */ +int Fts5tcl_Init(Tcl_Interp *interp){ + return TCL_OK; +} +#endif /* SQLITE_ENABLE_FTS5 */ +#endif /* SQLITE_TEST */ diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c new file mode 100644 index 0000000000000000000000000000000000000000..6f2d6e7ea27cda88b018cd9cc1ac0c7a926ea5d8 --- /dev/null +++ b/ext/fts5/fts5_test_mi.c @@ -0,0 +1,421 @@ +/* +** 2015 Aug 04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains test code only, it is not included in release +** versions of FTS5. It contains the implementation of an FTS5 auxiliary +** function very similar to the FTS4 function matchinfo(): +** +** https://www.sqlite.org/fts3.html#matchinfo +** +** Known differences are that: +** +** 1) this function uses the FTS5 definition of "matchable phrase", which +** excludes any phrases that are part of an expression sub-tree that +** does not match the current row. This comes up for MATCH queries +** such as: +** +** "a OR (b AND c)" +** +** In FTS4, if a single row contains instances of tokens "a" and "c", +** but not "b", all instances of "c" are considered matches. In FTS5, +** they are not (as the "b AND c" sub-tree does not match the current +** row. +** +** 2) For the values returned by 'x' that apply to all rows of the table, +** NEAR constraints are not considered. But for the number of hits in +** the current row, they are. +** +** This file exports a single function that may be called to register the +** matchinfo() implementation with a database handle: +** +** int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db); +*/ + + +#ifdef SQLITE_ENABLE_FTS5 + +#include "fts5.h" +#include +#include + +typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; + +#ifndef SQLITE_AMALGAMATION +typedef unsigned int u32; +#endif + +struct Fts5MatchinfoCtx { + int nCol; /* Number of cols in FTS5 table */ + int nPhrase; /* Number of phrases in FTS5 query */ + char *zArg; /* nul-term'd copy of 2nd arg */ + int nRet; /* Number of elements in aRet[] */ + u32 *aRet; /* Array of 32-bit unsigned ints to return */ +}; + + + +/* +** Return a pointer to the fts5_api pointer for database connection db. +** If an error occurs, return NULL and leave an error in the database +** handle (accessible using sqlite3_errcode()/errmsg()). +*/ +static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){ + sqlite3_stmt *pStmt = 0; + int rc; + + *ppApi = 0; + rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0); + (void)sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + } + + return rc; +} + + +/* +** Argument f should be a flag accepted by matchinfo() (a valid character +** in the string passed as the second argument). If it is not, -1 is +** returned. Otherwise, if f is a valid matchinfo flag, the value returned +** is the number of 32-bit integers added to the output array if the +** table has nCol columns and the query nPhrase phrases. +*/ +static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){ + int ret = -1; + switch( f ){ + case 'p': ret = 1; break; + case 'c': ret = 1; break; + case 'x': ret = 3 * nCol * nPhrase; break; + case 'y': ret = nCol * nPhrase; break; + case 'b': ret = ((nCol + 31) / 32) * nPhrase; break; + case 'n': ret = 1; break; + case 'a': ret = nCol; break; + case 'l': ret = nCol; break; + case 's': ret = nCol; break; + } + return ret; +} + +static int fts5MatchinfoIter( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + Fts5MatchinfoCtx *p, + int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*) +){ + int i; + int n = 0; + int rc = SQLITE_OK; + char f; + for(i=0; (f = p->zArg[i]); i++){ + rc = x(pApi, pFts, p, f, &p->aRet[n]); + if( rc!=SQLITE_OK ) break; + n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f); + } + return rc; +} + +static int fts5MatchinfoXCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + void *pUserData +){ + Fts5PhraseIter iter; + int iCol, iOff; + u32 *aOut = (u32*)pUserData; + int iPrev = -1; + + for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); + iCol>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[iCol*3+1]++; + if( iCol!=iPrev ) aOut[iCol*3 + 2]++; + iPrev = iCol; + } + + return SQLITE_OK; +} + +static int fts5MatchinfoGlobalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int rc = SQLITE_OK; + switch( f ){ + case 'p': + aOut[0] = p->nPhrase; + break; + + case 'c': + aOut[0] = p->nCol; + break; + + case 'x': { + int i; + for(i=0; inPhrase && rc==SQLITE_OK; i++){ + void *pPtr = (void*)&aOut[i * p->nCol * 3]; + rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb); + } + break; + } + + case 'n': { + sqlite3_int64 nRow; + rc = pApi->xRowCount(pFts, &nRow); + aOut[0] = (u32)nRow; + break; + } + + case 'a': { + sqlite3_int64 nRow = 0; + rc = pApi->xRowCount(pFts, &nRow); + if( nRow==0 ){ + memset(aOut, 0, sizeof(u32) * p->nCol); + }else{ + int i; + for(i=0; rc==SQLITE_OK && inCol; i++){ + sqlite3_int64 nToken; + rc = pApi->xColumnTotalSize(pFts, i, &nToken); + if( rc==SQLITE_OK){ + aOut[i] = (u32)((2*nToken + nRow) / (2*nRow)); + } + } + } + break; + } + + } + return rc; +} + +static int fts5MatchinfoLocalCb( + const Fts5ExtensionApi *pApi, + Fts5Context *pFts, + Fts5MatchinfoCtx *p, + char f, + u32 *aOut +){ + int i; + int rc = SQLITE_OK; + + switch( f ){ + case 'b': { + int iPhrase; + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; + for(i=0; inPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iCol; + for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); + iCol>=0; + pApi->xPhraseNextColumn(pFts, &iter, &iCol) + ){ + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); + } + } + + break; + } + + case 'x': + case 'y': { + int nMul = (f=='x' ? 3 : 1); + int iPhrase; + + for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; + + for(iPhrase=0; iPhrasenPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iOff, iCol; + for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); + iOff>=0; + pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) + ){ + aOut[nMul * (iCol + iPhrase * p->nCol)]++; + } + } + + break; + } + + case 'l': { + for(i=0; rc==SQLITE_OK && inCol; i++){ + int nToken; + rc = pApi->xColumnSize(pFts, i, &nToken); + aOut[i] = (u32)nToken; + } + break; + } + + case 's': { + int nInst; + + memset(aOut, 0, sizeof(u32) * p->nCol); + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &iPhrase, &iCol, &iOff); + iNextPhrase = iPhrase+1; + iNextOff = iOff+pApi->xPhraseSize(pFts, 0); + for(j=i+1; rc==SQLITE_OK && jxInst(pFts, j, &ip, &ic, &io); + if( ic!=iCol || io>iNextOff ) break; + if( ip==iNextPhrase && io==iNextOff ){ + nSeq++; + iNextPhrase = ip+1; + iNextOff = io + pApi->xPhraseSize(pFts, ip); + } + } + + if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq; + } + + break; + } + } + return rc; +} + +static Fts5MatchinfoCtx *fts5MatchinfoNew( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning error message */ + const char *zArg /* Matchinfo flag string */ +){ + Fts5MatchinfoCtx *p; + int nCol; + int nPhrase; + int i; + int nInt; + sqlite3_int64 nByte; + int rc; + + nCol = pApi->xColumnCount(pFts); + nPhrase = pApi->xPhraseCount(pFts); + + nInt = 0; + for(i=0; zArg[i]; i++){ + int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]); + if( n<0 ){ + char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return 0; + } + nInt += n; + } + + nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */ + + sizeof(u32) * nInt /* The p->aRet[] array */ + + (i+1); /* The p->zArg string */ + p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, nByte); + + p->nCol = nCol; + p->nPhrase = nPhrase; + p->aRet = (u32*)&p[1]; + p->nRet = nInt; + p->zArg = (char*)&p->aRet[nInt]; + memcpy(p->zArg, zArg, i); + + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(p); + p = 0; + } + + return p; +} + +static void fts5MatchinfoFunc( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + const char *zArg; + Fts5MatchinfoCtx *p; + int rc = SQLITE_OK; + + if( nVal>0 ){ + zArg = (const char*)sqlite3_value_text(apVal[0]); + }else{ + zArg = "pcx"; + } + + p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); + if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ + p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } + } + + if( rc==SQLITE_OK ){ + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + }else{ + /* No errors has occured, so return a copy of the array of integers. */ + int nByte = p->nRet * sizeof(u32); + sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT); + } +} + +int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ + int rc; /* Return code */ + fts5_api *pApi; /* FTS5 API functions */ + + /* Extract the FTS5 API pointer from the database handle. The + ** fts5_api_from_db() function above is copied verbatim from the + ** FTS5 documentation. Refer there for details. */ + rc = fts5_api_from_db(db, &pApi); + if( rc!=SQLITE_OK ) return rc; + + /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered + ** with this database handle, or an error (OOM perhaps?) has occurred. + ** + ** Also check that the fts5_api object is version 2 or newer. + */ + if( pApi==0 || pApi->iVersion<2 ){ + return SQLITE_ERROR; + } + + /* Register the implementation of matchinfo() */ + rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); + + return rc; +} + +#endif /* SQLITE_ENABLE_FTS5 */ diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c new file mode 100644 index 0000000000000000000000000000000000000000..994d304dc60edc064bf75d9f014ff522c71a8e2d --- /dev/null +++ b/ext/fts5/fts5_test_tok.c @@ -0,0 +1,484 @@ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts5tokenize" virtual table module. +** An fts5tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts5tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input HIDDEN, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) + +#include "fts5.h" +#include +#include + +typedef struct Fts5tokTable Fts5tokTable; +typedef struct Fts5tokCursor Fts5tokCursor; +typedef struct Fts5tokRow Fts5tokRow; + +/* +** Virtual table structure. +*/ +struct Fts5tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + fts5_tokenizer tok; /* Tokenizer functions */ + Fts5Tokenizer *pTok; /* Tokenizer instance */ +}; + +/* +** A container for a rows values. +*/ +struct Fts5tokRow { + char *zToken; + int iStart; + int iEnd; + int iPos; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts5tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + int iRowid; /* Current 'rowid' value */ + char *zInput; /* Input string */ + int nRow; /* Number of entries in aRow[] */ + Fts5tokRow *aRow; /* Array of rows to return */ +}; + +static void fts5tokDequote(char *z){ + char q = z[0]; + + if( q=='[' || q=='\'' || q=='"' || q=='`' ){ + int iIn = 1; + int iOut = 0; + if( q=='[' ) q = ']'; + + while( z[iIn] ){ + if( z[iIn]==q ){ + if( z[iIn+1]!=q ){ + /* Character iIn was the close quote. */ + iIn++; + break; + }else{ + /* Character iIn and iIn+1 form an escaped quote character. Skip + ** the input cursor past both and copy a single quote character + ** to the output buffer. */ + iIn += 2; + z[iOut++] = q; + } + }else{ + z[iOut++] = z[iIn++]; + } + } + + z[iOut] = '\0'; + } +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts5tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; i0 ){ + zModule = azDequote[0]; + } + + rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok); + if( rc==SQLITE_OK ){ + const char **azArg = (nDequote>1 ? (const char **)&azDequote[1] : 0); + int nArg = nDequote>0 ? nDequote-1 : 0; + rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pTab); + pTab = 0; + } + + *ppVtab = (sqlite3_vtab*)pTab; + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5tokTable *pTab = (Fts5tokTable *)pVtab; + if( pTab->pTok ){ + pTab->tok.xDelete(pTab->pTok); + } + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts5tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts5tokCursor *pCsr; + + pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts5tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts5tokOpenMethod(). +*/ +static void fts5tokResetCursor(Fts5tokCursor *pCsr){ + int i; + for(i=0; inRow; i++){ + sqlite3_free(pCsr->aRow[i].zToken); + } + sqlite3_free(pCsr->zInput); + sqlite3_free(pCsr->aRow); + pCsr->zInput = 0; + pCsr->aRow = 0; + pCsr->nRow = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + fts5tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + pCsr->iRowid++; + return SQLITE_OK; +} + +static int fts5tokCb( + void *pCtx, /* Pointer to Fts5tokCursor */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ +){ + Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx; + Fts5tokRow *pRow; + + if( (pCsr->nRow & (pCsr->nRow-1))==0 ){ + int nNew = pCsr->nRow ? pCsr->nRow*2 : 32; + Fts5tokRow *aNew; + aNew = (Fts5tokRow*)sqlite3_realloc64(pCsr->aRow, nNew*sizeof(Fts5tokRow)); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow)); + pCsr->aRow = aNew; + } + + pRow = &pCsr->aRow[pCsr->nRow]; + pRow->iStart = iStart; + pRow->iEnd = iEnd; + if( pCsr->nRow ){ + pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1); + } + pRow->zToken = sqlite3_malloc(nToken+1); + if( pRow->zToken==0 ) return SQLITE_NOMEM; + memcpy(pRow->zToken, pToken, nToken); + pRow->zToken[nToken] = 0; + pCsr->nRow++; + + return SQLITE_OK; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts5tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab); + + fts5tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->tok.xTokenize( + pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb + ); + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts5tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + return (pCsr->iRowid>pCsr->nRow); +} + +/* +** xColumn - Return a column value. +*/ +static int fts5tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1]; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pRow->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pRow->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pRow->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts5tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts5tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){ + static const sqlite3_module fts5tok_module = { + 0, /* iVersion */ + fts5tokConnectMethod, /* xCreate */ + fts5tokConnectMethod, /* xConnect */ + fts5tokBestIndexMethod, /* xBestIndex */ + fts5tokDisconnectMethod, /* xDisconnect */ + fts5tokDisconnectMethod, /* xDestroy */ + fts5tokOpenMethod, /* xOpen */ + fts5tokCloseMethod, /* xClose */ + fts5tokFilterMethod, /* xFilter */ + fts5tokNextMethod, /* xNext */ + fts5tokEofMethod, /* xEof */ + fts5tokColumnMethod, /* xColumn */ + fts5tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi); + return rc; +} + +#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c new file mode 100644 index 0000000000000000000000000000000000000000..b8a1136465837e1012bc26b11d1558a789553b82 --- /dev/null +++ b/ext/fts5/fts5_tokenize.c @@ -0,0 +1,1490 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + +#include "fts5Int.h" + +/************************************************************************** +** Start of ascii tokenizer implementation. +*/ + +/* +** For tokenizers with no "unicode" modifier, the set of token characters +** is the same as the set of ASCII range alphanumeric characters. +*/ +static unsigned char aAsciiTokenChar[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */ +}; + +typedef struct AsciiTokenizer AsciiTokenizer; +struct AsciiTokenizer { + unsigned char aTokenChar[128]; +}; + +static void fts5AsciiAddExceptions( + AsciiTokenizer *p, + const char *zArg, + int bTokenChars +){ + int i; + for(i=0; zArg[i]; i++){ + if( (zArg[i] & 0x80)==0 ){ + p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars; + } + } +} + +/* +** Delete a "ascii" tokenizer. +*/ +static void fts5AsciiDelete(Fts5Tokenizer *p){ + sqlite3_free(p); +} + +/* +** Create an "ascii" tokenizer. +*/ +static int fts5AsciiCreate( + void *pUnused, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; + AsciiTokenizer *p = 0; + UNUSED_PARAM(pUnused); + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + p = sqlite3_malloc(sizeof(AsciiTokenizer)); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + memset(p, 0, sizeof(AsciiTokenizer)); + memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); + for(i=0; rc==SQLITE_OK && i='A' && c<='Z' ) c += 32; + aOut[i] = c; + } +} + +/* +** Tokenize some text using the ascii tokenizer. +*/ +static int fts5AsciiTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int iUnused, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; + int rc = SQLITE_OK; + int ie; + int is = 0; + + char aFold[64]; + int nFold = sizeof(aFold); + char *pFold = aFold; + unsigned char *a = p->aTokenChar; + + UNUSED_PARAM(iUnused); + + while( isnFold ){ + if( pFold!=aFold ) sqlite3_free(pFold); + pFold = sqlite3_malloc64((sqlite3_int64)nByte*2); + if( pFold==0 ){ + rc = SQLITE_NOMEM; + break; + } + nFold = nByte*2; + } + asciiFold(pFold, &pText[is], nByte); + + /* Invoke the token callback */ + rc = xToken(pCtx, 0, pFold, nByte, is, ie); + is = ie+1; + } + + if( pFold!=aFold ) sqlite3_free(pFold); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +/************************************************************************** +** Start of unicode61 tokenizer implementation. +*/ + + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn>6)&0x1F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +#define FTS5_SKIP_UTF8(zIn) { \ + if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \ + while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + +typedef struct Unicode61Tokenizer Unicode61Tokenizer; +struct Unicode61Tokenizer { + unsigned char aTokenChar[128]; /* ASCII range token characters */ + char *aFold; /* Buffer to fold text into */ + int nFold; /* Size of aFold[] in bytes */ + int eRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int nException; + int *aiException; + + unsigned char aCategory[32]; /* True for token char categories */ +}; + +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */ +#define FTS5_REMOVE_DIACRITICS_NONE 0 +#define FTS5_REMOVE_DIACRITICS_SIMPLE 1 +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2 + +static int fts5UnicodeAddExceptions( + Unicode61Tokenizer *p, /* Tokenizer object */ + const char *z, /* Characters to treat as exceptions */ + int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */ +){ + int rc = SQLITE_OK; + int n = (int)strlen(z); + int *aNew; + + if( n>0 ){ + aNew = (int*)sqlite3_realloc64(p->aiException, + (n+p->nException)*sizeof(int)); + if( aNew ){ + int nNew = p->nException; + const unsigned char *zCsr = (const unsigned char*)z; + const unsigned char *zTerm = (const unsigned char*)&z[n]; + while( zCsraTokenChar[iCode] = (unsigned char)bTokenChars; + }else{ + bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]; + assert( (bToken==0 || bToken==1) ); + assert( (bTokenChars==0 || bTokenChars==1) ); + if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ + int i; + for(i=0; iiCode ) break; + } + memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int)); + aNew[i] = iCode; + nNew++; + } + } + } + p->aiException = aNew; + p->nException = nNew; + }else{ + rc = SQLITE_NOMEM; + } + } + + return rc; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Delete a "unicode61" tokenizer. +*/ +static void fts5UnicodeDelete(Fts5Tokenizer *pTok){ + if( pTok ){ + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok; + sqlite3_free(p->aiException); + sqlite3_free(p->aFold); + sqlite3_free(p); + } + return; +} + +static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){ + const char *z = zCat; + + while( *z ){ + while( *z==' ' || *z=='\t' ) z++; + if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){ + return SQLITE_ERROR; + } + while( *z!=' ' && *z!='\t' && *z!='\0' ) z++; + } + + sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar); + return SQLITE_OK; +} + +/* +** Create a "unicode61" tokenizer. +*/ +static int fts5UnicodeCreate( + void *pUnused, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; /* Return code */ + Unicode61Tokenizer *p = 0; /* New tokenizer object */ + + UNUSED_PARAM(pUnused); + + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + if( p ){ + const char *zCat = "L* N* Co"; + int i; + memset(p, 0, sizeof(Unicode61Tokenizer)); + + p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; + p->nFold = 64; + p->aFold = sqlite3_malloc64(p->nFold * sizeof(char)); + if( p->aFold==0 ){ + rc = SQLITE_NOMEM; + } + + /* Search for a "categories" argument */ + for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = (zArg[0] - '0'); + assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX + ); + } + }else + if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ + rc = fts5UnicodeAddExceptions(p, zArg, 1); + }else + if( 0==sqlite3_stricmp(azArg[i], "separators") ){ + rc = fts5UnicodeAddExceptions(p, zArg, 0); + }else + if( 0==sqlite3_stricmp(azArg[i], "categories") ){ + /* no-op */ + }else{ + rc = SQLITE_ERROR; + } + } + }else{ + rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + fts5UnicodeDelete((Fts5Tokenizer*)p); + p = 0; + } + *ppOut = (Fts5Tokenizer*)p; + } + return rc; +} + +/* +** Return true if, for the purposes of tokenizing with the tokenizer +** passed as the first argument, codepoint iCode is considered a token +** character (not a separator). +*/ +static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ + return ( + p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)] + ^ fts5UnicodeIsException(p, iCode) + ); +} + +static int fts5UnicodeTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int iUnused, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; + int rc = SQLITE_OK; + unsigned char *a = p->aTokenChar; + + unsigned char *zTerm = (unsigned char*)&pText[nText]; + unsigned char *zCsr = (unsigned char *)pText; + + /* Output buffer */ + char *aFold = p->aFold; + int nFold = p->nFold; + const char *pEnd = &aFold[nFold-6]; + + UNUSED_PARAM(iUnused); + + /* Each iteration of this loop gobbles up a contiguous run of separators, + ** then the next token. */ + while( rc==SQLITE_OK ){ + u32 iCode; /* non-ASCII codepoint read from input */ + char *zOut = aFold; + int is; + int ie; + + /* Skip any separator characters. */ + while( 1 ){ + if( zCsr>=zTerm ) goto tokenize_done; + if( *zCsr & 0x80 ) { + /* A character outside of the ascii range. Skip past it if it is + ** a separator character. Or break out of the loop if it is not. */ + is = zCsr - (unsigned char*)pText; + READ_UTF8(zCsr, zTerm, iCode); + if( fts5UnicodeIsAlnum(p, iCode) ){ + goto non_ascii_tokenchar; + } + }else{ + if( a[*zCsr] ){ + is = zCsr - (unsigned char*)pText; + goto ascii_tokenchar; + } + zCsr++; + } + } + + /* Run through the tokenchars. Fold them into the output buffer along + ** the way. */ + while( zCsrpEnd ){ + aFold = sqlite3_malloc64((sqlite3_int64)nFold*2); + if( aFold==0 ){ + rc = SQLITE_NOMEM; + goto tokenize_done; + } + zOut = &aFold[zOut - p->aFold]; + memcpy(aFold, p->aFold, nFold); + sqlite3_free(p->aFold); + p->aFold = aFold; + p->nFold = nFold = nFold*2; + pEnd = &aFold[nFold-6]; + } + + if( *zCsr & 0x80 ){ + /* An non-ascii-range character. Fold it into the output buffer if + ** it is a token character, or break out of the loop if it is not. */ + READ_UTF8(zCsr, zTerm, iCode); + if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ + non_ascii_tokenchar: + iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); + if( iCode ) WRITE_UTF8(zOut, iCode); + }else{ + break; + } + }else if( a[*zCsr]==0 ){ + /* An ascii-range separator character. End of token. */ + break; + }else{ + ascii_tokenchar: + if( *zCsr>='A' && *zCsr<='Z' ){ + *zOut++ = *zCsr + 32; + }else{ + *zOut++ = *zCsr; + } + zCsr++; + } + ie = zCsr - (unsigned char*)pText; + } + + /* Invoke the token callback */ + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); + } + + tokenize_done: + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; +} + +/************************************************************************** +** Start of porter stemmer implementation. +*/ + +/* Any tokens larger than this (in bytes) are passed through without +** stemming. */ +#define FTS5_PORTER_MAX_TOKEN 64 + +typedef struct PorterTokenizer PorterTokenizer; +struct PorterTokenizer { + fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */ + Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ + char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; +}; + +/* +** Delete a "porter" tokenizer. +*/ +static void fts5PorterDelete(Fts5Tokenizer *pTok){ + if( pTok ){ + PorterTokenizer *p = (PorterTokenizer*)pTok; + if( p->pTokenizer ){ + p->tokenizer_v2.xDelete(p->pTokenizer); + } + sqlite3_free(p); + } +} + +/* +** Create a "porter" tokenizer. +*/ +static int fts5PorterCreate( + void *pCtx, + const char **azArg, int nArg, + Fts5Tokenizer **ppOut +){ + fts5_api *pApi = (fts5_api*)pCtx; + int rc = SQLITE_OK; + PorterTokenizer *pRet; + void *pUserdata = 0; + const char *zBase = "unicode61"; + fts5_tokenizer_v2 *pV2 = 0; + + if( nArg>0 ){ + zBase = azArg[0]; + } + + pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + if( pRet ){ + memset(pRet, 0, sizeof(PorterTokenizer)); + rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); + }else{ + rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + int nArg2 = (nArg>0 ? nArg-1 : 0); + const char **az2 = (nArg2 ? &azArg[1] : 0); + memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2)); + rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer); + } + + if( rc!=SQLITE_OK ){ + fts5PorterDelete((Fts5Tokenizer*)pRet); + pRet = 0; + } + *ppOut = (Fts5Tokenizer*)pRet; + return rc; +} + +typedef struct PorterContext PorterContext; +struct PorterContext { + void *pCtx; + int (*xToken)(void*, int, const char*, int, int, int); + char *aBuf; +}; + +typedef struct PorterRule PorterRule; +struct PorterRule { + const char *zSuffix; + int nSuffix; + int (*xCond)(char *zStem, int nStem); + const char *zOutput; + int nOutput; +}; + +#if 0 +static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){ + int ret = -1; + int nBuf = *pnBuf; + PorterRule *p; + + for(p=aRule; p->zSuffix; p++){ + assert( strlen(p->zSuffix)==p->nSuffix ); + assert( strlen(p->zOutput)==p->nOutput ); + if( nBufnSuffix ) continue; + if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break; + } + + if( p->zSuffix ){ + int nStem = nBuf - p->nSuffix; + if( p->xCond==0 || p->xCond(aBuf, nStem) ){ + memcpy(&aBuf[nStem], p->zOutput, p->nOutput); + *pnBuf = nStem + p->nOutput; + ret = p - aRule; + } + } + + return ret; +} +#endif + +static int fts5PorterIsVowel(char c, int bYIsVowel){ + return ( + c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y') + ); +} + +static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){ + int i; + int bCons = bPrevCons; + + /* Scan for a vowel */ + for(i=0; i 0) */ +static int fts5Porter_MGt0(char *zStem, int nStem){ + return !!fts5PorterGobbleVC(zStem, nStem, 0); +} + +/* porter rule condition: (m > 1) */ +static int fts5Porter_MGt1(char *zStem, int nStem){ + int n; + n = fts5PorterGobbleVC(zStem, nStem, 0); + if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ + return 1; + } + return 0; +} + +/* porter rule condition: (m = 1) */ +static int fts5Porter_MEq1(char *zStem, int nStem){ + int n; + n = fts5PorterGobbleVC(zStem, nStem, 0); + if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ + return 1; + } + return 0; +} + +/* porter rule condition: (*o) */ +static int fts5Porter_Ostar(char *zStem, int nStem){ + if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){ + return 0; + }else{ + int i; + int mask = 0; + int bCons = 0; + for(i=0; i 1 and (*S or *T)) */ +static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){ + assert( nStem>0 ); + return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t') + && fts5Porter_MGt1(zStem, nStem); +} + +/* porter rule condition: (*v*) */ +static int fts5Porter_Vowel(char *zStem, int nStem){ + int i; + for(i=0; i0) ){ + return 1; + } + } + return 0; +} + + +/************************************************************************** +*************************************************************************** +** GENERATED CODE STARTS HERE (mkportersteps.tcl) +*/ + +static int fts5PorterStep4(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'c': + if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 'e': + if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'i': + if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 'l': + if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 'n': + if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt1(aBuf, nBuf-5) ){ + *pnBuf = nBuf - 5; + } + }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt1(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'o': + if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_MGt1(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + } + } + break; + + case 's': + if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 't': + if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'u': + if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'v': + if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'z': + if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt1(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep1B2(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ate", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + case 'b': + if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ble", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + case 'i': + if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){ + memcpy(&aBuf[nBuf-2], "ize", 3); + *pnBuf = nBuf - 2 + 3; + ret = 1; + } + break; + + } + return ret; +} + + +static int fts5PorterStep2(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ate", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){ + if( fts5Porter_MGt0(aBuf, nBuf-6) ){ + memcpy(&aBuf[nBuf-6], "tion", 4); + *pnBuf = nBuf - 6 + 4; + } + } + break; + + case 'c': + if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ence", 4); + *pnBuf = nBuf - 4 + 4; + } + }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ance", 4); + *pnBuf = nBuf - 4 + 4; + } + } + break; + + case 'e': + if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ize", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 'g': + if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "log", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 'l': + if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "ble", 3); + *pnBuf = nBuf - 3 + 3; + } + }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "al", 2); + *pnBuf = nBuf - 4 + 2; + } + }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ent", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "e", 1); + *pnBuf = nBuf - 3 + 1; + } + }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ous", 3); + *pnBuf = nBuf - 5 + 3; + } + } + break; + + case 'o': + if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ize", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ate", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ate", 3); + *pnBuf = nBuf - 4 + 3; + } + } + break; + + case 's': + if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ive", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ful", 3); + *pnBuf = nBuf - 7 + 3; + } + }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){ + if( fts5Porter_MGt0(aBuf, nBuf-7) ){ + memcpy(&aBuf[nBuf-7], "ous", 3); + *pnBuf = nBuf - 7 + 3; + } + } + break; + + case 't': + if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ive", 3); + *pnBuf = nBuf - 5 + 3; + } + }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){ + if( fts5Porter_MGt0(aBuf, nBuf-6) ){ + memcpy(&aBuf[nBuf-6], "ble", 3); + *pnBuf = nBuf - 6 + 3; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep3(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'a': + if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + memcpy(&aBuf[nBuf-4], "ic", 2); + *pnBuf = nBuf - 4 + 2; + } + } + break; + + case 's': + if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){ + if( fts5Porter_MGt0(aBuf, nBuf-4) ){ + *pnBuf = nBuf - 4; + } + } + break; + + case 't': + if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ic", 2); + *pnBuf = nBuf - 5 + 2; + } + }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "ic", 2); + *pnBuf = nBuf - 5 + 2; + } + } + break; + + case 'u': + if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + } + } + break; + + case 'v': + if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + *pnBuf = nBuf - 5; + } + } + break; + + case 'z': + if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){ + if( fts5Porter_MGt0(aBuf, nBuf-5) ){ + memcpy(&aBuf[nBuf-5], "al", 2); + *pnBuf = nBuf - 5 + 2; + } + } + break; + + } + return ret; +} + + +static int fts5PorterStep1B(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + + case 'e': + if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_MGt0(aBuf, nBuf-3) ){ + memcpy(&aBuf[nBuf-3], "ee", 2); + *pnBuf = nBuf - 3 + 2; + } + }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){ + if( fts5Porter_Vowel(aBuf, nBuf-2) ){ + *pnBuf = nBuf - 2; + ret = 1; + } + } + break; + + case 'n': + if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){ + if( fts5Porter_Vowel(aBuf, nBuf-3) ){ + *pnBuf = nBuf - 3; + ret = 1; + } + } + break; + + } + return ret; +} + +/* +** GENERATED CODE ENDS HERE (mkportersteps.tcl) +*************************************************************************** +**************************************************************************/ + +static void fts5PorterStep1A(char *aBuf, int *pnBuf){ + int nBuf = *pnBuf; + if( aBuf[nBuf-1]=='s' ){ + if( aBuf[nBuf-2]=='e' ){ + if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s') + || (nBuf>3 && aBuf[nBuf-3]=='i' ) + ){ + *pnBuf = nBuf-2; + }else{ + *pnBuf = nBuf-1; + } + } + else if( aBuf[nBuf-2]!='s' ){ + *pnBuf = nBuf-1; + } + } +} + +static int fts5PorterCb( + void *pCtx, + int tflags, + const char *pToken, + int nToken, + int iStart, + int iEnd +){ + PorterContext *p = (PorterContext*)pCtx; + + char *aBuf; + int nBuf; + + if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through; + aBuf = p->aBuf; + nBuf = nToken; + memcpy(aBuf, pToken, nBuf); + + /* Step 1. */ + fts5PorterStep1A(aBuf, &nBuf); + if( fts5PorterStep1B(aBuf, &nBuf) ){ + if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){ + char c = aBuf[nBuf-1]; + if( fts5PorterIsVowel(c, 0)==0 + && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] + ){ + nBuf--; + }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){ + aBuf[nBuf++] = 'e'; + } + } + } + + /* Step 1C. */ + if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){ + aBuf[nBuf-1] = 'i'; + } + + /* Steps 2 through 4. */ + fts5PorterStep2(aBuf, &nBuf); + fts5PorterStep3(aBuf, &nBuf); + fts5PorterStep4(aBuf, &nBuf); + + /* Step 5a. */ + assert( nBuf>0 ); + if( aBuf[nBuf-1]=='e' ){ + if( fts5Porter_MGt1(aBuf, nBuf-1) + || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1)) + ){ + nBuf--; + } + } + + /* Step 5b. */ + if( nBuf>1 && aBuf[nBuf-1]=='l' + && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) + ){ + nBuf--; + } + + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); + + pass_through: + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); +} + +/* +** Tokenize using the porter tokenizer. +*/ +static int fts5PorterTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int flags, + const char *pText, int nText, + const char *pLoc, int nLoc, + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) +){ + PorterTokenizer *p = (PorterTokenizer*)pTokenizer; + PorterContext sCtx; + sCtx.xToken = xToken; + sCtx.pCtx = pCtx; + sCtx.aBuf = p->aBuf; + return p->tokenizer_v2.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb + ); +} + +/************************************************************************** +** Start of trigram implementation. +*/ +typedef struct TrigramTokenizer TrigramTokenizer; +struct TrigramTokenizer { + int bFold; /* True to fold to lower-case */ + int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */ +}; + +/* +** Free a trigram tokenizer. +*/ +static void fts5TriDelete(Fts5Tokenizer *p){ + sqlite3_free(p); +} + +/* +** Allocate a trigram tokenizer. +*/ +static int fts5TriCreate( + void *pUnused, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; + TrigramTokenizer *pNew = 0; + UNUSED_PARAM(pUnused); + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + int i; + pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->bFold = 1; + pNew->iFoldParam = 0; + + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); + } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } + }else{ + rc = SQLITE_ERROR; + } + } + + if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ + rc = SQLITE_ERROR; + } + + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } + } + } + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Trigram tokenizer tokenize routine. +*/ +static int fts5TriTokenize( + Fts5Tokenizer *pTok, + void *pCtx, + int unusedFlags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + TrigramTokenizer *p = (TrigramTokenizer*)pTok; + int rc = SQLITE_OK; + char aBuf[32]; + char *zOut = aBuf; + int ii; + const unsigned char *zIn = (const unsigned char*)pText; + const unsigned char *zEof = (zIn ? &zIn[nText] : 0); + u32 iCode = 0; + int aStart[3]; /* Input offset of each character in aBuf[] */ + + UNUSED_PARAM(unusedFlags); + + /* Populate aBuf[] with the characters for the first trigram. */ + for(ii=0; ii<3; ii++){ + do { + aStart[ii] = zIn - (const unsigned char*)pText; + if( zIn>=zEof ) return SQLITE_OK; + READ_UTF8(zIn, zEof, iCode); + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + WRITE_UTF8(zOut, iCode); + } + + /* At the start of each iteration of this loop: + ** + ** aBuf: Contains 3 characters. The 3 characters of the next trigram. + ** zOut: Points to the byte following the last character in aBuf. + ** aStart[3]: Contains the byte offset in the input text corresponding + ** to the start of each of the three characters in the buffer. + */ + assert( zIn<=zEof ); + while( 1 ){ + int iNext; /* Start of character following current tri */ + const char *z1; + + /* Read characters from the input up until the first non-diacritic */ + do { + iNext = zIn - (const unsigned char*)pText; + if( zIn>=zEof ){ + iCode = 0; + break; + } + READ_UTF8(zIn, zEof, iCode); + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + + /* Pass the current trigram back to fts5 */ + rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext); + if( iCode==0 || rc!=SQLITE_OK ) break; + + /* Remove the first character from buffer aBuf[]. Append the character + ** with codepoint iCode. */ + z1 = aBuf; + FTS5_SKIP_UTF8(z1); + memmove(aBuf, z1, zOut - z1); + zOut -= (z1 - aBuf); + WRITE_UTF8(zOut, iCode); + + /* Update the aStart[] array */ + aStart[0] = aStart[1]; + aStart[1] = aStart[2]; + aStart[2] = iNext; + } + + return rc; +} + +/* +** Argument xCreate is a pointer to a constructor function for a tokenizer. +** pTok is a tokenizer previously created using the same method. This function +** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB +** indicating the style of pattern matching that the tokenizer can support. +** In practice, this is: +** +** "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB +** "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE +** all other tokenizers - FTS5_PATTERN_NONE +*/ +int sqlite3Fts5TokenizerPattern( + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), + Fts5Tokenizer *pTok +){ + if( xCreate==fts5TriCreate ){ + TrigramTokenizer *p = (TrigramTokenizer*)pTok; + if( p->iFoldParam==0 ){ + return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + } + } + return FTS5_PATTERN_NONE; +} + +/* +** Return true if the tokenizer described by p->azArg[] is the trigram +** tokenizer. This tokenizer needs to be loaded before xBestIndex is +** called for the first time in order to correctly handle LIKE/GLOB. +*/ +int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){ + return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram")); +} + + +/* +** Register all built-in tokenizers with FTS5. +*/ +int sqlite3Fts5TokenizerInit(fts5_api *pApi){ + struct BuiltinTokenizer { + const char *zName; + fts5_tokenizer x; + } aBuiltin[] = { + { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, + { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, + { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, + }; + + int rc = SQLITE_OK; /* Return code */ + int i; /* To iterate through builtin functions */ + + for(i=0; rc==SQLITE_OK && ixCreateTokenizer(pApi, + aBuiltin[i].zName, + (void*)pApi, + &aBuiltin[i].x, + 0 + ); + } + if( rc==SQLITE_OK ){ + fts5_tokenizer_v2 sPorter = { + 2, + fts5PorterCreate, + fts5PorterDelete, + fts5PorterTokenize + }; + rc = pApi->xCreateTokenizer_v2(pApi, + "porter", + (void*)pApi, + &sPorter, + 0 + ); + } + return rc; +} diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c new file mode 100644 index 0000000000000000000000000000000000000000..cc164a456916254cb0208a123561bb8d49ef9eec --- /dev/null +++ b/ext/fts5/fts5_unicode2.c @@ -0,0 +1,781 @@ +/* +** 2012-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + + +#include + + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int fts5_remove_diacritic(int c, int bComplex){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, + }; +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +int sqlite3Fts5UnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( eRemoveDiacritic ){ + ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2); + } + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} + + +int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ + aArray[0] = 1; + switch( zCat[0] ){ + case 'C': + switch( zCat[1] ){ + case 'c': aArray[1] = 1; break; + case 'f': aArray[2] = 1; break; + case 'n': aArray[3] = 1; break; + case 's': aArray[4] = 1; break; + case 'o': aArray[31] = 1; break; + case '*': + aArray[1] = 1; + aArray[2] = 1; + aArray[3] = 1; + aArray[4] = 1; + aArray[31] = 1; + break; + default: return 1; } + break; + + case 'L': + switch( zCat[1] ){ + case 'l': aArray[5] = 1; break; + case 'm': aArray[6] = 1; break; + case 'o': aArray[7] = 1; break; + case 't': aArray[8] = 1; break; + case 'u': aArray[9] = 1; break; + case 'C': aArray[30] = 1; break; + case '*': + aArray[5] = 1; + aArray[6] = 1; + aArray[7] = 1; + aArray[8] = 1; + aArray[9] = 1; + aArray[30] = 1; + break; + default: return 1; } + break; + + case 'M': + switch( zCat[1] ){ + case 'c': aArray[10] = 1; break; + case 'e': aArray[11] = 1; break; + case 'n': aArray[12] = 1; break; + case '*': + aArray[10] = 1; + aArray[11] = 1; + aArray[12] = 1; + break; + default: return 1; } + break; + + case 'N': + switch( zCat[1] ){ + case 'd': aArray[13] = 1; break; + case 'l': aArray[14] = 1; break; + case 'o': aArray[15] = 1; break; + case '*': + aArray[13] = 1; + aArray[14] = 1; + aArray[15] = 1; + break; + default: return 1; } + break; + + case 'P': + switch( zCat[1] ){ + case 'c': aArray[16] = 1; break; + case 'd': aArray[17] = 1; break; + case 'e': aArray[18] = 1; break; + case 'f': aArray[19] = 1; break; + case 'i': aArray[20] = 1; break; + case 'o': aArray[21] = 1; break; + case 's': aArray[22] = 1; break; + case '*': + aArray[16] = 1; + aArray[17] = 1; + aArray[18] = 1; + aArray[19] = 1; + aArray[20] = 1; + aArray[21] = 1; + aArray[22] = 1; + break; + default: return 1; } + break; + + case 'S': + switch( zCat[1] ){ + case 'c': aArray[23] = 1; break; + case 'k': aArray[24] = 1; break; + case 'm': aArray[25] = 1; break; + case 'o': aArray[26] = 1; break; + case '*': + aArray[23] = 1; + aArray[24] = 1; + aArray[25] = 1; + aArray[26] = 1; + break; + default: return 1; } + break; + + case 'Z': + switch( zCat[1] ){ + case 'l': aArray[27] = 1; break; + case 'p': aArray[28] = 1; break; + case 's': aArray[29] = 1; break; + case '*': + aArray[27] = 1; + aArray[28] = 1; + aArray[29] = 1; + break; + default: return 1; } + break; + + + default: + return 1; + } + return 0; +} + +static u16 aFts5UnicodeBlock[] = { + 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760, + 1760, 1760, 1760, 1760, 1760, 1763, 1765, + }; +static u16 aFts5UnicodeMap[] = { + 0, 32, 33, 36, 37, 40, 41, 42, 43, 44, + 45, 46, 48, 58, 60, 63, 65, 91, 92, 93, + 94, 95, 96, 97, 123, 124, 125, 126, 127, 160, + 161, 162, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 180, 181, 182, 184, 185, + 186, 187, 188, 191, 192, 215, 216, 223, 247, 248, + 256, 312, 313, 329, 330, 377, 383, 385, 387, 388, + 391, 394, 396, 398, 402, 403, 405, 406, 409, 412, + 414, 415, 417, 418, 423, 427, 428, 431, 434, 436, + 437, 440, 442, 443, 444, 446, 448, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 477, 478, 496, + 497, 498, 499, 500, 503, 505, 506, 564, 570, 572, + 573, 575, 577, 580, 583, 584, 592, 660, 661, 688, + 706, 710, 722, 736, 741, 748, 749, 750, 751, 768, + 880, 884, 885, 886, 890, 891, 894, 900, 902, 903, + 904, 908, 910, 912, 913, 931, 940, 975, 977, 978, + 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072, + 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369, + 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473, + 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545, + 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611, + 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758, + 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791, + 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984, + 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075, + 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210, + 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369, + 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416, + 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482, + 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519, + 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561, + 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622, + 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677, + 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749, + 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790, + 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869, + 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902, + 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947, + 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006, + 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059, + 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134, + 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199, + 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263, + 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302, + 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402, + 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458, + 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544, + 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655, + 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737, + 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773, + 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860, + 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896, + 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967, + 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046, + 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153, + 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190, + 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229, + 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295, + 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704, + 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888, + 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743, + 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906, + 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068, + 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107, + 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160, + 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435, + 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480, + 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679, + 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754, + 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824, + 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978, + 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043, + 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098, + 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168, + 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288, + 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406, + 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616, + 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976, + 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033, + 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118, + 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141, + 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184, + 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219, + 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249, + 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275, + 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317, + 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413, + 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459, + 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484, + 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500, + 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523, + 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597, + 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623, + 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972, + 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180, + 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665, + 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091, + 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, + 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, + 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, + 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, + 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, + 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, + 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, + 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, + 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, + 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, + 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, + 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, + 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, + 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, + 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, + 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, + 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, + 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, + 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, + 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, + 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, + 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, + 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, + 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, + 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, + 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, + 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, + 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, + 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, + 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, + 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, + 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, + 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, + 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, + 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, + 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, + 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, + 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, + 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, + 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, + 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, + 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, + 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, + 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, + 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, + 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, + 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, + 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, + 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, + 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, + 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263, + 311, 320, 373, 377, 394, 400, 464, 509, 640, 672, + 768, 800, 816, 833, 834, 842, 896, 927, 928, 968, + 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103, + 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432, + 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623, + 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912, + 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178, + 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285, + 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416, + 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760, + 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216, + 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, + 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, + 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, + 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, + 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, + 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, + 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, + 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, + 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, + 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, + 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, + 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, + 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, + 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, + 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, + 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, + 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488, + 1, 32, 256, 0, 65533, + }; +static u16 aFts5UnicodeData[] = { + 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53, + 49, 85, 333, 85, 121, 85, 841, 54, 53, 50, + 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61, + 53, 151, 58, 53, 56, 58, 39, 52, 57, 34, + 58, 56, 58, 57, 79, 56, 37, 85, 56, 47, + 39, 51, 111, 53, 745, 57, 233, 773, 57, 261, + 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126, + 126, 73, 69, 137, 37, 73, 37, 105, 101, 73, + 37, 73, 37, 190, 158, 37, 126, 126, 73, 37, + 126, 94, 37, 39, 94, 69, 135, 41, 40, 37, + 41, 40, 37, 41, 40, 37, 542, 37, 606, 37, + 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37, + 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582, + 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596, + 158, 38, 56, 94, 38, 101, 53, 88, 41, 53, + 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105, + 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541, + 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38, + 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76, + 53, 76, 53, 44, 871, 103, 85, 162, 121, 85, + 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684, + 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58, + 204, 70, 76, 58, 140, 71, 333, 103, 90, 39, + 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333, + 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300, + 38, 108, 38, 172, 501, 807, 108, 53, 39, 359, + 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268, + 138, 44, 74, 39, 236, 327, 76, 85, 333, 53, + 38, 199, 231, 44, 74, 263, 71, 711, 231, 39, + 135, 44, 39, 106, 140, 74, 74, 44, 39, 42, + 71, 103, 76, 333, 71, 87, 207, 58, 55, 76, + 42, 199, 71, 711, 231, 71, 71, 71, 44, 106, + 76, 76, 108, 44, 135, 39, 333, 76, 103, 44, + 76, 42, 295, 103, 711, 231, 71, 167, 44, 39, + 106, 172, 76, 42, 74, 44, 39, 71, 76, 333, + 53, 55, 44, 74, 263, 71, 711, 231, 71, 167, + 44, 39, 42, 44, 42, 140, 74, 74, 44, 44, + 42, 71, 103, 76, 333, 58, 39, 207, 44, 39, + 199, 103, 135, 71, 39, 71, 71, 103, 391, 74, + 44, 74, 106, 106, 44, 39, 42, 333, 111, 218, + 55, 58, 106, 263, 103, 743, 327, 167, 39, 108, + 138, 108, 140, 76, 71, 71, 76, 333, 239, 58, + 74, 263, 103, 743, 327, 167, 44, 39, 42, 44, + 170, 44, 74, 74, 76, 74, 39, 71, 76, 333, + 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106, + 44, 39, 42, 71, 76, 333, 207, 58, 199, 74, + 583, 775, 295, 39, 231, 44, 106, 108, 44, 266, + 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268, + 53, 333, 85, 71, 39, 71, 39, 39, 135, 231, + 103, 39, 39, 71, 135, 44, 71, 204, 76, 39, + 167, 38, 204, 333, 135, 39, 122, 501, 58, 53, + 122, 76, 218, 333, 335, 58, 44, 58, 44, 58, + 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42, + 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90, + 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76, + 74, 76, 39, 333, 213, 199, 74, 76, 135, 108, + 39, 106, 71, 234, 103, 140, 423, 44, 74, 76, + 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41, + 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319, + 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151, + 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551, + 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108, + 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76, + 42, 236, 266, 44, 74, 364, 117, 38, 117, 55, + 39, 44, 333, 335, 213, 49, 149, 108, 61, 333, + 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138, + 76, 106, 74, 44, 202, 108, 58, 85, 333, 967, + 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76, + 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44, + 74, 268, 202, 332, 44, 333, 333, 245, 38, 213, + 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44, + 74, 231, 333, 245, 346, 300, 314, 76, 42, 967, + 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415, + 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159, + 266, 268, 74, 76, 181, 333, 103, 333, 967, 198, + 85, 277, 108, 53, 428, 42, 236, 135, 44, 135, + 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260, + 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265, + 261, 265, 197, 201, 261, 41, 41, 41, 94, 229, + 265, 453, 261, 264, 261, 264, 261, 264, 165, 69, + 137, 40, 56, 37, 120, 101, 69, 137, 40, 120, + 133, 69, 137, 120, 261, 169, 120, 101, 69, 137, + 40, 88, 381, 162, 209, 85, 52, 51, 54, 84, + 51, 54, 52, 277, 59, 60, 162, 61, 309, 52, + 51, 149, 80, 117, 57, 54, 50, 373, 57, 53, + 48, 341, 61, 162, 194, 47, 38, 207, 121, 54, + 50, 38, 335, 121, 54, 50, 422, 855, 428, 139, + 44, 107, 396, 90, 41, 154, 41, 90, 37, 105, + 69, 105, 37, 58, 41, 90, 57, 169, 218, 41, + 58, 41, 58, 41, 58, 137, 58, 37, 137, 37, + 135, 37, 90, 69, 73, 185, 94, 101, 58, 57, + 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186, + 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018, + 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666, + 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217, + 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57, + 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281, + 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69, + 254, 105, 37, 94, 37, 94, 165, 70, 105, 37, + 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221, + 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231, + 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52, + 51, 117, 52, 51, 53, 52, 51, 309, 49, 85, + 49, 53, 52, 51, 85, 52, 51, 54, 50, 54, + 50, 54, 50, 54, 50, 181, 38, 341, 81, 858, + 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 90, + 54, 50, 54, 50, 54, 50, 54, 50, 49, 54, + 82, 58, 302, 140, 74, 49, 166, 90, 110, 38, + 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887, + 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178, + 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274, + 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38, + 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333, + 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798, + 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69, + 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382, + 70, 37, 231, 44, 103, 44, 135, 44, 743, 74, + 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74, + 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333, + 903, 268, 85, 743, 364, 74, 53, 935, 108, 42, + 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333, + 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263, + 44, 42, 333, 149, 519, 38, 199, 122, 39, 42, + 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44, + 39, 71, 38, 85, 359, 42, 76, 74, 85, 39, + 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74, + 44, 74, 44, 74, 53, 42, 44, 333, 39, 39, + 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399, + 229, 165, 39, 44, 327, 57, 423, 167, 39, 71, + 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55, + 58, 524, 245, 54, 50, 53, 236, 53, 81, 80, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 85, 54, 50, 149, + 112, 117, 149, 49, 54, 50, 54, 50, 54, 50, + 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34, + 117, 55, 117, 54, 50, 53, 57, 53, 49, 85, + 333, 85, 121, 85, 841, 54, 53, 50, 56, 48, + 56, 837, 54, 57, 50, 57, 54, 50, 53, 54, + 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199, + 103, 87, 57, 56, 58, 87, 58, 153, 90, 98, + 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455, + 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575, + 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263, + 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71, + 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799, + 71, 39, 108, 76, 140, 135, 103, 871, 108, 44, + 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615, + 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655, + 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34, + 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149, + 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383, + 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182, + 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898, + 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236, + 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837, + 841, 229, 581, 841, 837, 41, 73, 41, 73, 137, + 265, 133, 37, 229, 357, 841, 837, 73, 137, 265, + 233, 837, 73, 137, 169, 41, 233, 837, 841, 837, + 841, 837, 841, 837, 841, 837, 841, 837, 841, 901, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71, + 39, 39, 327, 135, 39, 39, 39, 39, 39, 39, + 103, 71, 39, 39, 39, 39, 39, 39, 71, 39, + 135, 231, 135, 135, 39, 327, 551, 103, 167, 551, + 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946, + 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210, + 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266, + 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351, + 34, 3074, 7692, 63, 63, + }; + +int sqlite3Fts5UnicodeCategory(u32 iCode) { + int iRes = -1; + int iHi; + int iLo; + int ret; + u16 iKey; + + if( iCode>=(1<<20) ){ + return 0; + } + iLo = aFts5UnicodeBlock[(iCode>>16)]; + iHi = aFts5UnicodeBlock[1+(iCode>>16)]; + iKey = (iCode & 0xFFFF); + while( iHi>iLo ){ + int iTest = (iHi + iLo) / 2; + assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest; + } + } + + if( iRes<0 ) return 0; + if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0; + ret = aFts5UnicodeData[iRes] & 0x1F; + if( ret!=30 ) return ret; + return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9; +} + +void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ + int i = 0; + int iTbl = 0; + while( i<128 ){ + int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; + int n = (aFts5UnicodeData[iTbl] >> 5) + i; + for(; i<128 && i3 && n<=9 ); + return n; + } +} + + +/* +** Bitmasks used by sqlite3GetVarint(). These precomputed constants +** are defined here rather than simply putting the constant expressions +** inline in order to work around bugs in the RVT compiler. +** +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f +** +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 +*/ +#define SLOT_2_0 0x001fc07f +#define SLOT_4_2_0 0xf01fc07f + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +*/ +u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + + a = *p; + /* a: p0 (unmasked) */ + if (!(a&0x80)) + { + *v = a; + return 1; + } + + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + a &= 0x7f; + a = a<<7; + a |= b; + *v = a; + return 2; + } + + /* Verify that constants are precomputed correctly */ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_2_0; + b &= 0x7f; + b = b<<7; + a |= b; + *v = a; + return 3; + } + + /* CSE1 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_2_0; + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + *v = a; + return 4; + } + + /* a: p0<<14 | p2 (masked) */ + /* b: p1<<14 | p3 (unmasked) */ + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + b &= SLOT_2_0; + s = a; + /* s: p0<<14 | p2 (masked) */ + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + /* b &= (0x7f<<14)|(0x7f); */ + b = b<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 5; + } + + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + s = s<<7; + s |= b; + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + + p++; + b = b<<14; + b |= *p; + /* b: p1<<28 | p3<<14 | p5 (unmasked) */ + if (!(b&0x80)) + { + /* we can skip this cause it was (effectively) done above in calc'ing s */ + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + a &= SLOT_2_0; + a = a<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 6; + } + + p++; + a = a<<14; + a |= *p; + /* a: p2<<28 | p4<<14 | p6 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b<<7; + a |= b; + s = s>>11; + *v = ((u64)s)<<32 | a; + return 7; + } + + /* CSE2 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p3<<28 | p5<<14 | p7 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_4_2_0; + /* moved CSE2 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + s = s>>4; + *v = ((u64)s)<<32 | a; + return 8; + } + + p++; + a = a<<15; + a |= *p; + /* a: p4<<29 | p6<<15 | p8 (unmasked) */ + + /* moved CSE2 up */ + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ + b &= SLOT_2_0; + b = b<<8; + a |= b; + + s = s<<4; + b = p[-4]; + b &= 0x7f; + b = b>>3; + s |= b; + + *v = ((u64)s)<<32 | a; + + return 9; +} + +/* +** The variable-length integer encoding is as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** C = xxxxxxxx 8 bits of data +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** 28 bits - BBBA +** 35 bits - BBBBA +** 42 bits - BBBBBA +** 49 bits - BBBBBBA +** 56 bits - BBBBBBBA +** 64 bits - BBBBBBBBC +*/ + +#ifdef SQLITE_NOINLINE +# define FTS5_NOINLINE SQLITE_NOINLINE +#else +# define FTS5_NOINLINE +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data write will be between 1 and 9 bytes. The number +** of bytes written is returned. +** +** A variable-length integer consists of the lower 7 bits of each byte +** for all bytes that have the 8th bit set and one byte with the 8th +** bit clear. Except, if we get to the 9th byte, it stores the full +** 8 bits and is the last byte. +*/ +static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){ + int i, j, n; + u8 buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (u8)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + assert( n<=9 ); + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + +int sqlite3Fts5PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; + return 1; + } + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; + return 2; + } + return fts5PutVarint64(p,v); +} + + +int sqlite3Fts5GetVarintLen(u32 iVal){ +#if 0 + if( iVal<(1 << 7 ) ) return 1; +#endif + assert( iVal>=(1 << 7) ); + if( iVal<(1 << 14) ) return 2; + if( iVal<(1 << 21) ) return 3; + if( iVal<(1 << 28) ) return 4; + return 5; +} diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c new file mode 100644 index 0000000000000000000000000000000000000000..fb280567f4e00636543ebc49c53077af2885c9b4 --- /dev/null +++ b/ext/fts5/fts5_vocab.c @@ -0,0 +1,808 @@ +/* +** 2015 May 08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite virtual table module implementing direct access to an +** existing FTS5 index. The module may create several different types of +** tables: +** +** col: +** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col)); +** +** One row for each term/column combination. The value of $doc is set to +** the number of fts5 rows that contain at least one instance of term +** $term within column $col. Field $cnt is set to the total number of +** instances of term $term in column $col (in any row of the fts5 table). +** +** row: +** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term)); +** +** One row for each term in the database. The value of $doc is set to +** the number of fts5 rows that contain at least one instance of term +** $term. Field $cnt is set to the total number of instances of term +** $term in the database. +** +** instance: +** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY()); +** +** One row for each term instance in the database. +*/ + + +#include "fts5Int.h" + + +typedef struct Fts5VocabTable Fts5VocabTable; +typedef struct Fts5VocabCursor Fts5VocabCursor; + +struct Fts5VocabTable { + sqlite3_vtab base; + char *zFts5Tbl; /* Name of fts5 table */ + char *zFts5Db; /* Db containing fts5 table */ + sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* FTS5 global object for this database */ + int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */ + unsigned bBusy; /* True if busy */ +}; + +struct Fts5VocabCursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ + Fts5Table *pFts5; /* Associated FTS5 table */ + + int bEof; /* True if this cursor is at EOF */ + Fts5IndexIter *pIter; /* Term/rowid iterator object */ + void *pStruct; /* From sqlite3Fts5StructureRef() */ + + int nLeTerm; /* Size of zLeTerm in bytes */ + char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + int colUsed; /* Copy of sqlite3_index_info.colUsed */ + + /* These are used by 'col' tables only */ + int iCol; + i64 *aCnt; + i64 *aDoc; + + /* Output values used by all tables. */ + i64 rowid; /* This table's current rowid value */ + Fts5Buffer term; /* Current value of 'term' column */ + + /* Output values Used by 'instance' tables only */ + i64 iInstPos; + int iInstOff; +}; + +#define FTS5_VOCAB_COL 0 +#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_INSTANCE 2 + +#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" +#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" +#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset" + +/* +** Bits for the mask used as the idxNum value by xBestIndex/xFilter. +*/ +#define FTS5_VOCAB_TERM_EQ 0x0100 +#define FTS5_VOCAB_TERM_GE 0x0200 +#define FTS5_VOCAB_TERM_LE 0x0400 + +#define FTS5_VOCAB_COLUSED_MASK 0xFF + + +/* +** Translate a string containing an fts5vocab table type to an +** FTS5_VOCAB_XXX constant. If successful, set *peType to the output +** value and return SQLITE_OK. Otherwise, set *pzErr to an error message +** and return SQLITE_ERROR. +*/ +static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){ + int rc = SQLITE_OK; + char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1); + if( rc==SQLITE_OK ){ + sqlite3Fts5Dequote(zCopy); + if( sqlite3_stricmp(zCopy, "col")==0 ){ + *peType = FTS5_VOCAB_COL; + }else + + if( sqlite3_stricmp(zCopy, "row")==0 ){ + *peType = FTS5_VOCAB_ROW; + }else + if( sqlite3_stricmp(zCopy, "instance")==0 ){ + *peType = FTS5_VOCAB_INSTANCE; + }else + { + *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); + rc = SQLITE_ERROR; + } + sqlite3_free(zCopy); + } + + return rc; +} + + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5vocab") +** argv[1] -> database name +** argv[2] -> table name +** +** then: +** +** argv[3] -> name of fts5 table +** argv[4] -> type of fts5vocab table +** +** or, for tables in the TEMP schema only. +** +** argv[3] -> name of fts5 tables database +** argv[4] -> name of fts5 table +** argv[5] -> type of fts5vocab table +*/ +static int fts5VocabInitVtab( + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Pointer to Fts5Global object */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + const char *azSchema[] = { + "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")" + }; + + Fts5VocabTable *pRet = 0; + int rc = SQLITE_OK; /* Return code */ + int bDb; + + bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0); + + if( argc!=5 && bDb==0 ){ + *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); + rc = SQLITE_ERROR; + }else{ + int nByte; /* Bytes of space to allocate */ + const char *zDb = bDb ? argv[3] : argv[1]; + const char *zTab = bDb ? argv[4] : argv[3]; + const char *zType = bDb ? argv[5] : argv[4]; + int nDb = (int)strlen(zDb)+1; + int nTab = (int)strlen(zTab)+1; + int eType = 0; + + rc = fts5VocabTableType(zType, pzErr, &eType); + if( rc==SQLITE_OK ){ + assert( eType>=0 && eTypepGlobal = (Fts5Global*)pAux; + pRet->eType = eType; + pRet->db = db; + pRet->zFts5Tbl = (char*)&pRet[1]; + pRet->zFts5Db = &pRet->zFts5Tbl[nTab]; + memcpy(pRet->zFts5Tbl, zTab, nTab); + memcpy(pRet->zFts5Db, zDb, nDb); + sqlite3Fts5Dequote(pRet->zFts5Tbl); + sqlite3Fts5Dequote(pRet->zFts5Db); + } + } + + *ppVTab = (sqlite3_vtab*)pRet; + return rc; +} + + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5VocabInitVtab(). +*/ +static int fts5VocabConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5VocabCreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Implementation of the xBestIndex method. +** +** Only constraints of the form: +** +** term <= ? +** term == ? +** term >= ? +** +** are interpreted. Less-than and less-than-or-equal are treated +** identically, as are greater-than and greater-than-or-equal. +*/ +static int fts5VocabBestIndexMethod( + sqlite3_vtab *pUnused, + sqlite3_index_info *pInfo +){ + int i; + int iTermEq = -1; + int iTermGe = -1; + int iTermLe = -1; + int idxNum = (int)pInfo->colUsed; + int nArg = 0; + + UNUSED_PARAM(pUnused); + + assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed ); + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->iColumn==0 ){ /* term column */ + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i; + if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i; + } + } + + if( iTermEq>=0 ){ + idxNum |= FTS5_VOCAB_TERM_EQ; + pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg; + pInfo->estimatedCost = 100; + }else{ + pInfo->estimatedCost = 1000000; + if( iTermGe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_GE; + pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + if( iTermLe>=0 ){ + idxNum |= FTS5_VOCAB_TERM_LE; + pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg; + pInfo->estimatedCost = pInfo->estimatedCost / 2; + } + } + + /* This virtual table always delivers results in ascending order of + ** the "term" column (column 0). So if the user has requested this + ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the + ** sqlite3_index_info.orderByConsumed flag to tell the core the results + ** are already in sorted order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + pInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts5VocabOpenMethod( + sqlite3_vtab *pVTab, + sqlite3_vtab_cursor **ppCsr +){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; + Fts5Table *pFts5 = 0; + Fts5VocabCursor *pCsr = 0; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + char *zSql = 0; + + if( pTab->bBusy ){ + pVTab->zErrMsg = sqlite3_mprintf( + "recursive definition for %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + return SQLITE_ERROR; + } + zSql = sqlite3Fts5Mprintf(&rc, + "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", + pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl + ); + if( zSql ){ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc==SQLITE_ERROR ) rc = SQLITE_OK; + + pTab->bBusy = 1; + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + i64 iId = sqlite3_column_int64(pStmt, 0); + pFts5 = sqlite3Fts5TableFromCsrid(pTab->pGlobal, iId); + } + pTab->bBusy = 0; + + if( rc==SQLITE_OK ){ + if( pFts5==0 ){ + rc = sqlite3_finalize(pStmt); + pStmt = 0; + if( rc==SQLITE_OK ){ + pVTab->zErrMsg = sqlite3_mprintf( + "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + rc = SQLITE_ERROR; + } + }else{ + rc = sqlite3Fts5FlushToDisk(pFts5); + } + } + + if( rc==SQLITE_OK ){ + i64 nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor); + pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); + } + + if( pCsr ){ + pCsr->pFts5 = pFts5; + pCsr->pStmt = pStmt; + pCsr->aCnt = (i64*)&pCsr[1]; + pCsr->aDoc = &pCsr->aCnt[pFts5->pConfig->nCol]; + }else{ + sqlite3_finalize(pStmt); + } + + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + pCsr->rowid = 0; + sqlite3Fts5IterClose(pCsr->pIter); + sqlite3Fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + pCsr->pIter = 0; + sqlite3_free(pCsr->zLeTerm); + pCsr->nLeTerm = -1; + pCsr->zLeTerm = 0; + pCsr->bEof = 0; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + fts5VocabResetCursor(pCsr); + sqlite3Fts5BufferFree(&pCsr->term); + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + } + return rc; +} + +static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){ + int eDetail = pCsr->pFts5->pConfig->eDetail; + int rc = SQLITE_OK; + Fts5IndexIter *pIter = pCsr->pIter; + i64 *pp = &pCsr->iInstPos; + int *po = &pCsr->iInstOff; + + assert( sqlite3Fts5IterEof(pIter)==0 ); + assert( pCsr->bEof==0 ); + while( eDetail==FTS5_DETAIL_NONE + || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) + ){ + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + if( rc==SQLITE_OK ){ + rc = fts5VocabInstanceNewTerm(pCsr); + if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break; + } + if( rc ){ + pCsr->bEof = 1; + break; + } + } + + return rc; +} + +/* +** Advance the cursor to the next row in the table. +*/ +static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; + int nCol = pCsr->pFts5->pConfig->nCol; + int rc; + + rc = sqlite3Fts5StructureTest(pCsr->pFts5->pIndex, pCsr->pStruct); + if( rc!=SQLITE_OK ) return rc; + pCsr->rowid++; + + if( pTab->eType==FTS5_VOCAB_INSTANCE ){ + return fts5VocabInstanceNext(pCsr); + } + + if( pTab->eType==FTS5_VOCAB_COL ){ + for(pCsr->iCol++; pCsr->iColiCol++){ + if( pCsr->aDoc[pCsr->iCol] ) break; + } + } + + if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){ + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + assert( nTerm>=0 ); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + return SQLITE_OK; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + memset(pCsr->aCnt, 0, nCol * sizeof(i64)); + memset(pCsr->aDoc, 0, nCol * sizeof(i64)); + pCsr->iCol = 0; + + assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); + while( rc==SQLITE_OK ){ + int eDetail = pCsr->pFts5->pConfig->eDetail; + const u8 *pPos; int nPos; /* Position list */ + i64 iPos = 0; /* 64-bit position read from poslist */ + int iOff = 0; /* Current offset within position list */ + + pPos = pCsr->pIter->pData; + nPos = pCsr->pIter->nData; + + switch( pTab->eType ){ + case FTS5_VOCAB_ROW: + /* Do not bother counting the number of instances if the "cnt" + ** column is not being read (according to colUsed). */ + if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){ + while( iPosaCnt[] */ + pCsr->aCnt[0]++; + } + } + } + pCsr->aDoc[0]++; + break; + + case FTS5_VOCAB_COL: + if( eDetail==FTS5_DETAIL_FULL ){ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + if( iCol!=ii ){ + if( ii>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[ii]++; + iCol = ii; + } + pCsr->aCnt[ii]++; + } + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPos=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[iPos]++; + } + }else{ + assert( eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; + } + break; + + default: + assert( pTab->eType==FTS5_VOCAB_INSTANCE ); + break; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + } + if( pTab->eType==FTS5_VOCAB_INSTANCE ) break; + + if( rc==SQLITE_OK ){ + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( nTerm!=pCsr->term.n + || (nTerm>0 && memcmp(zTerm, pCsr->term.p, nTerm)) + ){ + break; + } + if( sqlite3Fts5IterEof(pCsr->pIter) ) break; + } + } + } + } + + if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ + for(/* noop */; pCsr->iColaDoc[pCsr->iCol]==0; pCsr->iCol++); + if( pCsr->iCol==nCol ){ + rc = FTS5_CORRUPT; + } + } + return rc; +} + +/* +** This is the xFilter implementation for the virtual table. +*/ +static int fts5VocabFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *zUnused, /* Unused */ + int nUnused, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eType = pTab->eType; + int rc = SQLITE_OK; + + int iVal = 0; + int f = FTS5INDEX_QUERY_SCAN; + const char *zTerm = 0; + int nTerm = 0; + + sqlite3_value *pEq = 0; + sqlite3_value *pGe = 0; + sqlite3_value *pLe = 0; + + UNUSED_PARAM2(zUnused, nUnused); + + fts5VocabResetCursor(pCsr); + if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK); + + if( pEq ){ + zTerm = (const char *)sqlite3_value_text(pEq); + nTerm = sqlite3_value_bytes(pEq); + f = FTS5INDEX_QUERY_NOTOKENDATA; + }else{ + if( pGe ){ + zTerm = (const char *)sqlite3_value_text(pGe); + nTerm = sqlite3_value_bytes(pGe); + } + if( pLe ){ + const char *zCopy = (const char *)sqlite3_value_text(pLe); + if( zCopy==0 ) zCopy = ""; + pCsr->nLeTerm = sqlite3_value_bytes(pLe); + pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + if( pCsr->zLeTerm==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); + } + } + } + + if( rc==SQLITE_OK ){ + Fts5Index *pIndex = pCsr->pFts5->pIndex; + rc = sqlite3Fts5IndexQuery(pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); + if( rc==SQLITE_OK ){ + pCsr->pStruct = sqlite3Fts5StructureRef(pIndex); + } + } + if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){ + rc = fts5VocabInstanceNewTerm(pCsr); + } + if( rc==SQLITE_OK && !pCsr->bEof + && (eType!=FTS5_VOCAB_INSTANCE + || pCsr->pFts5->pConfig->eDetail!=FTS5_DETAIL_NONE) + ){ + rc = fts5VocabNextMethod(pCursor); + } + + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + return pCsr->bEof; +} + +static int fts5VocabColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eDetail = pCsr->pFts5->pConfig->eDetail; + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; + i64 iVal = 0; + + if( iCol==0 ){ + sqlite3_result_text( + pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT + ); + }else if( eType==FTS5_VOCAB_COL ){ + assert( iCol==1 || iCol==2 || iCol==3 ); + if( iCol==1 ){ + if( eDetail!=FTS5_DETAIL_NONE ){ + const char *z = pCsr->pFts5->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + }else if( iCol==2 ){ + iVal = pCsr->aDoc[pCsr->iCol]; + }else{ + iVal = pCsr->aCnt[pCsr->iCol]; + } + }else if( eType==FTS5_VOCAB_ROW ){ + assert( iCol==1 || iCol==2 ); + if( iCol==1 ){ + iVal = pCsr->aDoc[0]; + }else{ + iVal = pCsr->aCnt[0]; + } + }else{ + assert( eType==FTS5_VOCAB_INSTANCE ); + switch( iCol ){ + case 1: + sqlite3_result_int64(pCtx, pCsr->pIter->iRowid); + break; + case 2: { + int ii = -1; + if( eDetail==FTS5_DETAIL_FULL ){ + ii = FTS5_POS2COLUMN(pCsr->iInstPos); + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + ii = (int)pCsr->iInstPos; + } + if( ii>=0 && iipFts5->pConfig->nCol ){ + const char *z = pCsr->pFts5->pConfig->azCol[ii]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + break; + } + default: { + assert( iCol==3 ); + if( eDetail==FTS5_DETAIL_FULL ){ + int ii = FTS5_POS2OFFSET(pCsr->iInstPos); + sqlite3_result_int(pCtx, ii); + } + break; + } + } + } + + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); + return SQLITE_OK; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. The +** rowid should be written to *pRowid. +*/ +static int fts5VocabRowidMethod( + sqlite3_vtab_cursor *pCursor, + sqlite_int64 *pRowid +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + *pRowid = pCsr->rowid; + return SQLITE_OK; +} + +int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ + static const sqlite3_module fts5Vocab = { + /* iVersion */ 2, + /* xCreate */ fts5VocabCreateMethod, + /* xConnect */ fts5VocabConnectMethod, + /* xBestIndex */ fts5VocabBestIndexMethod, + /* xDisconnect */ fts5VocabDisconnectMethod, + /* xDestroy */ fts5VocabDestroyMethod, + /* xOpen */ fts5VocabOpenMethod, + /* xClose */ fts5VocabCloseMethod, + /* xFilter */ fts5VocabFilterMethod, + /* xNext */ fts5VocabNextMethod, + /* xEof */ fts5VocabEofMethod, + /* xColumn */ fts5VocabColumnMethod, + /* xRowid */ fts5VocabRowidMethod, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindFunction */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 + }; + void *p = (void*)pGlobal; + + return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0); +} diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y new file mode 100644 index 0000000000000000000000000000000000000000..134125db1faa7212bea68855d76e8161a171fe74 --- /dev/null +++ b/ext/fts5/fts5parse.y @@ -0,0 +1,197 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + + +// All token codes are small integers with #defines that begin with "TK_" +%token_prefix FTS5_ + +// The type of the data attached to each token is Token. This is also the +// default type for non-terminals. +// +%token_type {Fts5Token} +%default_type {Fts5Token} + +// The generated parser function takes a 4th argument as follows: +%extra_argument {Fts5Parse *pParse} + +// This code runs whenever there is a syntax error +// +%syntax_error { + UNUSED_PARAM(yymajor); /* Silence a compiler warning */ + sqlite3Fts5ParseError( + pParse, "fts5: syntax error near \"%.*s\"",TOKEN.n,TOKEN.p + ); +} +%stack_overflow { + sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow"); +} + +// The name of the generated procedure that implements the parser +// is as follows: +%name sqlite3Fts5Parser + +// The following text is included near the beginning of the C source +// code file that implements the parser. +// +%include { +#include "fts5Int.h" +#include "fts5parse.h" + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +#define yytestcase(X) testcase(X) + +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENOTNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + +} // end %include + +%left OR. +%left AND. +%left NOT. +%left TERM. +%left COLON. + +input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); } +%destructor input { (void)pParse; } + +%type cnearset {Fts5ExprNode*} +%type expr {Fts5ExprNode*} +%type exprlist {Fts5ExprNode*} +%destructor cnearset { sqlite3Fts5ParseNodeFree($$); } +%destructor expr { sqlite3Fts5ParseNodeFree($$); } +%destructor exprlist { sqlite3Fts5ParseNodeFree($$); } + +%type colset {Fts5Colset*} +%destructor colset { sqlite3_free($$); } +%type colsetlist {Fts5Colset*} +%destructor colsetlist { sqlite3_free($$); } + +colset(A) ::= MINUS LCP colsetlist(X) RCP. { + A = sqlite3Fts5ParseColsetInvert(pParse, X); +} +colset(A) ::= LCP colsetlist(X) RCP. { A = X; } +colset(A) ::= STRING(X). { + A = sqlite3Fts5ParseColset(pParse, 0, &X); +} +colset(A) ::= MINUS STRING(X). { + A = sqlite3Fts5ParseColset(pParse, 0, &X); + A = sqlite3Fts5ParseColsetInvert(pParse, A); +} + +colsetlist(A) ::= colsetlist(Y) STRING(X). { + A = sqlite3Fts5ParseColset(pParse, Y, &X); } +colsetlist(A) ::= STRING(X). { + A = sqlite3Fts5ParseColset(pParse, 0, &X); +} + +expr(A) ::= expr(X) AND expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0); +} +expr(A) ::= expr(X) OR expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0); +} +expr(A) ::= expr(X) NOT expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0); +} + +expr(A) ::= colset(X) COLON LP expr(Y) RP. { + sqlite3Fts5ParseSetColset(pParse, Y, X); + A = Y; +} +expr(A) ::= LP expr(X) RP. {A = X;} +expr(A) ::= exprlist(X). {A = X;} + +exprlist(A) ::= cnearset(X). {A = X;} +exprlist(A) ::= exprlist(X) cnearset(Y). { + A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y); +} + +cnearset(A) ::= nearset(X). { + A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); +} +cnearset(A) ::= colset(X) COLON nearset(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); + sqlite3Fts5ParseSetColset(pParse, A, X); +} + + +%type nearset {Fts5ExprNearset*} +%type nearphrases {Fts5ExprNearset*} +%destructor nearset { sqlite3Fts5ParseNearsetFree($$); } +%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); } + +nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); } +nearset(A) ::= CARET phrase(Y). { + sqlite3Fts5ParseSetCaret(Y); + A = sqlite3Fts5ParseNearset(pParse, 0, Y); +} +nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. { + sqlite3Fts5ParseNear(pParse, &X); + sqlite3Fts5ParseSetDistance(pParse, Y, &Z); + A = Y; +} + +nearphrases(A) ::= phrase(X). { + A = sqlite3Fts5ParseNearset(pParse, 0, X); +} +nearphrases(A) ::= nearphrases(X) phrase(Y). { + A = sqlite3Fts5ParseNearset(pParse, X, Y); +} + +/* +** The optional ", " at the end of the NEAR() arguments. +*/ +neardist_opt(A) ::= . { A.p = 0; A.n = 0; } +neardist_opt(A) ::= COMMA STRING(X). { A = X; } + +/* +** A phrase. A set of primitives connected by "+" operators. Examples: +** +** "the" + "quick brown" + fo * +** "the quick brown fo" * +** the+quick+brown+fo* +*/ +%type phrase {Fts5ExprPhrase*} +%destructor phrase { sqlite3Fts5ParsePhraseFree($$); } + +phrase(A) ::= phrase(X) PLUS STRING(Y) star_opt(Z). { + A = sqlite3Fts5ParseTerm(pParse, X, &Y, Z); +} +phrase(A) ::= STRING(Y) star_opt(Z). { + A = sqlite3Fts5ParseTerm(pParse, 0, &Y, Z); +} + +/* +** Optional "*" character. +*/ +%type star_opt {int} +star_opt(A) ::= STAR. { A = 1; } +star_opt(A) ::= . { A = 0; } diff --git a/ext/fts5/mkportersteps.tcl b/ext/fts5/mkportersteps.tcl new file mode 100644 index 0000000000000000000000000000000000000000..b6214c6bf73f8edf9600e26502a534e0e2a02a39 --- /dev/null +++ b/ext/fts5/mkportersteps.tcl @@ -0,0 +1,222 @@ +# +# 2014 Jun 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#------------------------------------------------------------------------- +# +# This script generates the implementations of the following C functions, +# which are part of the porter tokenizer implementation: +# +# static int fts5PorterStep1B(char *aBuf, int *pnBuf); +# static int fts5PorterStep1B2(char *aBuf, int *pnBuf); +# static int fts5PorterStep2(char *aBuf, int *pnBuf); +# static int fts5PorterStep3(char *aBuf, int *pnBuf); +# static int fts5PorterStep4(char *aBuf, int *pnBuf); +# + +set O(Step1B2) { + { at {} ate 1 } + { bl {} ble 1 } + { iz {} ize 1 } +} + +set O(Step1B) { + { "eed" fts5Porter_MGt0 "ee" 0 } + { "ed" fts5Porter_Vowel "" 1 } + { "ing" fts5Porter_Vowel "" 1 } +} + +set O(Step2) { + { "ational" fts5Porter_MGt0 "ate" } + { "tional" fts5Porter_MGt0 "tion" } + { "enci" fts5Porter_MGt0 "ence" } + { "anci" fts5Porter_MGt0 "ance" } + { "izer" fts5Porter_MGt0 "ize" } + { "logi" fts5Porter_MGt0 "log" } + { "bli" fts5Porter_MGt0 "ble" } + { "alli" fts5Porter_MGt0 "al" } + { "entli" fts5Porter_MGt0 "ent" } + { "eli" fts5Porter_MGt0 "e" } + { "ousli" fts5Porter_MGt0 "ous" } + { "ization" fts5Porter_MGt0 "ize" } + { "ation" fts5Porter_MGt0 "ate" } + { "ator" fts5Porter_MGt0 "ate" } + { "alism" fts5Porter_MGt0 "al" } + { "iveness" fts5Porter_MGt0 "ive" } + { "fulness" fts5Porter_MGt0 "ful" } + { "ousness" fts5Porter_MGt0 "ous" } + { "aliti" fts5Porter_MGt0 "al" } + { "iviti" fts5Porter_MGt0 "ive" } + { "biliti" fts5Porter_MGt0 "ble" } +} + +set O(Step3) { + { "icate" fts5Porter_MGt0 "ic" } + { "ative" fts5Porter_MGt0 "" } + { "alize" fts5Porter_MGt0 "al" } + { "iciti" fts5Porter_MGt0 "ic" } + { "ical" fts5Porter_MGt0 "ic" } + { "ful" fts5Porter_MGt0 "" } + { "ness" fts5Porter_MGt0 "" } +} + +set O(Step4) { + { "al" fts5Porter_MGt1 "" } + { "ance" fts5Porter_MGt1 "" } + { "ence" fts5Porter_MGt1 "" } + { "er" fts5Porter_MGt1 "" } + { "ic" fts5Porter_MGt1 "" } + { "able" fts5Porter_MGt1 "" } + { "ible" fts5Porter_MGt1 "" } + { "ant" fts5Porter_MGt1 "" } + { "ement" fts5Porter_MGt1 "" } + { "ment" fts5Porter_MGt1 "" } + { "ent" fts5Porter_MGt1 "" } + { "ion" fts5Porter_MGt1_and_S_or_T "" } + { "ou" fts5Porter_MGt1 "" } + { "ism" fts5Porter_MGt1 "" } + { "ate" fts5Porter_MGt1 "" } + { "iti" fts5Porter_MGt1 "" } + { "ous" fts5Porter_MGt1 "" } + { "ive" fts5Porter_MGt1 "" } + { "ize" fts5Porter_MGt1 "" } +} + +proc sort_cb {lhs rhs} { + set L [string range [lindex $lhs 0] end-1 end-1] + set R [string range [lindex $rhs 0] end-1 end-1] + string compare $L $R +} + +proc create_step_function {name data} { + + set T(function) { +static int fts5Porter${name}(char *aBuf, int *pnBuf){ + int ret = 0; + int nBuf = *pnBuf; + switch( aBuf[nBuf-2] ){ + ${switchbody} + } + return ret; +} + } + + set T(case) { + case '${k}': + ${ifstmts} + break; + } + + set T(if_0_0_0) { + if( ${match} ){ + *pnBuf = nBuf - $n; + } + } + set T(if_1_0_0) { + if( ${match} ){ + if( ${cond} ){ + *pnBuf = nBuf - $n; + } + } + } + set T(if_0_1_0) { + if( ${match} ){ + ${memcpy} + *pnBuf = nBuf - $n + $nRep; + } + } + set T(if_1_1_0) { + if( ${match} ){ + if( ${cond} ){ + ${memcpy} + *pnBuf = nBuf - $n + $nRep; + } + } + } + set T(if_1_0_1) { + if( ${match} ){ + if( ${cond} ){ + *pnBuf = nBuf - $n; + ret = 1; + } + } + } + set T(if_0_1_1) { + if( ${match} ){ + ${memcpy} + *pnBuf = nBuf - $n + $nRep; + ret = 1; + } + } + set T(if_1_1_1) { + if( ${match} ){ + if( ${cond} ){ + ${memcpy} + *pnBuf = nBuf - $n + $nRep; + ret = 1; + } + } + } + + set switchbody "" + + foreach I $data { + set k [string range [lindex $I 0] end-1 end-1] + lappend aCase($k) $I + } + foreach k [lsort [array names aCase]] { + set ifstmts "" + foreach I $aCase($k) { + set zSuffix [lindex $I 0] ;# Suffix text for this rule + set zRep [lindex $I 2] ;# Replacement text for rule + set xCond [lindex $I 1] ;# Condition callback (or "") + + set n [string length $zSuffix] + set nRep [string length $zRep] + + set match "nBuf>$n && 0==memcmp(\"$zSuffix\", &aBuf\[nBuf-$n\], $n)" + set memcpy "memcpy(&aBuf\[nBuf-$n\], \"$zRep\", $nRep);" + set cond "${xCond}(aBuf, nBuf-$n)" + + set bMemcpy [expr {$nRep>0}] + set bCond [expr {$xCond!=""}] + set bRet [expr {[llength $I]>3 && [lindex $I 3]}] + + set t $T(if_${bCond}_${bMemcpy}_${bRet}) + lappend ifstmts [string trim [subst -nocommands $t]] + } + + set ifstmts [join $ifstmts "else "] + + append switchbody [subst -nocommands $T(case)] + } + + + puts [subst -nocommands $T(function)] +} + + +puts [string trim { +/************************************************************************** +*************************************************************************** +** GENERATED CODE STARTS HERE (mkportersteps.tcl) +*/ +}] +foreach step [array names O] { + create_step_function $step $O($step) +} +puts [string trim { +/* +** GENERATED CODE ENDS HERE (mkportersteps.tcl) +*************************************************************************** +**************************************************************************/ +}] + + + diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl new file mode 100644 index 0000000000000000000000000000000000000000..8ea87dbdd14a4adbc01626fd8a997f26de51fffc --- /dev/null +++ b/ext/fts5/test/fts5_common.tcl @@ -0,0 +1,708 @@ +# 2014 Dec 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. .. test] +} +source $testdir/tester.tcl + +ifcapable !fts5 { + proc return_if_no_fts5 {} { + finish_test + return -code return + } + return +} else { + proc return_if_no_fts5 {} {} +} + +catch { + sqlite3_fts5_may_be_corrupt 0 + reset_db +} + +proc fts5_test_poslist {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xInstCount]} {incr i} { + lappend res [string map {{ } .} [$cmd xInst $i]] + } + set res +} + +proc fts5_test_poslist2 {cmd} { + set res [list] + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + $cmd xPhraseForeach $i c o { + lappend res $i.$c.$o + } + } + + #set res + sort_poslist $res +} + +proc fts5_test_insttoken {cmd iInst iToken} { + $cmd xInstToken $iInst $iToken +} + +proc fts5_test_collist {cmd} { + set res [list] + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + $cmd xPhraseColumnForeach $i c { lappend res $i.$c } + } + + set res +} + +proc fts5_collist {cmd iPhrase} { + set res [list] + $cmd xPhraseColumnForeach $iPhrase c { lappend res $c } + set res +} + +proc fts5_test_columnsize {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnSize $i] + } + set res +} + +proc fts5_columntext {cmd iCol} { + $cmd xColumnText $iCol +} +proc fts5_columnlocale {cmd iCol} { + $cmd xColumnLocale $iCol +} + +proc fts5_test_columntext {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnText $i] + } + set res +} + +proc fts5_test_columnlocale {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnLocale $i] + } + set res +} + +proc fts5_test_columntotalsize {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnTotalSize $i] + } + set res +} + +proc test_append_token {varname token iStart iEnd} { + upvar $varname var + lappend var $token + return "SQLITE_OK" +} +proc fts5_test_tokenize {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + set tokens [list] + $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens] + lappend res $tokens + } + set res +} + +proc fts5_test_rowcount {cmd} { + $cmd xRowCount +} + +proc fts5_test_rowid {cmd} { + $cmd xRowid +} + +proc test_queryphrase_cb {cnt cmd} { + upvar $cnt L + for {set i 0} {$i < [$cmd xInstCount]} {incr i} { + foreach {ip ic io} [$cmd xInst $i] break + set A($ic) 1 + } + foreach ic [array names A] { + lset L $ic [expr {[lindex $L $ic] + 1}] + } +} +proc fts5_test_queryphrase {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + set cnt [list] + for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 } + $cmd xQueryPhrase $i [list test_queryphrase_cb cnt] + lappend res $cnt + } + set res +} + +proc fts5_queryphrase {cmd iPhrase} { + set cnt [list] + for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 } + $cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt] + set cnt +} + +proc fts5_test_phrasecount {cmd} { + $cmd xPhraseCount +} + +proc fts5_test_all {cmd} { + set res [list] + lappend res columnsize [fts5_test_columnsize $cmd] + lappend res columntext [fts5_test_columntext $cmd] + lappend res columntotalsize [fts5_test_columntotalsize $cmd] + lappend res poslist [fts5_test_poslist $cmd] + lappend res tokenize [fts5_test_tokenize $cmd] + lappend res rowcount [fts5_test_rowcount $cmd] + set res +} + +proc fts5_aux_test_functions {db} { + foreach f { + fts5_test_columnsize + fts5_test_columntext + fts5_test_columnlocale + fts5_test_columntotalsize + fts5_test_poslist + fts5_test_poslist2 + fts5_test_collist + fts5_test_insttoken + fts5_test_tokenize + fts5_test_rowcount + fts5_test_rowid + fts5_test_all + + fts5_test_queryphrase + fts5_test_phrasecount + fts5_columntext + fts5_columnlocale + fts5_queryphrase + fts5_collist + } { + sqlite3_fts5_create_function $db $f $f + } +} + +proc fts5_segcount {tbl} { + set N 0 + foreach n [fts5_level_segs $tbl] { incr N $n } + set N +} + +proc fts5_level_segs {tbl} { + set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10" + set ret [list] + foreach L [lrange [db one $sql] 1 end] { + lappend ret [expr [llength $L] - 3] + } + set ret +} + +proc fts5_level_segids {tbl} { + set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10" + set ret [list] + foreach L [lrange [db one $sql] 1 end] { + set lvl [list] + foreach S [lrange $L 3 end] { + regexp {id=([1234567890]*)} $S -> segid + lappend lvl $segid + } + lappend ret $lvl + } + set ret +} + +proc fts5_rnddoc {n} { + set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]" + } + set doc +} + +#------------------------------------------------------------------------- +# Usage: +# +# nearset aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2... +# +# This command is used to test if a document (set of column values) matches +# the logical equivalent of a single FTS5 NEAR() clump and, if so, return +# the equivalent of an FTS5 position list. +# +# Parameter $aCol is passed a list of the column values for the document +# to test. Parameters $phrase1 and so on are the phrases. +# +# The result is a list of phrase hits. Each phrase hit is formatted as +# three integers separated by "." characters, in the following format: +# +# . . +# +# Options: +# +# -near N (NEAR distance. Default 10) +# -col C (List of column indexes to match against) +# -pc VARNAME (variable in caller frame to use for phrase numbering) +# -dict VARNAME (array in caller frame to use for synonyms) +# +proc nearset {aCol args} { + + # Process the command line options. + # + set O(-near) 10 + set O(-col) {} + set O(-pc) "" + set O(-dict) "" + + set nOpt [lsearch -exact $args --] + if {$nOpt<0} { error "no -- option" } + + # Set $lPhrase to be a list of phrases. $nPhrase its length. + set lPhrase [lrange $args [expr $nOpt+1] end] + set nPhrase [llength $lPhrase] + + foreach {k v} [lrange $args 0 [expr $nOpt-1]] { + if {[info exists O($k)]==0} { error "unrecognized option $k" } + set O($k) $v + } + + if {$O(-pc) == ""} { + set counter 0 + } else { + upvar $O(-pc) counter + } + + if {$O(-dict)!=""} { upvar $O(-dict) aDict } + + for {set j 0} {$j < [llength $aCol]} {incr j} { + for {set i 0} {$i < $nPhrase} {incr i} { + set A($j,$i) [list] + } + } + + # Loop through each column of the current row. + for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} { + + # If there is a column filter, test whether this column is excluded. If + # so, skip to the next iteration of this loop. Otherwise, set zCol to the + # column value and nToken to the number of tokens that comprise it. + if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue + set zCol [lindex $aCol $iCol] + set nToken [llength $zCol] + + # Each iteration of the following loop searches a substring of the + # column value for phrase matches. The last token of the substring + # is token $iLast of the column value. The first token is: + # + # iFirst = ($iLast - $O(-near) - 1) + # + # where $sz is the length of the phrase being searched for. A phrase + # counts as matching the substring if its first token lies on or before + # $iLast and its last token on or after $iFirst. + # + # For example, if the query is "NEAR(a+b c, 2)" and the column value: + # + # "x x x x A B x x C x" + # 0 1 2 3 4 5 6 7 8 9" + # + # when (iLast==8 && iFirst=5) the range will contain both phrases and + # so both instances can be added to the output poslists. + # + set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] + for { } {$iLast < $nToken} {incr iLast} { + + catch { array unset B } + + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + set p [lindex $lPhrase $iPhrase] + set nPm1 [expr {[llength $p] - 1}] + set iFirst [expr $iLast - $O(-near) - [llength $p]] + + for {set i $iFirst} {$i <= $iLast} {incr i} { + set lCand [lrange $zCol $i [expr $i+$nPm1]] + set bMatch 1 + foreach tok $p term $lCand { + if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break } + } + if {$bMatch} { lappend B($iPhrase) $i } + } + + if {![info exists B($iPhrase)]} break + } + + if {$iPhrase==$nPhrase} { + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)] + set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)] + } + } + } + } + + set res [list] + #puts [array names A] + + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} { + foreach a $A($iCol,$iPhrase) { + lappend res "$counter.$iCol.$a" + } + } + incr counter + } + + #puts "$aCol -> $res" + sort_poslist $res +} + +proc nearset_match {aDictVar tok term} { + if {[string match $tok $term]} { return 1 } + + upvar $aDictVar aDict + if {[info exists aDict($tok)]} { + foreach s $aDict($tok) { + if {[string match $s $term]} { return 1 } + } + } + return 0; +} + +#------------------------------------------------------------------------- +# Usage: +# +# sort_poslist LIST +# +# Sort a position list of the type returned by command [nearset] +# +proc sort_poslist {L} { + lsort -command instcompare $L +} +proc instcompare {lhs rhs} { + foreach {p1 c1 o1} [split $lhs .] {} + foreach {p2 c2 o2} [split $rhs .] {} + + set res [expr $c1 - $c2] + if {$res==0} { set res [expr $o1 - $o2] } + if {$res==0} { set res [expr $p1 - $p2] } + + return $res +} + +#------------------------------------------------------------------------- +# Logical operators used by the commands returned by fts5_tcl_expr(). +# +proc AND {args} { + foreach a $args { + if {[llength $a]==0} { return [list] } + } + sort_poslist [concat {*}$args] +} +proc OR {args} { + sort_poslist [concat {*}$args] +} +proc NOT {a b} { + if {[llength $b]>0} { return [list] } + return $a +} + +#------------------------------------------------------------------------- +# This command is similar to [split], except that it also provides the +# start and end offsets of each token. For example: +# +# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8} +# + +proc gobble_whitespace {textvar} { + upvar $textvar t + regexp {([ ]*)(.*)} $t -> space t + return [string length $space] +} + +proc gobble_text {textvar wordvar} { + upvar $textvar t + upvar $wordvar w + regexp {([^ ]*)(.*)} $t -> w t + return [string length $w] +} + +proc fts5_tokenize_split {text} { + set token "" + set ret [list] + set iOff [gobble_whitespace text] + while {[set nToken [gobble_text text word]]} { + lappend ret $word $iOff [expr $iOff+$nToken] + incr iOff $nToken + incr iOff [gobble_whitespace text] + } + + set ret +} + +#------------------------------------------------------------------------- +# +proc foreach_detail_mode {prefix script} { + set saved $::testprefix + foreach d [list full col none] { + set s [string map [list %DETAIL% $d] $script] + set ::detail $d + set ::testprefix "$prefix-$d" + reset_db + uplevel $s + unset ::detail + } + set ::testprefix $saved +} + +proc detail_check {} { + if {$::detail != "none" && $::detail!="full" && $::detail!="col"} { + error "not in foreach_detail_mode {...} block" + } +} +proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } +proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } +proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } + +proc foreach_tokenizer_mode {prefix script} { + set saved $::testprefix + foreach {d mapping} { + "" {} + "-origintext" {, tokenize="origintext unicode61", tokendata=1} + } { + set s [string map [list %TOKENIZER% $mapping] $script] + set ::testprefix "$prefix$d" + reset_db + sqlite3_fts5_register_origintext db + uplevel $s + } + set ::testprefix $saved +} + +#------------------------------------------------------------------------- +# Convert a poslist of the type returned by fts5_test_poslist() to a +# collist as returned by fts5_test_collist(). +# +proc fts5_poslist2collist {poslist} { + set res [list] + foreach h $poslist { + regexp {(.*)\.[1234567890]+} $h -> cand + lappend res $cand + } + set res [lsort -command fts5_collist_elem_compare -unique $res] + return $res +} + +# Comparison function used by fts5_poslist2collist to sort collist entries. +proc fts5_collist_elem_compare {a b} { + foreach {a1 a2} [split $a .] {} + foreach {b1 b2} [split $b .] {} + + if {$a1==$b1} { return [expr $a2 - $b2] } + return [expr $a1 - $b1] +} + + +#-------------------------------------------------------------------------- +# Construct and return a tcl list equivalent to that returned by the SQL +# query executed against database handle [db]: +# +# SELECT +# rowid, +# fts5_test_poslist($tbl), +# fts5_test_collist($tbl) +# FROM $tbl('$expr') +# ORDER BY rowid $order; +# +proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} { + + # Figure out the set of columns in the FTS5 table. This routine does + # not handle tables with UNINDEXED columns, but if it did, it would + # have to be here. + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + + set d "" + if {$aDictVar != ""} { + upvar $aDictVar aDict + set d aDict + } + + set cols "" + foreach e $lCols { append cols ", '$e'" } + set tclexpr [db one [subst -novar { + SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] ) + }]] + + set res [list] + db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { + set cols [list] + foreach col $lCols { lappend cols $x($col) } + + set ::pc 0 + set rowdata [eval $tclexpr] + if {$rowdata != ""} { + lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata] + } + } + + set res +} + +#------------------------------------------------------------------------- +# Similar to [fts5_query_data], but omit the collist field. +# +proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} { + set res [list] + + if {$aDictVar!=""} { + upvar $aDictVar aDict + set dict aDict + } else { + set dict "" + } + + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { + lappend res $rowid $poslist + } + set res +} + +proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} { + set res [list] + + if {$aDictVar!=""} { + upvar $aDictVar aDict + set dict aDict + } else { + set dict "" + } + + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { + lappend res $rowid $collist + } + set res +} + +#------------------------------------------------------------------------- +# + +# This command will only work inside a [foreach_detail_mode] block. It tests +# whether or not expression $expr run on FTS5 table $tbl is supported by +# the current mode. If so, 1 is returned. If not, 0. +# +# detail=full (all queries supported) +# detail=col (all but phrase queries and NEAR queries) +# detail=none (all but phrase queries, NEAR queries, and column filters) +# +proc fts5_expr_ok {expr tbl} { + + if {![detail_is_full]} { + set nearset "nearset_rc" + if {[detail_is_col]} { set nearset "nearset_rf" } + + set ::expr_not_ok 0 + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + + set cols "" + foreach e $lCols { append cols ", '$e'" } + set ::pc 0 + set tclexpr [db one [subst -novar { + SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] ) + }]] + eval $tclexpr + if {$::expr_not_ok} { return 0 } + } + + return 1 +} + +# Helper for [fts5_expr_ok] +proc nearset_rf {aCol args} { + set idx [lsearch -exact $args --] + if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { + set ::expr_not_ok 1 + } + list +} + +# Helper for [fts5_expr_ok] +proc nearset_rc {aCol args} { + nearset_rf $aCol {*}$args + if {[lsearch $args -col]>=0} { + set ::expr_not_ok 1 + } + list +} + +proc dump {tname} { + execsql_pp "SELECT * FROM ${tname}_idx" + execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data" +} + +#------------------------------------------------------------------------- +# Code for a simple Tcl tokenizer that supports synonyms at query time. +# +proc tclnum_tokenize {mode tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags == $mode && [info exists ::tclnum_syn($w)]} { + foreach s $::tclnum_syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } + } + } +} + +proc tclnum_create {args} { + set mode query + if {[llength $args]} { + set mode [lindex $args 0] + } + if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" } + return [list tclnum_tokenize $mode] +} + +proc fts5_tclnum_register {db} { + foreach SYNDICT { + {zero 0} + {one 1 i} + {two 2 ii} + {three 3 iii} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} + + {a1 a2 a3 a4 a5 a6 a7 a8 a9} + {b1 b2 b3 b4 b5 b6 b7 b8 b9} + {c1 c2 c3 c4 c5 c6 c7 c8 c9} + } { + foreach s $SYNDICT { + set o [list] + foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} + set ::tclnum_syn($s) $o + } + } + sqlite3_fts5_create_tokenizer db tclnum tclnum_create +} +# +# End of tokenizer code. +#------------------------------------------------------------------------- + diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test new file mode 100644 index 0000000000000000000000000000000000000000..bcad9e72411b9d73466230414cc346eafefeaca8 --- /dev/null +++ b/ext/fts5/test/fts5aa.test @@ -0,0 +1,664 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aa + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $::testprefix { +foreach_tokenizer_mode $::testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + SELECT name, sql FROM sqlite_master; +} { + t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} + t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} + t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} + t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} + t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} + t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} +} + +do_execsql_test 1.1 { + DROP TABLE t1; + SELECT name, sql FROM sqlite_master; +} {} + +#------------------------------------------------------------------------- +# + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); +} +do_execsql_test 2.1 { + INSERT INTO t1 VALUES('a b c', 'd e f'); +} + +do_test 2.2 { + execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } +} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/} + +foreach w {a b c d e f} { + do_execsql_test 2.3.$w.asc { + SELECT rowid FROM t1 WHERE t1 MATCH $w; + } {1} + do_execsql_test 2.3.$w.desc { + SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC; + } {1} +} + +do_execsql_test 2.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); + PRAGMA integrity_check; + PRAGMA integrity_check(t1); +} {ok ok} + + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); +} +foreach {i x y} { + 1 {g f d b f} {h h e i a} + 2 {f i g j e} {i j c f f} + 3 {e e i f a} {e h f d f} + 4 {h j f j i} {h a c f j} + 5 {d b j c g} {f e i b e} + 6 {a j a e e} {j d f d e} + 7 {g i j c h} {j d h c a} + 8 {j j i d d} {e e d f b} + 9 {c j j d c} {h j i f g} + 10 {b f h i a} {c f b b j} +} { + do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) } + do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') } + do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok + if {[set_test_counter errors]} break +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} +foreach {i x y} { + 1 {g f d b f} {h h e i a} + 2 {f i g j e} {i j c f f} + 3 {e e i f a} {e h f d f} + 4 {h j f j i} {h a c f j} + 5 {d b j c g} {f e i b e} + 6 {a j a e e} {j d f d e} + 7 {g i j c h} {j d h c a} + 8 {j j i d d} {e e d f b} + 9 {c j j d c} {h j i f g} + 10 {b f h i a} {c f b b j} +} { + do_execsql_test 4.$i.1 { INSERT INTO t1 VALUES($x, $y) } + do_execsql_test 4.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') } + if {[set_test_counter errors]} break +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} +foreach {i x y} { + 1 {dd abc abc abc abcde} {aaa dd ddd ddd aab} + 2 {dd aab d aaa b} {abcde c aaa aaa aaa} + 3 {abcde dd b b dd} {abc abc d abc ddddd} + 4 {aaa abcde dddd dddd abcde} {abc b b abcde abc} + 5 {aab dddd d dddd c} {ddd abcde dddd abcde c} + 6 {ddd dd b aab abcde} {d ddddd dddd c abc} + 7 {d ddddd ddd c abcde} {c aab d abcde ddd} + 8 {abcde aaa aab c c} {ddd c dddd b aaa} + 9 {abcde aab ddddd c aab} {dddd dddd b c dd} + 10 {ddd abcde dddd dd c} {dddd c c d abcde} +} { + do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) } + do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok + if {[set_test_counter errors]} break +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +do_execsql_test 6.1 { + INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a'); + REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d'); +} + +do_execsql_test 6.2 { + INSERT INTO t1(t1) VALUES('integrity-check') +} + +do_execsql_test 6.3 { + REPLACE INTO t1(rowid, x, y) VALUES('22', 'l l l', 'l l l'); +} + +do_execsql_test 6.4 { + REPLACE INTO t1(x, y) VALUES('x y z', 'x y z'); +} + +do_execsql_test 6.5 { + INSERT INTO t1(t1) VALUES('integrity-check') +} + +do_execsql_test 6.6 { + SELECT rowid, * FROM t1; +} { + 22 {l l l} {l l l} + 23 {x y z} {x y z} +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +expr srand(0) +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y,z); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +proc doc {} { + set v [list aaa aab abc abcde b c d dd ddd dddd ddddd] + set ret [list] + for {set j 0} {$j < 20} {incr j} { + lappend ret [lindex $v [expr int(rand()*[llength $v])]] + } + return $ret +} + +proc dump_structure {} { + db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} { + foreach lvl [lrange $t 1 end] { + set seg [string repeat . [expr [llength $lvl]-2]] + puts "[lrange $lvl 0 1] $seg" + } + } +} + +for {set i 1} {$i <= 10} {incr i} { + do_test 7.$i { + for {set j 0} {$j < 10} {incr j} { + set x [doc] + set y [doc] + set z [doc] + set rowid [expr int(rand() * 100)] + execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) } + } + execsql { INSERT INTO t1(t1) VALUES('integrity-check'); } + } {} + if {[set_test_counter errors]} break +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3"); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +do_execsql_test 8.1 { + INSERT INTO t1 VALUES('the quick brown fox'); + INSERT INTO t1(t1) VALUES('integrity-check'); +} + + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db + +expr srand(0) + +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3"); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +proc doc {} { + set v [list aaa aab abc abcde b c d dd ddd dddd ddddd] + set ret [list] + for {set j 0} {$j < 20} {incr j} { + lappend ret [lindex $v [expr int(rand()*[llength $v])]] + } + return $ret +} + +proc dump_structure {} { + db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} { + foreach lvl [lrange $t 1 end] { + set seg [string repeat . [expr [llength $lvl]-2]] + puts "[lrange $lvl 0 1] $seg" + } + } +} + +for {set i 1} {$i <= 10} {incr i} { + do_test 9.$i { + for {set j 0} {$j < 100} {incr j} { + set x [doc] + set y [doc] + set z [doc] + set rowid [expr int(rand() * 100)] + execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) } + } + execsql { INSERT INTO t1(t1) VALUES('integrity-check'); } + } {} + if {[set_test_counter errors]} break +} + + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); +} +set d10 { + 1 {g f d b f} {h h e i a} + 2 {f i g j e} {i j c f f} + 3 {e e i f a} {e h f d f} + 4 {h j f j i} {h a c f j} + 5 {d b j c g} {f e i b e} + 6 {a j a e e} {j d f d e} + 7 {g i j c h} {j d h c a} + 8 {j j i d d} {e e d f b} + 9 {c j j d c} {h j i f g} + 10 {b f h i a} {c f b b j} +} +foreach {rowid x y} $d10 { + do_execsql_test 10.1.$rowid.1 { INSERT INTO t1 VALUES($x, $y) } + do_execsql_test 10.1.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') } +} +foreach rowid {5 9 8 1 2 4 10 7 3 5 6} { + do_execsql_test 10.2.$rowid.1 { DELETE FROM t1 WHERE rowid = $rowid } + do_execsql_test 10.2.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') } +} +foreach {rowid x y} $d10 { + do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) } + do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') } +} + +do_execsql_test 10.4.1 { DELETE FROM t1 } +do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } + +#------------------------------------------------------------------------- +# +do_catchsql_test 11.1 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%); +} {1 {reserved fts5 column name: rank}} +do_catchsql_test 11.2 { + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%); +} {1 {reserved fts5 table name: rank}} +do_catchsql_test 11.3 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%); +} {1 {reserved fts5 column name: rowid}} + +#------------------------------------------------------------------------- +# +do_execsql_test 12.1 { + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); +} {} + +do_catchsql_test 12.2 { + SELECT t2 FROM t2 WHERE t2 MATCH '*stuff' +} {1 {unknown special query: stuff}} + +do_test 12.3 { + set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] + string is integer $res +} {1} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); +} {} + +do_execsql_test 13.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'o'; +} {1 2} + +do_execsql_test 13.4 { + DELETE FROM t1 WHERE rowid=2; +} {} + +do_execsql_test 13.5 { + SELECT rowid FROM t1 WHERE t1 MATCH 'o'; +} {1} + +do_execsql_test 13.6 { + SELECT rowid FROM t1 WHERE t1 MATCH '""'; +} {} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH d(x,y) AS ( + SELECT NULL, 'xyz xyz xyz xyz xyz xyz' + UNION ALL + SELECT NULL, 'xyz xyz xyz xyz xyz xyz' FROM d + ) + INSERT INTO t1 SELECT * FROM d LIMIT 200; +} + +do_execsql_test 15.x { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_test 14.2 { + set nRow 0 + db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { + db eval { + BEGIN; + CREATE TABLE t2(a, b); + ROLLBACK; + } + incr nRow + } + set nRow +} {200} + +do_test 14.3 { + set nRow 0 + db eval { BEGIN; } + db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { + db eval { + SAVEPOINT aaa; + CREATE TABLE t2(a, b); + ROLLBACK TO aaa; + RELEASE aaa; + } + incr nRow + } + set nRow +} {200} + +do_execsql_test 15.0 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 15.1 { + UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1; +} +do_catchsql_test 15.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +do_execsql_test 16.1 { + CREATE VIRTUAL TABLE n1 USING fts5(a); + INSERT INTO n1 VALUES('a b c d'); +} + +proc funk {} { + db eval { UPDATE n1_config SET v=50 WHERE k='version' } + set fd [db incrblob main n1_data block 10] + fconfigure $fd -translation binary +# puts -nonewline $fd "\x44\x45" + close $fd +} +db func funk funk + +# This test case corrupts the structure record within the first invocation +# of function funk(). Which used to cause the bm25() function to throw an +# exception. But since bm25() can now used the cached structure record, +# it never sees the corruption introduced by funk() and so the following +# statement no longer fails. +# +do_catchsql_test 16.2 { + SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' +} {0 {{} -1e-06 {}}} +# {1 {SQL logic error}} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 17.1 { + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%); + INSERT INTO b2 VALUES('a'); + INSERT INTO b2 VALUES('b'); + INSERT INTO b2 VALUES('c'); +} + +do_test 17.2 { + set res [list] + db eval { SELECT * FROM b2 ORDER BY rowid ASC } { + lappend res [execsql { SELECT * FROM b2 ORDER BY rowid ASC }] + } + set res +} {{a b c} {a b c} {a b c}} + +if {[string match n* %DETAIL%]==0} { + reset_db + sqlite3_fts5_register_origintext db + do_execsql_test 17.3 { + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); + INSERT INTO c2 VALUES('x x x', 'x x x'); + SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; + } {1} +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 17.1 { + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%); + INSERT INTO uio VALUES(NULL); + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + INSERT INTO uio SELECT NULL FROM uio; + SELECT count(*) FROM uio; +} {256} + +do_execsql_test 17.2 { + SELECT count(*) FROM uio WHERE rowid BETWEEN 8 AND 17 +} {10} +do_execsql_test 17.3 { + SELECT rowid FROM uio WHERE rowid BETWEEN 8 AND 17 +} {8 9 10 11 12 13 14 15 16 17} +do_execsql_test 17.4 { + SELECT rowid FROM uio WHERE rowid BETWEEN 8 AND 17 ORDER BY rowid DESC +} {17 16 15 14 13 12 11 10 9 8} +do_execsql_test 17.5 { + SELECT count(*) FROM uio +} {256} + +do_execsql_test 17.6 { + INSERT INTO uio(rowid) VALUES(9223372036854775807); + INSERT INTO uio(rowid) VALUES(-9223372036854775808); + SELECT count(*) FROM uio; +} {258} +do_execsql_test 17.7 { + SELECT min(rowid), max(rowid) FROM uio; +} {-9223372036854775808 9223372036854775807} + +do_execsql_test 17.8 { + INSERT INTO uio DEFAULT VALUES; + SELECT min(rowid), max(rowid), count(*) FROM uio; +} {-9223372036854775808 9223372036854775807 259} + +do_execsql_test 17.9 { + SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10; +} {-9223372036854775808 9 10} + +#-------------------------------------------------------------------- +# +do_execsql_test 18.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%); + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1 VALUES('abc*', NULL); + INSERT INTO t2 VALUES(1, 'abcdefg'); +} +do_execsql_test 18.2 { + SELECT t1.rowid, t2.rowid FROM t1, t2 WHERE t2 MATCH t1.a AND t1.rowid = t2.c +} {1 1} +do_execsql_test 18.3 { + SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c +} {1 1} + +#-------------------------------------------------------------------- +# fts5 table in the temp schema. +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 19.0 { + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t1 VALUES('x y z'); + INSERT INTO t1 VALUES('w x 1'); + SELECT rowid FROM t1 WHERE t1 MATCH 'x'; +} {1 2} + +#-------------------------------------------------------------------- +# Test that 6 and 7 byte varints can be read. +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%); +} +set ::ids [list \ + 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] +] +do_test 20.1 { + foreach id $::ids { + execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') } + } + execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } +} $::ids + +#-------------------------------------------------------------------- +# Test that a DROP TABLE may be executed within a transaction that +# writes to an FTS5 table. +# +do_execsql_test 21.0 { + CREATE TEMP TABLE t8(a, b); + CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%); +} + +do_execsql_test 21.1 { + BEGIN; + INSERT INTO ft VALUES('a b c'); + DROP TABLE t8; + COMMIT; +} + +do_execsql_test 22.0 { + CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t9(rowid, x) VALUES(2, 'bbb'); + BEGIN; + INSERT INTO t9(rowid, x) VALUES(1, 'aaa'); + DELETE FROM t9 WHERE rowid = 2; + INSERT INTO t9(rowid, x) VALUES(3, 'bbb'); + COMMIT; +} + +do_execsql_test 22.1 { + SELECT rowid FROM t9('a*') +} {1} + +#------------------------------------------------------------------------- +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%); + CREATE TABLE t11(x); +} +do_execsql_test 23.1 { + SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL; +} +do_execsql_test 23.2 { + SELECT * FROM t11, t10 WHERE t10.rowid IS NULL; +} + +#------------------------------------------------------------------------- +do_execsql_test 24.0 { + CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%); + INSERT INTO t12 VALUES('aaaa'); +} +do_execsql_test 24.1 { + BEGIN; + DELETE FROM t12 WHERE rowid=1; + SELECT * FROM t12('aaaa'); + INSERT INTO t12 VALUES('aaaa'); + END; +} +execsql_pp { + SELECT rowid, hex(block) FROM t12_data +} +do_execsql_test 24.2 { + INSERT INTO t12(t12) VALUES('integrity-check'); +} +do_execsql_test 24.3 { + SELECT * FROM t12('aaaa'); +} {aaaa} + +#------------------------------------------------------------------------- +do_execsql_test 25.0 { + CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%); +} +do_execsql_test 25.1 { + BEGIN; + INSERT INTO t13 VALUES('AAAA'); +SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*'); + + END; +} + + +} +} + +expand_all_sql db +finish_test diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test new file mode 100644 index 0000000000000000000000000000000000000000..7e312286f3a5b00718b0cd5ec49b7ef58a4149a1 --- /dev/null +++ b/ext/fts5/test/fts5ab.test @@ -0,0 +1,300 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ab + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1 VALUES('hello', 'world'); + INSERT INTO t1 VALUES('one two', 'three four'); + INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); +} + +do_execsql_test 1.1 { + SELECT * FROM t1 ORDER BY rowid DESC; +} { forty five {one two} {three four} hello world } + +do_execsql_test 1.2 { + SELECT rowid FROM t1 ORDER BY rowid DESC; +} {45 2 1} + +do_execsql_test 1.3 { + SELECT rowid FROM t1 ORDER BY rowid ASC; +} {1 2 45} + +do_execsql_test 1.4 { + SELECT * FROM t1 WHERE rowid=2; +} {{one two} {three four}} + +do_execsql_test 1.5 { + SELECT * FROM t1 WHERE rowid=2.01; +} {} + +do_execsql_test 1.6 { + SELECT * FROM t1 WHERE rowid=1.99; +} {} + +#------------------------------------------------------------------------- + +reset_db +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES('three'); +} + +do_catchsql_test 2.2 { + SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND' +} {1 {fts5: syntax error near "AND"}} + +do_execsql_test 2.3 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'two' } {2 two} +do_execsql_test 2.4 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'three' } {3 three} +do_execsql_test 2.5 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'one' } {1 one} + +do_execsql_test 2.6 { + INSERT INTO t1 VALUES('a b c d e f g'); + INSERT INTO t1 VALUES('b d e a a a i'); + INSERT INTO t1 VALUES('x y z b c c c'); +} + +foreach {tn expr res} { + 1 a {5 4} + 2 b {6 5 4} + 3 c {6 4} + 4 d {5 4} + 5 e {5 4} + 6 f {4} + 7 g {4} + 8 x {6} + 9 y {6} + 10 z {6} +} { + do_execsql_test 2.7.$tn.1 { + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC + } $res + do_execsql_test 2.7.$tn.2 { + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC + } [lsort -integer $res] +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a,b); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +foreach {tn a b} { + 1 {abashed abandons abase abash abaft} {abases abased} + 2 {abasing abases abaft abated abandons} {abases abandoned} + 3 {abatement abash abash abated abase} {abasements abashing} + 4 {abaft abasements abase abasement abasing} {abasement abases} + 5 {abaft abashing abatement abash abasements} {abandons abandoning} + 6 {aback abate abasements abashes abandoned} {abasement abased} + 7 {abandons abated abased aback abandoning} {abases abandoned} + 8 {abashing abases abasement abaft abashing} {abashed abate} + 9 {abash abase abate abashing abashed} {abandon abandoned} + 10 {abate abandoning abandons abasement aback} {abandon abandoning} +} { + do_execsql_test 3.1.$tn.1 { INSERT INTO t1 VALUES($a, $b) } + do_execsql_test 3.1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') } +} + +foreach {tn expr res} { + 1 {abash} {9 5 3 1} + 2 {abase} {9 4 3 1} + 3 {abase + abash} {1} + 4 {abash + abase} {9} + 5 {abaft + abashing} {8 5} + 6 {abandon + abandoning} {10} + 7 {"abashing abases abasement abaft abashing"} {8} +} { + do_execsql_test 3.2.$tn { + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC + } $res +} + +do_execsql_test 3.3 { + SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)' +} {6} + +foreach {tn expr res} { + 1 {abash} {1 3 5 9} + 2 {abase} {1 3 4 9} + 3 {abase + abash} {1} + 4 {abash + abase} {9} + 5 {abaft + abashing} {5 8} + 6 {abandon + abandoning} {10} + 7 {"abashing abases abasement abaft abashing"} {8} +} { + do_execsql_test 3.4.$tn { + SELECT rowid FROM t1 WHERE t1 MATCH $expr + } $res +} + +#------------------------------------------------------------------------- +# Documents with more than 2M tokens. +# + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%); +} +foreach {tn doc} [list \ + 1 [string repeat {a x } 1500000] \ + 2 "[string repeat {a a } 1500000] x" \ +] { + do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) } +} + +do_execsql_test 4.3 { + SELECT rowid FROM s1 WHERE s1 MATCH 'x' +} {1 2} + +if {[detail_is_full]} { + do_execsql_test 4.4 { + SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' + } {1 2} +} + +do_execsql_test 4.5.1 { + SELECT rowid FROM s1 WHERE s1 MATCH 'a AND x' +} {1 2} + +do_execsql_test 4.5.2 { + SELECT rowid FROM s1 WHERE s1 MATCH 'a x' +} {1 2} + +#------------------------------------------------------------------------- +# Check that a special case of segment promotion works. The case is where +# a new segment is written to level L, but the oldest segment within level +# (L-2) is larger than it. +# +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); + INSERT INTO s2(s2, rank) VALUES('pgsz', 32); + INSERT INTO s2(s2, rank) VALUES('automerge', 0); +} + +proc rnddoc {n} { + set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]] + } + set doc +} +db func rnddoc rnddoc + +do_test 5.1 { + for {set i 1} {$i <= 65} {incr i} { + execsql { INSERT INTO s2 VALUES(rnddoc(10)) } + } + for {set i 1} {$i <= 63} {incr i} { + execsql { DELETE FROM s2 WHERE rowid = $i } + } + fts5_level_segs s2 +} {0 8} + +do_test 5.2 { + execsql { + INSERT INTO s2(s2, rank) VALUES('automerge', 8); + } + for {set i 0} {$i < 7} {incr i} { + execsql { INSERT INTO s2 VALUES(rnddoc(50)) } + } + fts5_level_segs s2 +} {8 0 0} + +# Test also the other type of segment promotion - when a new segment is written +# that is larger than segments immediately following it. +do_test 5.3 { + execsql { + DROP TABLE s2; + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); + INSERT INTO s2(s2, rank) VALUES('pgsz', 32); + INSERT INTO s2(s2, rank) VALUES('automerge', 0); + } + + for {set i 1} {$i <= 16} {incr i} { + execsql { INSERT INTO s2 VALUES(rnddoc(5)) } + } + fts5_level_segs s2 +} {0 1} + +do_test 5.4 { + execsql { INSERT INTO s2 VALUES(rnddoc(160)) } + fts5_level_segs s2 +} {2 0} + +#------------------------------------------------------------------------- +# +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%); + BEGIN; + INSERT INTO s3 VALUES('a b c'); + INSERT INTO s3 VALUES('A B C'); +} + +do_execsql_test 6.1.1 { + SELECT rowid FROM s3 WHERE s3 MATCH 'a' +} {1 2} + +do_execsql_test 6.1.2 { + SELECT rowid FROM s3 WHERE s3 MATCH 'a' ORDER BY rowid DESC +} {2 1} + +do_execsql_test 6.2 { + COMMIT; +} + +do_execsql_test 6.3 { + SELECT rowid FROM s3 WHERE s3 MATCH 'a' +} {1 2} + +do_test 6.4 { + db close + sqlite3 db test.db + execsql { + BEGIN; + INSERT INTO s3(s3) VALUES('optimize'); + ROLLBACK; + } +} {} + +#------------------------------------------------------------------------- +# +set doc [string repeat "a b c " 500] +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + INSERT INTO x1 VALUES($doc); +} + +} ;# foreach_detail_mode... + + +finish_test diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test new file mode 100644 index 0000000000000000000000000000000000000000..4628e909c16d16cbd2b4e075fc431de6da032ecb --- /dev/null +++ b/ext/fts5/test/fts5ac.test @@ -0,0 +1,278 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ac + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +set data { + 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} + 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} + 2 {l s z j k i m p s} {l w e j t j e e i t w r o p o} + 3 {x g y m y m h p} {k j j b r e y y a k y} + 4 {q m a i y i z} {o w a g k x g j m w e u k} + 5 {k o a w y b s z} {s g l m m l m g p} + 6 {d a q i z h b l c p k j g k} {p x u j x t v c z} + 7 {f d a g o c t i} {w f c x l d r k i j} + 8 {y g w u b q p o m j y b p a e k} {r i d k y w o z q m a t p} + 9 {r k o m c c j s x m x m x m q r} {y r c a q d z k n x n} + 10 {k j q m g q a j d} {d d e z g w h c d o o g x d} + 11 {j z u m o y q j f w e e w t r j w} {g m o r x n t n w i f g l z f} + 12 {s y w a w d o h x m k} {c w k z b p o r a} + 13 {u t h x e g s k n g i} {f j w g c s r} + 14 {b f i c s u z t k} {c k q s j u i z o} + 15 {n a f n u s w h y n s i q e w} {x g e g a s s h n} + 16 {k s q e j n p} {t r j f t o e k k l m i} + 17 {g d t u w r o p m n m n p h b o u} {h s w o s l j e} + 18 {f l q y q q g e e x j r} {n b r r g e i r t x q k} + 19 {f i r g o a w e p i l o a w} {e k r z t d g h g i b d i e m} + 20 {l d u u f p y} {g o m m u x m g l j t t x x u} + 21 {m c d k x i c z l} {m i a i e u h} + 22 {w b f o c g x y j} {z d w x d f h i p} + 23 {w u i u x t c h k i b} {b y k h b v r t g j} + 24 {h f d j s w s b a p k} {a q y u z e y m m j q r} + 25 {d i x y x x k i y f s d j h z p n} {l l q m e t c w g y h t s v g} + 26 {g s q w t d k x g f m j p k y} {r m b x e l t d} + 27 {j l s q u g y v e c l o} {m f l m m m h g x x l n c} + 28 {c t j g v r s b z j} {l c f y d t q n} + 29 {e x z y w i h l} {b n b x e y q e n u m} + 30 {g y y h j b w r} {q b q f u s k c k g r} + 31 {g u l x l b r c m z b u c} {k g t b x k x n t e z d h o} + 32 {w g v l z f b z h p s c v h} {g e w v m h k r g w a r f q} + 33 {c g n f u d o y o b} {e y o h x x y y i z s b h a j} + 34 {v y h c q u u s q y x x k s q} {d n r m y k n t i r n w e} + 35 {o u c x l e b t a} {y b a x y f z x r} + 36 {x p h l j a a u u j h} {x o f s z m b c q p} + 37 {k q t i c a q n m v v} {v r z e f m y o} + 38 {r w t t t t r v v o e p g h} {l w x a g a u h y} + 39 {o p v g v b a g o} {j t q c r b b g y z} + 40 {f s o r o d t h q f x l} {r d b m k i f s t d l m y x j w} + 41 {t m o t m f m f} {i p i q j v n v m b q} + 42 {t x w a r l w d t b c o d o} {a h f h w z d n s} + 43 {t u q c d g p q x j o l c x c} {m n t o z z j a y} + 44 {v d i i k b f s z r v r z y} {g n q y s x x m b x c l w} + 45 {p v v a c s z y e o l} {m v t u d k m k q b d c v z r} + 46 {f y k l d r q w r s t r e} {h m v r r l r r t f q e x y} + 47 {w l n l t y x} {n h s l a f c h u f l x x m v n o} + 48 {t n v i k e b p z p d j j l i o} {i v z p g u e j s i k n h w d c} + 49 {z v x p n l t a j c} {e j l e n c e t a d} + 50 {w u b x u i v h a i y m m r p m s} {s r h d o g z y f f x e} + 51 {d c c x b c a x g} {p r a j v u y} + 52 {f w g r c o d l t u e z h i} {j l l s s b j m} + 53 {p m t f k i x} {u v y a z g w v v m x h i} + 54 {l c z g l o j i c d e b} {b f v y w u i b e i y} + 55 {r h c x f x a d s} {z x y k f l r b q c v} + 56 {v x x c y h z x b g m o q n c} {h n b i t g h a q b c o r u} + 57 {d g l o h t b s b r} {n u e p t i m u} + 58 {t d y e t d c w u o s w x f c h} {i o s v y b r d r} + 59 {l b a p q n d r} {k d c c d n y q h g a o p e x} + 60 {f r z v m p k r} {x x r i s b a g f c} + 61 {s a z i e r f i w c n y v z t k s} {y y i r y n l s b w i e k n} + 62 {n x p r e x q r m v i b y} {f o o z n b s r q j} + 63 {y j s u j x o n r q t f} {f v k n v x u s o a d e f e} + 64 {u s i l y c x q} {r k c h p c h b o s s u s p b} + 65 {m p i o s h o} {s w h u n d m n q t y k b w c} + 66 {l d f g m x x x o} {s w d d f b y j j h h t i y p j o} + 67 {c b m h f n v w n h} {i r w i e x r w l z p x u g u l s} + 68 {y a h u h i m a y q} {d d r x h e v q n z y c j} + 69 {c x f d x o n p o b r t b l p l} {m i t k b x v f p t m l l y r o} + 70 {u t l w w m s} {m f m o l t k o p e} + 71 {f g q e l n d m z x q} {z s i i i m f w w f n g p e q} + 72 {n l h a v u o d f j d e x} {v v s l f g d g r a j x i f z x} + 73 {x v m v f i g q e w} {r y s j i k m j j e d g r n o i f} + 74 {g d y n o h p s y q z j d w n h w} {x o d l t j i b r d o r y} + 75 {p g b i u r b e q d v o a g w m k} {q y z s f q o h} + 76 {u z a q u f i f f b} {b s p b a a d x r r i q f} + 77 {w h h z t h p o a h h e e} {h w r p h k z v y f r x} + 78 {c a r k i a p u x} {f w l p t e m l} + 79 {q q u k o t r k z} {f b m c w p s s o z} + 80 {t i g v y q s r x m r x z e f} {x o j w a u e y s j c b u p p r o} + 81 {n j n h r l a r e o z w e} {v o r r j a v b} + 82 {i f i d k w d n h} {o i d z i z l m w s b q v u} + 83 {m d g q q b k b w f q q p p} {j m q f b y c i z k y q p l e a} + 84 {m x o n y f g} {y c n x n q j i y c l h b r q z} + 85 {v o z l n p c} {g n j n t b b x n c l d a g j v} + 86 {z n a y f b t k k t d b z a v} {r p c n r u k u} + 87 {b q t x z e c w} {q a o a l o a h i m j r} + 88 {j f h o x x a z g b a f a m i b} {j z c z y x e x w t} + 89 {t c t p r s u c q n} {z x l i k n f q l n t} + 90 {w t d q j g m r f k n} {l e w f w w a l y q k i q t p c t} + 91 {c b o k l i c b s j n m b l} {y f p q o w g} + 92 {f y d j o q t c c q m f j s t} {f h e d y m o k} + 93 {k x j r m a d o i z j} {r t t t f e b r x i v j v g o} + 94 {s f e a e t i h h d q p z t q} {b k m k w h c} + 95 {h b n j t k i h o q u} {w n g i t o k c a m y p f l x c p} + 96 {f c x p y r b m o l m o a} {p c a q s u n n x d c f a o} + 97 {u h h k m n k} {u b v n u a o c} + 98 {s p e t c z d f n w f} {l s f j b l c e s h} + 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} +} + +foreach {tn2 sql} { + 1 {} + 2 {BEGIN} +} { + reset_db + fts5_aux_test_functions db + + do_execsql_test 1.$tn2.0 { + CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%); + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); + } + + execsql $sql + + do_test 1.$tn2.1.1 { + foreach {id x y} $data { + execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } + } + execsql { INSERT INTO xx(xx) VALUES('integrity-check') } + } {} + + + #------------------------------------------------------------------------- + # + do_execsql_test 1.$tn2.integrity { + INSERT INTO xx(xx) VALUES('integrity-check'); + } + + #------------------------------------------------------------------------- + # + foreach {tn expr} { + 1.1 "a AND b" + 1.2 "a OR b" + 1.3 "o" + 1.4 "b q" + 1.5 "e a e" + 1.6 "m d g q q b k b w f q q p p" + 1.7 "l o o l v v k" + 1.8 "a" + 1.9 "b" + 1.10 "c" + 1.11 "no" + 1.12 "L O O L V V K" + 1.13 "a AND b AND c" + 1.14 "x:a" + + 2.1 "x:a" + 2.2 "y:a" + 2.3 "x:b" + 2.4 "y:b" + + 3.1 "{x}:a" + 3.2 "{y}:a" + 3.3 "{x}:b" + 3.4 "{y}:b" + + 4.1 "{x y}:a" + 4.2 "{y x}:a" + 4.3 "{x x}:b" + 4.4 "{y y}:b" + + 5.1 {{"x" "y"}:a} + 5.2 {{"y" x}:a} + 5.3 {{x "x"}:b} + 5.4 {{"y" y}:b} + + 6.1 "b + q" + 6.2 "e + a + e" + 6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" + 6.4 "l + o + o + l + v + v + k" + 6.5 "L + O + O + L + V + V + K" + + 7.1 "a+b AND c" + 7.2 "d+c AND u" + 7.3 "d+c AND u+d" + 7.4 "a+b OR c" + 7.5 "d+c OR u" + 7.6 "d+c OR u+d" + + 8.1 "NEAR(a b)" + 8.2 "NEAR(r c)" + 8.2 { NEAR(r c, 5) } + 8.3 { NEAR(r c, 3) } + 8.4 { NEAR(r c, 2) } + 8.5 { NEAR(r c, 0) } + 8.6 { NEAR(a b c) } + 8.7 { NEAR(a b c, 8) } + 8.8 { x : NEAR(r c) } + 8.9 { y : NEAR(r c) } + + 9.1 { NEAR(r c) } + 9.2 { NEAR(r c, 5) } + 9.3 { NEAR(r c, 3) } + 9.4 { NEAR(r c, 2) } + 9.5 { NEAR(r c, 0) } + 9.6 { NEAR(a b c) } + 9.7 { NEAR(a b c, 8) } + 9.8 { x : NEAR(r c) } + 9.9 { y : NEAR(r c) } + 9.10 { x : "r c" } + 9.11 { y : "r c" } + 9.12 { a AND b } + 9.13 { a AND b AND c } + 9.14a { a } + 9.14b { a OR b } + 9.15 { a OR b AND c } + 9.16 { c AND b OR a } + 9.17 { c AND (b OR a) } + 9.18 { c NOT (b OR a) } + 9.19 { (c NOT b) OR (a AND d) } + } { + + if {[fts5_expr_ok $expr xx]==0} { + do_test 1.$tn2.$tn.OMITTED { list } [list] + continue + } + + set res [fts5_query_data $expr xx] + do_execsql_test 1.$tn2.$tn.[llength $res].asc { + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) + FROM xx WHERE xx match $expr + } $res + + set res [fts5_query_data $expr xx DESC] + do_execsql_test 1.$tn2.$tn.[llength $res].desc { + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) + FROM xx WHERE xx match $expr ORDER BY 1 DESC + } $res + } +} + +} + +do_execsql_test 2.1 { + SELECT fts5_expr_tcl('a AND b'); +} {{AND [nearset -- {a}] [nearset -- {b}]}} + +do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0} +do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2} + +foreach {tn expr tclexpr} { + 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} +} { + do_execsql_test 2.3.$tn { + SELECT fts5_expr_tcl($expr, 'N $x') + } [list $tclexpr] +} + +finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test new file mode 100644 index 0000000000000000000000000000000000000000..27806a4c0c20146b72f29b662db35d318c4be682 --- /dev/null +++ b/ext/fts5/test/fts5ad.test @@ -0,0 +1,243 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# More specifically, the focus is on testing prefix queries, both with and +# without prefix indexes. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ad + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); + INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); + INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is'); + INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the'); +} {} + +foreach {tn match res} { + 1 {c*} {1} + 2 {i*} {3 2} + 3 {t*} {3 1} + 4 {r*} {3 1} +} { + do_execsql_test 1.$tn { + SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid DESC + } $res +} + +foreach {tn match res} { + 5 {c*} {1} + 6 {i*} {2 3} + 7 {t*} {1 3} + 8 {r*} {1 3} +} { + do_execsql_test 1.$tn { + SELECT rowid FROM yy WHERE yy MATCH $match + } $res +} + +foreach {T create} { + 2 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + + 3 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + + 4 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + } + + 5 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + } + +} { + + do_test $T.1 { + execsql { DROP TABLE IF EXISTS t1 } + execsql $create + } {} + + do_test $T.1 { + foreach {rowid a b} { + 0 {fghij uvwxyz klmn pq uvwx} {klmn f fgh uv fghij klmno} + 1 {uv f abcd abcd fghi} {pq klm uv uv fgh uv a} + 2 {klmn klm pqrs fghij uv} {f k uvw ab abcd pqr uv} + 3 {ab pqrst a fghi ab pqr fg} {k klmno a fg abcd} + 4 {abcd pqrst uvwx a fgh} {f klmno fghij kl pqrst} + 5 {uvwxyz k abcde u a} {uv k k kl klmn} + 6 {uvwxyz k klmn pqrst uv} {fghi pqrs abcde u k} + 7 {uvwxy klmn u p pqrst fgh} {p f fghi abcd uvw kl uv} + 8 {f klmno pqrst uvwxy pqrst} {uv abcde klm pq pqr} + 9 {f abcde a uvwxyz pqrst} {fghij abc k uvwx pqr fghij uvwxy} + 10 {ab uv f fg pqrst uvwxy} {fgh p uv k abc klm uvw} + 11 {pq klmno a uvw abcde uvwxyz} {fghij pq uvwxyz pqr fghi} + 12 {fgh u pq fgh uvw} {uvw pqr f uvwxy uvwx} + 13 {uvwx klmn f fgh abcd pqr} {uvw k fg uv klm abcd} + 14 {ab uvwx pqrst pqr uvwxyz pqrs} {uvwxyz abcde ab ab uvw abcde} + 15 {abc abcde uvwxyz abc kl k pqr} {klm k k klmno u fgh} + 16 {fghi abcd fghij uv uvwxyz ab uv} {klmn pqr a uvw fghi} + 17 {abc pqrst fghi uvwx uvw klmn fghi} {ab fg pqr pqrs p} + 18 {pqr kl a fghij fgh fg kl} {pqr uvwxyz uvw abcd uvwxyz} + 19 {fghi fghi pqr kl fghi f} {klmn u u klmno klmno} + 20 {abc pqrst klmno kl pq uvwxy} {abc k fghi pqrs klm} + 21 {a pqr uvwxyz uv fghi a fgh} {abc pqrs pqrst pq klm} + 22 {klm abc uvwxyz klm pqrst} {fghij k pq pqr u klm fghij} + 23 {p klm uv p a a} {uvwxy klmn uvw abcde pq} + 24 {uv fgh fg pq uvwxy u uvwxy} {pqrs a uvw p uvwx uvwxyz fg} + 25 {fghij fghi klmn abcd pq kl} {fghi abcde pqrs abcd fgh uvwxy} + 26 {pq fgh a abc klmno klmn} {fgh p k p fg fghij} + 27 {fg pq kl uvwx fghij pqrst klmn} {abcd uvw abcd fghij f fghij} + 28 {uvw fghi p fghij pq fgh uvwx} {k fghij abcd uvwx pqr fghi} + 29 {klm pq abcd pq f uvwxy} {pqrst p fghij pqr p} + 30 {ab uvwx fg uvwx klmn klm} {klmn klmno fghij klmn klm} + 31 {pq k pqr abcd a pqrs} {abcd abcd uvw a abcd klmno ab} + 32 {pqrst u abc pq klm} {abc kl uvwxyz fghij u fghi p} + 33 {f uvwxy u k f uvw uvwx} {pqrs uvw fghi fg pqrst klm} + 34 {pqrs pq fghij uvwxyz pqr} {ab abc abc uvw f pq f} + 35 {uvwxy ab uvwxy klmno kl pqrs} {abcde uvw pqrs uvwx k k} + 36 {uvwxyz k ab abcde abc uvw} {uvw abcde uvw klmn uv klmn} + 37 {k kl uv abcde uvwx fg u} {u abc uvwxy k fg abcd} + 38 {fghi pqrst fghi pqr pqrst uvwx} {u uv uvwx fghi abcde} + 39 {k pqrst k uvw fg pqrst fghij} {uvwxy ab kl klmn uvwxyz abcde} + 40 {fg uvwxy pqrs klmn uvwxyz klm p} {k uv ab fghij fgh k pqrs} + 41 {uvwx abc f pq uvwxy k} {ab uvwxyz abc f fghij} + 42 {uvwxy klmno uvwxyz uvwxyz pqrst} {uv kl kl klmno k f abcde} + 43 {abcde ab pqrs fg f fgh} {abc fghij fghi k k} + 44 {uvw abcd a ab pqrst klmn fg} {pqrst u uvwx pqrst fghij f pqrst} + 45 {uvwxy p kl uvwxyz ab pqrst fghi} {abc f pqr fg a k} + 46 {u p f a fgh} {a kl pq uv f} + 47 {pqrs abc fghij fg abcde ab a} {p ab uv pqrs kl fghi abcd} + 48 {abcde uvwxy pqrst uv abc pqr uvwx} {uvwxy klm uvwxy uvwx k} + 49 {fgh klm abcde klmno u} {a f fghij f uvwxyz abc u} + 50 {uv uvw uvwxyz uvwxyz uv ab} {uvwx pq fg u k uvwxy} + 51 {uvwxy pq p kl fghi} {pqrs fghi pqrs abcde uvwxyz ab} + 52 {pqr p uvwxy kl pqrs klmno fghij} {ab abcde abc pqrst pqrs uv} + 53 {fgh pqrst p a klmno} {ab ab pqrst pqr kl pqrst} + 54 {abcd klm ab uvw a fg u} {f pqr f abcd uv} + 55 {u fg uvwxyz k uvw} {abc pqrs f fghij fg pqrs uvwxy} + 56 {klm fg p fghi fg a} {uv a fghi uvwxyz a fghi} + 57 {uvwxy k abcde fgh f fghi} {f kl klmn f fghi klm} + 58 {klm k fgh uvw fgh fghi} {klmno uvwx u pqrst u} + 59 {fghi pqr pqrst p uvw fghij} {uv pqrst pqrs pq fghij klm} + 60 {uvwx klm uvwxy uv klmn} {p a a abc klmn ab k} + 61 {uvwxy uvwx klm uvwx klm} {pqrs ab ab uvwxyz fg} + 62 {kl uv uv uvw fg kl k} {abcde uvw fgh uvwxy klm} + 63 {a abc fgh u klm abcd} {fgh pqr uv klmn fghij} + 64 {klmn k klmn klmno pqrs pqr} {fg kl abcde klmno uvwxy kl pq} + 65 {uvwxyz klm fghi abc abcde kl} {uvwxy uvw uvwxyz uvwxyz pq pqrst} + 66 {pq klm abc pqrst fgh f} {u abcde pqrst abcde fg} + 67 {u pqrst kl u uvw klmno} {u pqr pqrs fgh u p} + 68 {abc fghi uvwxy fgh k pq} {uv p uvwx uvwxyz ab} + 69 {klmno f uvwxyz uvwxy klmn fg ab} {fgh kl a pqr abcd pqr} + 70 {fghi pqrst pqrst uv a} {uvwxy k p uvw uvwx a} + 71 {a fghij f p uvw} {klm fg abcd abcde klmno pqrs} + 72 {uv uvwx uvwx uvw klm} {uv fghi klmno uvwxy uvw} + 73 {kl uvwxy ab f pq klm u} {uvwxy klmn klm abcd pq fg k} + 74 {uvw pqrst abcd uvwxyz ab} {fgh fgh klmn abc pq} + 75 {uvwxyz klm pq abcd klmno pqr uvwxyz} {kl f a fg pqr klmn} + 76 {uvw uvwxy pqr k pqrst kl} {uvwxy abc uvw uvw u} + 77 {fgh klm u uvwxyz f uvwxy abcde} {uv abcde klmno u u ab} + 78 {klmno abc pq pqr fgh} {p uv abcd fgh abc u k} + 79 {fg pqr uvw pq uvwx} {uv uvw fghij pqrs fg p} + 80 {abcd pqrs uvwx uvwxy uvwx} {u uvw pqrst pqr abcde pqrs kl} + 81 {uvwxyz klm pq uvwxy fghij} {p pq klm fghij u a a} + 82 {uvwx k uvwxyz klmno pqrst kl} {abcde p f pqrst abcd uvwxyz p} + 83 {abcd abcde klm pqrst uvwxyz} {uvw pqrst u p uvwxyz a pqrs} + 84 {k klm abc uv uvwxy klm klmn} {k abc pqr a abc p kl} + 85 {klmn abcd pqrs p pq klm a} {klmn kl ab uvw pq} + 86 {klmn a pqrs abc uvw pqrst} {a pqr kl klm a k f} + 87 {pqrs ab uvwx uvwxy a pqr f} {fg klm uvwx pqr pqr} + 88 {klmno ab k kl u uvwxyz} {uv kl uvw fghi uv uvw} + 89 {pq fghi pqrst klmn uvwxy abc pqrs} {fg f f fg abc abcde klm} + 90 {kl a k fghi uvwx fghi u} {ab uvw pqr fg a p abc} + 91 {uvwx pqrs klmno ab fgh uvwx} {pqr uvwx abc kl f klmno kl} + 92 {fghij pq pqrs fghij f pqrst} {u abcde fg pq pqr fgh k} + 93 {fgh u pqrs abcde klmno abc} {abc fg pqrst pqr abcde} + 94 {uvwx p abc f pqr p} {k pqrs kl klm abc fghi klm} + 95 {kl p klmno uvwxyz klmn} {fghi ab a fghi pqrs kl} + 96 {pqr fgh pq uvwx a} {uvw klm klmno fg uvwxy uvwx} + 97 {fg abc uvwxyz fghi pqrst pq} {abc k a ab abcde f} + 98 {uvwxy fghi uvwxy u abcde abcde uvw} {klmn uvwx pqrs uvw uvwxy abcde} + 99 {pq fg fghi uvwx uvwx fghij uvwxy} {klmn klmn f abc fg a} + } { + execsql { + INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b); + } + } + } {} + + proc prefix_query {prefixlist} { + set ret [list] + db eval {SELECT rowid, a, b FROM t1 ORDER BY rowid DESC} { + set bMatch 1 + foreach pref $prefixlist { + if { [lsearch -glob $a $pref]<0 && [lsearch -glob $b $pref]<0 } { + set bMatch 0 + break + } + } + if {$bMatch} { lappend ret $rowid } + } + return $ret + } + + do_execsql_test $T.integrity { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + foreach {bAsc sql} { + 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} + 0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC} + } { + foreach {tn prefix} { + 1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} + 6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*} + 11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*} + 16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*} + 21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*} + 27 {x*} + 28 {a f*} 29 {a* f*} 30 {a* fghij*} + } { + set res [prefix_query $prefix] + if {$bAsc} { + set res [lsort -integer -increasing $res] + } + set n [llength $res] + do_execsql_test $T.$bAsc.$tn.$n $sql $res + } + } + + catchsql COMMIT +} + +} + +finish_test diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test new file mode 100644 index 0000000000000000000000000000000000000000..205a59a69ffe1d798f090d6ce35d2a91ec456af4 --- /dev/null +++ b/ext/fts5/test/fts5ae.test @@ -0,0 +1,311 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ae + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('hello', 'world'); + SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; +} {1} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('world', 'hello'); + SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; +} {1 2} + +do_execsql_test 1.3 { + INSERT INTO t1 VALUES('world', 'world'); + SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; +} {1 2} + +do_execsql_test 1.4.1 { + INSERT INTO t1 VALUES('hello', 'hello'); +} + +do_execsql_test 1.4.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; +} {1 2 4} + +fts5_aux_test_functions db + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e'); + INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p'); +} + +do_execsql_test 2.1 { + SELECT rowid, fts5_test_poslist(t2) FROM t2 + WHERE t2 MATCH 'm' ORDER BY rowid; +} { + 1 {0.0.5 0.1.0 0.1.2} + 2 {0.0.7 0.1.5} +} + +do_execsql_test 2.2 { + SELECT rowid, fts5_test_poslist(t2) FROM t2 + WHERE t2 MATCH 'u OR q' ORDER BY rowid; +} { + 1 {0.0.0} + 2 {1.0.2 1.0.10} +} + +if {[detail_is_full]} { + do_execsql_test 2.3 { + SELECT rowid, fts5_test_poslist(t2) FROM t2 + WHERE t2 MATCH 'y:o' ORDER BY rowid; + } { + 1 {0.1.3 0.1.7} + } +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t'); + INSERT INTO t3 VALUES( 'r c', ''); +} + +if {[detail_is_full]} { + do_execsql_test 3.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; + } { + 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} + } + + do_execsql_test 3.2 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; + } { + 2 {0.0.0 1.0.1} + } +} + +do_execsql_test 3.3 { + INSERT INTO t3 + VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); + SELECT rowid, fts5_test_poslist(t3) + FROM t3 WHERE t3 MATCH 'a OR b AND c'; +} { + 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2} + 3 0.0.5 +} + +#------------------------------------------------------------------------- +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO t4 + VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); +} + +do_execsql_test 4.1 { + SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c'; +} { + 1 0.0.5 +} + +#------------------------------------------------------------------------- +# Test that the xColumnSize() and xColumnAvgsize() APIs work. +# +reset_db +fts5_aux_test_functions db + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); + INSERT INTO t5 VALUES('', 'a'); + INSERT INTO t5 VALUES('a', ''); +} +do_execsql_test 5.2 { + SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {1 0} + 2 {0 1} + 1 {4 6} +} + +do_execsql_test 5.3 { + SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {a {}} + 2 {{} a} + 1 {{a b c d} {e f g h i j}} +} + +do_execsql_test 5.4 { + SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {5 7} + 2 {5 7} + 1 {5 7} +} + +do_execsql_test 5.5 { + INSERT INTO t5 VALUES('x y z', 'v w x y z'); + SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {8 12} + 2 {8 12} + 1 {8 12} +} + +#------------------------------------------------------------------------- +# Test the xTokenize() API +# +reset_db +fts5_aux_test_functions db +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); + INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); +} + +do_execsql_test 6.2 { + SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*' +} { + 1 {{there are more} {things in heaven and earth}} + 2 {{horatio than are} {dreamt of in your philosophy}} +} + +#------------------------------------------------------------------------- +# Test the xQueryPhrase() API +# +reset_db +fts5_aux_test_functions db +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%); +} +do_test 7.2 { + foreach {x y} { + {q i b w s a a e l o} {i b z a l f p t e u} + {b a z t a l o x d i} {b p a d b f h d w y} + {z m h n p p u i e g} {v h d v b x j j c z} + {a g i m v a u c b i} {p k s o t l r t b m} + {v v c j o d a s c p} {f f v o k p o f o g} + } { + execsql {INSERT INTO t7 VALUES($x, $y)} + } + execsql { SELECT count(*) FROM t7 } +} {5} + +foreach {tn q res} { + 1 a {{4 2}} + 2 b {{3 4}} + 3 c {{2 1}} + 4 d {{2 2}} + 5 {a AND b} {{4 2} {3 4}} + 6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}} +} { + do_execsql_test 7.3.$tn { + SELECT fts5_test_queryphrase(t7) FROM t7 WHERE t7 MATCH $q LIMIT 1 + } [list $res] +} + +do_execsql_test 7.4 { + SELECT fts5_test_rowcount(t7) FROM t7 WHERE t7 MATCH 'a'; +} {5 5 5 5} + +#do_execsql_test 7.4 { +# SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a'; +#} {5 5 5 5} +# + +#------------------------------------------------------------------------- +# +do_test 8.1 { + execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) } + foreach {rowid x y} { + 0 {A o} {o o o C o o o o o o o o} + 1 {o o B} {o o o C C o o o o o o o} + 2 {A o o} {o o o o D D o o o o o o} + 3 {o B} {o o o o o D o o o o o o} + 4 {E o G} {H o o o o o o o o o o o} + 5 {F o G} {I o J o o o o o o o o o} + 6 {E o o} {H o J o o o o o o o o o} + 7 {o o o} {o o o o o o o o o o o o} + 9 {o o o} {o o o o o o o o o o o o} + } { + execsql { INSERT INTO t8(rowid, x, y) VALUES($rowid, $x, $y) } + } +} {} + +foreach {tn q res} { + 1 {a} {0 2} + 2 {b} {3 1} + 3 {c} {1 0} + 4 {d} {2 3} + 5 {g AND (e OR f)} {5 4} + 6 {j AND (h OR i)} {5 6} +} { + do_execsql_test 8.2.$tn.1 { + SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8); + } $res + + do_execsql_test 8.2.$tn.2 { + SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank; + } $res + + do_execsql_test 8.2.$tn.3 { + SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank; + } $res +} + +#------------------------------------------------------------------------- +# Test xPhraseCount() for some different queries. +# +do_test 9.1 { + execsql { CREATE VIRTUAL TABLE t9 USING fts5(x) } + foreach x { + "a b c" "d e f" + } { + execsql { INSERT INTO t9 VALUES($x) } + } +} {} + +foreach {tn q cnt} { + 1 {a AND b} 2 + 2 {a OR b} 2 + 3 {a OR b OR c} 3 + 4 {NEAR(a b)} 2 +} { + do_execsql_test 9.2.$tn { + SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1 + } $cnt +} + +} + +finish_test diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test new file mode 100644 index 0000000000000000000000000000000000000000..9c95ef2daaccc1684108794115141734babfd56b --- /dev/null +++ b/ext/fts5/test/fts5af.test @@ -0,0 +1,226 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# More specifically, the tests in this file focus on the built-in +# snippet() function. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5af + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); +} + +proc do_snippet_test {tn doc match res} { + + uplevel #0 [list set v1 $doc] + uplevel #0 [list set v2 $match] + + do_execsql_test $tn.1 { + DELETE FROM t1; + INSERT INTO t1 VALUES($v1, NULL); + SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2; + } [list $res] + + do_execsql_test $tn.2 { + DELETE FROM t1; + INSERT INTO t1 VALUES(NULL, $v1); + SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2; + } [list $res] + + do_execsql_test $tn.3 { + DELETE FROM t1; + INSERT INTO t1 VALUES($v1, NULL); + SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2 + ORDER BY rank DESC; + } [list $res] + + +} + + +foreach {tn doc res} { + + 1.1 {X o o o o o o} {[X] o o o o o o} + 1.2 {o X o o o o o} {o [X] o o o o o} + 1.3 {o o X o o o o} {o o [X] o o o o} + 1.4 {o o o X o o o} {o o o [X] o o o} + 1.5 {o o o o X o o} {o o o o [X] o o} + 1.6 {o o o o o X o} {o o o o o [X] o} + 1.7 {o o o o o o X} {o o o o o o [X]} + + 2.1 {X o o o o o o o} {[X] o o o o o o...} + 2.2 {o X o o o o o o} {o [X] o o o o o...} + 2.3 {o o X o o o o o} {o o [X] o o o o...} + 2.4 {o o o X o o o o} {o o o [X] o o o...} + 2.5 {o o o o X o o o} {o o o o [X] o o...} + 2.6 {o o o o o X o o} {o o o o o [X] o...} + 2.7 {o o o o o o X o} {o o o o o o [X]...} + 2.8 {o o o o o o o X} {...o o o o o o [X]} + + 2.9 {o o o o o o o X o} {...o o o o o [X] o} + 2.10 {o o o o o o o X o o} {...o o o o [X] o o} + 2.11 {o o o o o o o X o o o} {...o o o [X] o o o} + 2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...} + + + 3.1 {X o o o o o o o o} {[X] o o o o o o...} + 3.2 {o X o o o o o o o} {o [X] o o o o o...} + 3.3 {o o X o o o o o o} {o o [X] o o o o...} + 3.4 {o o o X o o o o o} {o o o [X] o o o...} + + 3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...} + 3.6 {o o o o o o o o X o o o} {...o o o [X] o o o} + 3.7 {o o o o o o o o o X o o} {...o o o o [X] o o} + 3.8 {o o o o o o o o o o X o} {...o o o o o [X] o} + 3.9 {o o o o o o o o o o o X} {...o o o o o o [X]} + + 4.1 {X o o o o o X o o} {[X] o o o o o [X]...} + 4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...} + 4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]} + + 5.1 {X o o o o X o o o} {[X] o o o o [X] o...} + 5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...} + 5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o} + 5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]} + + 6.1 {X o o o X o o o} {[X] o o o [X] o o...} + 6.2 {o X o o o X o o o} {o [X] o o o [X] o...} + 6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...} + 6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o} + 6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]} + + 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} + 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} + 7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...} + 7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o} + 7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o} + 7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]} + + 8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o} + {...o o [X] [X] [X] o o...} + 8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} + {...o o [X] o o o o...} + 8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} + {o o o o [X] o o...} +} { + do_snippet_test 1.$tn $doc X $res +} + +if {[detail_is_full]} { + foreach {tn doc res} { + 1.1 {X Y o o o o o} {[X Y] o o o o o} + 1.2 {o X Y o o o o} {o [X Y] o o o o} + 1.3 {o o X Y o o o} {o o [X Y] o o o} + 1.4 {o o o X Y o o} {o o o [X Y] o o} + 1.5 {o o o o X Y o} {o o o o [X Y] o} + 1.6 {o o o o o X Y} {o o o o o [X Y]} + + 2.1 {X Y o o o o o o} {[X Y] o o o o o...} + 2.2 {o X Y o o o o o} {o [X Y] o o o o...} + 2.3 {o o X Y o o o o} {o o [X Y] o o o...} + 2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o} + 2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o} + 2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o} + 2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]} + + 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} + 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} + 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} + 3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...} + 3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o} + 3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o} + 3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o} + 3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]} + } { + do_snippet_test 2.$tn $doc "X + Y" $res + } +} + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13'); + SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz'); +} { + {1 2 3 4 5...} +} + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE p1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO p1 VALUES( + 'x a a a a a a a a a a', + 'a a a a a a a a a a a a a a a a a a a x' + ); +} +do_execsql_test 5.1 { + SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); +} {{[x] a a a a a...}} + +do_execsql_test 5.2 { + SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x'); +} {{[x] a a a a a}} +do_execsql_test 5.3 { + SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x'); +} {{x] a a a a a...}} +do_execsql_test 5.4 { + SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x'); +} {{[x a a a a a...}} +do_execsql_test 5.5 { + SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x OR ""'); +} {{[x a a a a a...}} +do_execsql_test 5.6 { + SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x OR ' || x'DB'); +} {{[x a a a a a...}} + +} ;# foreach_detail_mode + +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(colA, colB); + INSERT INTO t1 VALUES('A B C', 'D E F'); +} + +do_execsql_test 6.1 { + SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} ...[B]...} +breakpoint +do_execsql_test 6.2 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} {D E...}} +do_execsql_test 6.3 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} {D...}} + +do_execsql_test 6.1 { + SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} [A]...} +breakpoint +do_execsql_test 6.2 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} {D E...}} +do_execsql_test 6.3 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} {D...}} + + + +finish_test diff --git a/ext/fts5/test/fts5ag.test b/ext/fts5/test/fts5ag.test new file mode 100644 index 0000000000000000000000000000000000000000..42cd9137847c4d21936478871f86a1cdeac557a4 --- /dev/null +++ b/ext/fts5/test/fts5ag.test @@ -0,0 +1,144 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ag + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# This file attempts to verify that the extension APIs work with +# "ORDER BY rank" queries. This is done by comparing the results of +# the fts5_test() function when run with queries of the form: +# +# ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC] +# +# and +# +# ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC] +# + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%); +} + +do_test 1.1 { + foreach {x y z} { + {j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w} + {m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v} + {m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n} + {j e m v k p e c j m} {m p v z d x l n i a} {v p u p m t p q i f} + {v r w l e e t d z p} {c s b w k m n k o u} {w g y f v w v w v p} + {k d g o u j p z n o} {t g e q l z i g b j} {f i q q j y h b g h} + {j s w x o t j b t m} {v a v v r t x c q a} {r t k x w u l h a g} + {j y b i u d e m d w} {y s o j h i n a u p} {n a g b u c w e b m} + {b c k s c w j p w b} {m o c o w o b d q q} {n t y o y z y r z e} + {p n q l e l h z q c} {n s e i h c v b b u} {m p d i t a o o f f} + {k c o n v e z l b m} {s m n i n s d e s u} {t a u e q d a o u c} + {h d t o i a g b b p} {k x c i g f g b b k} {x f i v n a n n j i} + {f z k r b u s k z e} {n z v z w l e r h t} {t i s v v a v p n s} + {k f e c t z r e f d} {f m g r c w q k b v} {v y s y f r b f e f} + {z r c t d q q h x b} {u c g z n z u v s s} {y t n f f x b f d x} + {u n p n u t i m e j} {p j j d m f k p m z} {d o l v c o e a h w} + {h o q w t f v i c y} {c q u n r z s l l q} {z x a q w s b w s y} + {y m s x k i m n x c} {b i a n v h z n k a} {w l q p b h h g d y} + {z v s j f p v l f w} {c s b i z e k i g c} {x b v d w j f e d z} + {r k k j e o m k g b} {h b d c h m y b t u} {u j s h k z c u d y} + {v h i v s y z i k l} {d t m w q w c a z p} {r s e s x v d w k b} + {u r e q j y h o o s} {x x z r x y t f j s} {k n h x i i u e c v} + {q l f d a p w l q o} {y z q w j o p b o v} {s u h z h f d f n l} + {q o e o x x l g q i} {j g m h q q w c d b} {o m d h w a g b f n} + {m x k t s s y l v a} {j x t c a u w b w g} {n f j b v x y p u t} + {u w k a q b u w k w} {a h j u o w f s k p} {j o f s h y t j h g} + {x v b l m t l m h l} {t p y i y i q b q a} {k o o z w a c h c f} + {j g c d k w b d t v} {a k v c m a v h v p} {i c a i j g h l j h} + {l m v l c z j b p b} {z p z f l n k i b a} {j v q k g i x g i b} + {m c i w u z m i s z} {i z r f n l q z k w} {x n b p b q r g i z} + {d g i o o x l f x d} {r t m f b n q y c b} {i u g k w x n m p o} + {t o s i q d z x d t} {v a k s q z j c o o} {z f n n r l y w v v} + {w k h d t l j g n n} {r z m v y b l n c u} {v b v s c l n k g v} + {m a g r a b u u n z} {u y l h v w v k b f} {x l p g i s j f x v} + {v s g x k z a k a r} {l t g v j q l k p l} {f h n a x t v s t y} + {z u v u x p s j y t} {g b q e e g l n w g} {e n p j i g j f u r} + {q z l t w o l m p e} {t s g h r p r o t z} {y b f a o n u m z g} + {d t w n y b o g f o} {d a j e r l g g s h} {d z e l w q l t h f} + {f l u w q v x j a h} {f n u l l d m h h w} {d x c c e r o d q j} + {b y f q s q f u l g} {u z w l f d b i a g} {m v q b g u o z e z} + {h z p t s e x i v m} {l h q m e o x x x j} {e e d n p r m g j f} + {k h s g o n s d a x} {u d t t s j o v h a} {z r b a e u v o e s} + {m b b g a f c p a t} {w c m j o d b l g e} {f p j p m o s y v j} + {c r n h d w c a b l} {s g e u s d n j b g} {b o n a x a b x y l} + {r h u x f c d z n o} {x y l g u m i i w d} {t f h b z v r s r g} + {t i o r b v g g p a} {d x l u q k m o s u} {j f h t u n z u k m} + {g j t y d c n j y g} {w e s k v c w i g t} {g a h r g v g h r o} + {e j l a q j g i n h} {d z k c u p n u p p} {t u e e v z v r r g} + {l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j} + {i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a} + } { + execsql { INSERT INTO t1 VALUES($x, $y, $z) } + } + set {} {} +} {} + +fts5_aux_test_functions db + +proc do_fts5ag_test {tn E} { + set q1 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank} + set q2 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)} + + set res [execsql $q1] + set expected [execsql $q2] + uplevel [list do_test $tn.1 [list set {} $res] $expected] + + append q1 " DESC" + append q2 " DESC" + + set res [execsql $q1] + set expected [execsql $q2] + uplevel [list do_test $tn.2 [list set {} $res] $expected] +} + +foreach {tn expr} { + 2.1 a + 2.2 b + 2.3 c + 2.4 d + + 3.0 {a AND b} + 3.1 {a OR b} + 3.2 {b OR c AND d} +} { + do_fts5ag_test $tn $expr +} + +if {[detail_is_full]} { + foreach {tn expr} { + 4.1 {"m m"} + 4.2 {e + s} + 4.3 {NEAR(c d)} + } { + do_fts5ag_test $tn $expr + } +} + +} ;# foreach_detail_mode + + +finish_test diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test new file mode 100644 index 0000000000000000000000000000000000000000..bf9c9e9dbc9368875d078561d67df3489878a245 --- /dev/null +++ b/ext/fts5/test/fts5ah.test @@ -0,0 +1,181 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# TESTRUNNER: slow + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ah + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +#------------------------------------------------------------------------- +# This file contains tests for very large doclists. +# + +set Y [list] +set W [list] +do_test 1.0 { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) } + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) } + set v {w w w w w w w w w w w w w w w w w w w w} + execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) } + for {set i 1} {$i <= 10000} {incr i} { + set v {x x x x x x x x x x x x x x x x x x x x} + if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i} + if {($i % 1577)==0} {lset v 5 W ; lappend W $i} + execsql { INSERT INTO t1 VALUES($v) } + } + set v {w w w w w w w w w w w w w w w w w w w w} + execsql { INSERT INTO t1 VALUES($v) } +} {} + +do_execsql_test 1.1.1 { + SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' +} [lsort -integer -incr $W] + +do_execsql_test 1.1.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'x* AND w*' +} [lsort -integer -incr $W] + +do_execsql_test 1.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' +} [lsort -integer -incr $Y] + +do_execsql_test 1.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +proc reads {} { + db one {SELECT t1 FROM t1 WHERE t1 MATCH '*reads'} +} + +proc execsql_reads {sql} { + set nRead [reads] + execsql $sql + expr [reads] - $nRead +} + +do_test 1.4 { + set nRead [reads] + execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } + set nReadX [expr [reads] - $nRead] + #puts -nonewline "(nReadX=$nReadX)" + if {[detail_is_full]} { set expect 1000 } + if {[detail_is_col]} { set expect 250 } + if {[detail_is_none]} { set expect 80 } + + expr $nReadX>$expect +} {1} + +do_test 1.5 { + set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }] + set bwd [execsql_reads { + SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC + }] + expr {$bwd < $fwd + 12} +} {1} + +foreach {tn q res} " + 1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x' } [list $W] + 2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } [list $W] + 3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W] + 4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y] +" { + if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue + + if {[detail_is_full]} { set ratio 8 } + if {[detail_is_col]} { set ratio 4 } + if {[detail_is_none]} { set ratio 2 } + + do_test 1.6.$tn.1 { + set n [execsql_reads $q] + #puts -nonewline "(n=$n nReadX=$nReadX)" + expr {$n < ($nReadX / $ratio)} + } {1} + + do_test 1.6.$tn.2 { + set n [execsql_reads "$q ORDER BY rowid DESC"] + #puts -nonewline "(n=$n nReadX=$nReadX)" + expr {$n < ($nReadX / $ratio)} + } {1} + + do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res] + do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res] +} + +#------------------------------------------------------------------------- +# Now test that adding range constraints on the rowid field reduces the +# number of pages loaded from disk. +# +foreach {tn fraction tail cnt} { + 1 0.6 {rowid > 5000} 5000 + 2 0.2 {rowid > 9000} 1000 + 3 0.2 {rowid < 1000} 999 + 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 + 5 0.6 {rowid >= 5000} 5001 + 6 0.2 {rowid >= 9000} 1001 + 7 0.2 {rowid <= 1000} 1000 + 8 0.6 {rowid > '5000'} 5000 + 9 0.2 {rowid > '9000'} 1000 + 10 0.1 {rowid = 444} 1 +} { + set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail" + set n [execsql_reads $q] + set ret [llength [execsql $q]] + + # Because the position lists for 'x' are quite long in this db, the + # advantage is a bit smaller in detail=none mode. Update $fraction to + # reflect this. + if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] } + + do_test "1.7.$tn.asc.(n=$n ret=$ret)" { + expr {$n < ($fraction*$nReadX) && $ret==$cnt} + } {1} + + set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC" + set n [execsql_reads $q] + set ret [llength [execsql $q]] + do_test "1.7.$tn.desc.(n=$n ret=$ret)" { + expr {$n < 2*$fraction*$nReadX && $ret==$cnt} + } {1} +} + +do_execsql_test 1.8.1 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text'; +} {10000} +do_execsql_test 1.8.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; +} {10000} + +do_execsql_test 1.8.3 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text'; +} {4999} +do_execsql_test 1.8.4 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text'; +} {0} + +do_catchsql_test 1.9 { + SELECT * FROM t1('*xy'); +} {1 {unknown special query: xy}} + +} ;# foreach_detail_mode + +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} + +finish_test diff --git a/ext/fts5/test/fts5ai.test b/ext/fts5/test/fts5ai.test new file mode 100644 index 0000000000000000000000000000000000000000..a6576d3afc2110dc35d59c20c7a4babcf0837af4 --- /dev/null +++ b/ext/fts5/test/fts5ai.test @@ -0,0 +1,63 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, it tests transactions and savepoints +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ai + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%); +} {} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('d e f'); + SAVEPOINT one; + INSERT INTO t1 VALUES('g h i'); + SAVEPOINT two; + INSERT INTO t1 VALUES('j k l'); + ROLLBACK TO one; + INSERT INTO t1 VALUES('m n o'); + SAVEPOINT two; + INSERT INTO t1 VALUES('p q r'); + RELEASE one; + SAVEPOINT one; + INSERT INTO t1 VALUES('s t u'); + ROLLBACK TO one; + COMMIT; +} + +do_execsql_test 1.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 1.3 { + SAVEPOINT one; + INSERT INTO t1 VALUES('v w x'); + ROLLBACK TO one; +} +} + + +finish_test diff --git a/ext/fts5/test/fts5aj.test b/ext/fts5/test/fts5aj.test new file mode 100644 index 0000000000000000000000000000000000000000..e802306b388230a6e777ce83fa82e9e50810f6a9 --- /dev/null +++ b/ext/fts5/test/fts5aj.test @@ -0,0 +1,68 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, this tests that, provided the amount of data remains +# constant, the FTS index does not grow indefinitely as rows are inserted +# and deleted, +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aj + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc doc {} { + set dict [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + set res [list] + for {set i 0} {$i < 20} {incr i} { + lappend res [lindex $dict [expr int(rand() * 26)]] + } + set res +} + +proc structure {} { + set val [db one {SELECT fts5_decode(rowid,block) FROM t1_data WHERE rowid=10}] + foreach lvl [lrange $val 1 end] { + lappend res [expr [llength $lvl]-2] + } + set res +} + +expr srand(0) +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); +} + +for {set iTest 0} {$iTest < 50000} {incr iTest} { + if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } } + set new [doc] + execsql { INSERT INTO t1 VALUES($new) } + if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] } + if {0==($iTest % 1000)} { + set sz [db one {SELECT count(*) FROM t1_data}] + set s [structure] + do_execsql_test 1.$iTest.$sz.{$s} { + INSERT INTO t1(t1) VALUES('integrity-check') + } + } +} + +do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') } + + +finish_test diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test new file mode 100644 index 0000000000000000000000000000000000000000..253f14fc7986913a93690832285f3262f22eb605 --- /dev/null +++ b/ext/fts5/test/fts5ak.test @@ -0,0 +1,183 @@ +# 2014 November 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, the auxiliary function "highlight". +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ak + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); + INSERT INTO ft1 VALUES('i d d a g i b g d d'); + INSERT INTO ft1 VALUES('h d b j c c g a c a'); + INSERT INTO ft1 VALUES('e j a e f h b f h h'); + INSERT INTO ft1 VALUES('j f h d g h i b d f'); + INSERT INTO ft1 VALUES('d c j d c j b c g e'); + INSERT INTO ft1 VALUES('i a d e g j g d a a'); + INSERT INTO ft1 VALUES('j f c e d a h j d b'); + INSERT INTO ft1 VALUES('i c c f a d g h j e'); + INSERT INTO ft1 VALUES('i d i g c d c h b f'); + INSERT INTO ft1 VALUES('g d a e h a b c f j'); + + CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%); + INSERT INTO ft2 VALUES('a b c d e f g h i j'); +} + +do_execsql_test 1.2 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e'; +} { + {[e] j a [e] f h b f h h} + {d c j d c j b c g [e]} + {i a d [e] g j g d a a} + {j f c [e] d a h j d b} + {i c c f a d g h j [e]} + {g d a [e] h a b c f j} +} + +do_execsql_test 1.3 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e' +} { + {[e] j a [e] f h b f h h} + {d c j d c j b c g [e]} + {i a d [e] g j g d a a} + {j f c [e] d a h j d b} + {i c c f a d g h j [e]} + {g d a [e] h a b c f j} +} + +do_execsql_test 1.4 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d' +} { + {a b c [d] e [f] g h i j} +} + +do_execsql_test 1.5 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f' +} { + {a b c [d] e [f] g h i j} +} + +#------------------------------------------------------------------------- +# Tests below this point require detail=full. +#------------------------------------------------------------------------- +if {[detail_is_full]==0} continue + + +do_execsql_test 2.1 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; +} { + {[h d] b j c c g a c a} + {j f [h d] g h i b d f} +} + +do_execsql_test 2.2 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; +} { + {i [d d] a g i b g [d d]} +} + +do_execsql_test 2.3 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; +} { + {i [d d] a g i b g [d d]} +} + +do_execsql_test 2.4 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' +} {{a [b c d e] f g h i j}} + +do_execsql_test 2.5 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' +} { + {a [b c d] [e f g] h i j} +} + +do_execsql_test 2.6 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' +} { + {a [b c d] e f g h i j} +} + +do_execsql_test 2.7 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' +} { + {a [b c d e] f g h i j} +} + +#------------------------------------------------------------------------- +# The example from the docs. +# +do_execsql_test 3.1 { + -- Assuming this: + CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%); + INSERT INTO ft VALUES('a b c x c d e'); + INSERT INTO ft VALUES('a b c c d e'); + INSERT INTO ft VALUES('a b c d e'); + + -- The following SELECT statement returns these three rows: + -- '[a b c] x [c d e]' + -- '[a b c] [c d e]' + -- '[a b c d e]' + SELECT highlight(ft, 0, '[', ']') FROM ft WHERE ft MATCH 'a+b+c AND c+d+e'; +} { + {[a b c] x [c d e]} + {[a b c] [c d e]} + {[a b c d e]} +} + +do_execsql_test 3.2 { + SELECT highlight(ft, 0, NULL, NULL) FROM ft WHERE ft MATCH 'a+b+c AND c+d+e'; +} { + {a b c x c d e} + {a b c c d e} + {a b c d e} +} + +} + +# 2023-04-06 https://sqlite.org/forum/forumpost/cae4367d9b +# +# This is not a test of FTS5, but rather a test of the of what happens to +# prepared statements that encounter SQLITE_SCHEMA while other prepared +# statements are running. The original problem POC used FTS5, and so +# is seems reasonable to put the test here. +# +# The vdbeaux24.test module in TH3 also tests this same behavior but +# without requiring FTS5 or an other extension. +# +reset_db +db null NULL +do_execsql_test 4.0 { + CREATE TABLE t5(a PRIMARY KEY); + INSERT INTO t5 VALUES(0); + CREATE VIRTUAL TABLE t6 USING fts5(0); + DELETE FROM t6; + CREATE TABLE t7(x); + WITH cte(a) AS ( + SELECT a FROM t5 + WHERE ((0,0) IN (SELECT 0, LAG(0) OVER (PARTITION BY 0) FROM t6), 0) + < (a,0) + ) + SELECT max(a) FROM cte; +} NULL + +finish_test diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test new file mode 100644 index 0000000000000000000000000000000000000000..7187ad67c7f358690fdc7c70f8b510938400d2b0 --- /dev/null +++ b/ext/fts5/test/fts5al.test @@ -0,0 +1,309 @@ +# 2014 November 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, this function tests the %_config table. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5al + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); + SELECT * FROM ft1_config; +} {version 4} + +do_execsql_test 1.2 { + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); + SELECT * FROM ft1_config; +} {pgsz 32 version 4} + +do_execsql_test 1.3 { + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); + SELECT * FROM ft1_config; +} {pgsz 64 version 4} + +#-------------------------------------------------------------------------- +# Test the logic for parsing the rank() function definition. +# +foreach {tn defn} { + 1 "fname()" + 2 "fname(1)" + 3 "fname(1,2)" + 4 "fname(null,NULL,nUlL)" + 5 " fname ( null , NULL , nUlL ) " + 6 "fname('abc')" + 7 "fname('a''bc')" + 8 "fname('''abc')" + 9 "fname('abc''')" + + 7 "fname( 'a''bc' )" + 8 "fname('''abc' )" + 9 "fname( 'abc''' )" + + 10 "fname(X'1234ab')" + + 11 "myfunc(1.2)" + 12 "myfunc(-1.0)" + 13 "myfunc(.01,'abc')" +} { + do_execsql_test 2.1.$tn { + INSERT INTO ft1(ft1, rank) VALUES('rank', $defn); + } +} + +foreach {tn defn} { + 1 "" + 2 "fname" + 3 "fname(X'234ab')" + 4 "myfunc(-1.,'abc')" +} { + do_test 2.2.$tn { + catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) } + } {1 {SQL logic error}} +} + +#------------------------------------------------------------------------- +# Assorted tests of the tcl interface for creating extension functions. +# + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1 VALUES('q w e r t y'); + INSERT INTO t1 VALUES('y t r e w q'); +} + +proc argtest {cmd args} { return $args } +sqlite3_fts5_create_function db argtest argtest + +do_execsql_test 3.2.1 { + SELECT argtest(t1, 123) FROM t1 WHERE t1 MATCH 'q' +} {123 123} + +do_execsql_test 3.2.2 { + SELECT argtest(t1, 123, 456) FROM t1 WHERE t1 MATCH 'q' +} {{123 456} {123 456}} + +proc rowidtest {cmd} { $cmd xRowid } +sqlite3_fts5_create_function db rowidtest rowidtest + +do_execsql_test 3.3.1 { + SELECT rowidtest(t1) FROM t1 WHERE t1 MATCH 'q' +} {1 2} + +proc insttest {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xInstCount]} {incr i} { + lappend res [$cmd xInst $i] + } + set res +} +sqlite3_fts5_create_function db insttest insttest + +do_execsql_test 3.4.1 { + SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q' +} { + {{0 0 0}} + {{0 0 5}} +} + +if {[detail_is_full]} { + do_execsql_test 3.4.2 { + SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' + } { + {{1 0 1}} + {{0 0 2} {1 0 4}} + } +} + +proc coltest {cmd} { + list [$cmd xColumnSize 0] [$cmd xColumnText 0] +} +sqlite3_fts5_create_function db coltest coltest + +do_execsql_test 3.5.1 { + SELECT coltest(t1) FROM t1 WHERE t1 MATCH 'q' +} { + {6 {q w e r t y}} + {6 {y t r e w q}} +} + +#------------------------------------------------------------------------- +# Tests for remapping the "rank" column. +# +# 4.1.*: Mapped to a function with no arguments. +# 4.2.*: Mapped to a function with one or more arguments. +# + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e'); + INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q'); + INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t'); + INSERT INTO t2 VALUES('h j g t r e l s g s', 'k q k c i i c k n s'); + INSERT INTO t2 VALUES('b l k h d n n n m i', 'p t i a r b t q o l'); + INSERT INTO t2 VALUES('k r i l j b g i p a', 't q c h a i m g n l'); + INSERT INTO t2 VALUES('a e c q n m o m d g', 'l c t g i s q g q e'); + INSERT INTO t2 VALUES('b o j h f o g b p e', 'r t l h s b g i c p'); + INSERT INTO t2 VALUES('s q k f q b j g h f', 'n m a o p e i e k t'); + INSERT INTO t2 VALUES('o q g g q c o k a b', 'r t k p t f t h p c'); +} + +proc firstinst {cmd} { + foreach {p c o} [$cmd xInst 0] {} + expr $c*100 + $o +} +sqlite3_fts5_create_function db firstinst firstinst + +do_execsql_test 4.1.1 { + SELECT rowid, firstinst(t2) FROM t2 WHERE t2 MATCH 'a' ORDER BY rowid ASC +} { + 1 0 2 4 3 6 5 103 + 6 9 7 0 9 102 10 8 +} + +do_execsql_test 4.1.2 { + SELECT rowid, rank FROM t2 + WHERE t2 MATCH 'a' AND rank MATCH 'firstinst()' + ORDER BY rowid ASC +} { + 1 0 2 4 3 6 5 103 + 6 9 7 0 9 102 10 8 +} + +do_execsql_test 4.1.3 { + SELECT rowid, rank FROM t2 + WHERE t2 MATCH 'a' AND rank MATCH 'firstinst()' + ORDER BY rank DESC +} { + 5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0 +} + +do_execsql_test 4.1.4 { + INSERT INTO t2(t2, rank) VALUES('rank', 'firstinst()'); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rowid ASC +} { + 1 0 2 4 3 6 5 103 + 6 9 7 0 9 102 10 8 +} + +do_execsql_test 4.1.5 { + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rank DESC +} { + 5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0 +} + +do_execsql_test 4.1.6 { + INSERT INTO t2(t2, rank) VALUES('rank', 'firstinst ( ) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rank DESC +} { + 5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0 +} + +proc rowidplus {cmd ival} { + expr [$cmd xRowid] + $ival +} +sqlite3_fts5_create_function db rowidplus rowidplus + +if {[detail_is_full]} { + do_execsql_test 4.2.1 { + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' + } { + 10 110 + } + do_execsql_test 4.2.2 { + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' + } { + 10 121 + } + + do_execsql_test 4.2.3 { + SELECT rowid, rank FROM t2 + WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' + } { + 10 122 + } +} + +proc rowidmod {cmd imod} { + expr [$cmd xRowid] % $imod +} +sqlite3_fts5_create_function db rowidmod rowidmod +do_execsql_test 4.3.1 { + CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%); + INSERT INTO t3 VALUES('a one'); + INSERT INTO t3 VALUES('a two'); + INSERT INTO t3 VALUES('a three'); + INSERT INTO t3 VALUES('a four'); + INSERT INTO t3 VALUES('a five'); + INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()'); +} + +do_execsql_test 4.3.2 { + SELECT * FROM t3 + WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)' + ORDER BY rank ASC +} { + {a four} {a one} {a five} {a two} {a three} +} + +do_execsql_test 4.3.3 { + SELECT *, rank FROM t3 + WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' + ORDER BY rank ASC +} { + {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 +} + +do_execsql_test 4.3.4 { + SELECT * FROM t3('a', 'rowidmod(4)') ORDER BY rank ASC; +} { + {a four} {a one} {a five} {a two} {a three} +} + +do_execsql_test 4.3.5 { + SELECT *, rank FROM t3('a', 'rowidmod(3)') ORDER BY rank ASC +} { + {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 +} + +do_catchsql_test 4.4.3 { + SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' +} {1 {no such function: xyz}} +do_catchsql_test 4.4.4 { + SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL +} {1 {parse error in rank function: }} + +# Check that the second and subsequent rank= constraints are ignored. +# +do_catchsql_test 4.3.3 { + SELECT *, rank FROM t3 + WHERE t3 MATCH 'a' AND + rank MATCH 'nosuch()' AND + rank MATCH 'rowidmod(3)' + ORDER BY rank ASC +} {1 {unable to use function MATCH in the requested context}} + +} ;# foreach_detail_mode + + +finish_test diff --git a/ext/fts5/test/fts5alter.test b/ext/fts5/test/fts5alter.test new file mode 100644 index 0000000000000000000000000000000000000000..bb5f78dc86f09559d12ef9cab1f1494ccd430618 --- /dev/null +++ b/ext/fts5/test/fts5alter.test @@ -0,0 +1,101 @@ +# 2015 Jun 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on renaming FTS5 tables using the +# "ALTER TABLE ... RENAME TO ..." command +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5alter + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Test renaming regular, contentless and columnsize=0 FTS5 tables. +# +do_execsql_test 1.1.0 { + CREATE VIRTUAL TABLE "a x" USING fts5(a, x); + INSERT INTO "a x" VALUES('a a a', 'x x x'); + ALTER TABLE "a x" RENAME TO "x y"; +} +do_execsql_test 1.1.1 { + SELECT * FROM "x y"; + SELECT rowid FROM "x y" WHERE "x y" MATCH 'a' +} {{a a a} {x x x} 1} + +do_execsql_test 1.2.0 { + CREATE VIRTUAL TABLE "one/two" USING fts5(one, columnsize=0); + INSERT INTO "one/two"(rowid, one) VALUES(456, 'd d d'); + ALTER TABLE "one/two" RENAME TO "three/four"; +} +do_execsql_test 1.2.1 { + SELECT * FROM "three/four"; + SELECT rowid FROM "three/four" WHERE "three/four" MATCH 'd' +} {{d d d} 456} + +do_execsql_test 1.3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(val, content=''); + INSERT INTO t1(rowid, val) VALUES(-1, 'drop table'); + INSERT INTO t1(rowid, val) VALUES(-2, 'drop view'); + ALTER TABLE t1 RENAME TO t2; +} +do_execsql_test 1.3.1 { + SELECT rowid, * FROM t2; + SELECT rowid FROM t2 WHERE t2 MATCH 'table' +} {-2 {} -1 {} -1} + +#------------------------------------------------------------------------- +# Test renaming an FTS5 table within a transaction. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE zz USING fts5(a); + INSERT INTO zz(rowid, a) VALUES(-56, 'a b c'); + BEGIN; + INSERT INTO zz(rowid, a) VALUES(-22, 'a b c'); + ALTER TABLE zz RENAME TO yy; + SELECT rowid FROM yy WHERE yy MATCH 'a + b + c'; + COMMIT; +} {-56 -22} + +do_execsql_test 2.2 { + BEGIN; + ALTER TABLE yy RENAME TO ww; + INSERT INTO ww(rowid, a) VALUES(-11, 'a b c'); + SELECT rowid FROM ww WHERE ww MATCH 'a + b + c'; +} {-56 -22 -11} + +do_execsql_test 2.3 { + ROLLBACK; + SELECT rowid FROM yy WHERE yy MATCH 'a + b + c'; +} {-56 -22} + +#------------------------------------------------------------------------- + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE abc USING fts5(a); + INSERT INTO abc(rowid, a) VALUES(1, 'a'); + BEGIN; + INSERT INTO abc(rowid, a) VALUES(2, 'a'); +} +do_execsql_test 3.2 { + SELECT rowid FROM abc WHERE abc MATCH 'a'; +} {1 2} + +do_execsql_test 3.3 { + COMMIT; + SELECT rowid FROM abc WHERE abc MATCH 'a'; +} {1 2} + +finish_test diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test new file mode 100644 index 0000000000000000000000000000000000000000..b771af912e30161cb1a221af254d6c45b0f5ff56 --- /dev/null +++ b/ext/fts5/test/fts5auto.test @@ -0,0 +1,344 @@ +# 2015 May 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file contains automatically generated tests for various types +# of MATCH expressions. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5auto + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set data { + -4026076 + {n x w k b p x b n t t d s} {f j j s p j o} + {w v i y r} {i p y s} + {a o q v e n q r} {q v g u c y a z y} + 3995120 + {c} {e e w d t} + {x c p f w r s m l r b f d} {g g u e} + {s n u t d v p d} {b k v p m f} + -2913881 + {k m} {a} + {w r j z n s l} {m j i w d t w e l} + {z n c} {v f b m} + 174082 + {j} {q l w u k e q v r i} + {j l} {u v w r s p e l} + {p i k j k q c t g u s} {g u y s m h q k g t e s o r} + 3207399 + {e t} {} + {p} {y v r b e k h d e v} + {t m w z b g q t s d d h} {o n v u i t o y k j} + 182399 + {} {m o s o x d y f a x j z} + {x n z r c d} {n r x i r} + {s v s} {a u} + 768994 + {e u t q v z q k j p u f j p} {y c b} + {p s d} {k n w p m p p} + {u o x s d} {f s g r d b d r m m m z y} + 3931037 + {c j p x e} {c n k t h z o i} + {} {r r p j k x w q} + {o r d z d} {x} + 3105748 + {p x r u} {x i s w o t o g x m z i w} + {q x m z} {h c j w b l y w x c o} + {m b k v} {t v q i s a d x} + -2501642 + {o u d n w o m o o s n t r h} {k p e u y p e z d j r y g} + {v b b h d d q y j q j} {a m w d t} + {y e f n} {a k x i x} + -1745680 + {z u w j f d b f} {j w i c g u d w e} + {m f p v m a s p v c o s} {s c r z o t w l b e a q} + {m k q} {k b a v o} + -932328 + {r v i u m q d r} {f z u v h c m r f g} + {r x r} {k p i d h h w h z u a x} + {k m j p} {h l j a e u c i q x x f x g} + -3923818 + {t t p b n u i h e c k} {m z} + {v u d c} {v y y j s g} + {o a f k k q p h g x e n z x} {h d w c o l} + -2145922 + {z z l f a l g e d c d h} {j b j p k o o u b q} + {d i g q t f d r h k} {n w g j c x r p t y f l c t} + {d o c u k f o} {r y s x z s p p h g t p y c} + 4552917 + {j w j y h l k u} {n a} + {y h w c n k} {b} + {w} {z l r t s i m v c y} + 2292008 + {q v q j w y y x u t} {r q z n h a b o} + {d q y} {y v o e j} + {} {a b h c d l p d x} + 1407892 + {n j j u q d o a u c f} {r d b w o q n g} + {d e v w s} {v d v o u o x s l s j z y} + {j y w h i f g i h m} {v n z b n y} + -4412544 + {g h h r s} {h e r e} + {n q s} {o p z r m l l t} + {p} {f s u o b j} + 1209110 + {o a a z t t u h j} {z z i r k r} + {i c x q w g v o x z i z p} {q o g k i n z x e d v w v} + {p f v b g f e d n p u c y k} {q z z a i p a a s r e z} + 3448977 + {i v} {l u x t b o k} + {f h u v p} {k a o y j} + {d m k c j} {v c e r u e f i t} + -4703774 + {d h v w u z r e h x o l t} {p s f y w y r q d a m w} + {c h g c g j j f t b i c q} {s e} + {c t q j g f} {v n r w y r a g e j d} + 2414151 + {s o o s d s k q b f q v p e} {j r o b t o p d l o o x} + {d d k t v e} {} + {t v o d w} {w e q w h y c y y i j b a m} + -3342407 + {m c h n e p d o c r w n t} {j d k s p q l} + {t g s r w x j l r z r} {h} + {r q v x i r a n h s} {m y p b v w r a u o g q r} + -993951 + {l n p u o j d x t u u c o j} {k r n a r e k v i t o e} + {q f t t a a c z v f} {o n m p v f o e n} + {h z h i p s b j z h} {i t w m k c u g n i} + 1575251 + {} {z s i j d o x j a r t} + {h g j u j n v e n z} {p z j n n f} + {s q q f d w r l y i z d o m} {b a n d h t b y g h d} + 4263668 + {q g t h f s} {s g x p f q z i s o f l i} + {q k} {w v h a x n a r b} + {m j a h o b i x k r w z q u} {m t r g j o e q t m p u l} + 2487819 + {m w g x r n e u t s r} {b x a t u u j c r n} + {j} {w f j r e e y l p} + {o u h b} {o c a c a b v} + 167966 + {o d b s d o a u m o x y} {c} + {r w d o b v} {z e b} + {i n z a f g z o} {m u b a g} + 1948599 + {n r g q d j s} {n k} + {l b p d v t k h y y} {u m k e c} + {t b n y o t b} {j w c i r x x} + 2941631 + {l d p l b g f} {e k e} + {p j} {m c s w t b k n l d x} + {f o v y v l} {c w p s w j w c u t y} + 3561104 + {d r j j r j i g p} {u} + {g r j q} {z l p d s n f c h t d c v z} + {w r c f s x z y} {g f o k g g} + -2223281 + {y e t j j z f p o m m z} {h k o g o} + {m x a t} {l q x l} + {r w k d l s y b} {q g k b} + -4502874 + {k k b x k l f} {r} + {} {q m z b k h k u n e z} + {z q g y m y u} {} + 1757599 + {d p z j y u r} {z p l q w j t j} + {n i r x r y j} {} + {h} {w t d q c x z z x e e} + -4809589 + {} {z p x u h i i n g} + {w q s u d b f x n} {l y k b b r x t i} + {n d v j q o t o d p z e} {u r y u v u c} + 1068408 + {y e} {e g s k e w t p v o b k} + {z c m s} {r u r u h n h b p q g b} + {j k b l} {m c d t s r s q a d b o f} + -1972554 + {m s w} {d k v s a r k p a r i v} + {g j z k p} {y k c v r e u o q f i b a} + {i p i} {c z w c y b n z i v} + -2052385 + {} {x e u f f g n c i x n e i e} + {} {p s w d x p g} + {} {s j a h n} + 2805981 + {m x g c w o e} {k g u y r y i u e g g} + {f k j v t x p h x k u} {w i} + {b l f z f v t n} {i u d o d p h s m u} + 2507621 + {} {u b n l x f n j t} + {u r x l h} {h r l m r} + {d y e n b s q v t k n q q} {x l t v w h a s k} + -3138375 + {e o f j y x u w v e w z} {r d q g k n n v r c z n e w} + {l y i q z k j p u f q s k} {c i l l i m a a g a z r x f} + {a v k h m q z b y n z} {q g w c y r r o a} + -457971 + {j x a w e c s h f l f} {q} + {j f v j u m d q r v v} {x n v a w} + {i e h d h f u w t t z} {v s u l s v o v i k n e} + 2265221 + {z t c y w n y r t} {n b a x s} + {q w a v} {a b s d x i g w t e z h} + {t l} {j k r w f f y j o k u} + -3941280 + {r x t o z} {f j n z k} + {t x e b t d b k w i s} {j t y h i h} + {y q g n g s u v c z j z n g} {n n g t l p h} + 2084745 + {z d z d} {j} + {o e k t b k a z l w} {o p i h k c x} + {c r b t i j f} {z e n m} + 1265843 + {} {j s g j j x u y} + {u q t f} {g o g} + {w o j e d} {w q n a c t q x j} + -2941116 + {i n c u o} {f b} + {o m s q d o z a q} {f s v o b b} + {o a z c h r} {j e w h b f z} + -1265441 + {p g z q v a o a x a} {s t h} + {w i p o c} {s n d g f z w q o d v v l j} + {y f b i a s v} {u m o z k k s t s d p b l p} + -1989158 + {r i c n} {r e w w i n z} + {q u s y b w u g y g f o} {y} + {d} {j x i b x u y d c p v a h} + 2391989 + {b n w x w f q h p i} {e u b b i n a i o c d g} + {v a z o i e n l x l r} {r u f o r k w m d w} + {k s} {r f e j q p w} +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f); +} {} + +fts5_aux_test_functions db + +proc do_auto_test {tn tbl expr} { + foreach order {asc desc} { + set res [fts5_poslist_data $expr $tbl $order] + set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]" + + set ::autotest_expr $expr + do_execsql_test $testname [subst -novar { + SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl] + WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order] + }] $res + } +} + +#------------------------------------------------------------------------- +# + +for {set fold 0} {$fold < 3} {incr fold} { + switch $fold { + 0 { set map {} } + 1 { set map { + a a b a c b d b e c f c g d h d + i e j e k f l f m g g g o h p h + q i r i s j t j u k v k w l x l + y m z m + }} + + 2 { set map { + a a b a c a d a e a f a g a h a + i b j b k b l b m b g b o b p b + q c r c s c t c u c v c w c x c + }} + } + + execsql { + BEGIN; + DELETE FROM tt; + } + foreach {rowid a b c d e f} [string map $map $data] { + if {$rowid==-4703774} { + execsql { + INSERT INTO tt(rowid, a, b, c, d, e, f) + VALUES($rowid, $a, $b, $c, $d, $e, $f) + } + } + } + execsql COMMIT + + + foreach {tn expr} { + A.1 { {a} : x } + A.2 { {a b} : x } + A.3 { {a b f} : x } + A.4 { {f a b} : x } + A.5 { {f a b} : x y } + A.6 { {f a b} : x + y } + A.7 { {c a b} : x + c } + A.8 { {c d} : "l m" } + A.9 { {c e} : "l m" } + A.10 { {a b c a b c a b c f f e} : "l m" } + + B.1 { a NOT b } + B.2 { a NOT a:b } + B.3 { a OR (b AND c) } + B.4 { a OR (b AND {a b c}:c) } + B.5 { a OR "b c" } + B.6 { a OR b OR c } + + C.1 { a OR (b AND "b c") } + C.2 { a OR (b AND "z c") } + } { + do_auto_test 3.$fold.$tn tt $expr + } +} + +proc replace_elems {list args} { + set ret $list + foreach {idx elem} $args { + set ret [lreplace $ret $idx $idx $elem] + } + set ret +} + +#------------------------------------------------------------------------- +# +set bigdoc [string trim [string repeat "a " 1000]] +do_test 4.0 { + set a [replace_elems $bigdoc 50 x 950 x] + set b [replace_elems $bigdoc 20 y 21 x 887 x 888 y] + set c [replace_elems $bigdoc 1 z 444 z 789 z] + execsql { + CREATE VIRTUAL TABLE yy USING fts5(c1, c2, c3); + INSERT INTO yy(rowid, c1, c2, c3) VALUES(-56789, $a, $b, $c); + INSERT INTO yy(rowid, c1, c2, c3) VALUES(250, $a, $b, $c); + } +} {} + +foreach {tn expr} { + 1 x + 2 y + 3 z + + 4 {c1 : x} 5 {c2 : x} 6 {c3 : x} + 7 {c1 : y} 8 {c2 : y} 9 {c3 : y} + 10 {c1 : z} 11 {c2 : z} 12 {c3 : z} +} { + do_auto_test 4.$tn yy $expr +} + + + +finish_test diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test new file mode 100644 index 0000000000000000000000000000000000000000..960dbc5117c6b256fed9278ff640f4974930e703 --- /dev/null +++ b/ext/fts5/test/fts5aux.test @@ -0,0 +1,404 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the auxiliary function APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aux + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc inst {cmd i} { + $cmd xInst $i +} +sqlite3_fts5_create_function db inst inst + +proc colsize {cmd i} { + $cmd xColumnSize $i +} +sqlite3_fts5_create_function db colsize colsize + +proc totalsize {cmd i} { + $cmd xColumnTotalSize $i +} +sqlite3_fts5_create_function db totalsize totalsize + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1 VALUES('one two', 'two one zero'); + INSERT INTO f1 VALUES('one one', 'one one one'); +} + +do_catchsql_test 1.1 { + SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} +do_catchsql_test 1.2 { + SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two'; +} {0 {{0 0 1}}} +do_catchsql_test 1.3 { + SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two'; +} {0 {{0 1 0}}} +do_catchsql_test 1.4 { + SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} + +do_catchsql_test 2.1 { + SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} +do_execsql_test 2.2 { + SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero'; +} {2 3} +do_execsql_test 2.3 { + SELECT colsize(f1, -1) FROM f1 WHERE f1 MATCH 'zero'; +} {5} + +do_execsql_test 2.4.1 { + SELECT totalsize(f1, -1) FROM f1 WHERE f1 MATCH 'zero'; +} {10} +do_execsql_test 2.4.2 { + SELECT totalsize(f1, 0) FROM f1 WHERE f1 MATCH 'zero'; +} {4} +do_execsql_test 2.4.3 { + SELECT totalsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero'; +} {6} +do_catchsql_test 2.4.4 { + SELECT totalsize(f1, 2) FROM f1 WHERE f1 MATCH 'zero'; +} {1 SQLITE_RANGE} + +#------------------------------------------------------------------------- +# Test the xSet and xGetAuxdata APIs with a NULL destructor. +# +proc prevrowid {add cmd} { + set res [$cmd xGetAuxdataInt 0] + set r [$cmd xRowid] + $cmd xSetAuxdataInt $r + return [expr $res + $add] +} +sqlite3_fts5_create_function db prevrowid [list prevrowid 0] +sqlite3_fts5_create_function db prevrowid1 [list prevrowid 1] + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE e5 USING fts5(x); + INSERT INTO e5 VALUES('a b c'); + INSERT INTO e5 VALUES('d e f'); + INSERT INTO e5 VALUES('a b c'); + INSERT INTO e5 VALUES('d e f'); + INSERT INTO e5 VALUES('a b c'); +} + +do_execsql_test 3.1 { + SELECT prevrowid(e5) || '+' || rowid FROM e5 WHERE e5 MATCH 'c' +} {0+1 1+3 3+5} + +do_execsql_test 3.2 { + SELECT prevrowid(e5) || '+' || prevrowid1(e5) || '+' || rowid + FROM e5 WHERE e5 MATCH 'e' +} {0+1+2 2+3+4} + +#------------------------------------------------------------------------- +# Test that if the xQueryPhrase callback returns other than SQLITE_OK, +# the query is abandoned. And that if it returns an error code other than +# SQLITE_DONE, the error is propagated back to the caller. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE e7 USING fts5(x); + INSERT INTO e7 VALUES('a x a'); + INSERT INTO e7 VALUES('b x b'); + INSERT INTO e7 VALUES('c x c'); + INSERT INTO e7 VALUES('d x d'); + INSERT INTO e7 VALUES('e x e'); +} + +proc xCallback {rowid code cmd} { + set r [$cmd xRowid] + lappend ::cb $r + if {$r==$rowid} { return $code } + return "" +} + +proc phrasequery {cmd code} { + set ::cb [list] + $cmd xQueryPhrase 1 [list xCallback [$cmd xRowid] $code] + set ::cb +} + +sqlite3_fts5_create_function db phrasequery phrasequery + +do_execsql_test 4.1 { + SELECT phrasequery(e7, 'SQLITE_OK') FROM e7 WHERE e7 MATCH 'c x' +} {{1 2 3 4 5}} + +do_execsql_test 4.2 { + SELECT phrasequery(e7, 'SQLITE_DONE') FROM e7 WHERE e7 MATCH 'c x' +} {{1 2 3}} + +do_catchsql_test 4.3 { + SELECT phrasequery(e7, 'SQLITE_ERROR') FROM e7 WHERE e7 MATCH 'c x' +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +# Auxiliary function calls with many cursors in the global cursor list. +# +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE e9 USING fts5(y); + INSERT INTO e9(rowid, y) VALUES(1, 'i iii'); + INSERT INTO e9(rowid, y) VALUES(2, 'ii iv'); + INSERT INTO e9(rowid, y) VALUES(3, 'ii'); + INSERT INTO e9(rowid, y) VALUES(4, 'i iv'); + INSERT INTO e9(rowid, y) VALUES(5, 'iii'); +} + +proc my_rowid {cmd} { $cmd xRowid } +sqlite3_fts5_create_function db my_rowid my_rowid + +foreach {var q} { + s1 i + s2 ii + s3 iii + s4 iv +} { + set sql "SELECT my_rowid(e9) FROM e9 WHERE e9 MATCH '$q'" + set $var [sqlite3_prepare db $sql -1 dummy] +} + +do_test 5.1.1 { sqlite3_step $s1 ; sqlite3_column_int $s1 0 } 1 +do_test 5.1.2 { sqlite3_step $s2 ; sqlite3_column_int $s2 0 } 2 +do_test 5.1.3 { sqlite3_step $s3 ; sqlite3_column_int $s3 0 } 1 +do_test 5.1.4 { sqlite3_step $s4 ; sqlite3_column_int $s4 0 } 2 + +do_test 5.2.1 { sqlite3_step $s1 ; sqlite3_column_int $s1 0 } 4 +do_test 5.2.2 { sqlite3_step $s2 ; sqlite3_column_int $s2 0 } 3 +do_test 5.2.3 { sqlite3_step $s3 ; sqlite3_column_int $s3 0 } 5 +do_test 5.2.4 { sqlite3_step $s4 ; sqlite3_column_int $s4 0 } 4 + +sqlite3_finalize $s1 +sqlite3_finalize $s2 +sqlite3_finalize $s3 +sqlite3_finalize $s4 + +#------------------------------------------------------------------------- +# Passing an invalid first argument to an auxiliary function is detected. +# +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE e11 USING fts5(y, z); + INSERT INTO e11(rowid, y, z) VALUES(1, 'a b', 45); + INSERT INTO e11(rowid, y, z) VALUES(2, 'b c', 46); +} + +do_catchsql_test 6.1 { + SELECT my_rowid(z) FROM e11 WHERE e11 MATCH 'b' +} {1 {no such cursor: 45}} + +do_catchsql_test 6.2 { + SELECT my_rowid(y) FROM e11 WHERE e11 MATCH 'b' +} {1 {no such cursor: 0}} + +#------------------------------------------------------------------------- +# Test passing an out-of-range phrase number to xPhraseSize (should +# return 0). +# +proc my_phrasesize {cmd iPhrase} { $cmd xPhraseSize $iPhrase } +sqlite3_fts5_create_function db my_phrasesize my_phrasesize + +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('a b c'); +} +do_execsql_test 7.2 { + SELECT + my_phrasesize(t1, -1), + my_phrasesize(t1, 0), + my_phrasesize(t1, 1), + my_phrasesize(t1, 2) + FROM t1 WHERE t1 MATCH 'a OR b+c' +} {0 1 2 0} + +#------------------------------------------------------------------------- +# +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); +} + +foreach {tn lRow res} { + 4 {"a a a" "b" "a d"} {"[a] [a] [a]" "[a] d"} + 1 {"b d" "a b"} {"[b] [d]" "[a] b"} + 2 {"d b" "a d"} {"[d] [b]" "[a] d"} + 3 {"a a d"} {"[a] [a] d"} +} { + execsql { DELETE FROM x1 } + foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } } + do_execsql_test 8.$tn { + SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)'; + } $res +} + +#------------------------------------------------------------------------- +# Test the built-in bm25() demo. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('a', NULL); -- 1 + INSERT INTO t1 VALUES('a', NULL); -- 2 + INSERT INTO t1 VALUES('a', NULL); -- 3 + INSERT INTO t1 VALUES('a', NULL); -- 4 + INSERT INTO t1 VALUES('a', NULL); -- 5 + INSERT INTO t1 VALUES('a', NULL); -- 6 + INSERT INTO t1 VALUES('a', NULL); -- 7 + INSERT INTO t1 VALUES('a', NULL); -- 8 + INSERT INTO t1 VALUES(NULL, 'a a b'); -- 9 + INSERT INTO t1 VALUES(NULL, 'b b a'); -- 10 +} + +do_execsql_test 9.2 { + SELECT rowid FROM t1('a AND b') ORDER BY rank; +} { + 10 9 +} + +do_execsql_test 9.3 { + SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank; +} { + 9 10 +} + +#------------------------------------------------------------------------- +# Test that aux. functions may not be used in aggregate queries. +# +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, z); + INSERT INTO t1 VALUES('a', 'one two', 1); + INSERT INTO t1 VALUES('b', 'two three', 2); + INSERT INTO t1 VALUES('c', 'three four', 1); + INSERT INTO t1 VALUES('d', 'four five', 2); + INSERT INTO t1 VALUES('e', 'five six', 1); + INSERT INTO t1 VALUES('f', 'six seven', 2); +} + +proc firstcol {cmd} { $cmd xColumnText 0 } +sqlite3_fts5_create_function db firstcol firstcol + +do_execsql_test 10.1.1 { + SELECT firstcol(t1) FROM t1 +} {a b c d e f} +do_execsql_test 10.1.2 { + SELECT group_concat(x, '.') FROM t1 +} {a.b.c.d.e.f} + +do_catchsql_test 10.1.3 { + SELECT group_concat(firstcol(t1), '.') FROM t1 +} {1 {unable to use function firstcol in the requested context}} + +do_catchsql_test 10.1.4 { + SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid +} {1 {unable to use function firstcol in the requested context}} + +#------------------------------------------------------------------------- +# Test that xInstCount() works from within an xPhraseQuery() callback. +# +reset_db + +proc xCallback {cmd} { + incr ::hitcount [$cmd xInstCount] + return SQLITE_OK +} +proc fts5_hitcount {cmd} { + set ::hitcount 0 + $cmd xQueryPhrase 0 xCallback + return $::hitcount +} +sqlite3_fts5_create_function db fts5_hitcount fts5_hitcount + +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE x1 USING fts5(z); + INSERT INTO x1 VALUES('one two three'); + INSERT INTO x1 VALUES('one two one three one'); + INSERT INTO x1 VALUES('one two three'); +} + +do_execsql_test 11.2 { + SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1; +} {5} + +#------------------------------------------------------------------------- +# Test that xColumnText returns SQLITE_RANGE when it should. +# +reset_db +fts5_aux_test_functions db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + INSERT INTO t1 VALUES('one', 'two', 'three'); + INSERT INTO t1 VALUES('one', 'one', 'one'); + INSERT INTO t1 VALUES('two', 'two', 'two'); + INSERT INTO t1 VALUES('three', 'three', 'three'); +} + +do_catchsql_test 12.1.1 { + SELECT fts5_columntext(t1, -1) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 3) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 1) FROM t1('one AND two'); +} {0 two} + +do_catchsql_test 12.2.1 { + SELECT fts5_queryphrase(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.2 { + SELECT fts5_queryphrase(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.3 { + SELECT fts5_queryphrase(t1, 1) FROM t1('one AND two'); +} {0 {{1 2 1}}} + +do_catchsql_test 12.3.1 { + SELECT fts5_collist(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.2 { + SELECT fts5_collist(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.3 { + SELECT fts5_collist(t1, 1) FROM t1('one AND two'); +} {0 1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +do_catchsql_test 13.2 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {0 {{a b c} {d e f}}} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +do_catchsql_test 13.4 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {1 {SQL logic error}} + +finish_test diff --git a/ext/fts5/test/fts5aux2.test b/ext/fts5/test/fts5aux2.test new file mode 100644 index 0000000000000000000000000000000000000000..2352970ec742aa5e2f3d2e279d2a0e3dc9ca2f64 --- /dev/null +++ b/ext/fts5/test/fts5aux2.test @@ -0,0 +1,71 @@ +# 2024 June 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the auxiliary function APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aux + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('a b', 'c d'); + INSERT INTO x1 VALUES('d e', 'a b'); + INSERT INTO x1 VALUES('a b', 'e f'); + INSERT INTO x1 VALUES('d e', 'c d'); +} + +fts5_aux_test_functions db +do_execsql_test 1.1 { + SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2 +} [list [list {*}{ + columnsize {2 2} + columntext {{d e} {a b}} + columntotalsize {8 8} + poslist {} + tokenize {{d e} {a b}} + rowcount 4 +}]] + +do_execsql_test 1.2 { + SELECT fts5_test_columntext(x1) FROM x1 +} { + {{a b} {c d}} + {{d e} {a b}} + {{a b} {e f}} + {{d e} {c d}} +} + +do_execsql_test 1.3 { + SELECT fts5_test_rowid(x1) FROM x1 +} { + 1 2 3 4 +} +do_execsql_test 1.4 { + SELECT fts5_test_phrasecount(x1) FROM x1 +} { + 0 0 0 0 +} +do_catchsql_test 1.5 { + SELECT fts5_queryphrase(x1, 0) FROM x1 +} {1 SQLITE_RANGE} +do_execsql_test 1.6 { + SELECT fts5_test_rowcount(x1) FROM x1 +} {4 4 4 4} + + +finish_test diff --git a/ext/fts5/test/fts5auxdata.test b/ext/fts5/test/fts5auxdata.test new file mode 100644 index 0000000000000000000000000000000000000000..7f99fed3168dcadab8792346281ceba34fe6b756 --- /dev/null +++ b/ext/fts5/test/fts5auxdata.test @@ -0,0 +1,114 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the fts5 xSetAuxdata() and xGetAuxdata() APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5auxdata + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1(rowid, a, b) VALUES(1, 'a', 'b1'); + INSERT INTO f1(rowid, a, b) VALUES(2, 'a', 'b2'); + INSERT INTO f1(rowid, a, b) VALUES(3, 'a', 'b3'); + INSERT INTO f1(rowid, a, b) VALUES(4, 'a', 'b4'); + INSERT INTO f1(rowid, a, b) VALUES(5, 'a', 'b5'); +} + +proc aux_function_1 {cmd tn} { + switch [$cmd xRowid] { + 1 { + do_test $tn.1 [list $cmd xGetAuxdata 0 ] {} + $cmd xSetAuxdata "one" + } + + 2 { + do_test $tn.2 [list $cmd xGetAuxdata 0 ] {one} + $cmd xSetAuxdata "two" + } + + 3 { + do_test $tn.3 [list $cmd xGetAuxdata 0 ] {two} + } + + 4 { + do_test $tn.4 [list $cmd xGetAuxdata 1 ] {two} + } + + 5 { + do_test $tn.5 [list $cmd xGetAuxdata 0 ] {} + } + } +} + +sqlite3_fts5_create_function db aux_function_1 aux_function_1 +db eval { + SELECT aux_function_1(f1, 1) FROM f1 WHERE f1 MATCH 'a' + ORDER BY rowid ASC +} + +proc aux_function_2 {cmd tn inst} { + if {$inst == "A"} { + switch [$cmd xRowid] { + 1 { + do_test $tn.1.$inst [list $cmd xGetAuxdata 0 ] {} + $cmd xSetAuxdata "one $inst" + } + 2 { + do_test $tn.2.$inst [list $cmd xGetAuxdata 0 ] "one $inst" + $cmd xSetAuxdata "two $inst" + } + 3 { + do_test $tn.3.$inst [list $cmd xGetAuxdata 0 ] "two $inst" + } + 4 { + do_test $tn.4.$inst [list $cmd xGetAuxdata 1 ] "two $inst" + } + 5 { + do_test $tn.5.$inst [list $cmd xGetAuxdata 0 ] {} + } + } + } else { + switch [$cmd xRowid] { + 1 { + do_test $tn.1.$inst [list $cmd xGetAuxdata 0 ] "one A" + } + 2 { + do_test $tn.2.$inst [list $cmd xGetAuxdata 0 ] "two A" + } + 3 { + do_test $tn.3.$inst [list $cmd xGetAuxdata 0 ] "two A" + } + 4 { + do_test $tn.4.$inst [list $cmd xGetAuxdata 0 ] {} + } + 5 { + do_test $tn.5.$inst [list $cmd xGetAuxdata 0 ] {} + } + } + } +} + +sqlite3_fts5_create_function db aux_function_2 aux_function_2 +db eval { + SELECT aux_function_2(f1, 2, 'A'), aux_function_2(f1, 2, 'B') + FROM f1 WHERE f1 MATCH 'a' + ORDER BY rowid ASC +} + +finish_test diff --git a/ext/fts5/test/fts5bigid.test b/ext/fts5/test/fts5bigid.test new file mode 100644 index 0000000000000000000000000000000000000000..ae20ec641e0b0f21f55a1bbcf2e86d1aaf96ec94 --- /dev/null +++ b/ext/fts5/test/fts5bigid.test @@ -0,0 +1,62 @@ +# 2023 May 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5bigid + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set nRow 20000 + +proc do_ascdesc_test {tn query} { + set ::lAsc [db eval { SELECT rowid FROM x1($query) }] + set ::lDesc [db eval { SELECT rowid FROM x1($query) ORDER BY rowid DESC }] + do_test $tn.1 { lsort -integer $::lAsc } $::lAsc + do_test $tn.2 { lsort -integer -decr $::lDesc } $::lDesc + do_test $tn.3 { lsort -integer $::lDesc } $::lAsc +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); +} + +do_test 1.1 { + for {set ii 0} {$ii < $nRow} {incr ii} { + db eval { + REPLACE INTO x1(rowid, a) VALUES(random(), 'movement at the station'); + } + } +} {} + +do_ascdesc_test 1.2 "the" + +do_execsql_test 1.3 { + DELETE FROM x1 +} + +do_test 1.4 { + for {set ii 0} {$ii < $nRow} {incr ii} { + db eval { + INSERT INTO x1(rowid, a) VALUES( + $ii + 0x6FFFFFFFFFFFFFFF, 'movement at the station' + ); + } + } +} {} + +do_ascdesc_test 1.5 "movement" + +finish_test diff --git a/ext/fts5/test/fts5bigpl.test b/ext/fts5/test/fts5bigpl.test new file mode 100644 index 0000000000000000000000000000000000000000..9e3d86c0e69ce566d4ca12dc6be12aa0cddc6f26 --- /dev/null +++ b/ext/fts5/test/fts5bigpl.test @@ -0,0 +1,63 @@ +# 2015 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on really large position lists. Those that require +# 4 or 5 byte position-list size varints. Because of the amount of memory +# required, these tests only run on 64-bit platforms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5bigpl + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if { $tcl_platform(wordSize)<8 } { + finish_test + return +} + +do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x) } + +do_test 1.1 { + foreach t {a b c d e f g h i j} { + set doc [string repeat "$t " 1200000] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +do_test 1.2 { + execsql { DELETE FROM t1 } + foreach t {"a b" "b a" "c d" "d c"} { + set doc [string repeat "$t " 600000] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + + +# 5-byte varint. This test takes 30 seconds or so on a 2014 workstation. +# The generated database is roughly 635MiB. +# +do_test 2.1...slow { + execsql { DELETE FROM t1 } + foreach t {a} { + set doc [string repeat "$t " 150000000] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +finish_test diff --git a/ext/fts5/test/fts5bigtok.test b/ext/fts5/test/fts5bigtok.test new file mode 100644 index 0000000000000000000000000000000000000000..f74ec8f665ffc9a44fe90b7b023af52bed379873 --- /dev/null +++ b/ext/fts5/test/fts5bigtok.test @@ -0,0 +1,66 @@ +# 2016 Jan 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5bigtok +return_if_no_fts5 + +proc rndterm {} { + set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + set l [lindex $L [expr int(rand() * [llength $L])]] + string repeat $l [expr int(rand() * 5) + 60] +} + +proc rnddoc {n} { + set res [list] + for {set i 0} {$i < $n} {incr i} { + lappend res [rndterm] + } + set res +} + +foreach_detail_mode $::testprefix { + db func rnddoc rnddoc + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, row); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 ) + INSERT INTO t1 SELECT rnddoc(3) FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 ) + INSERT INTO t1 SELECT rnddoc(3) FROM s; + } + + foreach v [db eval {SELECT term FROM t1vocab}] { + set res [db eval {SELECT rowid FROM t1($v)}] + do_execsql_test 1.[string range $v 0 0] { + SELECT rowid FROM t1($v) ORDER BY rowid DESC + } [lsort -integer -decr $res] + } + + do_execsql_test 2.0 { + INSERT INTO t1(t1) VALUES('optimize'); + } + + foreach v [db eval {SELECT term FROM t1vocab}] { + set res [db eval {SELECT rowid FROM t1($v)}] + do_execsql_test 2.[string range $v 0 0] { + SELECT rowid FROM t1($v) ORDER BY rowid DESC + } [lsort -integer -decr $res] + } +} + +finish_test diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test new file mode 100644 index 0000000000000000000000000000000000000000..93485541042ee6eeab5fb752ffd6e81ba92d2302 --- /dev/null +++ b/ext/fts5/test/fts5blob.test @@ -0,0 +1,166 @@ +# 2024 July 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file verifies that: +# +# * blob values may be written to locale=0 tables. +# +# * blob values - other than fts5_locale() values - may not be written +# to locale=0 tables. This is an SQLITE_MISMATCH error +# +# * blob values may be returned by queries on the external-content table +# of a locale=0 table. +# +# * blob values not may be returned by queries on the external-content +# table of a locale=1 table, apart from fts5_locale() blobs. This is an +# SQLITE_MISMATCH error. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5blob + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Test that blobs may be stored in normal locale=0 tables. +# +foreach {tn enc} { + 1 utf8 + 2 utf16 +} { + reset_db + fts5_aux_test_functions db + + execsql "PRAGMA encoding = $enc" + + execsql " + CREATE VIRTUAL TABLE t1 USING fts5(x, y); + " + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546'); + INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546'); + INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz'); + } + + do_execsql_test 1.$tn.1 { + SELECT rowid, quote(x), quote(y) FROM t1 + } { + 1 555 X'0000000041424320444546' + 2 666 X'41424300444546' + 3 777 'xyz' + } + + do_execsql_test 1.$tn.2 { + DELETE FROM t1 WHERE rowid=2; + DELETE FROM t1 WHERE rowid=1; + } + + do_execsql_test 1.$tn.3 { + PRAGMA integrity_check; + } {ok} +} + +#-------------------------------------------------------------------------- +# Test that a blob may be stored and retrieved in an unindexed column of +# a regular table with locale=1. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1); + INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546'); +} + +do_execsql_test 2.1 { + select rowid, x, quote(y) FROM t1 +} { + 12 twelve X'0000000041424320444546' +} + +#-------------------------------------------------------------------------- +# Test that blobs may not be written to any type of table with locale=1 +# set. Except, they may be written to UNINDEXED columns. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {0 {}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {0 {}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {0 {}} + + +#-------------------------------------------------------------------------- +# Test that fts5_locale() values may not be written to any type of table +# without locale=1 set. Even to an UNINDEXED column. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=); + + CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.4 { + INSERT INTO x4(rowid, a, b, c) + VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +foreach {tn sql} { + 1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') } + 2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') } + 3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') } +} { + do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}} +} + + +finish_test + + diff --git a/ext/fts5/test/fts5cat.test b/ext/fts5/test/fts5cat.test new file mode 100644 index 0000000000000000000000000000000000000000..71e2abe3aece0eb9d3908117c42d08c8198b7804 --- /dev/null +++ b/ext/fts5/test/fts5cat.test @@ -0,0 +1,76 @@ +# 2016 Jan 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5cat + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'"); + INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names'); +} + +do_execsql_test 1.1 { + SELECT rowid FROM t1('option'); +} {1} + +do_execsql_test 1.2 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row); + + CREATE VIRTUAL TABLE t3 USING fts5( + x, tokenize="unicode61 categories 'L* N* Co Mn'" + ); + CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row); + + CREATE VIRTUAL TABLE t4 USING fts5( + x, tokenize="unicode61 categories 'L* N* Co M*'" + ); + CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row); + + INSERT INTO t2 VALUES ('สนามกีฬา'); + INSERT INTO t3 VALUES ('สนามกีฬา'); + INSERT INTO t4 VALUES ('สนามกีฬา'); +} + +do_execsql_test 1.3 { + SELECT * FROM t2t +} {สนามก 1 1 ฬา 1 1} + +do_execsql_test 1.4 { + SELECT * FROM t3t +} {สนามกีฬา 1 1} + +do_execsql_test 1.5 { + SELECT * FROM t4t +} {สนามกีฬา 1 1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 " + CREATE VIRTUAL TABLE x1 USING fts5(c, + tokenize=\"unicode61 categories ' \t'\"); +" + +do_catchsql_test 2.1 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tMYZ'\"); +" {1 {error in tokenizer constructor}} + +do_catchsql_test 2.2 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tXYZ'\"); +" {1 {error in tokenizer constructor}} + + +finish_test diff --git a/ext/fts5/test/fts5circref.test b/ext/fts5/test/fts5circref.test new file mode 100644 index 0000000000000000000000000000000000000000..ea992195afbe01c528c9ce84a0cd657965898bb8 --- /dev/null +++ b/ext/fts5/test/fts5circref.test @@ -0,0 +1,80 @@ +# 2018 Dec 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5circref + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE tt USING fts5(a); + SELECT name FROM sqlite_master ORDER BY 1; +} { + tt tt_config tt_content tt_data tt_docsize tt_idx +} +db_save_and_close + +foreach {tn schema sql} { + 1 { + CREATE TRIGGER tr1 AFTER INSERT ON tt_config BEGIN + SELECT * FROM tt; + END; + } { + INSERT INTO tt(tt, rank) VALUES('usermerge', 4); + } + + 2 { + CREATE TRIGGER tr1 AFTER INSERT ON tt_docsize BEGIN + SELECT * FROM tt; + END; + } { + INSERT INTO tt(a) VALUES('one two three'); + } + + 3 { + CREATE TRIGGER tr1 AFTER INSERT ON tt_content BEGIN + SELECT * FROM tt; + END; + } { + INSERT INTO tt(a) VALUES('one two three'); + } + + 4 { + CREATE TRIGGER tr1 AFTER INSERT ON tt_data BEGIN + SELECT * FROM tt; + END; + } { + INSERT INTO tt(a) VALUES('one two three'); + } + + 5 { + CREATE TRIGGER tr1 AFTER INSERT ON tt_idx BEGIN + SELECT * FROM tt; + END; + } { + INSERT INTO tt(a) VALUES('one two three'); + } +} { + db_restore_and_reopen + do_execsql_test 1.1.$tn.1 $schema + do_catchsql_test 1.1.$tn.2 $sql {1 {SQL logic error}} + db close +} + + +finish_test diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test new file mode 100644 index 0000000000000000000000000000000000000000..e5429572c58e0ecce1e57b4eccb69626aec29ab4 --- /dev/null +++ b/ext/fts5/test/fts5colset.test @@ -0,0 +1,105 @@ +# 2016 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5colset + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $::testprefix { + if {[detail_is_none]} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 + } + + foreach {tn q res} { + 1 "a" {1 2 3 4} + 2 "{a} : a" {1} + 3 "-{a} : a" {2 3 4} + 4 "- {a c} : a" {2 4} + 5 " - {d d c} : a" {1 2} + 6 "- {d c b a} : a" {} + 7 "-{\"a\"} : b" {1 2 3} + 8 "- c : a" {1 2 4} + 9 "-c : a" {1 2 4} + 10 "-\"c\" : a" {1 2 4} + } { + do_execsql_test 1.$tn { + SELECT rowid FROM t1($q) + } $res + } + + foreach {tn q res} { + 0 {{a} : (a AND ":")} {} + 1 "{a b c} : (a AND d)" {2 3} + 2 "{a b c} : (a AND b:d)" {3} + 3 "{a b c} : (a AND d:d)" {} + 4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4} + 5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3} + 6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {} + 7 "{a b c} : (b:a AND c:b)" {2} + } { + do_execsql_test 2.$tn { + SELECT rowid FROM t1($q) + } $res + } + + foreach {tn w res} { + 0 "a MATCH 'a'" {1} + 1 "b MATCH 'a'" {2} + 2 "b MATCH '{a b c} : a'" {2} + 3 "b MATCH 'a OR b'" {1 2} + 4 "b MATCH 'a OR a:b'" {2} + 5 "b MATCH 'a OR b:b'" {1 2} + } { + do_execsql_test 3.$tn " + SELECT rowid FROM t1 WHERE $w + " $res + } + + do_catchsql_test 4.1 { + SELECT * FROM t1 WHERE rowid MATCH 'a' + } {1 {no query solution}} +} + +#------------------------------------------------------------------------- +# Confirm that the expression parser creates the same expression tree +# for: +# +# {a b} : (abc AND def) +# -{c d} : (abc AND def) +# +# Assuming that the table columns are (a, b, c, d). +# +do_execsql_test 5.1 { + SELECT fts5_expr('abcd AND cdef'); +} {{"abcd" AND "cdef"}} +do_execsql_test 5.2 { + SELECT fts5_expr('{a b} : (abcd AND cdef)', 'a', 'b', 'c', 'd'); +} {{{a b} : "abcd" AND {a b} : "cdef"}} +do_execsql_test 5.3 { + SELECT fts5_expr('-{c d} : (abcd AND cdef)', 'a', 'b', 'c', 'd'); +} {{{a b} : "abcd" AND {a b} : "cdef"}} + + +finish_test diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test new file mode 100644 index 0000000000000000000000000000000000000000..7af49184b8eb315f3b363ce52fa6bd960371d191 --- /dev/null +++ b/ext/fts5/test/fts5columnsize.test @@ -0,0 +1,150 @@ +# 2015 Jun 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on fts5 tables with the columnsize=0 option. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5columnsize + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Check that the option can be parsed and that the %_docsize table is +# only created if it is set to true. +# +foreach {tn outcome stmt} { + 1 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0) } + 2 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=1) } + 3 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='0') } + 4 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='1') } + 5 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='') } + 6 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=2) } + 7 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0, columnsize=1) } + 8 1 { CREATE VIRTUAL TABLE t1 USING fts5(x) } + 9 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=11) } +} { + execsql { + DROP TABLE IF EXISTS t1; + } + if {$outcome==2} { + do_catchsql_test 1.$tn.1 $stmt {1 {malformed columnsize=... directive}} + } else { + do_execsql_test 1.$tn.2 $stmt + do_execsql_test 1.$tn.3 { + SELECT count(*) FROM sqlite_master WHERE name = 't1_docsize' + } $outcome + } +} + +#------------------------------------------------------------------------- +# Run tests on a table with no %_content or %_docsize backing store. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, columnsize=0, content=''); +} +do_catchsql_test 2.1 { + INSERT INTO t2 VALUES('a b c d e f'); +} {1 {datatype mismatch}} +do_execsql_test 2.2 { + INSERT INTO t2(rowid, x) VALUES(1, 'c d e f'); + INSERT INTO t2(rowid, x) VALUES(2, 'c d e f g h'); + INSERT INTO t2(rowid, x) VALUES(3, 'a b c d e f g h'); +} {} +do_execsql_test 2.3 { + SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::'; + SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::'; + SELECT rowid FROM t2 WHERE t2 MATCH 'h'; +} {3 :: 1 2 3 :: 2 3} +do_execsql_test 2.4 { + INSERT INTO t2(t2, rowid, x) VALUES('delete', 2, 'c d e f g h'); + SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::'; + SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::'; + SELECT rowid FROM t2 WHERE t2 MATCH 'h'; +} {3 :: 1 3 :: 3} +do_execsql_test 2.5 { + INSERT INTO t2(t2) VALUES('delete-all'); + SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::'; + SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::'; + SELECT rowid FROM t2 WHERE t2 MATCH 'h'; +} {:: ::} +do_execsql_test 2.6 { + INSERT INTO t2(rowid, x) VALUES(1, 'o t t f'); + INSERT INTO t2(rowid, x) VALUES(2, 'f s s e'); + INSERT INTO t2(rowid, x) VALUES(3, 'n t e t'); +} + +do_catchsql_test 2.7.1 { + SELECT rowid FROM t2 +} {1 {t2: table does not support scanning}} +do_catchsql_test 2.7.2 { + SELECT rowid FROM t2 WHERE rowid=2 +} {1 {t2: table does not support scanning}} +do_catchsql_test 2.7.3 { + SELECT rowid FROM t2 WHERE rowid BETWEEN 1 AND 3 +} {1 {t2: table does not support scanning}} + +do_execsql_test 2.X { + DROP TABLE t2 +} + +#------------------------------------------------------------------------- +# Test the xColumnSize() API +# +fts5_aux_test_functions db + +do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE t3 USING fts5(x, y UNINDEXED, z, columnsize=0); + INSERT INTO t3 VALUES('a a', 'b b b', 'c'); + INSERT INTO t3 VALUES('x a x', 'b b b y', ''); +} +do_execsql_test 3.1.1 { + SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a' +} { + 1 {2 0 1} 2 {3 0 0} +} +do_execsql_test 3.1.2 { + INSERT INTO t3 VALUES(NULL, NULL, 'a a a a'); + DELETE FROM t3 WHERE rowid = 1; + SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a' +} { + 2 {3 0 0} 3 {0 0 4} +} + +do_execsql_test 3.2.0 { + CREATE VIRTUAL TABLE t4 USING fts5(x, y UNINDEXED, z, columnsize=0, content=''); + INSERT INTO t4(rowid, x, y, z) VALUES(1, 'a a', 'b b b', 'c'); + INSERT INTO t4(rowid, x, y, z) VALUES(2, 'x a x', 'b b b y', ''); +} +do_execsql_test 3.2.1 { + SELECT rowid, fts5_test_columnsize(t4) FROM t4 WHERE t4 MATCH 'a' +} { + 1 {-1 0 -1} 2 {-1 0 -1} +} + +#------------------------------------------------------------------------- +# Test the integrity-check +# +do_execsql_test 4.1.1 { + CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0); + INSERT INTO t5 VALUES('1 2 3 4'); + INSERT INTO t5 VALUES('2 4 6 8'); +} + +do_execsql_test 4.1.2 { + INSERT INTO t5(t5) VALUES('integrity-check'); +} + +finish_test diff --git a/ext/fts5/test/fts5config.test b/ext/fts5/test/fts5config.test new file mode 100644 index 0000000000000000000000000000000000000000..28f3146ea3f7b6cf742a544b282e2e21ea5850d1 --- /dev/null +++ b/ext/fts5/test/fts5config.test @@ -0,0 +1,265 @@ +# 2015 Jan 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on the code in fts5_config.c, which is largely concerned +# with parsing the various configuration and CREATE TABLE options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5config + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Try different types of quote characters. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5('a', "b", [c], `d`); + PRAGMA table_info = t1; +} { + 0 a {} 0 {} 0 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 +} + +#------------------------------------------------------------------------- +# Syntax errors in the prefix= option. +# +foreach {tn opt} { + 1 {prefix=x} + 2 {prefix='x'} + 3 {prefix='$'} + 4 {prefix='1,2,'} + 5 {prefix=',1'} + 6 {prefix='1,2,3...'} + 7 {prefix='1,2,3xyz'} +} { + set res [list 1 {malformed prefix=... directive}] + do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res +} + +#------------------------------------------------------------------------- +# Syntax errors in the 'rank' option. +# +foreach {tn val} { + 1 "f1(xyz)" + 2 "f1(zyx)" + 3 "f1(nzz)" + 4 "f1(x'!!')" + 5 "f1(x':;')" + 6 "f1(x'[]')" + 7 "f1(x'{}')" + 8 "f1('abc)" +} { + do_catchsql_test 3.$tn { + INSERT INTO t1(t1, rank) VALUES('rank', $val); + } {1 {SQL logic error}} +} + +#------------------------------------------------------------------------- +# The parsing of SQL literals specified as part of 'rank' options. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE zzz USING fts5(one); + INSERT INTO zzz VALUES('a b c'); +} +proc first {cmd A} { return $A } +sqlite3_fts5_create_function db first first + +foreach {tn arg} { + 1 "123" + 2 "'01234567890ABCDEF'" + 3 "x'0123'" + 4 "x'ABCD'" + 5 "x'0123456789ABCDEF'" + 6 "x'0123456789abcdef'" + 7 "22.5" + 8 "-91.5" + 9 "-.5" + 10 "''''" + 11 "+.5" +} { + set func [string map {' ''} "first($arg)"] + do_execsql_test 4.1.$tn " + INSERT INTO zzz(zzz, rank) VALUES('rank', '$func'); + SELECT rank IS $arg FROM zzz WHERE zzz MATCH 'a + b + c' + " 1 +} + +do_execsql_test 4.2 { + INSERT INTO zzz(zzz, rank) VALUES('rank', 'f1()'); +} {} + +#------------------------------------------------------------------------- +# Misquoting in tokenize= and other options. +# +do_catchsql_test 5.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii"); +} {1 {parse error in tokenize directive}} + +do_catchsql_test 5.2 { + CREATE VIRTUAL TABLE xx USING fts5(x, [y[]); +} {0 {}} + +do_catchsql_test 5.3 { + CREATE VIRTUAL TABLE yy USING fts5(x, [y]]); +} {1 {unrecognized token: "]"}} + +#------------------------------------------------------------------------- +# Errors in prefix= directives. +# +do_catchsql_test 6.2 { + CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001'); +} {1 {prefix length out of range (max 999)}} +do_catchsql_test 6.3 { + CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000'); +} {1 {prefix length out of range (max 999)}} +do_catchsql_test 6.4 { + CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1 , 1000000'); +} {1 {prefix length out of range (max 999)}} + +#------------------------------------------------------------------------- +# Duplicate tokenize= and other options. +# +do_catchsql_test 7.1 { + CREATE VIRTUAL TABLE abc USING fts5(a, tokenize=porter, tokenize=ascii); +} {1 {multiple tokenize=... directives}} +do_catchsql_test 7.2 { + CREATE VIRTUAL TABLE abc USING fts5(a, content=porter, content=ascii); +} {1 {multiple content=... directives}} +do_catchsql_test 7.3 { + CREATE VIRTUAL TABLE abc USING fts5(a, content_rowid=porter, content_rowid=a); +} {1 {multiple content_rowid=... directives}} + +#------------------------------------------------------------------------- +# Unrecognized option. +# +do_catchsql_test 8.0 { + CREATE VIRTUAL TABLE abc USING fts5(a, nosuchoption=123); +} {1 {unrecognized option: "nosuchoption"}} +do_catchsql_test 8.1 { + CREATE VIRTUAL TABLE abc USING fts5(a, "nosuchoption"=123); +} {1 {parse error in ""nosuchoption"=123"}} + +#------------------------------------------------------------------------- +# Errors in: +# +# 9.1.* 'pgsz' options. +# 9.2.* 'automerge' options. +# 9.3.* 'crisismerge' options. +# 9.4.* a non-existant option. +# 9.5.* 'hashsize' options. +# +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE abc USING fts5(a, b); +} {} +do_catchsql_test 9.1.1 { + INSERT INTO abc(abc, rank) VALUES('pgsz', -5); +} {1 {SQL logic error}} +do_catchsql_test 9.1.2 { + INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000); +} {1 {SQL logic error}} +do_catchsql_test 9.1.3 { + INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67); +} {1 {SQL logic error}} + +do_catchsql_test 9.2.1 { + INSERT INTO abc(abc, rank) VALUES('automerge', -5); +} {1 {SQL logic error}} +do_catchsql_test 9.2.2 { + INSERT INTO abc(abc, rank) VALUES('automerge', 50000000); +} {1 {SQL logic error}} +do_catchsql_test 9.2.3 { + INSERT INTO abc(abc, rank) VALUES('automerge', 66.67); +} {1 {SQL logic error}} +do_execsql_test 9.2.4 { + INSERT INTO abc(abc, rank) VALUES('automerge', 1); +} {} + +do_catchsql_test 9.3.1 { + INSERT INTO abc(abc, rank) VALUES('crisismerge', -5); +} {1 {SQL logic error}} +do_catchsql_test 9.3.2 { + INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67); +} {1 {SQL logic error}} +do_execsql_test 9.3.3 { + INSERT INTO abc(abc, rank) VALUES('crisismerge', 1); +} {} +do_execsql_test 9.3.4 { + INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000); +} {} + +do_catchsql_test 9.4.1 { + INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1); +} {1 {SQL logic error}} + +do_catchsql_test 9.5.1 { + INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer'); +} {1 {SQL logic error}} +do_catchsql_test 9.5.2 { + INSERT INTO abc(abc, rank) VALUES('hashsize', -500000); +} {1 {SQL logic error}} +do_catchsql_test 9.5.3 { + INSERT INTO abc(abc, rank) VALUES('hashsize', 500000); +} {0 {}} + +#------------------------------------------------------------------------- +# Too many prefix indexes. Maximum allowed is 31. +# +foreach {tn spec} { + 1 {prefix="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"} + 2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"} +} { + set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)" + do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}} +} + +#------------------------------------------------------------------------- +# errors in the detail= option. +# +foreach {tn opt} { + 1 {detail=x} + 2 {detail='x'} + 3 {detail='$'} + 4 {detail='1,2,'} + 5 {detail=',1'} + 6 {detail=''} +} { + set res [list 1 {malformed detail=... directive}] + do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res +} + +do_catchsql_test 12.1 { + INSERT INTO t1(t1, rank) VALUES('rank', NULL);; +} {1 {SQL logic error}} + +#------------------------------------------------------------------------- +# errors in the 'usermerge' option +# +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE tt USING fts5(ttt); +} +foreach {tn val} { + 1 -1 + 2 4.2 + 3 17 + 4 1 +} { + set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)" + do_catchsql_test 13.$tn $sql {1 {SQL logic error}} +} + +finish_test diff --git a/ext/fts5/test/fts5conflict.test b/ext/fts5/test/fts5conflict.test new file mode 100644 index 0000000000000000000000000000000000000000..b5bf0a1160d6db9af9fe1ad5bc349f6c66915cbf --- /dev/null +++ b/ext/fts5/test/fts5conflict.test @@ -0,0 +1,108 @@ +# 2015 October 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5conflict + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, a, b); + CREATE VIRTUAL TABLE ft USING fts5(a, b, content=t1, content_rowid=x); +} + +do_execsql_test 1.1 { + REPLACE INTO ft(rowid, a, b) VALUES(1, 'a b c', 'a b c'); + REPLACE INTO t1 VALUES(1, 'a b c', 'a b c'); +} + +do_execsql_test 1.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.0 { + CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c); + CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content=tbl, content_rowid=a); + CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN + INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c); + END; + CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN + INSERT INTO fts_idx(fts_idx, rowid, b, c) + VALUES('delete', old.a, old.b, old.c); + END; + CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN + INSERT INTO fts_idx(fts_idx, rowid, b, c) + VALUES('delete', old.a, old.b, old.c); + INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c); + END; +} + +do_execsql_test 2.1 { + PRAGMA recursive_triggers = 1; + INSERT INTO tbl VALUES(1, 'x y z', '1 2 3'); + INSERT INTO tbl VALUES(10, 'x y z', '1 2 3'); + INSERT INTO tbl VALUES(100, 'x 1 z', '1 y 3'); + + UPDATE tbl SET b = '1 2 x' WHERE rowid=10; + REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1'); + DELETE FROM tbl WHERE a=100; + + INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Tests for OR IGNORE conflict handling. +# +reset_db +foreach_detail_mode $::testprefix { + + do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(xyz, detail=%DETAIL%); + + BEGIN; + INSERT INTO t1(rowid, xyz) VALUES(13, 'thirteen documents'); + INSERT INTO t1(rowid, xyz) VALUES(14, 'fourteen documents'); + INSERT INTO t1(rowid, xyz) VALUES(15, 'fifteen documents'); + COMMIT; + } + + set db_cksum [cksum] + foreach {tn sql} { + 1 { + INSERT OR IGNORE INTO t1(rowid, xyz) VALUES(14, 'new text'); + } + 2 { + UPDATE OR IGNORE t1 SET rowid=13 WHERE rowid=15; + } + 3 { + INSERT OR IGNORE INTO t1(rowid, xyz) + SELECT 13, 'some text' + UNION ALL + SELECT 14, 'some text' + UNION ALL + SELECT 15, 'some text' + } + } { + do_execsql_test 3.1.$tn.1 $sql + do_test 3.1.$tn.2 { cksum } $db_cksum + } + +} + + +finish_test diff --git a/ext/fts5/test/fts5connect.test b/ext/fts5/test/fts5connect.test new file mode 100644 index 0000000000000000000000000000000000000000..46077340ca2c804876460914bc2ca619e9bae092 --- /dev/null +++ b/ext/fts5/test/fts5connect.test @@ -0,0 +1,246 @@ +# 2017 August 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + + + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5connect + +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# The tests in this file test the outcome of a schema-reset happening +# within the xConnect() method of an FTS5 table. At one point this +# was causing a problem in SQLite. Each test proceeds as follows: +# +# 1. Connection [db] opens the db and reads from some unrelated, non-FTS5 +# table causing SQLite to load the db schema into memory. +# +# 2. Connection [db2] opens the db and modifies the db schema. +# +# 3. Connection [db] reads or writes an existing fts5 table. That the +# schema has been modified is detected inside the fts5 xConnect() +# callback that is invoked by sqlite3_prepare(). +# +# 4. Verify that the statement in 3 has worked. SQLite should detect +# that the schema has changed and successfully prepare the +# statement against the new schema. +# +# Test plan: +# +# 1.*: Trigger the xConnect()/schema-reset using statements executed +# directly against an FTS5 table. +# +# 2.*: Using various statements executed by various BEFORE triggers. +# +# 3.*: Using various statements executed by various AFTER triggers. +# +# 4.*: Using various statements executed by various INSTEAD OF triggers. +# + + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b); + CREATE TABLE abc(x INTEGER PRIMARY KEY); + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); + + INSERT INTO ft1 VALUES('one', 'two'); + INSERT INTO ft1 VALUES('three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM ft1" {one two three four} + 2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft1" {five six three four} + 4 "INSERT INTO ft1 VALUES('seven', 'eight')" {} + 5 "SELECT * FROM ft1" {five six three four seven eight} + 6 "DELETE FROM ft1 WHERE rowid=2" {} + 7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft1" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 1.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 1.$tn.2 $sql $res + + do_execsql_test 1.$tn.3 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); + } +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts5(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE log(txt); + + CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN + INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b); + INSERT INTO log VALUES('insert'); + END; + + CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN + DELETE FROM ft2 WHERE rowid = old.rowid; + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN + UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid; + INSERT INTO log VALUES('update'); + END; + + INSERT INTO t2 VALUES('one', 'two'); + INSERT INTO t2 VALUES('three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM t2" {one two three four} + 2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft2" {five six three four} + 4 "INSERT INTO t2 VALUES('seven', 'eight')" {} + 5 "SELECT * FROM ft2" {five six three four seven eight} + 6 "DELETE FROM t2 WHERE rowid=2" {} + 7 "UPDATE t2 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft2" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 2.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 2.$tn.2 $sql $res + + do_execsql_test 2.$tn.3 { + INSERT INTO ft2(ft2) VALUES('integrity-check'); + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft3 USING fts5(a, b); + CREATE TABLE t3(a, b); + + CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN + INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b); + INSERT INTO log VALUES('insert'); + END; + + CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN + DELETE FROM ft3 WHERE rowid = old.rowid; + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN + UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid; + INSERT INTO log VALUES('update'); + END; + + INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two'); + INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM t3" {one two three four} + 2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft3" {five six three four} + 4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {} + 5 "SELECT * FROM ft3" {five six three four seven eight} + 6 "DELETE FROM t3 WHERE rowid=2" {} + 7 "UPDATE t3 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft3" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 3.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 3.$tn.2 $sql $res + + do_execsql_test 3.$tn.3 { + INSERT INTO ft3(ft3) VALUES('integrity-check'); + } +} + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft4 USING fts5(a, b); + CREATE VIEW v4 AS SELECT rowid, * FROM ft4; + + CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN + INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b); + INSERT INTO log VALUES('insert'); + END; + + CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN + DELETE FROM ft4 WHERE rowid = old.rowid; + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN + UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid; + INSERT INTO log VALUES('update'); + END; + + INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two'); + INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM ft4" {one two three four} + 2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft4" {five six three four} + 4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {} + 5 "SELECT * FROM ft4" {five six three four seven eight} + 6 "DELETE FROM v4 WHERE rowid=2" {} + 7 "UPDATE v4 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft4" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 4.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 4.$tn.2 $sql $res + + do_execsql_test 4.$tn.3 { + INSERT INTO ft3(ft3) VALUES('integrity-check'); + } +} + +finish_test diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test new file mode 100644 index 0000000000000000000000000000000000000000..05b5cc611345f8b98cd4d19387b27349c657d492 --- /dev/null +++ b/ext/fts5/test/fts5content.test @@ -0,0 +1,368 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5content + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Contentless tables +# +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b, content=''); + INSERT INTO f1(rowid, a, b) VALUES(1, 'one', 'o n e'); + INSERT INTO f1(rowid, a, b) VALUES(2, 'two', 't w o'); + INSERT INTO f1(rowid, a, b) VALUES(3, 'three', 't h r e e'); +} + +do_execsql_test 1.2 { + SELECT rowid FROM f1 WHERE f1 MATCH 'o'; +} {1 2} + +do_execsql_test 1.3 { + INSERT INTO f1(a, b) VALUES('four', 'f o u r'); + SELECT rowid FROM f1 WHERE f1 MATCH 'o'; +} {1 2 4} + +do_execsql_test 1.4 { + SELECT rowid, a, b FROM f1 WHERE f1 MATCH 'o'; +} {1 {} {} 2 {} {} 4 {} {}} + +do_execsql_test 1.5 { + SELECT rowid, highlight(f1, 0, '[', ']') FROM f1 WHERE f1 MATCH 'o'; +} {1 {} 2 {} 4 {}} + +do_execsql_test 1.6 { + SELECT rowid, highlight(f1, 0, '[', ']') IS NULL FROM f1 WHERE f1 MATCH 'o'; +} {1 1 2 1 4 1} + +do_execsql_test 1.7 { + SELECT rowid, snippet(f1, -1, '[', ']', '...', 5) IS NULL + FROM f1 WHERE f1 MATCH 'o'; +} {1 1 2 1 4 1} + +do_execsql_test 1.8 { + SELECT rowid, snippet(f1, 1, '[', ']', '...', 5) IS NULL + FROM f1 WHERE f1 MATCH 'o'; +} {1 1 2 1 4 1} + +do_execsql_test 1.9 { + SELECT rowid FROM f1; +} {1 2 3 4} + +do_execsql_test 1.10 { + SELECT * FROM f1; +} {{} {} {} {} {} {} {} {}} + +do_execsql_test 1.11 { + SELECT rowid, a, b FROM f1 ORDER BY rowid ASC; +} {1 {} {} 2 {} {} 3 {} {} 4 {} {}} + +do_execsql_test 1.12 { + SELECT a IS NULL FROM f1; +} {1 1 1 1} + +do_catchsql_test 1.13 { + DELETE FROM f1 WHERE rowid = 2; +} {1 {cannot DELETE from contentless fts5 table: f1}} + +do_catchsql_test 1.14 { + UPDATE f1 SET a = 'a b c' WHERE rowid = 2; +} {1 {cannot UPDATE contentless fts5 table: f1}} + +do_execsql_test 1.15 { + INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o'); +} {} + +do_execsql_test 1.16 { + SELECT rowid FROM f1 WHERE f1 MATCH 'o'; +} {1 4} + +do_execsql_test 1.17 { + SELECT rowid FROM f1; +} {1 3 4} + +#------------------------------------------------------------------------- +# External content tables +# +reset_db +do_execsql_test 2.1 { + -- Create a table. And an external content fts5 table to index it. + CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c); + CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content='tbl', content_rowid='a'); + + -- Triggers to keep the FTS index up to date. + CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN + INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c); + END; + CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN + INSERT INTO fts_idx(fts_idx, rowid, b, c) + VALUES('delete', old.a, old.b, old.c); + END; + CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN + INSERT INTO fts_idx(fts_idx, rowid, b, c) + VALUES('delete', old.a, old.b, old.c); + INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c); + END; +} + +do_execsql_test 2.2 { + INSERT INTO tbl VALUES(1, 'one', 'o n e'); + INSERT INTO tbl VALUES(NULL, 'two', 't w o'); + INSERT INTO tbl VALUES(3, 'three', 't h r e e'); +} + +do_execsql_test 2.3 { + INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); +} + +do_execsql_test 2.4 { + DELETE FROM tbl WHERE rowid=2; + INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); +} + +do_execsql_test 2.5 { + UPDATE tbl SET c = c || ' x y z'; + INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); +} + +do_execsql_test 2.6 { + SELECT * FROM fts_idx WHERE fts_idx MATCH 't AND x'; +} {three {t h r e e x y z}} + +do_execsql_test 2.7 { + SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx + WHERE fts_idx MATCH 't AND x'; +} {{[t] h r e e [x] y z}} + +#------------------------------------------------------------------------- +# Quick tests of the 'delete-all' command. +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t3 USING fts5(x, content=''); + INSERT INTO t3 VALUES('a b c'); + INSERT INTO t3 VALUES('d e f'); +} + +do_execsql_test 3.2 { + SELECT count(*) FROM t3_docsize; + SELECT count(*) FROM t3_data; +} {2 4} + +do_execsql_test 3.3 { + INSERT INTO t3(t3) VALUES('delete-all'); + SELECT count(*) FROM t3_docsize; + SELECT count(*) FROM t3_data; +} {0 2} + +do_execsql_test 3.4 { + INSERT INTO t3 VALUES('a b c'); + INSERT INTO t3 VALUES('d e f'); + SELECT rowid FROM t3 WHERE t3 MATCH 'e'; +} {2} + +do_execsql_test 3.5 { + SELECT rowid FROM t3 WHERE t3 MATCH 'c'; +} {1} + +do_execsql_test 3.6 { + SELECT count(*) FROM t3_docsize; + SELECT count(*) FROM t3_data; +} {2 4} + +do_execsql_test 3.7 { + CREATE VIRTUAL TABLE t4 USING fts5(x); +} {} +do_catchsql_test 3.8 { + INSERT INTO t4(t4) VALUES('delete-all'); +} {1 {'delete-all' may only be used with a contentless or external content fts5 table}} + +#------------------------------------------------------------------------- +# Test an external content table with a more interesting schema. +# +do_execsql_test 4.1 { + CREATE TABLE x2(a, "key col" PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO x2 VALUES('a b', 1, 'c d' , 'e f'); + INSERT INTO x2 VALUES('x y', -40, 'z z' , 'y x'); + + CREATE VIRTUAL TABLE t2 USING fts5(a, c, content=x2, content_rowid='key col'); + INSERT INTO t2(t2) VALUES('rebuild'); +} + +do_execsql_test 4.2 { SELECT rowid FROM t2 } {-40 1} +do_execsql_test 4.3 { SELECT rowid FROM t2 WHERE t2 MATCH 'c'} {} +do_execsql_test 4.4 { SELECT rowid FROM t2 WHERE t2 MATCH 'a'} {1} +do_execsql_test 4.5 { SELECT rowid FROM t2 WHERE t2 MATCH 'x'} {-40} + +do_execsql_test 4.6 { INSERT INTO t2(t2) VALUES('integrity-check') } {} + +do_execsql_test 4.7 { + DELETE FROM x2 WHERE "key col" = 1; + INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f'); + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {} +do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40} + +#------------------------------------------------------------------------- +# Test that if the 'rowid' field of a 'delete' is not an integer, no +# changes are made to the FTS index. +# +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=); + INSERT INTO t5(rowid, a, b) VALUES(-1, 'one', 'two'); + INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four'); + INSERT INTO t5(rowid, a, b) VALUES( 1, 'five', 'six'); +} + +set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}] + +do_execsql_test 5.1 { + INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four'); + SELECT md5sum(id, block) FROM t5_data; +} $::checksum + + +#------------------------------------------------------------------------- +# Check that a contentless table can be dropped. +# +reset_db +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, content=""); + SELECT name FROM sqlite_master; +} {xx xx_data xx_idx xx_docsize xx_config} +do_execsql_test 6.2 { + DROP TABLE xx; + SELECT name FROM sqlite_master; +} {} + +#--------------------------------------------------------------------------- +# Check that an fts5 table cannot be its own content table. +# +reset_db +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, c=t1 ); + INSERT INTO t1( a ) VALUES('abc'); +} +do_catchsql_test 7.1.2 { + SELECT * FROM t1; +} {1 {recursively defined fts5 content table}} +do_catchsql_test 7.1.3 { + SELECT * FROM t1('abc'); +} {1 {recursively defined fts5 content table}} +do_catchsql_test 7.1.4 { + SELECT count(*) FROM t1; +} {1 {recursively defined fts5 content table}} +do_catchsql_test 7.1.5 { + SELECT * FROM t1('abc') ORDER BY rank; +} {1 {recursively defined fts5 content table}} + +reset_db +do_execsql_test 7.2.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, c=t2 ); + CREATE VIRTUAL TABLE t2 USING fts5(a, c=t1 ); + INSERT INTO t1( a ) VALUES('abc'); +} +do_catchsql_test 7.2.2 { + SELECT * FROM t1; +} {1 {recursively defined fts5 content table}} +do_catchsql_test 7.2.3 { + SELECT * FROM t1('abc'); +} {1 {recursively defined fts5 content table}} +do_catchsql_test 7.2.4 { + SELECT count(*) FROM t1; +} {1 {recursively defined fts5 content table}} +do_catchsql_test 7.2.5 { + SELECT * FROM t1('abc') ORDER BY rank; +} {1 {recursively defined fts5 content table}} + +#--------------------------------------------------------------------------- +# Check that if the content table is a view, and that view contains an +# error, a reasonable error message is returned if the user tries to +# read from the view via the fts5 table. +# +reset_db +do_execsql_test 8.1 { + CREATE VIEW a1 AS + SELECT 1 AS r, text_value(1) AS t + UNION ALL + SELECT 2 AS r, text_value(2) AS t; + + CREATE VIRTUAL TABLE t1 USING fts5(t, content='a1', content_rowid='r'); +} + +foreach {tn sql} { + 1 "SELECT * FROM t1" + 2 "INSERT INTO t1(t1) VALUES('rebuild')" + 3 "SELECT * FROM t1 WHERE rowid=1" +} { + do_catchsql_test 8.2.$tn $sql {1 {no such function: text_value}} +} + +proc text_value {i} { + if {$i==1} { return "one" } + if {$i==2} { return "two" } + return "many" +} +db func text_value text_value + +do_execsql_test 8.3.1 { SELECT * FROM t1 } {one two} +do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') } +do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one} +do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one two three'); + INSERT INTO t1 VALUES(2, 'one two three'); + + CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a); + INSERT INTO ft(ft) VALUES('rebuild'); +} + +do_execsql_test 9.2 { + SELECT rowid, b FROM ft('two'); +} { + 1 {one two three} + 2 {one two three} +} + +do_execsql_test 9.3 { + DELETE FROM t1 WHERE a=2; +} + +do_catchsql_test 9.4 { + SELECT rowid FROM ft('two'); +} {0 {1 2}} + +do_catchsql_test 9.5 { + SELECT * FROM ft('two'); +} {1 {fts5: missing row 2 from content table 'main'.'t1'}} + +fts5_aux_test_functions db + +do_catchsql_test 9.6 { + SELECT rowid, fts5_columntext(ft, 0) FROM ft('two'); +} {1 SQLITE_CORRUPT_VTAB} + +finish_test + diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test new file mode 100644 index 0000000000000000000000000000000000000000..991e9888fca37b244792ba74dcfa9f15cfb0d528 --- /dev/null +++ b/ext/fts5/test/fts5contentless.test @@ -0,0 +1,290 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Check that it is not possible to specify "contentless_delete=1" for +# anything other than a contentless table. +# +set res(0) {0 {}} +set res(1) {1 {contentless_delete=1 requires a contentless table}} +foreach {tn sql bError} { + 1 "(a, b, contentless_delete=1)" 1 + 2 "(a, b, contentless_delete=1, content=abc)" 1 + 3 "(a, b, contentless_delete=1, content=)" 0 + 4 "(content=, contentless_delete=1, a)" 0 + 5 "(content='', contentless_delete=1, hello)" 0 +} { + execsql { BEGIN } + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError) + execsql { ROLLBACK } +} + +# Check that it is not possible to specify "contentless_delete=1" +# along with columnsize=1. +# +set res(0) {0 {}} +set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}} +foreach {tn sql bError} { + 2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 +} { + execsql { BEGIN } + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError) + execsql { ROLLBACK } +} + +# Check that if contentless_delete=1 is specified, then the "origin" +# column is added to the %_docsize table. +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(c, content=''); + CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1); +} +do_execsql_test 3.1 { + SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize'); +} { + {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} + {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)} +} + +do_execsql_test 3.2.1 { + SELECT hex(block) FROM x1_data WHERE id=10 +} {00000000000000} +do_execsql_test 3.2.2 { + SELECT hex(block) FROM x2_data WHERE id=10 +} {00000000FF000001000000} + +do_execsql_test 3.3 { + INSERT INTO x2 VALUES('first text'); + INSERT INTO x2 VALUES('second text'); +} +do_execsql_test 3.4 { + SELECT id, origin FROM x2_docsize +} {1 1 2 2} +do_execsql_test 3.5 { + SELECT level, segment, loc1, loc2 FROM fts5_structure( + (SELECT block FROM x2_data WHERE id=10) + ) +} { + 0 0 1 1 + 0 1 2 2 +} +do_execsql_test 3.6 { + INSERT INTO x2(x2) VALUES('optimize'); +} +do_execsql_test 3.7 { + SELECT level, segment, loc1, loc2 FROM fts5_structure( + (SELECT block FROM x2_data WHERE id=10) + ) +} { + 1 0 1 2 +} + +do_execsql_test 3.8 { + DELETE FROM x2 WHERE rowid=2; +} + +do_execsql_test 3.9 { + SELECT rowid FROM x2('text') +} {1} + +#-------------------------------------------------------------------------- +reset_db +proc document {n} { + set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + set ret [list] + for {set ii 0} {$ii < $n} {incr ii} { + lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set ret +} + +set nRow 1000 + +do_execsql_test 4.0 { + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 100); +} +do_test 4.1 { + for {set ii 0} {$ii < $nRow} {incr ii} { + set doc [document 6] + execsql { + INSERT INTO t1 VALUES($doc); + INSERT INTO ft VALUES($doc); + } + } +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.2.$v { set L1 } $L2 +} + +do_test 4.3 { + for {set ii 1} {$ii < $nRow} {incr ii 2} { + execsql { + DELETE FROM ft WHERE rowid=$ii; + DELETE FROM t1 WHERE rowid=$ii; + } + } +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.4.$v { set L1 } $L2 +} + +do_execsql_test 4.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.6.$v { set L1 } $L2 +} + +#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(rowid, x) VALUES(1, 'one two three'); + INSERT INTO ft(rowid, x) VALUES(2, 'one two four'); + INSERT INTO ft(rowid, x) VALUES(3, 'one two five'); + INSERT INTO ft(rowid, x) VALUES(4, 'one two seven'); + INSERT INTO ft(rowid, x) VALUES(5, 'one two eight'); +} + +do_execsql_test 5.1 { + DELETE FROM ft WHERE rowid=2 +} + +do_execsql_test 5.2 { + SELECT rowid FROM ft +} {1 3 4 5} + +do_catchsql_test 5.3 { + UPDATE ft SET x='four six' WHERE rowid=3 +} {0 {}} + +do_execsql_test 5.4 { + SELECT rowid FROM ft('one'); +} {1 4 5} + +do_execsql_test 5.5 { + REPLACE INTO ft(rowid, x) VALUES(3, 'four six'); + SELECT rowid FROM ft('one'); +} {1 4 5} + +do_execsql_test 5.6 { + REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven'); + SELECT rowid FROM ft('one'); +} {1 4 5 6} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(rowid, x) VALUES(1, 'one two three'); + INSERT INTO ft(rowid, x) VALUES(2, 'one two four'); +} + +do_test 6.1 { + db eval { SELECT rowid FROM ft('one two') } { + if {$rowid==1} { + db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') } + } + } +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); +} + +set lRowid [list -450 0 1 2 42] + +do_test 7.1 { + execsql BEGIN + foreach r $lRowid { + execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); } + } + execsql COMMIT +} {} + +do_test 7.2 { + execsql BEGIN + foreach r $lRowid { + execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); } + } + execsql COMMIT +} {} + +do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {} +do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 8.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {1 {'delete' may not be used with a contentless_delete=1 table}} + +do_execsql_test 8.2 { + BEGIN; + INSERT INTO ft(rowid, x) VALUES(3, 'four four four'); + DELETE FROM ft WHERE rowid=3; + COMMIT; + SELECT rowid FROM ft('four'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 9.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {0 {}} + +do_catchsql_test 9.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2); +} {1 {malformed contentless_delete=... directive}} + +do_catchsql_test 9.3 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11); +} {1 {malformed contentless_delete=... directive}} + +finish_test diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test new file mode 100644 index 0000000000000000000000000000000000000000..248534bce482cf683fa4de84cc63cc4cee30ada6 --- /dev/null +++ b/ext/fts5/test/fts5contentless2.test @@ -0,0 +1,207 @@ +# 2023 July 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc vocab {} { + list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp +} + +proc document {nToken} { + set doc [list] + set vocab [vocab] + for {set ii 0} {$ii < $nToken} {incr ii} { + lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set doc +} +db func document document + +proc contains {doc token} { + expr {[lsearch $doc $token]>=0} +} +db func contains contains + +proc do_compare_tables_test {tn} { + uplevel [list do_test $tn { + foreach v [vocab] { + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }] + set l2 [execsql { SELECT rowid FROM t2($v) }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 1]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 0]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set l1 [execsql { + SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC + }] + set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + } + set {} {} + } {}] +} + +proc lshuffle {in} { + set L [list] + set ret [list] + foreach elem $in { lappend L [list [expr rand()] $elem] } + foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] } + set ret +} + +expr srand(0) + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5( + doc, prefix=2, content=, contentless_delete=1 + ); + + CREATE TABLE t1(doc); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE rowid = old.rowid; + END; +} + +set SMALLEST64 -9223372036854775808 +set LARGEST64 9223372036854775807 + +foreach {tn r1 r2} { + 1 0 50 + 2 $SMALLEST64 $SMALLEST64+50 + 3 $LARGEST64-50 $LARGEST64 + 4 -50 -1 +} { + set r1 [expr $r1] + set r2 [expr $r2] + + do_test 1.1.$tn { + execsql BEGIN + for {set ii $r1} {$ii <= $r2} {incr ii} { + execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); } + } + execsql COMMIT + } {} +} +do_test 1.2 { + db eval { SELECT rowid, doc FROM t1 } { + execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) } + } +} {} + +foreach {tn rowid} { + 1 $SMALLEST64 + 2 0 + 3 -5 + 4 -30 + 5 $LARGEST64 + 6 $LARGEST64-1 +} { + set rowid [expr $rowid] + do_execsql_test 1.3.$tn.1 { + DELETE FROM t1 WHERE rowid=$rowid + } + do_compare_tables_test 1.3.$tn.2 +} + +set iTest 1 +foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] { + if {($iTest % 50)==0} { + execsql { INSERT INTO t2(t2) VALUES('optimize') } + } + if {($iTest % 5)==0} { + execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) } + } + do_execsql_test 1.4.$iTest.1($r) { + DELETE FROM t1 WHERE rowid=$r + } + do_compare_tables_test 1.4.$iTest.2 + incr iTest +} + +do_execsql_test 1.5 { + SELECT * FROM t1 +} {} + +#------------------------------------------------------------------------- +reset_db +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s; +} + +do_execsql_test 2.1 { + BEGIN; + DELETE FROM t2 WHERE rowid=32; + DELETE FROM t2 WHERE rowid=64; + DELETE FROM t2 WHERE rowid=96; + DELETE FROM t2 WHERE rowid=128; + DELETE FROM t2 WHERE rowid=160; + DELETE FROM t2 WHERE rowid=192; + COMMIT; +} + +do_execsql_test 2.2 { + SELECT * FROM t2('128'); +} {} + +#------------------------------------------------------------------------- + +foreach {tn step} { + 1 3 + 2 7 + 3 15 +} { + set step [expr $step] + + reset_db + db func document document + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1); + INSERT INTO t2(t2, rank) VALUES('pgsz', 100); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s; + } + do_execsql_test 3.$tn.1 { + DELETE FROM t2 WHERE (rowid % $step)==0 + } + do_execsql_test 3.$tn.2 { + SELECT * FROM t2( $step * 5 ) + } {} +} + + + +finish_test diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test new file mode 100644 index 0000000000000000000000000000000000000000..693840da82e7e541d34e8156cc6f2be92b5ec5fe --- /dev/null +++ b/ext/fts5/test/fts5contentless3.test @@ -0,0 +1,195 @@ +# 2023 July 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + BEGIN; + INSERT INTO ft VALUES('one one one'); + INSERT INTO ft VALUES('two two two'); + INSERT INTO ft VALUES('three three three'); + INSERT INTO ft VALUES('four four four'); + INSERT INTO ft VALUES('five five five'); + INSERT INTO ft VALUES('six six six'); + INSERT INTO ft VALUES('seven seven seven'); + INSERT INTO ft VALUES('eight eight eight'); + INSERT INTO ft VALUES('nine nine nine'); + COMMIT; + + DELETE FROM ft WHERE rowid=3; +} + +proc myhex {hex} { binary decode hex $hex } +db func myhex myhex + +do_execsql_test 1.1 { + UPDATE ft_data SET block = + myhex('04000000 00000001' || + '01020304 01020304 01020304 01020304' || + '01020304 01020304 01020304 01020304' + ) + WHERE id = (SELECT max(id) FROM ft_data); +} + +do_execsql_test 1.2 { + DELETE FROM ft WHERE rowid=1 +} + +do_execsql_test 1.3 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.3 { + UPDATE ft_data SET block = + myhex('08000000 00000001' || + '0000000001020304 0000000001020304 0000000001020304 0000000001020304' || + '0000000001020304 0000000001020304 0000000001020304 0000000001020304' + ) + WHERE id = (SELECT max(id) FROM ft_data); +} + +do_execsql_test 1.4 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.5 { + DELETE FROM ft WHERE rowid=4 +} + +do_execsql_test 1.6 { + UPDATE ft_data SET block = myhex('04000000 00000000') + WHERE id = (SELECT max(id) FROM ft_data); +} +do_execsql_test 1.7 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.8 { + UPDATE ft_data SET block = myhex('04000000 00000000') + WHERE id = (SELECT max(id) FROM ft_data); +} +do_execsql_test 1.9 { + DELETE FROM ft WHERE rowid=8 +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + INSERT INTO ft VALUES('one one one'); + INSERT INTO ft VALUES('two two two'); + INSERT INTO ft VALUES('three three three'); + INSERT INTO ft VALUES('four four four'); + INSERT INTO ft VALUES('five five five'); + INSERT INTO ft VALUES('six six six'); + INSERT INTO ft VALUES('seven seven seven'); + INSERT INTO ft VALUES('eight eight eight'); + INSERT INTO ft VALUES('nine nine nine'); +} + +do_execsql_test 2.1 { + INSERT INTO ft(ft) VALUES('optimize'); +} +do_execsql_test 2.2 { + SELECT count(*) FROM ft_data +} {3} +do_execsql_test 2.3 { + DELETE FROM ft WHERE rowid=5 +} +do_execsql_test 2.4 { + SELECT count(*) FROM ft_data +} {4} + +# Check that an 'optimize' works (rewrites the index) if there is a single +# segment with one or more tombstone hash pages. +do_execsql_test 2.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} +do_execsql_test 2.6 { + SELECT count(*) FROM ft_data +} {3} + +# Check that an 'optimize' is a no-op if there is a single segment +# and no tombstone hash pages. +do_execsql_test 2.7 { + INSERT INTO ft(ft) VALUES('optimize'); + SELECT rowid FROM ft_data; +} [db eval {SELECT rowid FROM ft_data}] + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s; + INSERT INTO ft(ft) VALUES('optimize'); +} + +do_execsql_test 3.1 { + SELECT count(*) FROM ft_data +} {200} + +do_execsql_test 3.2 { + DELETE FROM ft WHERE (rowid % 50)==0; + SELECT count(*) FROM ft_data; +} {203} + +do_execsql_test 3.3 { + INSERT INTO ft(ft, rank) VALUES('merge', 500); + SELECT rowid FROM ft_data; +} [db eval {SELECT rowid FROM ft_data}] + +do_execsql_test 3.4 { + INSERT INTO ft(ft, rank) VALUES('merge', -1000); + SELECT count(*) FROM ft_data; +} {197} + +do_execsql_test 3.5 { + DELETE FROM ft WHERE (rowid % 50)==1; + SELECT count(*) FROM ft_data; +} {200} + +do_execsql_test 3.6 { + SELECT level, segment, npgtombstone FROM fts5_structure( + (SELECT block FROM ft_data WHERE id=10) + ) +} {1 0 3} + +do_test 3.6 { + while 1 { + set nChange [db total_changes] + execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) } + if {([db total_changes] - $nChange)<2} break + } +} {} + +do_execsql_test 3.7 { + SELECT level, segment, npgtombstone FROM fts5_structure( + (SELECT block FROM ft_data WHERE id=10) + ) +} {2 0 0} + + +finish_test diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test new file mode 100644 index 0000000000000000000000000000000000000000..7fdf8c4b013045ce7e85d2279cac0bf2751e9dcb --- /dev/null +++ b/ext/fts5/test/fts5contentless4.test @@ -0,0 +1,247 @@ +# 2023 July 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc document {n} { + set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + set ret [list] + for {set ii 0} {$ii < $n} {incr ii} { + lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set ret +} +db func document document + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 240); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO ft SELECT document(12) FROM s; +} + +do_execsql_test 1.1 { + INSERT INTO ft(ft) VALUES('optimize'); +} + +do_execsql_test 1.2 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 0 1000 0} + +do_execsql_test 1.3 { + DELETE FROM ft WHERE rowid < 50 +} + +do_execsql_test 1.4 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 0 1000 49} + +do_execsql_test 1.5 { + DELETE FROM ft WHERE rowid < 1000 +} + +do_execsql_test 1.6 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {1 0 1 0} + +#-------------------------------------------------------------------------- +reset_db +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); +} + +do_test 2.1 { + for {set ii 0} {$ii < 5000} {incr ii} { + execsql { INSERT INTO ft VALUES( document(12) ) } + } +} {} + +do_execsql_test 2.2 { + SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {5000} + +for {set ii 5000} {$ii >= 0} {incr ii -100} { + do_execsql_test 2.3.$ii { + DELETE FROM ft WHERE rowid > $ii + } + do_execsql_test 2.3.$ii.2 { + SELECT + CAST((total(nentry) - total(nentrytombstone)) AS integer) + FROM + fts5_structure( (SELECT block FROM ft_data WHERE id=10) ) + } $ii +} + +execsql_pp { + SELECT * FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} + +do_test 2.4 { + for {set ii 0} {$ii < 5000} {incr ii} { + execsql { INSERT INTO ft VALUES( document(12) ) } + } +} {} + +for {set ii 1} {$ii <= 5000} {incr ii 10} { + do_execsql_test 2.3.$ii { + DELETE FROM ft WHERE rowid = $ii; + INSERT INTO ft VALUES( document(12) ); + INSERT INTO ft(ft, rank) VALUES('merge', -10); + } + + do_execsql_test 2.3.$ii.2 { + SELECT + CAST((total(nentry) - total(nentrytombstone)) AS integer) + FROM + fts5_structure( (SELECT block FROM ft_data WHERE id=10) ) + } 5000 +} + +#------------------------------------------------------------------------- +reset_db +db func document document +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT document(12) FROM s; +} + +do_catchsql_test 3.1 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text'); +} {1 {SQL logic error}} +do_catchsql_test 3.2 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 50); +} {0 {}} +do_execsql_test 3.3 { + SELECT * FROM ft_config WHERE k='deletemerge' +} {deletemerge 50} +do_catchsql_test 3.4 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 101); +} {0 {}} +do_execsql_test 3.5 { + SELECT * FROM ft_config WHERE k='deletemerge' +} {deletemerge 101} + +do_execsql_test 3.6 { + DELETE FROM ft WHERE rowid<95 +} + +do_execsql_test 3.7 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {94 100} + +do_execsql_test 3.8 { + DELETE FROM ft WHERE rowid=95 +} + +do_execsql_test 3.9 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {95 100} + +do_execsql_test 3.10 { + DELETE FROM ft; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT document(12) FROM s; + INSERT INTO ft(ft, rank) VALUES('deletemerge', 50); +} + +do_execsql_test 3.11 { + DELETE FROM ft WHERE rowid<95 +} + +do_execsql_test 3.12 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 6} + +#------------------------------------------------------------------------- +reset_db +db func document document +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1); + INSERT INTO x1(x1, rank) VALUES('usermerge', 16); + INSERT INTO x1(x1, rank) VALUES('deletemerge', 40); + INSERT INTO x1 VALUES('one'); + INSERT INTO x1 VALUES('two'); + INSERT INTO x1 VALUES('three'); + INSERT INTO x1 VALUES('four'); + INSERT INTO x1 VALUES('five'); + INSERT INTO x1 VALUES('six'); + INSERT INTO x1 VALUES('seven'); + INSERT INTO x1 VALUES('eight'); + INSERT INTO x1 VALUES('nine'); + INSERT INTO x1 VALUES('ten'); +} + +do_execsql_test 4.1 { + SELECT level, segment FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) +} { + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 +} + +for {set ii 1} {$ii < 4} {incr ii} { + do_execsql_test 4.2.$ii { + DELETE FROM x1 WHERE rowid = $ii; + INSERT INTO x1(x1, rank) VALUES('merge', 5); + SELECT level, segment FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) + } { + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 + } +} + +do_execsql_test 4.3 { + DELETE FROM x1 WHERE rowid = $ii; + INSERT INTO x1(x1, rank) VALUES('merge', 5); + SELECT level, segment, nentry FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) +} { + 1 0 6 +} + +finish_test diff --git a/ext/fts5/test/fts5contentless5.test b/ext/fts5/test/fts5contentless5.test new file mode 100644 index 0000000000000000000000000000000000000000..86d075328624f1db2b52a93310eac5418181a96a --- /dev/null +++ b/ext/fts5/test/fts5contentless5.test @@ -0,0 +1,111 @@ +# 2023 August 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless5 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +unset -nocomplain res + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1); + INSERT INTO t1 VALUES('A', 'B', 'C'); + INSERT INTO t1 VALUES('D', 'E', 'F'); + INSERT INTO t1 VALUES('G', 'H', 'I'); +} + +do_execsql_test 1.01 { + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES('x', 'y'); +} + +# explain_i "UPDATE t1 SET a='a' WHERE t1.rowid=1" +#breakpoint +#explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL" + +#breakpoint +#explain_i "UPDATE t1 SET a='a' WHERE b IS NULL AND rowid=?" + +foreach {tn up err} { + 1 "UPDATE t1 SET a='a', b='b', c='c' WHERE rowid=1" 0 + 2 "UPDATE t1 SET a='a', b='b' WHERE rowid=1" 1 + 3 "UPDATE t1 SET b='b', c='c' WHERE rowid=1" 1 + 4 "UPDATE t1 SET a='a', c='c' WHERE rowid=1" 1 + 5 "UPDATE t1 SET a='a', c='c' WHERE t1.rowid=1 AND b IS NULL" 1 + 6 "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1" 1 + 7 "UPDATE t1 SET a='a', b='b', c='c' FROM t2 WHERE t1.rowid=1" 0 +} { + + set res(0) {0 {}} + set res(1) {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: t1}} + do_catchsql_test 1.$tn $up $res($err) +} + +#------------------------------------------------------------------------- +reset_db + +proc random {n} { expr {abs(int(rand()*$n))} } +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} +proc vocab {} { + list abc def ghi jkl mno pqr stu vwx yza +} +proc term {} { + select_one [vocab] +} +proc document {} { + set nTerm [expr [random 3] + 7] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, contentless_delete=1, content=''); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); +} + +do_test 2.1 { + for {set ii 1} {$ii < 12} {incr ii} { + db transaction { + for {set jj 0} {$jj < 10} {incr jj} { + set doc [document] + execsql { INSERT INTO ft VALUES($doc); } + } + } + } +} {} + +do_test 2.2 { + foreach r [db eval {SELECT rowid FROM ft}] { + execsql { DELETE FROM ft WHERE rowid=$r } + } +} {} + +set doc [document] +do_execsql_test 2.3 { + INSERT INTO ft VALUES($doc) +} + + +finish_test diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test new file mode 100644 index 0000000000000000000000000000000000000000..0abd8b86de09264c4e62d569abe8c6a389b38dc3 --- /dev/null +++ b/ext/fts5/test/fts5corrupt.test @@ -0,0 +1,125 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that the FTS5 'integrity-check' command detects +# inconsistencies (corruption) in the on-disk backing tables. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +do_test 1.1 { + db transaction { + for {set i 1} {$i < 200} {incr i} { + set doc [list [string repeat x $i] [string repeat y $i]] + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } + } + fts5_level_segs t1 +} {1} +db_save + +do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') } +set segid [lindex [fts5_level_segids t1] 0] + +sqlite3_db_config db DEFENSIVE 0 +do_test 1.3 { + execsql { + DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4); + } + catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {1 {database disk image is malformed}} +do_execsql_test 1.3b { + PRAGMA integrity_check(t1); +} {{malformed inverted index for FTS5 table main.t1}} + + +do_test 1.4 { + db_restore_and_reopen + sqlite3_db_config db DEFENSIVE 0 + execsql { + UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE + rowid = fts5_rowid('segment', $segid, 4); + } + catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {1 {database disk image is malformed}} + +db_restore_and_reopen +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} + + +#-------------------------------------------------------------------- +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2(t2, rank) VALUES('pgsz', 64); +} +db func rnddoc fts5_rnddoc +do_test 2.1 { + for {set i 0} {$i < 500} {incr i} { + execsql { INSERT INTO t2 VALUES(rnddoc(50)) } + } + execsql { INSERT INTO t2(t2) VALUES('integrity-check') } +} {} + +#-------------------------------------------------------------------- +# A mundane test - missing row in the %_content table. +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(x); + INSERT INTO t3 VALUES('one o'); + INSERT INTO t3 VALUES('two e'); + INSERT INTO t3 VALUES('three o'); + INSERT INTO t3 VALUES('four e'); + INSERT INTO t3 VALUES('five o'); +} +do_execsql_test 3.1 { + SELECT * FROM t3 WHERE t3 MATCH 'o' +} {{one o} {three o} {five o}} +sqlite3_db_config db DEFENSIVE 0 +do_catchsql_test 3.1 { + DELETE FROM t3_content WHERE rowid = 3; + SELECT * FROM t3 WHERE t3 MATCH 'o'; +} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}} + +#-------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('one two three'); + INSERT INTO t2 VALUES('four five six'); + INSERT INTO t2 VALUES('seven eight nine'); + INSERT INTO t2 VALUES('ten eleven twelve'); +} +do_execsql_test 4.1 { + SELECT hex(block) FROM t2_data WHERE id=1; +} {040C} +do_execsql_test 4.2 { + UPDATE t2_data SET block = X'0402' WHERE id=1 +} +breakpoint +do_catchsql_test 4.3 { + DELETE FROM t2 WHERE rowid=3 +} {1 {database disk image is malformed}} + +finish_test diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test new file mode 100644 index 0000000000000000000000000000000000000000..6b4d6d411f8f82087d4973919701710c8cdbb480 --- /dev/null +++ b/ext/fts5/test/fts5corrupt2.test @@ -0,0 +1,277 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that FTS5 handles corrupt databases (i.e. internal +# inconsistencies in the backing tables) correctly. In this case +# "correctly" means without crashing. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +sqlite3_fts5_may_be_corrupt 1 + +# Create a simple FTS5 table containing 100 documents. Each document +# contains 10 terms, each of which start with the character "x". +# +expr srand(0) +db func rnddoc fts5_rnddoc +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) + INSERT INTO t1 SELECT rnddoc(10) FROM ii; +} +set mask [expr 31 << 31] + +if 0 { + +# Test 1: +# +# For each page in the t1_data table, open a transaction and DELETE +# the t1_data entry. Then run: +# +# * an integrity-check, and +# * unless the deleted block was a b-tree node, a query for "t1 MATCH 'x*'" +# +# and check that the corruption is detected in both cases. The +# rollback the transaction. +# +# Test 2: +# +# Same thing, except instead of deleting a row from t1_data, replace its +# blob content with integer value 14. +# +foreach {tno stmt} { + 1 { DELETE FROM t1_data WHERE rowid=$rowid } + 2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid } +} { + set tn 0 + foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] { + incr tn + #if {$tn!=224} continue + + do_test 1.$tno.$tn.1.$rowid { + execsql { BEGIN } + execsql $stmt + catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } {1 {database disk image is malformed}} + + if {($rowid & $mask)==0} { + # Node is a leaf node, not a b-tree node. + do_catchsql_test 1.$tno.$tn.2.$rowid { + SELECT rowid FROM t1 WHERE t1 MATCH 'x*' + } {1 {database disk image is malformed}} + } + + do_execsql_test 1.$tno.$tn.3.$rowid { + ROLLBACK; + INSERT INTO t1(t1) VALUES('integrity-check'); + } {} + } +} + +} + +# Using the same database as the 1.* tests. +# +# Run N-1 tests, where N is the number of bytes in the rightmost leaf page +# of the fts index. For test $i, truncate the rightmost leafpage to $i +# bytes. Then test both the integrity-check detects the corruption. +# +# Also tested is that "MATCH 'x*'" does not crash and sometimes reports +# corruption. It may not report the db as corrupt because truncating the +# final leaf to some sizes may create a valid leaf page. +# +set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] +set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}] +set all [db eval {SELECT rowid FROM t1}] +sqlite3_db_config db DEFENSIVE 0 +unset -nocomplain res +for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { + do_execsql_test 2.$i.1 { + BEGIN; + UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid; + } + + do_catchsql_test 2.$i.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } {1 {database disk image is malformed}} + + do_test 2.$i.3 { + set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}] + expr { + $res=="1 {database disk image is malformed}" + || $res=="0 {$all}" + } + } 1 + + do_execsql_test 2.$i.4 { + ROLLBACK; + INSERT INTO t1(t1) VALUES('integrity-check'); + } {} +} + +#------------------------------------------------------------------------- +# Test that corruption in leaf page headers is detected by queries that use +# doclist-indexes. +# +set doc "A B C D E F G H I J " +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x3 USING fts5(tt); + INSERT INTO x3(x3, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000) + INSERT INTO x3 + SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii; +} + +foreach {tn hdr} { + 1 "\x00\x00\x00\x00" + 2 "\xFF\xFF\xFF\xFF" + 3 "\x44\x45" +} { + set tn2 0 + set nCorrupt 0 + set nCorrupt2 0 + foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] { + if {$rowid & $mask} continue + incr tn2 + do_test 3.$tn.$tn2.1 { + execsql BEGIN + + set fd [db incrblob main x3_data block $rowid] + fconfigure $fd -translation binary + set existing [read $fd [string length $hdr]] + seek $fd 0 + puts -nonewline $fd $hdr + close $fd + + set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}] + if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + set {} 1 + } {1} + + if {($tn2 % 10)==0 && $existing != $hdr} { + do_test 3.$tn.$tn2.2 { + catchsql { INSERT INTO x3(x3) VALUES('integrity-check') } + } {1 {database disk image is malformed}} + do_execsql_test 3.$tn.$tn2.3 { + PRAGMA integrity_check(x3); + } {{malformed inverted index for FTS5 table main.x3}} + } + + execsql ROLLBACK + } + + do_test 3.$tn.x { expr $nCorrupt>0 } 1 +} + +#-------------------------------------------------------------------- +# +set doc "A B C D E F G H I J " +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x4 USING fts5(tt); + INSERT INTO x4(x4, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) + INSERT INTO x4 + SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii; +} + +foreach {tn nCut} { + 1 1 + 2 10 +} { + set tn2 0 + set nCorrupt 0 + foreach rowid [db eval {SELECT rowid FROM x4_data WHERE rowid>10}] { + if {$rowid & $mask} continue + incr tn2 + do_test 4.$tn.$tn2 { + execsql { + BEGIN; + UPDATE x4_data SET block = substr(block, 1, length(block)-$nCut) + WHERE id = $rowid; + } + + set res [catchsql { + SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC + }] + if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + set {} 1 + } {1} + + execsql ROLLBACK + } + + # do_test 4.$tn.x { expr $nCorrupt>0 } 1 +} + +set doc [string repeat "A B C " 1000] +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE x5 USING fts5(tt); + INSERT INTO x5(x5, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) + INSERT INTO x5 SELECT $doc FROM ii; +} + +foreach {tn hdr} { + 1 "\x00\x01" +} { + set tn2 0 + set nCorrupt 0 + foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] { + if {$rowid & $mask} continue + incr tn2 + do_test 5.$tn.$tn2 { + execsql BEGIN + + set fd [db incrblob main x5_data block $rowid] + fconfigure $fd -translation binary + puts -nonewline $fd $hdr + close $fd + + catchsql { INSERT INTO x5(x5) VALUES('integrity-check') } + set {} {} + } {} + + execsql ROLLBACK + } +} + +#-------------------------------------------------------------------- +reset_db +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE x5 USING fts5(tt); + INSERT INTO x5 VALUES('a'); + INSERT INTO x5 VALUES('a a'); + INSERT INTO x5 VALUES('a a a'); + INSERT INTO x5 VALUES('a a a a'); + + UPDATE x5_docsize SET sz = X'' WHERE id=3; +} +proc colsize {cmd i} { + $cmd xColumnSize $i +} +sqlite3_fts5_create_function db colsize colsize + +do_catchsql_test 6.2 { + SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a' +} {1 SQLITE_CORRUPT_VTAB} + + +sqlite3_fts5_may_be_corrupt 0 +finish_test diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test new file mode 100644 index 0000000000000000000000000000000000000000..3e8b0377cc74c1cd77844b5cf3c02265c6cacd8a --- /dev/null +++ b/ext/fts5/test/fts5corrupt3.test @@ -0,0 +1,15684 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that FTS5 handles corrupt databases (i.e. internal +# inconsistencies in the backing tables) correctly. In this case +# "correctly" means without crashing. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +sqlite3_fts5_may_be_corrupt 1 +database_may_be_corrupt + +proc create_t1 {} { + expr srand(0) + db func rnddoc fts5_rnddoc + db eval { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) + INSERT INTO t1 SELECT rnddoc(10) FROM ii; + } +} + +# Create a simple FTS5 table containing 100 documents. Each document +# contains 10 terms, each of which start with the character "x". +# +do_test 1.0 { create_t1 } {} + +do_test 1.1 { + # Pick out the rowid of the right-most b-tree leaf in the new segment. + set rowid [db one { + SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1 + }] + set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}] + set {} {} +} {} + +sqlite3_db_config db DEFENSIVE 0 +for {set i 0} {$i < $L} {incr i} { + do_test 1.2.$i { + catchsql { + BEGIN; + UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid; + INSERT INTO t1(t1) VALUES('integrity-check'); + } + } {1 {database disk image is malformed}} + catchsql ROLLBACK +} + +#------------------------------------------------------------------------- +# Test that trailing bytes appended to the averages record are ignored. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES(rnddoc(10)); + INSERT INTO t2 VALUES(rnddoc(10)); + SELECT length(block) FROM t2_data WHERE id=1; +} {2} +do_execsql_test 2.2 { + UPDATE t2_data SET block = block || 'abcd' WHERE id=1; + SELECT length(block) FROM t2_data WHERE id=1; +} {6} +do_execsql_test 2.2 { + INSERT INTO t2 VALUES(rnddoc(10)); + SELECT length(block) FROM t2_data WHERE id=1; +} {2} + + +#------------------------------------------------------------------------- +# Test that missing leaf pages are recognized as corruption. +# +reset_db +do_test 3.0 { create_t1 } {} +sqlite3_db_config db DEFENSIVE 0 + +do_execsql_test 3.1 { + SELECT count(*) FROM t1_data; +} {105} + +proc do_3_test {tn} { + set i 0 + foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] { + incr i + do_test $tn.$i { + db eval BEGIN + db eval {DELETE FROM t1_data WHERE rowid = $::rowid} + list [ + catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg + ] $msg + } {1 {database disk image is malformed}} + catch { db eval ROLLBACK } + } +} + +do_3_test 3.2 + +do_execsql_test 3.3 { + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1(t1) VALUES('optimize'); +} {} + +do_3_test 3.4 + +do_test 3.5 { + execsql { + DELETE FROM t1; + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); + } + for {set i 0} {$i < 1000} {incr i} { + set rnd [expr int(rand() * 1000)] + set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]] + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } +} {} + +do_3_test 3.6 + +do_test 3.7 { + execsql { + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1(t1) VALUES('optimize'); + } +} {} + +do_3_test 3.8 + +do_test 3.9 { + execsql { + DELETE FROM t1; + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + for {set i 0} {$i < 100} {incr i} { + set rnd [expr int(rand() * 100)] + set doc "x[string repeat $rnd 20]" + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } +} {} + +do_3_test 3.10 + +#------------------------------------------------------------------------- +# Test that segments that end unexpectedly are identified as corruption. +# +reset_db +sqlite3_db_config db DEFENSIVE 0 +do_test 4.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + for {set i 0} {$i < 100} {incr i} { + set rnd [expr int(rand() * 100)] + set doc "x[string repeat $rnd 20]" + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } + execsql { INSERT INTO t1(t1) VALUES('optimize') } +} {} + +set nErr 0 +for {set i 1} {1} {incr i} { + set struct [db one {SELECT block FROM t1_data WHERE id=10}] + binary scan $struct c* var + set end [lindex $var end] + if {$end<=$i} break + lset var end [expr $end - $i] + set struct [binary format c* $var] + + db close + sqlite3 db test.db + sqlite3_db_config db DEFENSIVE 0 + + db eval { + BEGIN; + UPDATE t1_data SET block = $struct WHERE id=10; + } + do_test 4.1.$i { + incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }] + set {} {} + } {} + catch { db eval ROLLBACK } +} +do_test 4.1.x { expr $nErr>45 } 1 + +#------------------------------------------------------------------------- +# + +# The first argument passed to this command must be a binary blob +# containing an FTS5 leaf page. This command returns a copy of this +# blob, with the pgidx of the leaf page replaced by a single varint +# containing value $iVal. +# +proc rewrite_pgidx {blob iVal} { + binary scan $blob SS off1 szLeaf + if {$iVal<0 || $iVal>=128} { + error "$iVal out of range!" + } else { + set pgidx [binary format c $iVal] + } + + binary format a${szLeaf}a* $blob $pgidx +} + +reset_db +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 40); + BEGIN; + INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj'); + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1(x1) VALUES('optimize'); + COMMIT; +} + +#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b } +# +db func rewrite_pgidx rewrite_pgidx +set i 0 +foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] { + foreach val {2 100} { + do_test 5.2.$val.[incr i] { + catchsql { + BEGIN; + UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid; + SELECT rowid FROM x1 WHERE x1 MATCH 'xa*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xb*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xc*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xd*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xe*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xf*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xg*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xh*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xi*'; + } + set {} {} + } {} + catch { db eval ROLLBACK } + } +} + +#------------------------------------------------------------------------ +# +reset_db +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 6.1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('bbbbb ccccc'); + SELECT quote(block) FROM t1_data WHERE rowid>100; +} {X'000000180630626262626201020201056363636363010203040A'} +do_execsql_test 6.1.1 { + UPDATE t1_data SET block = + X'000000180630626262626201020201056161616161010203040A' + WHERE rowid>100; +} +do_catchsql_test 6.1.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------- +reset_db +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 6.2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 VALUES('aa bb cc dd ee'); + SELECT pgno, quote(term) FROM t1_idx; +} {2 X'' 4 X'3064'} +do_execsql_test 6.2.1 { + UPDATE t1_idx SET term = X'3065' WHERE pgno=4; +} +do_catchsql_test 6.2.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------- +reset_db +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 6.3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('abc abcdef abcdefghi'); + SELECT quote(block) FROM t1_data WHERE id>100; +} {X'0000001C043061626301020204036465660102030703676869010204040808'} +do_execsql_test 6.3.1 { + BEGIN; + UPDATE t1_data SET block = + X'0000001C043061626301020204036465660102035003676869010204040808' + ------------------------------------------^^--------------------- + WHERE id>100; +} +do_catchsql_test 6.3.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 6.3.3 { + ROLLBACK; + BEGIN; + UPDATE t1_data SET block = + X'0000001C043061626301020204036465660102030750676869010204040808' + --------------------------------------------^^------------------- + WHERE id>100; +} +do_catchsql_test 6.3.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 6.3.4 { + ROLLBACK; + BEGIN; + UPDATE t1_data SET block = + X'0000001C043061626301020204036465660102030707676869010204040850' + --------------------------------------------------------------^^- + WHERE id>100; +} +do_catchsql_test 6.3.5 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 6.3.6 { + ROLLBACK; + BEGIN; + UPDATE t1_data SET block = + X'0000001C503061626301020204036465660102030707676869010204040808' + ----------^^----------------------------------------------------- + WHERE id>100; +} +do_catchsql_test 6.3.5 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + + +#------------------------------------------------------------------------ +# +reset_db +proc rnddoc {n} { + set map [list a b c d] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc "x[lindex $map [expr int(rand()*4)]]" + } + set doc +} + +db func rnddoc rnddoc +do_test 7.0 { + execsql { + CREATE VIRTUAL TABLE t5 USING fts5(x); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5(t5) VALUES('optimize'); + } +} {} + +sqlite3_db_config db DEFENSIVE 0 +do_test 7.1 { + foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] { + db eval BEGIN + db eval {DELETE FROM t5_data WHERE rowid = $i} + set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ] + if {$r != "1 {database disk image is malformed}"} { error $r } + db eval ROLLBACK + } +} {} + +#------------------------------------------------------------------------ +# Corruption within the structure record. +# +reset_db +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y); + INSERT INTO t1 VALUES('one', 'two'); +} + +sqlite3_db_config db DEFENSIVE 0 +do_test 9.1.1 { + set blob "12345678" ;# cookie + append blob "0105" ;# 1 level, total of 5 segments + append blob "06" ;# write counter + append blob "0002" ;# first level has 0 segments merging, 2 other. + append blob "450108" ;# first segment + execsql "REPLACE INTO t1_data VALUES(10, X'$blob')" +} {} +do_catchsql_test 9.1.2 { + SELECT * FROM t1('one AND two'); +} {1 {database disk image is malformed}} + +do_test 9.2.1 { + set blob "12345678" ;# cookie + append blob "0205" ;# 2 levels, total of 5 segments + append blob "06" ;# write counter + append blob "0001" ;# first level has 0 segments merging, 1 other. + append blob "450108" ;# first segment + execsql "REPLACE INTO t1_data VALUES(10, X'$blob')" +} {} +do_catchsql_test 9.2.2 { + SELECT * FROM t1('one AND two'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 10.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename c9.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 06 36 b0 a0 10 21 ck.....ft..6...! +| 4064: d6 f7 07 46 96 d6 97 a6 05 01 03 00 10 03 03 0f ...F............ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 heblet1t1CREATE +| page 8 offset 28672 +| 0: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 16: 20 55 53 49 4e 47 20 66 74 73 35 28 63 6f 6e 74 USING fts5(cont +| 32: 65 6e 74 29 0d 00 00 00 03 0f bd 00 0f e8 0f ef ent)............ +| 48: 0f bd 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| end c9.db + }] +} {} +do_catchsql_test 10.1 { + SELECT * FROM t1 WHERE t1 MATCH 'abandon'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 11.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c10b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 30 38 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..08...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 44 d9 (id INTEGER PRD. +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 06 0f 59 00 0f e8 0f ef 0f bd 0f b0 ......Y......... +| 16: 0f 73 0f 59 00 00 00 00 00 00 00 00 00 00 00 00 .s.Y............ +| 3920: 00 00 00 00 00 00 00 00 00 13 84 80 80 80 80 04 ................ +| 3936: 03 01 2a 0a 00 00 00 00 01 02 02 00 02 01 01 01 ..*............. +| 3952: 02 01 01 36 84 80 80 80 80 03 03 05 66 00 40 00 ...6........f.@. +| 3968: 00 00 01 00 00 00 29 07 30 61 63 74 69 76 65 04 ......).0active. +| 3984: 02 02 02 03 74 6f 6d 06 02 02 05 02 69 63 07 02 ....tom.....ic.. +| 4000: 02 01 06 62 6f 6f 6d 65 72 05 02 02 04 0b 08 07 ...boomer....... +| 4016: 06 84 80 80 80 80 02 03 01 10 01 07 07 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 02 0f f3 00 0f fa 0f f3 00 00 00 00 ................ +| 4080: 00 00 00 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 07 0f b6 00 0f f6 0f ec 0f e0 0f d5 ................ +| 16: 0f ca 0f c1 0f b6 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 09 07 03 00 19 61 74 6f 6d 69 ...........atomi +| 4032: 63 07 06 03 00 15 61 74 6f 6d 09 05 03 00 19 62 c.....atom.....b +| 4048: 6f 6f 6d 65 72 09 04 03 00 19 61 63 74 69 76 65 oomer.....active +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 07 0f d6 00 0f fa 0f f4 0f ee 0f e8 ................ +| 16: 0f e2 0f dc 0f d6 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 04 07 03 00 0e 01 04 06 03 00 ................ +| 4064: 0e 01 04 05 03 00 0e 01 04 04 03 00 0e 01 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c10b.db +}]} {} + +# This returns SQLITE_CONSTRAINT instead of SQLITE_CORRUPT. The problem is +# that the corrupted structure-record leads fts5 to try to use a segment-id +# that is already in use. This is caught by the PRIMARY KEY constraint on +# the %_idx table. +# +do_catchsql_test 11.1 { + UPDATE t1 SET content='abc' WHERE content='boomer'; +} {1 {constraint failed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 12.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c2.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f d8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 20 01 01 01 01 ...$....... .... +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 3f e0 ..............?. +| 16: a0 30 30 01 b6 16 26 16 e6 46 f6 e0 80 20 30 01 .00...&..F... 0. +| 32: 76 16 26 16 67 40 80 10 30 01 76 16 26 16 36 b0 v.&.g@..0.v.&.6. +| 48: d0 00 00 00 30 fe e0 00 ff a0 ff 40 fe 00 00 00 ....0......@.... +| page 5 offset 16384 +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c2.db +}]} {} + +do_catchsql_test 11.1 { + SELECT * FROM t1 WHERE t1 MATCH 'abandon'; +} {1 {database disk image is malformed}} + +do_catchsql_test 11.2 { + INSERT INTO t1(t1, rank) VALUES('merge', 500); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 13.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c13.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 4f 69 64 ATE TABLE 't1Oid +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 eb 00 00 00 01 01 01 00 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 01 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f2 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c13.db +SELECT * FROM t1 WHERE t1 MATCH 'abandon'; +}]} {} + +do_catchsql_test 13.1 { + SELECT * FROM t1 WHERE t1 MATCH 'abandon'; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 14.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c14b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 30 38 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..08...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 0f ef 00 04 0f 18 00 0f e8 0f 18 0f bd 0f 2c ..............., +| 3856: 00 00 00 00 00 00 00 00 12 0a 03 00 2a 00 00 00 ............*... +| 3872: 00 01 02 02 00 02 01 01 01 02 01 01 81 09 88 80 ................ +| 3888: 80 80 80 01 04 00 82 16 00 00 00 79 06 30 61 62 ...........y.0ab +| 3904: 61 63 6b 08 02 07 04 04 6e 64 6f 6e 08 02 05 02 ack.....ndon.... +| 3920: 05 63 74 69 76 65 04 02 02 04 02 0b 02 04 6c 70 .ctive........lp +| 3936: 68 61 08 04 02 0a 02 03 74 6b 6d 06 02 02 03 02 ha......tkm..... +| 3952: 6f 6d 08 02 09 05 02 69 63 07 02 02 01 06 62 61 om.....ic.....ba +| 3968: 63 6b 75 70 08 02 04 02 05 6f 6f 6d 65 72 05 02 ckup.....oomer.. +| 3984: 02 01 0c 63 68 61 6e 6e 65 62 6f 6f 6d 65 72 08 ...channeboomer. +| 4000: 02 08 07 01 6c 08 02 03 01 04 74 65 73 74 08 02 ....l.....test.. +| 4016: 06 04 0a 09 0d 0a 08 07 07 0b 0a 11 06 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 03 9a 07 05 01 03 00 10 08 11 00 on.............. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 02 0f f3 00 0f fa 0f f3 00 00 00 00 ................ +| 4080: 00 00 00 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 08 0f 6a 00 0f f6 0f ec 0f e0 0f d5 ......j......... +| 16: 0f ca 0f c1 0f b6 0f 6a 00 00 00 00 00 00 00 00 .......j........ +| 3936: 00 00 00 00 00 00 00 00 00 00 4a 08 04 00 81 19 ..........J..... +| 3952: 61 6c 70 68 61 20 63 68 61 6e 6e 65 6c 20 62 61 alpha channel ba +| 3968: 63 6b 75 70 20 61 62 61 6e 64 6f 6e 20 74 65 73 ckup abandon tes +| 3984: 74 20 61 62 61 63 6b 20 63 68 61 6e 6e 65 62 6f t aback channebo +| 4000: 6f 6d 65 72 20 61 74 6f 6d 20 61 6c 70 68 61 20 omer atom alpha +| 4016: 61 63 74 69 76 65 09 07 03 00 19 61 74 6f 6d 69 active.....atomi +| 4032: 63 07 06 03 00 15 61 74 6b 6d 09 05 03 00 19 62 c.....atkm.....b +| 4048: 6f 6f 6d 65 72 09 04 03 00 19 61 63 74 69 76 65 oomer.....active +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 08 0f d0 00 0f fa 0f f4 0f ee 0f e8 ................ +| 16: 0f e2 0f dc 0f d6 0f d0 00 00 00 00 00 00 00 00 ................ +| 4048: 04 08 03 00 0e 0a 04 07 03 00 0e 01 04 06 03 00 ................ +| 4064: 0e 01 04 05 03 00 0e 01 04 04 03 00 0e 01 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c14b.db +}]} {} + +do_catchsql_test 14.1 { + INSERT INTO t1(t1) VALUES('optimize'); +} {1 {database disk image is malformed}} + +#--------------------------------------------------------------------------- +# +reset_db +do_test 15.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename c16.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 00 0f f6 0f ec ..!!...tabl..... +| 3680: 0f e0 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ..sizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 00 02 22 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 04 67 90 38 2a 07 05 01 03 00 10 03 03 0f on.g.8*......... +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| page 8 offset 28672 +| 0: 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 31 5f .......-tablet1_ +| 16: 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 54 45 idxt1_idx.CREATE +| 32: 20 54 41 42 4c 45 20 27 74 31 5f 66 17 42 03 30 TABLE 't1_f.B.0 +| 48: 01 00 00 10 10 04 02 02 00 00 00 00 00 00 00 00 ................ +| 64: 70 00 00 00 00 00 00 00 00 00 00 00 70 00 00 00 p...........p... +| end c16.db +}]} {} + +do_catchsql_test 15.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {/*malformed database schema*/} + +#--------------------------------------------------------------------------- +# +reset_db +do_test 16.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c17.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 00 02 22 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 41 01 ...$..........A. +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c17.db +}]} {} + +do_catchsql_test 16.1 { +INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 17.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c18.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 00 02 22 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 0a aa aa aa aa aa aa aa aa aa ...$............ +| page 3 offset 8192 +| 0: aa aa aa aa aa aa aa aa aa aa aa aa aa aa aa aa ................ +| 16: aa aa aa aa aa aa aa aa 00 00 10 10 10 00 10 10 ................ +| 32: 10 10 a0 00 00 00 10 ff a0 00 ff 52 05 64 95 25 ...........R.d.% +| 48: 45 54 14 c2 05 44 14 24 c4 52 07 43 12 05 55 34 ET...D.$.R.C..U4 +| 64: 94 e4 72 06 67 47 33 52 86 36 f6 e7 46 56 e7 42 ..r.gG3R.6..FV.B +| 80: 90 d0 00 00 00 30 fb d0 00 fe 80 fe f0 fb 00 00 .....0.......... +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c18.db +}]} {} + +do_catchsql_test 17.1 { + SELECT * FROM t1 WHERE t1 MATCH 'abandon'; +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 18.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c19b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 30 38 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..08...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 09 a6 00 06 09 22 00 0f e8 09 22 0f bd 0f 2c ..............., +| 16: 09 bd 09 3c 00 00 00 00 00 00 00 00 00 00 00 00 ...<............ +| 2336: 00 00 18 0a 03 00 36 00 00 00 00 01 04 04 00 04 ......6......... +| 2352: 01 01 01 02 01 01 03 01 01 04 01 01 63 90 80 80 ............c... +| 2368: 80 80 01 04 00 81 4a 00 00 00 56 06 30 61 62 61 ......J...V.0aba +| 2384: 63 6b 08 01 04 04 6e 64 6f 6e 03 01 05 01 02 05 ck....ndon...... +| 2400: 63 74 69 76 65 08 01 02 04 6c 70 68 61 08 01 02 ctive....lpha... +| 2416: 03 74 6f 6d 08 01 01 06 62 61 63 6b 75 70 08 01 .tom....backup.. +| 2432: 01 0c 63 68 61 6e 6e 65 62 6f 6f 6d 65 72 08 01 ..channeboomer.. +| 2448: 07 01 6c 08 01 01 04 74 65 73 74 08 01 04 09 0a ..l....test..... +| 2464: 09 08 07 0a 10 05 0f 18 00 17 30 00 00 00 00 01 ..........0..... +| 2480: 03 03 00 03 01 01 01 02 01 01 03 01 01 8a 53 8c ..............S. +| 2496: 80 80 80 80 01 04 00 95 2a 00 00 05 35 0d 30 30 ........*...5.00 +| 2512: 31 30 66 66 61 30 30 30 66 66 61 05 02 1c 02 49 10ffa000ffa....I +| 2528: 33 34 33 35 32 34 35 34 31 35 34 34 35 32 30 35 3435245415445205 +| 2544: 34 34 31 34 32 34 63 34 35 32 30 32 37 37 34 33 441424c452027743 +| 2560: 31 35 66 36 39 36 34 37 38 32 37 32 38 37 33 36 15f6964782728736 +| 2576: 35 36 37 36 39 36 34 32 63 32 30 37 34 36 35 37 56769642c2074657 +| 2592: 32 36 64 32 63 32 30 37 30 05 02 12 02 3b 36 31 26d2c2070....;61 +| 2608: 37 32 31 32 31 30 31 37 37 37 34 36 31 36 32 35 7212101777461625 +| 2624: 63 36 35 37 34 33 31 35 66 36 33 36 66 36 65 37 c6574315f636f6e7 +| 2640: 34 36 35 36 65 37 34 37 34 33 31 35 66 36 33 36 4656e7474315f636 +| 2656: 66 36 65 37 34 36 35 36 65 05 04 07 07 01 04 31 f6e74656e......1 +| 2672: 66 62 64 05 02 19 01 44 32 34 38 34 38 30 38 30 fbd....D24848080 +| 2688: 38 30 38 30 30 31 30 33 30 30 34 65 30 30 30 30 80800103004e0000 +| 2704: 30 30 31 65 30 36 33 30 36 31 36 32 36 31 36 33 001e063061626163 +| 2720: 36 62 30 31 30 32 30 32 30 34 30 32 36 36 37 34 6b01020204026674 +| 2736: 30 32 30 32 30 32 30 34 30 34 36 65 05 02 1a 02 02020204046e.... +| 2752: 03 66 65 72 05 02 1d 01 28 33 65 37 34 36 35 36 .fer....(3e74656 +| 2768: 65 37 34 32 39 30 64 30 30 30 30 30 30 30 33 30 e74290d000000030 +| 2784: 66 62 64 30 30 30 66 65 38 30 66 65 66 30 66 62 fbd000fe80fef0fb +| 2800: 64 05 02 18 01 4a 34 31 35 32 35 39 32 30 34 62 d....J415259204b +| 2816: 34 35 35 39 32 63 32 30 36 32 36 63 36 66 36 33 45592c20626c6f63 +| 2832: 36 62 32 30 34 32 34 63 34 66 34 32 32 39 33 61 6b20424c4f42293a +| 2848: 30 31 30 36 31 37 31 31 31 31 30 38 36 33 37 34 0106171111086374 +| 2864: 36 31 36 32 36 63 36 35 37 34 33 31 37 34 33 31 61626c6574317431 +| 2880: 05 02 16 02 49 33 35 32 34 35 34 31 35 34 34 35 ....I35245415445 +| 2896: 32 30 35 36 34 39 35 32 35 34 35 35 34 31 34 63 205649525455414c +| 2912: 32 30 35 34 34 31 34 32 34 63 34 35 32 30 37 34 205441424c452074 +| 2928: 33 31 6f 30 35 35 35 33 34 39 34 65 34 37 32 30 31o05553494e4720 +| 2944: 36 36 37 34 37 33 33 35 32 38 36 33 36 66 05 02 6674733528636f.. +| 2960: 17 02 49 35 32 30 35 34 34 31 34 32 34 63 34 35 ..I5205441424c45 +| 2976: 32 30 32 37 37 34 33 31 35 66 36 33 36 66 36 65 202774315f636f6e +| 2992: 37 34 36 35 36 65 37 34 32 37 32 38 36 39 36 34 74656e7427286964 +| 3008: 32 30 34 39 34 65 35 34 34 35 34 37 34 35 35 32 20494e5445474552 +| 3024: 32 30 35 30 35 32 34 39 34 64 34 31 05 02 0e 44 205052494d41...D +| 3040: 29 62 30 35 30 37 31 37 32 31 32 31 30 31 38 31 )b05071721210181 +| 3056: 30 31 37 34 36 31 36 32 36 63 36 35 37 34 33 31 017461626c657431 +| 3072: 35 66 36 34 36 66 36 33 37 33 05 02 09 01 4a 35 5f646f6373....J5 +| 3088: 32 34 35 34 31 35 34 34 35 32 30 35 34 34 31 34 2454154452054414 +| 3104: 32 34 63 34 35 32 30 32 37 37 34 33 31 35 66 36 24c45202774315f6 +| 3120: 34 36 31 37 34 36 31 32 37 32 38 36 39 36 34 32 4617461272869642 +| 3136: 30 34 39 34 65 35 34 34 35 34 37 34 35 35 32 32 0494e54454745522 +| 3152: 30 35 30 35 32 34 39 34 64 05 02 15 03 3a 35 39 05052494d....:59 +| 3168: 32 30 34 62 34 35 35 39 32 63 32 30 36 33 33 30 204b45592c206330 +| 3184: 32 39 36 39 30 33 30 37 31 37 31 39 31 39 30 31 2969030717191901 +| 3200: 38 31 32 64 37 34 36 31 36 32 36 63 36 35 37 34 812d7461626c6574 +| 3216: 33 31 35 66 36 39 79 79 05 02 0f 02 49 34 32 30 315f69yy....I420 +| 3232: 35 32 34 66 35 37 34 39 34 34 35 35 30 32 30 37 524f574944550207 +| 3248: 31 37 31 62 31 62 30 31 38 31 30 31 37 34 36 31 171b1b0181017461 +| 3264: 36 32 36 63 36 37 37 34 33 31 35 66 36 34 36 31 626c6774315f6461 +| 3280: 37 34 36 31 37 34 33 31 35 66 36 34 36 31 37 34 746174315f646174 +| 3296: 36 31 30 32 34 33 05 02 14 02 07 66 36 39 36 34 610243.....f6964 +| 3312: 37 38 05 02 11 01 4a 36 34 36 66 36 65 30 33 30 78....J646f6e030 +| 3328: 32 30 32 30 34 30 61 30 37 30 35 30 31 30 33 30 202040a070501030 +| 3344: 30 31 30 30 33 30 33 30 66 30 61 30 33 30 30 32 01003030f0a03002 +| 3360: 34 30 30 30 30 30 30 30 30 30 31 30 31 30 31 30 4000000000101010 +| 3376: 30 30 31 30 31 30 31 30 31 30 61 30 30 30 30 30 0010101010a00000 +| 3392: 30 05 02 1b 02 49 35 32 37 32 38 36 39 36 34 32 0....I5272869642 +| 3408: 30 34 39 34 65 35 34 34 35 34 37 34 35 35 32 32 0494e54454745522 +| 3424: 30 35 30 35 32 34 39 34 64 34 31 35 32 35 39 32 05052494d4152592 +| 3440: 30 34 62 34 35 35 39 32 63 32 30 37 33 37 61 32 04b45592c20737a2 +| 3456: 30 34 32 34 63 34 66 34 32 32 39 35 35 30 34 05 0424c4f42295504. +| 3472: 04 06 07 02 49 37 36 65 36 66 32 63 32 30 35 30 ....I76e6f2c2050 +| 3488: 35 32 34 39 34 64 34 31 35 32 35 39 32 30 34 62 52494d415259204b +| 3504: 34 35 35 39 32 38 37 33 36 35 36 37 36 39 36 34 4559287365676964 +| 3520: 32 63 32 30 37 34 36 35 37 32 36 64 32 39 32 39 2c207465726d2929 +| 3536: 32 30 35 37 34 39 35 34 34 38 34 66 35 35 05 02 20574954484f55.. +| 3552: 13 02 49 39 37 61 36 35 37 34 33 31 35 66 36 34 ..I97a6574315f64 +| 3568: 36 66 36 33 37 33 36 39 37 61 36 35 30 35 34 33 6f6373697a650543 +| 3584: 35 32 34 35 34 31 35 34 34 35 32 30 35 34 34 31 5245415445205441 +| 3600: 34 32 34 63 34 35 32 30 32 37 37 34 33 31 35 66 424c45202774315f +| 3616: 36 34 36 66 36 33 37 33 36 39 37 61 05 04 05 07 646f6373697a.... +| 3632: 01 0e 37 34 30 34 34 33 35 32 34 35 34 31 35 34 ..74044352454154 +| 3648: 05 04 08 07 02 49 36 32 39 32 30 35 37 34 39 35 .....I6292057495 +| 3664: 34 34 38 34 66 35 35 35 34 32 30 35 32 34 66 35 4484f555420524f5 +| 3680: 37 34 39 34 34 35 62 30 35 30 37 31 37 32 31 32 749445b050717212 +| 3696: 31 30 31 38 31 30 31 37 34 36 31 36 32 36 63 36 10181017461626c6 +| 3712: 35 37 34 33 31 35 66 36 34 36 66 36 33 37 33 05 574315f646f6373. +| 3728: 02 04 01 06 62 61 63 6b 75 70 05 02 1e 02 05 65 ....backup.....e +| 3744: 61 6d 65 72 05 02 02 02 05 6f 6f 6d 65 72 05 01 amer.....oomer.. +| 3760: 02 40 75 6d 6d 32 34 63 34 35 32 30 32 37 37 34 .@umm24c45202774 +| 3776: 33 31 35 66 36 33 36 66 36 65 36 36 36 39 36 37 315f636f6e666967 +| 3792: 32 37 32 38 36 62 32 30 35 30 35 32 34 39 34 64 27286b205052494d +| 3808: 34 31 35 32 35 39 32 30 34 62 34 35 35 39 32 63 415259204b45592c +| 3824: 32 30 05 02 03 01 04 79 65 6b 72 05 02 10 04 11 20.....yekr..... +| 3840: 4e 41 09 49 08 2d 4f 4e 4e 2e 4f 3f 4e 0c 4f 4f NA.I.-ONN.O?N.OO +| 3856: 4e 4f 14 4e 0b 0a 09 45 0f ef 00 14 2a 00 00 00 NO.N...E....*... +| 3872: 00 01 02 02 00 02 01 01 01 02 01 01 81 09 88 80 ................ +| 3888: 80 80 80 01 04 00 82 16 00 00 00 79 06 30 61 62 ...........y.0ab +| 3904: 61 63 6b 08 02 07 04 04 6e 64 6f 6e 08 02 05 02 ack.....ndon.... +| 3920: 05 63 74 69 76 65 04 02 02 04 02 0b 02 04 6c 70 .ctive........lp +| 3936: 68 61 08 04 02 0a 02 03 74 6b 6d 06 02 02 03 02 ha......tkm..... +| 3952: 6f 6d 08 02 09 05 02 69 63 07 02 02 01 06 62 61 om.....ic.....ba +| 3968: 63 6b 75 70 08 02 04 02 05 6f 6f 6d 65 72 05 02 ckup.....oomer.. +| 3984: 02 01 0c 63 68 61 6e 6e 65 62 6f 6f 6d 65 72 08 ...channeboomer. +| 4000: 02 08 07 01 6c 08 02 03 01 04 74 65 73 74 08 02 ....l.....test.. +| 4016: 06 04 0a 09 0d 0a 08 07 07 0b 0a 11 06 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 08 02 04 02 66 74 00 02 22 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 06 22 00 on.............. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 04 0f e5 00 0f fa 0f f3 0f ec 0f e5 ................ +| 4064: 00 00 00 00 00 06 04 01 0c 01 04 02 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 0f 68 00 06 08 98 00 0f f6 0f ec 0f d5 08 98 ..h............. +| 16: 0f c1 0f b6 0f 68 0f 68 00 00 00 00 00 00 00 00 .....h.h........ +| 2192: 00 00 00 00 00 00 00 00 8d 4d 05 04 00 9b 1f 62 .........M.....b +| 2208: 65 61 6d 65 72 20 62 75 6d 6d 32 34 63 34 35 32 eamer bumm24c452 +| 2224: 30 32 37 37 34 33 31 35 66 36 33 36 66 36 65 36 02774315f636f6e6 +| 2240: 36 36 39 36 37 32 37 32 38 36 62 32 30 35 30 35 6696727286b20505 +| 2256: 32 34 39 34 64 34 31 35 32 35 39 32 30 34 62 34 2494d415259204b4 +| 2272: 35 35 39 32 63 32 30 0a 37 36 32 39 32 30 35 37 5592c20.76292057 +| 2288: 34 39 35 34 34 38 34 66 35 35 35 34 32 30 35 32 4954484f55542052 +| 2304: 34 66 35 37 34 39 34 34 35 62 30 35 30 37 31 37 4f5749445b050717 +| 2320: 32 31 32 31 30 31 38 31 30 31 37 34 36 31 36 32 2121018101746162 +| 2336: 36 63 36 35 37 34 33 31 35 66 36 34 36 66 36 33 6c6574315f646f63 +| 2352: 37 33 0a 36 39 37 61 36 35 37 34 33 31 35 66 36 73.697a6574315f6 +| 2368: 34 36 66 36 33 37 33 36 39 37 61 36 35 30 35 34 46f6373697a65054 +| 2384: 33 35 32 34 35 34 31 35 34 34 35 32 30 35 34 34 3524541544520544 +| 2400: 31 34 32 34 63 34 35 32 30 32 37 37 34 33 31 35 1424c45202774315 +| 2416: 66 36 34 36 66 36 33 37 33 36 39 37 61 0a 36 35 f646f6373697a.65 +| 2432: 32 37 32 38 36 39 36 34 32 30 34 39 34 65 35 34 2728696420494e54 +| 2448: 34 35 34 37 34 35 35 32 32 30 35 30 35 32 34 39 4547455220505249 +| 2464: 34 64 34 31 35 32 35 39 32 30 34 62 34 35 35 39 4d415259204b4559 +| 2480: 32 63 32 30 37 33 37 61 32 30 34 32 34 63 34 66 2c20737a20424c4f +| 2496: 34 32 32 39 35 35 30 34 0a 30 36 31 37 32 31 32 42295504.0617212 +| 2512: 31 30 31 37 37 37 34 36 31 36 32 35 63 36 35 37 101777461625c657 +| 2528: 34 33 31 35 66 36 33 36 66 36 65 37 34 36 35 36 4315f636f6e74656 +| 2544: 65 37 34 37 34 33 31 35 66 36 33 36 66 36 65 37 e7474315f636f6e7 +| 2560: 34 36 35 36 65 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 4656e........... +| 2576: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 2592: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 2608: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 2624: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 2640: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 2656: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 37 34 30 .............740 +| 2672: 34 34 33 35 32 34 35 34 31 35 34 0a 34 35 32 30 44352454154.4520 +| 2688: 35 34 34 31 34 32 34 63 34 35 32 30 32 37 37 34 5441424c45202774 +| 2704: 33 31 35 66 36 33 36 66 36 65 37 34 36 35 36 65 315f636f6e74656e +| 2720: 37 34 32 37 32 38 36 39 36 34 32 30 34 39 34 65 742728696420494e +| 2736: 35 34 34 35 34 37 34 35 35 32 32 30 35 30 35 62 544547455220505b +| 2752: 30 35 30 37 31 37 32 31 32 31 30 31 38 31 30 31 0507172121018101 +| 2768: 37 34 36 31 36 32 36 63 36 35 37 34 33 31 35 66 7461626c6574315f +| 2784: 36 34 36 66 36 33 37 33 0a 36 39 37 61 36 35 37 646f6373.697a657 +| 2800: 34 33 31 35 66 36 34 36 66 36 33 37 33 36 39 37 4315f646f6373697 +| 2816: 61 36 35 30 35 34 33 35 32 34 35 34 31 35 34 34 a650543524541544 +| 2832: 35 32 30 35 34 34 31 34 32 34 63 34 35 32 30 32 5205441424c45202 +| 2848: 37 37 34 33 31 35 66 36 34 36 66 36 33 37 33 36 774315f646f63736 +| 2864: 39 37 61 0a 36 35 32 37 32 38 36 39 36 34 32 30 97a.652728696420 +| 2880: 34 39 34 65 35 34 34 35 34 37 34 35 35 32 32 30 494e544547455220 +| 2896: 35 30 35 32 34 39 34 64 34 31 35 32 35 39 32 30 5052494d41525920 +| 2912: 34 62 34 35 35 39 32 63 32 30 37 33 37 61 32 30 4b45592c20737a20 +| 2928: 34 32 34 63 34 66 34 32 32 39 35 35 30 34 0a 30 424c4f42295504.0 +| 2944: 36 31 37 32 31 32 31 30 31 37 37 37 34 36 31 36 6172121017774616 +| 2960: 32 35 63 36 35 37 34 33 31 35 66 36 33 36 66 36 25c6574315f636f6 +| 2976: 65 37 34 36 35 36 65 37 34 37 34 33 31 35 66 36 e74656e7474315f6 +| 2992: 33 36 66 36 65 37 34 36 35 36 65 0b 0b 0b 0b 0b 36f6e74656e..... +| 3008: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 3024: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 3040: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 3056: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 3072: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 3088: 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b 0b ................ +| 3104: 0b 0b 0b 37 34 30 34 34 33 35 32 34 35 34 31 35 ...7404435245415 +| 3120: 34 0a 34 35 32 30 35 34 34 31 34 32 34 63 34 35 4.45205441424c45 +| 3136: 32 30 32 37 37 34 33 31 35 66 36 33 36 66 36 65 202774315f636f6e +| 3152: 37 34 36 35 36 65 37 34 32 37 32 38 36 39 36 34 74656e7427286964 +| 3168: 32 30 34 39 34 65 35 34 34 35 34 37 34 35 35 32 20494e5445474552 +| 3184: 32 30 35 30 35 32 34 39 34 64 34 31 0a 35 32 35 205052494d41.525 +| 3200: 39 32 30 34 62 34 35 35 39 32 63 32 30 36 33 33 9204b45592c20633 +| 3216: 30 32 39 36 39 30 33 30 37 31 37 31 39 31 39 30 0296903071719190 +| 3232: 31 38 31 32 64 37 34 36 31 36 32 36 63 36 35 37 1812d7461626c657 +| 3248: 34 33 31 35 66 36 39 c3 bf c3 bf 7e c3 bf c3 89 4315f69....~.... +| 3264: 4b 52 c2 81 35 66 36 39 36 34 37 38 0a 30 33 34 KR..5f696478.034 +| 3280: 33 35 32 34 35 34 31 35 34 34 35 32 30 35 34 34 3524541544520544 +| 3296: 31 34 32 34 63 34 35 32 30 32 37 37 34 33 31 35 1424c45202774315 +| 3312: 66 36 39 36 34 37 38 32 37 32 38 37 33 36 35 36 f696478272873656 +| 3328: 37 36 39 36 34 32 63 32 30 37 34 36 35 37 32 36 769642c207465726 +| 3344: 64 32 63 32 30 37 30 0a 36 37 36 65 36 66 32 63 d2c2070.676e6f2c +| 3360: 32 30 35 30 35 32 34 39 34 64 34 31 35 32 35 39 205052494d415259 +| 3376: 32 30 34 62 34 35 35 39 32 38 37 33 36 35 36 37 204b455928736567 +| 3392: 36 39 36 34 32 63 32 30 37 34 36 35 37 32 36 64 69642c207465726d +| 3408: 32 39 32 39 32 30 35 37 34 39 35 34 34 38 34 66 292920574954484f +| 3424: 35 35 0a 35 34 32 30 35 32 34 66 35 37 34 39 34 55.5420524f57494 +| 3440: 34 35 35 30 32 30 37 31 37 31 62 31 62 30 31 38 4550207171b1b018 +| 3456: 31 30 31 37 34 36 31 36 32 36 63 36 37 37 34 33 1017461626c67743 +| 3472: 31 35 66 36 34 36 31 37 34 36 31 37 34 33 31 35 15f6461746174315 +| 3488: 66 36 34 36 31 37 34 36 31 30 32 34 33 0a 35 32 f646174610243.52 +| 3504: 34 35 34 31 35 34 34 35 32 30 35 34 34 31 34 32 4541544520544142 +| 3520: 34 63 34 35 32 30 32 37 37 34 33 31 35 66 36 34 4c45202774315f64 +| 3536: 36 31 37 34 36 31 32 37 32 38 36 39 36 34 32 30 6174612728696420 +| 3552: 34 39 34 65 35 34 34 35 34 37 34 35 35 32 32 30 494e544547455220 +| 3568: 35 30 35 32 34 39 34 64 0a 34 31 35 32 35 39 32 5052494d.4152592 +| 3584: 30 34 62 34 35 35 39 32 63 32 30 36 32 36 63 36 04b45592c20626c6 +| 3600: 66 36 33 36 62 32 30 34 32 34 63 34 66 34 32 32 f636b20424c4f422 +| 3616: 39 33 61 30 31 30 36 31 37 31 31 31 31 30 38 36 93a0106171111086 +| 3632: 33 37 34 36 31 36 32 36 63 36 35 37 34 33 31 37 37461626c6574317 +| 3648: 34 33 31 0a 34 33 35 32 34 35 34 31 35 34 34 35 431.435245415445 +| 3664: 32 30 35 36 34 39 35 32 35 34 35 35 34 31 34 63 205649525455414c +| 3680: 32 30 35 34 34 31 34 32 34 63 34 35 32 30 37 34 205441424c452074 +| 3696: 33 31 c3 94 30 35 35 35 33 34 39 34 65 34 37 32 31..05553494e472 +| 3712: 30 36 36 37 34 37 33 33 35 32 38 36 33 36 66 0a 06674733528636f. +| 3728: 33 65 37 34 36 35 36 65 37 34 32 39 30 64 30 30 3e74656e74290d00 +| 3744: 30 30 30 30 30 33 30 66 62 64 30 30 30 66 65 38 0000030fbd000fe8 +| 3760: 30 66 65 66 30 66 62 64 0a 5b 31 66 62 64 5d 32 0fef0fbd.[1fbd]2 +| 3776: 34 38 34 38 30 38 30 38 30 38 30 30 31 30 33 30 4848080808001030 +| 3792: 30 34 65 30 30 30 30 30 30 31 65 30 36 33 30 36 04e0000001e06306 +| 3808: 31 36 32 36 31 36 33 36 62 30 31 30 32 30 32 30 16261636b0102020 +| 3824: 34 30 32 36 36 37 34 30 32 30 32 30 32 30 34 30 4026674020202040 +| 3840: 34 36 65 0a 36 34 36 66 36 65 30 33 30 32 30 32 46e.646f6e030202 +| 3856: 30 34 30 61 30 37 30 35 30 31 30 33 30 30 31 30 040a070501030010 +| 3872: 30 33 30 33 30 66 30 61 30 33 30 30 32 34 30 30 03030f0a03002400 +| 3888: 30 30 30 30 30 30 30 31 30 31 30 31 30 30 30 31 0000000101010001 +| 3904: 30 31 30 31 30 31 30 61 30 30 30 30 30 30 0a 30 0101010a000000.0 +| 3920: 31 30 66 66 61 30 30 30 66 66 61 0a 5b 32 66 65 10ffa000ffa.[2fe +| 3936: 72 20 62 61 63 6b 75 70 0f ca 00 4e 81 1d 61 6c r backup...N..al +| 3952: 70 68 61 20 63 68 61 6e 6e 65 6c 20 c2 af 62 61 pha channel ..ba +| 3968: 63 6b 75 70 20 61 62 61 6e 64 6f 6e 20 74 65 73 ckup abandon tes +| 3984: 74 20 61 62 61 63 6b 20 63 68 61 6e 6e 65 62 6f t aback channebo +| 4000: 6f 6d 65 72 20 61 74 6f 6d 20 61 6c 70 68 61 20 omer atom alpha +| 4016: 61 63 74 69 76 65 09 07 03 00 19 61 74 6f 6d 69 active.....atomi +| 4032: 63 07 06 03 00 15 61 74 6b 6d 0f e0 00 0b 19 62 c.....atkm.....b +| 4048: 6f 6f 6d 65 72 09 04 03 00 19 61 63 74 69 76 65 oomer.....active +| 4064: 00 00 00 0c 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 66 21 66 74 08 01 03 00 17 61 62 61 63 6b .af!ft.....aback +| page 5 offset 16384 +| 0: 0d 0f ee 00 06 0f d6 00 0f fa 0f f4 0f e8 0f e2 ................ +| 16: 0f dc 0f d6 0f d0 0f d0 00 00 00 00 00 00 00 00 ................ +| 4048: 0f ee 00 06 0e 0a 04 07 03 00 0e 01 04 06 03 00 ................ +| 4064: 0e 01 04 05 03 00 0e 1d 04 04 03 00 0e 01 00 00 ................ +| 4080: 00 06 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c19b.db +}]} {} + +do_catchsql_test 18.1 { + INSERT INTO t1(t1) VALUES('optimize'); +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 19.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c20b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 30 38 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..08...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 0f 20 00 05 0e a0 00 0f e8 0e a0 0f bd 0f 34 .. ............4 +| 16: 0e b7 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3744: 15 0a 03 00 30 00 00 00 00 01 03 03 00 03 01 01 ....0........... +| 3760: 01 02 01 01 03 01 01 62 8c 80 80 80 80 01 04 00 .......b........ +| 3776: 81 48 00 00 00 55 06 30 61 62 61 63 6b 08 01 04 .H...U.0aback... +| 3792: 04 6e 64 6f 6e 03 01 05 01 02 05 63 74 69 76 65 .ndon......ctive +| 3808: 08 01 02 04 6c 70 68 61 08 01 02 03 74 6f 6d 08 ....lpha....tom. +| 3824: 01 01 06 62 61 63 6b 75 70 08 01 02 05 6f 6f 6d ...backup....oom +| 3840: 65 72 08 01 01 07 63 68 61 6e 6e 65 6c 08 01 01 er....channel... +| 3856: 04 74 65 73 74 08 01 04 09 0a 09 08 07 0a 09 0b .test........... +| 3872: 0f ef 00 14 2a 00 00 00 00 01 02 02 00 02 01 01 ....*........... +| 3888: 01 02 01 01 81 01 88 80 80 80 80 01 04 00 82 06 ................ +| 3904: 00 00 00 72 06 30 61 62 61 63 6b 08 02 07 04 04 ...r.0aback..... +| 3920: 6e 64 6f 6e 08 02 05 02 05 63 74 69 76 65 04 02 ndon.....ctive.. +| 3936: 02 04 02 0b 02 04 6c 70 68 61 08 04 02 0a 02 03 ......lpha...... +| 3952: 74 6f 6d 06 02 02 02 02 09 05 02 69 63 07 02 02 tom........ic... +| 3968: 01 06 62 61 63 6b 75 70 08 02 04 02 05 6f 6f 66 ..backup.....oof +| 3984: 65 72 05 02 02 04 03 6d 65 72 08 02 08 01 07 63 er.....mer.....c +| 4000: 68 61 6e 6e 65 6c 08 02 03 01 04 74 65 73 74 08 hannel.....test. +| 4016: 02 06 04 0a 09 0d 0a 0b 07 0b 0a 08 0c 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 66 04 00 22 74 00 02 22 04 04 6e 64 ck..f...t.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 06 06 00 on.............. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 0f e0 00 06 0f b6 00 0f f6 0f ec 0f d5 0f ca ................ +| 16: 0f c1 0f b6 0f 70 0f 70 00 00 00 00 00 00 00 00 .....p.p........ +| 3952: 0f e0 00 46 81 0d 61 6c 70 68 61 20 63 68 61 6e ...F..alpha chan +| 3968: 6e 65 6c 20 62 61 63 6b 75 70 20 61 62 61 6e 64 nel backup aband +| 3984: 6f 6e 20 74 65 73 74 20 61 62 61 63 6b 20 62 6f on test aback bo +| 4000: 6f 6d 65 72 20 61 74 6f 6d 20 61 6c 70 68 61 20 omer atom alpha +| 4016: 61 63 74 69 76 65 09 07 03 00 19 61 74 6f 6d 69 active.....atomi +| 4032: 63 07 06 03 00 15 61 74 6f 6d 09 05 03 00 19 62 c.....atom.....b +| 4048: 6f 6f 66 65 72 09 04 03 00 19 61 63 74 69 76 65 oofer.....active +| 4064: 00 00 00 0c 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 0f ee 00 06 0f d6 00 0f fa 0f f4 0f e8 0f e2 ................ +| 16: 0f dc 0f d6 0f d0 0f d0 00 00 00 00 00 00 00 00 ................ +| 4048: 0f ee 00 06 0e 0a 04 07 03 00 0e 01 04 06 03 00 ................ +| 4064: 0e 01 04 05 03 00 0e 01 04 04 03 00 0e 01 00 00 ................ +| 4080: 00 06 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 86 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 .eck....optimize +| end c20b.db +}]} {} + +do_catchsql_test 19.1 { + INSERT INTO t1(t1) VALUES('optimize'); +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 20.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-cf347c523f793c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 0e ee ee ee ee ee ee ee ee ...$............ +| page 3 offset 8192 +| 0: ee ee ee ee ee ee ee ee ee ee ee ee ee ee ee ee ................ +| 16: ee ee ee ee ee ee ee ee ee ee ee ee ee ee ee ee ................ +| 32: ee ee ee ee ee ee ee ee ee ee ee ee 00 10 10 10 ................ +| 48: 00 10 10 10 10 a0 00 00 00 10 ff a0 00 ff 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end crash-cf347c523f793c.db +}]} {} + +do_catchsql_test 20.1 { + SELECT * FROM t1 WHERE t1 MATCH 'abandon'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 21.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c22b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 30 38 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..08...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 0e 8e 00 06 0e 2f 00 0f e8 0e 2f 0f bd 0f 3b ....../..../...; +| 16: 0e a5 0e 49 00 00 00 00 00 00 00 00 00 00 00 00 ...I............ +| 3616: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 18 ................ +| 3632: 0a 03 00 36 00 00 00 00 01 04 04 00 04 01 01 01 ...6............ +| 3648: 02 01 01 03 01 01 04 01 01 3e 90 80 80 80 80 01 .........>...... +| 3664: 04 00 81 00 00 00 00 36 06 30 62 61 63 6b 75 05 .......6.0backu. +| 3680: 02 04 05 02 04 02 05 65 61 6d 65 72 05 02 02 05 .......eamer.... +| 3696: 02 02 02 05 6f 6f 6d 65 72 05 01 05 01 02 05 75 ....oomer......u +| 3712: 6d 6d 65 72 05 02 03 05 02 03 04 0d 0d 0b 0f 27 mmer...........' +| 3728: 00 17 30 00 00 00 00 01 03 03 00 03 01 01 01 02 ..0............. +| 3744: 01 01 03 01 01 7b 8c 80 80 80 80 01 04 00 81 7a ...............z +| 3760: 00 00 00 6d 06 30 61 62 61 63 6b 0d 02 07 04 04 ...m.0aback..... +| 3776: 6e 64 6f 6e 0d 02 05 02 05 63 74 69 76 65 09 02 ndon.....ctive.. +| 3792: 02 04 02 0b 02 04 6c 70 68 61 0d 04 02 0a 02 03 ......lpha...... +| 3808: 74 6f 6d 0b 02 02 02 02 09 05 02 69 63 0c 02 02 tom........ic... +| 3824: 01 06 62 61 63 6b 75 70 0d 02 04 02 05 6f 6f 6d ..backup.....oom +| 3840: 65 72 0a 02 02 03 02 08 01 07 63 68 61 6e 6e 65 er........channe +| 3856: 6c 0d 02 03 01 04 74 65 73 74 0d 02 06 04 0a 09 l.....test...... +| 3872: 0d 0a 0b 07 0b 0d 0c 0f ef 00 14 2a 00 00 00 00 ...........*.... +| 3888: 01 02 02 00 02 01 01 01 02 01 01 7b 88 80 80 80 ................ +| 3904: 80 01 04 00 81 7a 00 00 00 6d 06 30 61 62 61 63 .....z...m.0abac +| 3920: 6b 08 02 07 04 04 6e 64 6f 6e 08 02 05 02 05 63 k.....ndon.....c +| 3936: 74 69 76 65 04 02 02 04 02 0b 02 04 6c 70 68 61 tive........lpha +| 3952: 08 04 02 0a 02 03 74 6f 6d 06 02 02 02 02 09 05 ......tom....... +| 3968: 02 69 63 07 02 02 01 06 62 61 63 6b 75 70 08 02 .ic.....backup.. +| 3984: 04 02 05 6f 6f 6d 65 72 05 02 02 03 02 08 01 07 ...oomer........ +| 4000: 63 68 61 6e 6e 65 6c 08 02 03 01 04 74 65 73 74 channel.....test +| 4016: 08 02 06 04 0a 09 0d 0a 0b 07 0b 0d 0c 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 00 02 22 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 08 0a 07 05 01 03 00 10 0d 23 00 on............#. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 04 0f e5 00 0f fa 0f f3 0f ec 0f e5 ................ +| 4064: 00 00 00 00 00 06 04 01 0c 01 04 02 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 0f 5a 00 0d 0e ce 00 0f f6 0f ec 0f e0 0f d5 ..Z............. +| 16: 0e e7 0f c1 0f b6 0f 70 0f 65 0e ce 0f 51 0f 46 .......p.e...Q.F +| 32: 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3776: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 17 0a ................ +| 3792: 03 00 35 62 65 61 6d 65 72 20 62 75 6d 6d 65 72 ..5beamer bummer +| 3808: 20 62 61 63 6b 75 29 17 05 03 00 35 62 65 61 6d backu)....5beam +| 3824: 65 72 20 62 75 6d 6d 65 72 20 62 61 63 6b 75 29 er bummer backu) +| 3840: 44 0d 04 00 81 0d 61 6c 70 68 61 20 63 68 61 6e D.....alpha chan +| 3856: 6e 65 6c 20 62 61 63 6b 75 70 20 61 62 61 6e 64 nel backup aband +| 3872: 6f 6e 20 74 65 73 74 20 61 62 61 63 6b 20 62 6f on test aback bo +| 3888: 6f 6d 65 72 20 61 74 6f 6d 20 61 6c 70 68 61 20 omer atom alpha +| 3904: 61 63 74 69 76 65 09 0c 03 00 19 61 74 6f 6d 69 active.....atomi +| 3920: 63 07 0b 03 00 15 61 74 6f 6d 0f ca 00 0b 19 62 c.....atom.....b +| 3936: 6f 6f 6d 65 72 09 09 03 00 19 61 63 74 69 76 65 oomer.....active +| 3952: 44 08 04 00 81 0d 61 6c 70 68 61 20 63 68 61 6e D.....alpha chan +| 3968: 6e 65 6c 20 62 61 63 6b 75 70 20 61 62 61 6e 64 nel backup aband +| 3984: 6f 6e 20 74 65 73 74 20 61 62 61 63 6b 20 62 6f on test aback bo +| 4000: 6f 6d 65 72 20 61 74 6f 6d 20 61 6c 70 68 61 20 omer atom alpha +| 4016: 61 63 74 69 76 65 09 07 03 00 19 61 74 6f 6d 69 active.....atomi +| 4032: 63 07 06 03 00 15 61 74 6f 6d 00 00 00 0b 19 62 c.....atom.....b +| 4048: 6f 6f 6d 65 72 09 04 03 00 19 61 63 74 69 76 65 oomer.....active +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 0d 0f b2 00 0f fa 0f f4 0f ee 0f e8 ................ +| 16: 0f e2 0f dc 0f d6 0f d0 0f ca 0f c4 0f be 0f b8 ................ +| 32: 0f b2 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 04 0d 03 00 0e 0a 04 0c 03 00 0e 01 04 0b ................ +| 4032: 03 00 0e 01 04 0a 03 00 0e 03 04 09 03 00 0e 01 ................ +| 4048: 04 08 03 00 0e 0a 04 07 03 00 0e 01 04 06 03 00 ................ +| 4064: 0e 01 04 05 03 00 0e 03 04 04 03 00 0e 01 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end c22b.db +}]} {} + +do_catchsql_test 21.1 { + DELETE FROM t1 WHERE t1 MATCH 'ab*ndon'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 22.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename c22b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 02 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................ +| 48: 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 ................ +| 96: 00 2e 30 38 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..08...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 41 62 6c 65 74 31 5f 64 ..!!...tAblet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 75 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 uAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 0f 15 00 05 08 4e 00 0f e8 08 4e 0f bd 0f 29 ......N....N...) +| 16: 08 65 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .e.............. +| 2112: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 0a ................ +| 2128: 03 00 30 00 00 00 00 01 03 02 01 03 01 01 01 02 ..0............. +| 2144: 01 01 03 01 01 8d 28 8c 80 80 80 80 01 04 00 9a ......(......... +| 2160: 54 00 00 06 77 19 30 30 30 30 30 30 33 30 66 65 T...w.00000030fe +| 2176: 65 30 30 30 66 66 61 30 66 66 34 30 66 65 65 05 e000ffa0ff40fee. +| 2192: 02 19 02 0b 31 30 66 66 61 30 30 30 66 66 61 05 ....10ffa000ffa. +| 2208: 06 13 18 09 02 25 33 34 33 35 32 34 35 34 31 35 .....%3435245415 +| 2224: 34 34 35 32 30 35 34 34 31 34 32 34 63 34 35 32 445205441424c452 +| 2240: 30 32 37 37 34 33 31 35 66 36 39 05 02 07 02 37 02774315f69....7 +| 2256: 34 30 33 30 33 30 30 30 65 30 31 30 34 30 32 30 40303000e0104020 +| 2272: 33 30 30 30 65 30 31 30 34 30 31 30 33 30 30 30 3000e01040103000 +| 2288: 65 30 31 30 61 30 30 30 30 30 30 30 31 30 66 66 e010a000000010ff +| 2304: 34 30 30 30 66 66 34 05 02 1b 02 27 35 30 34 30 4000ff4....'5040 +| 2320: 39 30 63 30 31 30 32 30 64 30 30 30 30 30 30 30 90c01020d0000000 +| 2336: 33 30 66 65 30 30 30 30 66 66 36 30 66 65 63 30 30fe0000ff60fec0 +| 2352: 66 65 30 05 02 2c 27 02 66 30 05 02 16 02 1d 36 fe0..,'.f0.....6 +| 2368: 31 37 31 31 31 31 30 38 36 33 37 34 36 31 36 32 1711110863746162 +| 2384: 36 63 36 35 37 34 33 31 37 34 33 31 05 02 0d 02 6c6574317431.... +| 2400: 29 39 30 33 30 32 31 62 37 32 36 35 36 32 37 35 )903021b72656275 +| 2416: 36 34 36 31 37 34 36 31 37 34 33 31 35 66 36 34 6461746174315f64 +| 2432: 36 31 37 34 36 31 30 32 34 33 05 02 1f 02 43 61 6174610243....Ca +| 2448: 30 33 30 33 30 30 31 62 36 31 36 32 36 31 36 65 0303001b6162616e +| 2464: 36 34 36 66 36 65 30 38 30 32 30 33 30 30 31 37 646f6e0802030017 +| 2480: 36 31 36 32 36 31 36 36 37 34 30 38 30 31 30 33 6162616674080103 +| 2496: 30 30 31 37 36 31 36 32 36 31 36 33 36 62 30 64 0017616261636b0d +| 2512: 30 30 05 02 18 34 1d 33 36 62 30 31 30 32 30 32 00...4.36b010202 +| 2528: 30 34 30 32 36 36 37 34 30 32 30 32 30 32 30 34 0402667402020204 +| 2544: 30 34 36 65 05 02 2e 02 33 62 30 33 31 62 30 31 046e....3b031b01 +| 2560: 37 36 36 35 37 32 37 33 36 39 36 66 36 65 30 34 76657273696f6e04 +| 2576: 30 64 30 30 30 30 30 30 30 33 30 66 64 36 30 30 0d000000030fd600 +| 2592: 30 66 66 34 30 66 65 31 30 66 64 36 05 02 1d 01 0ff40fe10fd6.... +| 2608: 04 31 66 62 64 05 04 10 18 01 01 32 05 02 14 02 .1fbd......2.... +| 2624: 43 34 38 34 38 30 38 30 38 30 38 30 30 31 30 33 C484808080800103 +| 2640: 30 30 34 65 30 30 30 30 30 30 31 65 30 36 33 30 004e0000001e0630 +| 2656: 36 31 36 32 36 31 36 33 36 62 30 31 30 32 30 32 616261636b010202 +| 2672: 30 34 30 32 36 36 37 34 30 32 30 32 30 32 30 34 0402667402020204 +| 2688: 30 34 36 65 05 04 11 18 02 01 66 05 02 2a 03 02 046e......f..*.. +| 2704: 65 72 05 02 31 01 08 33 35 32 38 36 33 36 66 05 er..1..3528636f. +| 2720: 02 24 02 03 66 65 30 05 04 17 18 01 2b 34 31 35 .$..fe0.....+415 +| 2736: 32 35 39 32 30 34 62 34 35 35 39 32 63 32 30 36 259204b45592c206 +| 2752: 32 36 63 36 66 36 33 36 62 32 30 34 32 34 63 34 26c6f636b20424c4 +| 2768: 66 34 32 32 39 33 61 30 05 02 0c 2c 1f 31 30 36 f42293a0...,.106 +| 2784: 31 37 31 31 31 31 30 38 36 33 37 34 36 31 36 32 1711110863746162 +| 2800: 36 63 36 35 37 34 33 31 37 34 33 31 05 02 22 02 6c6574317431.... +| 2816: 40 33 35 32 34 35 34 31 35 34 34 35 32 30 35 36 @352454154452056 +| 2832: 34 39 35 32 35 34 37 35 34 31 34 63 32 30 35 34 49525475414c2054 +| 2848: 34 31 34 32 34 63 34 35 32 30 37 34 33 31 32 30 41424c4520743120 +| 2864: 35 35 35 33 34 39 34 65 34 37 32 30 36 36 37 34 5553494e47206674 +| 2880: 37 05 02 23 42 09 33 33 35 32 38 36 33 36 66 05 7..#B.33528636f. +| 2896: 02 0e 02 03 66 65 65 05 02 1a 01 34 35 32 34 35 ....fee....45245 +| 2912: 34 31 35 34 34 35 32 30 35 34 34 31 34 32 34 63 415445205441424c +| 2928: 34 35 32 30 32 37 37 34 33 31 35 66 36 34 36 31 45202774315f6461 +| 2944: 37 34 36 31 32 37 32 38 36 39 36 34 32 30 34 39 7461272869642049 +| 2960: 05 02 20 16 37 c3 b0 35 32 30 32 37 37 34 33 31 .. .7..520277431 +| 2976: 35 66 36 34 36 31 37 34 36 31 32 37 32 38 36 39 5f64617461272869 +| 2992: 36 34 32 30 34 39 34 65 35 34 34 35 34 37 34 35 6420494e54454745 +| 3008: 35 32 32 30 35 30 35 32 34 39 34 64 05 02 0b 03 52205052494d.... +| 3024: 48 35 39 32 30 34 62 34 35 35 39 32 63 32 30 36 H59204b45592c206 +| 3040: 33 33 30 32 39 36 39 30 33 30 30 30 31 35 37 35 3302969030001575 +| 3056: 33 31 31 39 32 36 64 37 34 36 31 36 32 36 63 36 311926d7461626c6 +| 3072: 35 37 34 33 31 35 66 36 39 36 34 37 38 37 34 33 574315f696478743 +| 3088: 31 35 66 36 39 36 34 37 38 05 02 06 02 38 34 32 15f696478....842 +| 3104: 30 35 32 34 66 35 37 34 39 34 34 35 35 30 32 30 0524f57494455020 +| 3120: 37 31 37 31 62 31 62 30 31 38 31 30 31 37 34 36 7171b1b018101746 +| 3136: 31 36 32 36 63 36 35 37 34 33 31 35 66 31 37 34 1626c6574315f174 +| 3152: 36 31 30 32 34 33 05 02 0a 02 03 66 66 34 05 02 610243.....ff4.. +| 3168: 1c 01 4a 36 34 36 66 36 65 30 33 30 32 30 32 30 ..J646f6e0302020 +| 3184: 34 30 61 30 37 30 35 30 31 30 33 30 30 31 30 30 40a0705010300100 +| 3200: 33 30 33 30 66 30 61 30 33 30 30 32 34 30 30 30 3030f0a030024000 +| 3216: 30 30 30 30 30 30 31 30 31 30 31 30 30 30 31 30 0000001010100010 +| 3232: 31 30 31 30 30 39 61 30 30 30 30 30 30 05 02 2f 101009a000000../ +| 3248: 42 09 31 30 61 30 30 30 30 30 30 05 02 28 08 43 B.10a000000..(.C +| 3264: 74 30 32 30 32 30 34 30 61 30 37 30 35 30 31 30 t0202040a0705010 +| 3280: 33 30 30 31 30 30 33 30 33 30 66 30 61 30 33 30 3001003030f0a030 +| 3296: 30 32 34 30 30 30 30 30 30 30 30 30 31 30 31 30 0240000000001010 +| 3312: 31 30 30 30 31 30 31 30 31 30 31 30 61 30 30 30 100010101010a000 +| 3328: 30 30 30 05 02 12 02 49 37 36 65 36 66 32 63 32 000....I76e6f2c2 +| 3344: 30 35 30 35 32 34 39 34 64 34 31 35 32 35 39 32 05052494d4152592 +| 3360: 30 34 62 34 35 35 39 32 38 37 33 36 35 36 37 36 04b4559287365676 +| 3376: 39 36 34 32 63 32 30 37 34 36 35 37 32 36 64 32 9642c207465726d2 +| 3392: 39 32 39 32 30 35 37 34 39 35 34 34 38 34 66 35 92920574954484f5 +| 3408: 35 05 02 09 02 4c 39 37 61 36 35 37 34 33 31 35 5....L97a6574315 +| 3424: 66 36 34 36 66 36 33 37 33 36 39 37 61 36 35 30 f646f6373697a650 +| 3440: 35 34 33 35 32 34 35 34 31 35 34 34 35 32 30 35 5435245415445205 +| 3456: 34 38 36 39 36 34 32 30 34 39 34 65 35 34 34 35 48696420494e5445 +| 3472: 34 37 34 35 35 32 32 30 35 30 35 32 34 39 34 34 4745522050524944 +| 3488: 64 31 05 02 05 02 27 65 37 34 36 35 36 65 37 34 d1....'e74656e74 +| 3504: 32 39 30 64 30 30 30 30 30 30 30 33 30 66 62 64 290d000000030fbd +| 3520: 30 30 30 62 65 38 30 66 65 66 30 66 62 64 05 02 000be80fef0fbd.. +| 3536: 0f 1e 0b 66 65 38 30 66 65 66 30 66 62 64 05 02 ...fe80fef0fbd.. +| 3552: 25 02 03 66 64 36 05 02 1e 01 4a 37 36 32 39 32 %..fd6....J76292 +| 3568: 30 35 37 34 39 35 34 34 38 34 66 35 35 35 34 32 0574954484f55542 +| 3584: 30 35 32 34 66 35 37 34 39 34 34 35 62 30 35 30 0524f5749445b050 +| 3600: 37 31 37 32 31 32 31 30 31 38 31 30 31 37 34 36 7172121018101746 +| 3616: 31 36 32 36 63 36 35 37 34 33 31 35 66 36 34 36 1626c6574315f646 +| 3632: 66 36 33 37 33 05 02 04 02 21 38 32 37 32 38 37 f6373....!827287 +| 3648: 33 36 35 36 37 36 39 36 34 32 63 32 30 37 34 36 3656769642c20746 +| 3664: 35 37 32 36 64 32 63 32 30 37 30 05 02 08 01 01 5726d2c2070..... +| 3680: 61 05 02 2b 01 06 62 61 63 6b 75 70 05 02 32 02 a..+..backup..2. +| 3696: 05 65 61 6d 65 72 05 02 02 02 05 6f 6f 6d 65 72 .eamer.....oomer +| 3712: 05 01 02 40 75 6d 6d 32 34 63 34 35 32 30 32 37 ...@umm24c452027 +| 3728: 37 34 33 31 35 66 36 33 36 66 36 65 36 36 36 39 74315f636f6e6669 +| 3744: 36 37 32 37 32 38 36 62 32 30 35 30 35 32 34 39 6727286b20505249 +| 3760: 34 64 34 31 35 32 35 39 32 30 34 62 34 35 35 39 4d415259204b4559 +| 3776: 32 63 32 30 05 02 03 01 15 65 35 34 34 35 34 37 2c20.....e544547 +| 3792: 34 35 35 32 32 30 35 30 35 32 34 39 34 64 05 02 4552205052494d.. +| 3808: 21 01 02 66 61 05 02 15 04 1d 12 2a 3c 2c 07 22 !..fa......*<,.. +| 3824: 2e 48 22 38 0a 06 49 06 07 0d 09 30 24 45 0e 08 .H.8..I....0$E.. +| 3840: 39 3c 4d 3d 08 4f 0e 48 4e 51 2c 10 08 4f 26 06 9.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 b2 04 55 85 44 54 e5 34 MIT LOAD..U.DT.4 +| 3216: 94 f4 e5 84 e4 f4 34 15 34 51 e1 f0 50 03 30 f1 ......4.4Q..P.0. +| 3232: 74 f4 d4 95 42 04 c4 f4 14 42 04 55 85 44 54 e5 t...B....B.U.DT. +| 3248: 34 94 f4 e5 85 25 45 24 94 d1 f1 e0 50 03 30 f1 4....%E$....P.0. +| 3264: 94 d4 15 82 04 d4 54 d4 f5 25 93 d3 53 03 03 03 ......T..%..S... +| 3280: 03 03 03 05 84 24 94 e4 15 25 91 f1 d0 50 03 30 .....$...%...P.0 +| 3296: f1 94 d4 15 82 04 d4 54 d4 f5 25 93 d3 53 03 03 .......T..%..S.. +| 3312: 03 03 03 03 05 84 e4 f4 34 15 34 51 e1 c0 50 03 ........4.4Q..P. +| 3328: 30 f1 74 d4 15 82 04 d4 54 d4 f5 25 93 d3 53 03 0.t.....T..%..S. +| 3344: 03 03 03 03 03 05 85 25 45 24 94 d1 81 b0 50 02 .......%E$....P. +| 3360: 50 f1 94 54 e4 14 24 c4 52 05 25 45 24 54 55 84 P..T..$.R.%E$TU. +| 3376: 24 94 e4 15 25 91 81 a0 50 02 50 f1 94 54 e4 14 $...%...P.P..T.. +| 3392: 24 c4 52 05 25 45 24 54 55 84 e4 f4 34 15 34 51 $.R.%E$TU...4.4Q +| 3408: 71 90 50 02 50 f1 74 54 e4 14 24 c4 52 05 25 45 q.P.P.tT..$.R.%E +| 3424: 24 54 55 85 25 45 24 94 d1 a1 80 50 02 90 f1 94 $TU.%E$....P.... +| 3440: 54 e4 14 24 c4 52 04 d4 54 d5 35 95 33 55 84 24 T..$.R..T.5.3U.$ +| 3456: 94 e4 15 25 91 a1 70 50 02 90 f1 94 54 e4 14 24 ...%..pP....T..$ +| 3472: c4 52 04 d4 54 d5 35 95 33 55 84 e4 f4 34 15 34 .R..T.5.3U...4.4 +| 3488: 51 91 60 50 02 90 f1 74 54 e4 14 24 c4 52 04 d4 Q.`P...tT..$.R.. +| 3504: 54 d5 35 95 33 55 85 25 45 24 94 d1 81 50 50 02 T.5.3U.%E$...PP. +| 3520: 50 f1 94 54 e4 14 24 c4 52 04 a5 34 f4 e3 15 84 P..T..$.R..4.... +| 3536: 24 94 e4 15 25 91 81 40 50 02 50 f1 94 54 e4 14 $...%..@P.P..T.. +| 3552: 24 c4 52 04 a5 34 f4 e3 15 84 e4 f4 34 15 34 51 $.R..4......4.4Q +| 3568: 71 30 50 02 50 f1 74 54 e4 14 24 c4 52 04 a5 34 q0P.P.tT..$.R..4 +| 3584: f4 e3 15 85 25 45 24 94 d1 a1 20 50 02 90 f1 94 ....%E$... P.... +| 3600: 54 e4 14 24 c4 52 04 74 54 f5 04 f4 c5 95 84 24 T..$.R.tT......$ +| 3616: 94 e4 15 25 91 a1 10 50 02 90 f1 94 54 e4 14 24 ...%...P....T..$ +| 3632: c4 52 04 74 54 f5 04 f4 c5 95 84 e4 f4 34 15 34 .R.tT........4.4 +| 3648: 51 91 00 50 02 90 f1 74 54 e4 14 24 c4 52 04 74 Q..P...tT..$.R.t +| 3664: 54 f5 04 f4 c5 95 85 25 45 24 94 d1 70 f0 50 02 T......%E$..p.P. +| 3680: 30 f1 94 54 e4 14 24 c5 20 46 54 53 35 58 42 49 0..T..$. FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 97 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 02 02 50 08 5f 17 44 45 42 55 47 CASE...P._.DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end crash-b87dfef02880fe.db +}]} {} + +do_catchsql_test 24.1 { + UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thread*'; +} {1 {database disk image is malformed}} + +do_catchsql_test 24.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 25.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-e3b1b19e4d4bcc.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 06 00 00 00 00 .....@ ........ +| 32: 00 00 00 00 00 00 00 06 00 00 00 04 00 00 00 00 ................ +| 48: 03 20 54 35 24 54 15 44 52 04 94 e4 44 55 82 07 . T5$T.DR...DU.. +| 64: 43 27 a2 04 f4 e2 07 43 22 87 a2 95 30 30 71 71 C'.....C....00qq +| 80: 11 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 .SQLite format 3 +| 96: 00 10 00 01 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 03 30 01 00 00 10 .....N.....0.... +| 128: 10 04 02 02 00 00 00 00 00 00 00 00 30 00 00 00 ............0... +| 144: 00 00 00 00 00 00 00 00 20 00 00 00 40 00 00 00 ........ ...@... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 d9 44 (id INTEGER PR.D +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f e0 fe 00 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 00 03 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 01 fe 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end crash-e3b1b19e4d4bcc.db +}]} {} + +do_catchsql_test 25.1 { + INSERT INTO t1(t1) VALUES('rebuild'); +} {1 {database disk image is malformed}} + +do_execsql_test 25.2 { + PRAGMA page_size=512; +} + +#-------------------------------------------------------------------------- +reset_db +do_test 26.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename c30b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 30 38 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ..08...........6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 0c 0f 00 05 09 fe 00 0f e6 09 fe 0c 94 0c 23 ...............# +| 16: 0a 15 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 2544: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 0a ................ +| 2560: 03 00 30 00 00 00 00 01 03 03 00 03 01 01 01 02 ..0............. +| 2576: 01 01 03 01 01 83 72 8c 80 80 80 80 01 04 00 87 ......r......... +| 2592: 68 00 00 01 e4 02 30 30 03 03 06 02 83 0f 30 30 h.....00......00 +| 2608: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2624: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2640: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2656: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2672: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2688: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2704: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2720: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2736: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2752: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2768: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2784: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2800: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2816: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2832: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2848: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2864: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2880: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2896: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2912: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2928: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2944: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2960: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2976: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2992: 30 30 30 30 30 30 30 30 30 30 30 30 30 03 06 01 0000000000000... +| 3008: 01 03 01 08 32 30 31 36 30 36 30 39 03 03 07 01 ....20160609.... +| 3024: 01 34 03 03 05 01 01 35 03 03 04 01 06 62 69 6e .4.....5.....bin +| 3040: 61 72 79 03 07 01 02 02 01 08 63 6f 6d 70 69 6c ary.......compil +| 3056: 65 72 03 03 02 01 03 67 63 63 03 03 03 01 01 78 er.....gcc.....x +| 3072: 03 07 01 01 02 04 06 83 17 0d 06 06 0d 0d 08 0f ................ +| 3088: ef 00 14 2a 00 00 00 00 01 02 02 00 02 01 01 01 ...*............ +| 3104: 02 01 01 6a 88 80 80 80 80 01 04 00 81 58 00 00 ...j.........X.. +| 3120: 00 5f 07 30 62 69 6e 61 72 79 0c 01 03 01 01 06 ._.0binary...... +| 3136: 65 6e 61 62 6c 65 0a 01 01 01 01 01 01 01 01 01 enable.......... +| 3152: 01 01 01 04 66 74 73 34 0a 01 01 01 01 01 04 01 ....fts4........ +| 3168: 35 0d 01 01 01 01 01 01 06 6e 6f 63 61 73 65 0b 5........nocase. +| 3184: 01 03 01 01 05 72 74 72 69 6d 0a 01 03 01 01 01 .....rtrim...... +| 3200: 78 0a 01 01 01 01 01 01 01 01 01 01 01 04 0c 14 x............... +| 3216: 0c 09 0c 0b 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56 ..threadsafe.W4V +| 3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02 .d.F...vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 1e 4e 1f 1e 00 D..@........N... +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 0e 5b 00 1e 0a 4d 00 0f d8 0f af 0a 4d 0f 74 ..[...M......M.t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e 40 0e 24 0e 08 .a.N./.....@.$.. +| 32: 0d ef 0d d5 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 ...........h.O.5 +| 48: 0d 1b 0c fb 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e ...........x.W.> +| 64: 0c 24 0c 0a 0c 0a 0c 0a 0c 0a 0c 0a 0c 0a 0c 0a .$.............. +| 2624: 00 00 00 00 00 00 00 00 00 00 00 00 00 83 3a 03 ..............:. +| 2640: 06 00 43 86 33 19 43 4f 4d 50 49 4c 45 52 3d 67 ..C.3.COMPILER=g +| 2656: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 2672: 39 58 27 30 30 30 30 30 30 30 30 30 30 30 30 30 9X'0000000000000 +| 2688: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2704: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2720: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2736: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2752: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2768: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2784: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2800: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2816: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2832: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2848: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2864: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2880: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2896: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2912: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2928: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2944: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2960: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2976: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 2992: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 3008: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 3024: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 3040: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 3056: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 3072: 30 30 30 27 42 49 4e 41 52 59 18 24 05 00 25 0f 000'BINARY.$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 B`-EMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 0f 86 00 94 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 00 00 00 7b 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 00 00 00 E FTS5XNOCASE... +| 3728: 62 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 b#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 00 00 00 4a 23 0f 19 45 4e 41 42 RTRIM...J#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 00 00 LE FTS4XBINARY.. +| 3776: 00 31 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 .1#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 00 00 00 18 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d TAT VT$.HNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 00 00 00 29 43 0f 19 43 4f 4d XRTRIM...)C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 0f 88 00 1e 0e e0 00 0f f8 0f f0 0f b0 0f e0 ................ +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f 80 0f 78 0f 70 .............x.p +| 32: 0f 68 0f 60 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 .h.`.X.P.H.@.8.0 +| 48: 0f 28 0f 20 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 .(. ............ +| 64: 0e e8 0e e0 0e e0 0e e0 0e e0 0e e0 0e e0 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 0f e8 00 28 12 02 01 01 ...........(.... +| 3984: 00 00 00 28 12 02 01 01 00 00 00 20 12 02 01 01 ...(....... .... +| 4000: 00 00 00 18 12 02 01 01 00 00 00 10 12 02 01 01 ................ +| 4016: 06 03 03 00 12 06 02 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 00 00 00 08 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end c30b.db +}]} {} + +do_catchsql_test 26.1 { + BEGIN; + INSERT INTO t1(t1) VALUES('rebuild'); + INSERT INTO t1(t1) VALUES('integrity-check'); + COMMIT; +} {0 {}} + +#-------------------------------------------------------------------------- +reset_db +do_test 27.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename timeout-2ca5b0658c98.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 20 01 02 02 02 01 02 02 01 06 64 iler. .........d +| 3408: 62 73 7c cc cc cc cc cc cc cc cc cc cc cc cc cc bs|............. +| 3424: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc ................ +| 3440: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc ................ +| 3456: cc cc cc cc cc cc c4 61 74 07 02 03 01 02 03 01 .......at....... +| 3472: 02 03 02 04 65 62 75 67 04 02 02 01 02 02 01 02 ....ebug........ +| 3488: 02 01 06 65 6e 61 62 6c 65 07 02 02 01 02 02 01 ...enable....... +| 3504: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3520: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3536: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 05 23 ...............# +| 3552: d6 76 36 32 d3 52 e3 42 e3 02 03 23 03 13 63 03 .v62.R.B...#..c. +| 3568: 63 03 95 84 e4 f4 34 15 34 52 60 10 50 04 30 f1 c.....4.4R`.P.0. +| 3584: 74 34 f4 d5 04 94 c4 55 23 d6 76 36 32 d3 52 e3 t4.....U#.v62.R. +| 3600: 42 e3 02 03 23 03 13 63 03 63 03 95 85 25 45 24 B...#..c.c...%E$ +| 3616: 94 d0 d0 00 00 02 40 ee 00 00 ff 80 ff 00 fe 80 ......@......... +| 3632: fe 00 fd 80 fd 00 fc 80 fc 00 fb 80 fb 00 fa 80 ................ +| 3648: fa 00 f9 80 f9 00 f8 80 f8 00 f7 80 f7 00 f6 80 ................ +| 3664: f6 00 f5 80 f5 00 f4 80 f4 00 f8 0f 30 0f 28 0f ............0.(. +| 3680: 20 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e ............... +| 3696: e0 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 5f e8 54 45 4e OMIT LOAD E_.TEN +| 3248: 53 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f SIONXRTRIM....3. +| 3264: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 31 81 .MAX MEMORY=501. +| 3280: 40 50 02 50 f1 94 54 e4 14 24 c4 52 04 a5 35 f4 @P.P..T..$.R..5. +| 3296: e3 15 84 e4 f4 34 15 34 51 71 30 50 02 50 f1 74 .....4.4Qq0P.P.t +| 3312: 54 e4 14 24 c4 52 04 a5 34 f4 e3 15 85 25 45 24 T..$.R..4....%E$ +| 3328: 94 d1 a1 20 50 02 90 f1 94 54 e4 14 24 c4 52 04 ... P....T..$.R. +| 3344: 74 54 f5 04 f4 c5 95 84 24 94 e4 15 25 91 a1 10 tT......$...%... +| 3360: 50 02 90 f1 94 54 e4 14 24 c4 52 04 74 54 f5 04 P....T..$.R.tT.. +| 3376: f4 c5 95 84 e4 f4 34 15 34 51 91 00 50 02 90 f1 ......4.4Q..P... +| 3392: 74 54 e4 14 24 c4 52 04 74 54 f5 04 f4 c5 95 85 tT..$.R.tT...... +| 3408: 25 45 24 94 d1 70 f0 50 02 30 f1 94 54 e4 14 24 %E$..p.P.0..T..$ +| 3424: c4 52 04 65 45 33 55 84 24 94 e4 15 25 91 70 e0 .R.eE3U.$...%.p. +| 3440: 50 02 30 f1 94 54 e4 14 24 c4 52 04 65 45 33 55 P.0..T..$.R.eE3U +| 3456: 84 e4 f4 34 15 34 51 60 d0 50 02 30 f1 74 54 e4 ...4.4Q`.P.0.tT. +| 3472: 14 24 c4 52 04 65 45 33 55 85 25 45 24 94 d1 70 .$.R.eE3U.%E$..p +| 3488: c0 50 02 30 f1 94 54 e4 14 24 c4 52 04 65 45 33 .P.0..T..$.R.eE3 +| 3504: 45 84 24 94 e4 15 25 91 70 b0 50 02 30 f1 94 54 E.$...%.p.P.0..T +| 3520: e4 14 24 c4 52 04 65 45 33 45 84 e4 f4 34 15 34 ..$.R.eE3E...4.4 +| 3536: 51 60 a0 50 02 30 f1 74 54 e4 14 24 c4 52 04 65 Q`.P.0.tT..$.R.e +| 3552: 45 33 45 85 25 45 24 94 d1 e0 90 50 03 10 f1 94 E3E.%E$....P.... +| 3568: 54 e4 42 4c 45 20 44 42 53 54 41 54 20 56 54 41 T.BLE DBSTAT VTA +| 3584: 42 58 42 49 4e 41 52 59 1e 08 05 00 31 0f 19 45 BXBINARY....1..E +| 3600: e4 14 24 c4 52 04 44 25 35 44 15 42 05 65 42 41 ..$.R.D%5D.B.eBA +| 3616: 54 84 e4 f4 34 15 34 51 d0 70 50 03 10 f1 74 54 T...4.4Q.pP...tT +| 3632: e4 14 24 c4 52 04 44 25 35 44 15 42 05 65 44 14 ..$.R.D%5D.B.eD. +| 3648: 25 85 25 45 24 94 d1 10 60 50 01 70 f1 94 44 54 %.%E$...`P.p..DT +| 3664: 25 54 75 84 24 94 e4 15 25 91 10 50 50 01 70 f1 %Tu.$...%..PP.p. +| 3680: 94 44 54 25 54 75 84 e4 f4 34 15 34 51 00 40 50 .DT%Tu...4.4Q.@P +| 3696: 01 70 f1 74 44 54 25 54 75 85 25 45 24 94 d2 70 .p.tDT%Tu.%E$..p +| 3712: 30 50 04 30 f1 94 34 f4 d5 04 94 c4 55 23 d6 76 0P.0..4.....U#.v +| 3728: 36 32 d3 52 e3 42 e3 02 03 23 03 13 63 03 63 03 62.R.B...#..c.c. +| 3744: 95 84 24 94 e4 15 25 92 70 20 50 04 30 f1 94 34 ..$...%.p P.0..4 +| 3760: f4 d5 04 94 c4 53 30 01 00 00 10 10 04 02 02 00 .....S0......... +| 3776: 00 00 00 00 00 00 00 80 00 00 00 20 00 00 00 10 ........... .... +| 3792: 00 00 00 90 00 00 00 40 00 00 00 00 00 00 00 00 .......@........ +| page 6 offset 20480 +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end timeout-2ca5b0658c98.db +}]} {} + +do_catchsql_test 27.1 { + DELETE FROM t1 WHERE a MATCH 'fts*'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 28.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-e2d47e0624a42c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 20 ....c.....g.... +| 4048: 10 16 80 10 60 10 20 30 10 16 90 10 60 10 20 40 ....`. 0....`. @ +| 4064: 40 60 60 60 80 80 70 10 30 01 40 30 90 00 90 00 @```..p.0.@0.... +| 4080: 00 01 12 40 00 00 00 00 10 10 10 00 10 10 10 10 ...@............ +| page 3 offset 8192 +| 0: a0 00 00 00 30 fe c0 00 ff a0 ff 30 fe 00 00 00 ....0......0.... +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 01 01 68 .d...e...f.....h +| 4048: 01 01 01 01 01 69 01 01 01 04 06 06 06 04 04 04 .....i.......... +| 4064: 06 06 07 01 03 00 14 03 09 09 09 0f 0a 03 00 24 ...............$ +| 4080: 00 00 00 00 01 01 01 00 01 01 01 01 0a 00 00 00 ................ +| page 8 offset 28672 +| 0: 01 0f fa 00 0f fa 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 1f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-e2d47e0624a42c.db +}]} {} + +do_catchsql_test 28.1 { + SELECT count( fts5_decode(id, block) ) FROM t2_data; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 29.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-e114c036e13dde.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 06 0d b6 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d b6 00 00 00 00 00 00 ...k............ +| 3504: 00 00 00 00 00 00 56 07 06 17 1f 1f 01 7d 74 61 ......V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c af 00 0f e6 0f ef 0c af 00 00 ................ +| 3232: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 86 ................ +| 3248: 2f 84 80 80 80 80 01 04 00 8c 62 00 00 03 12 02 /.........b..... +| 3264: 30 30 01 04 05 03 01 04 05 03 01 04 05 03 1f 02 00.............. +| 3280: 03 01 02 03 01 02 03 01 08 35 30 30 30 30 30 30 .........5000000 +| 3296: 30 1c 02 04 01 02 04 01 02 04 01 01 36 01 02 04 0...........6... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 05 63 6c 61 6e ............clan +| 3392: 67 01 02 03 01 02 03 01 02 03 02 07 6f 6d 70 69 g...........ompi +| 3408: 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 62 ler...........db +| 3424: 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 65 stat...........e +| 3440: 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 6e bug...........en +| 3456: 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 02 able............ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3520: 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 04 .....xtension... +| 3536: 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 03 ........fts4.... +| 3552: 01 02 01 02 03 04 01 35 0d 02 03 01 02 03 01 02 .......5........ +| 3568: 03 01 07 67 65 6f 70 6f 6c 79 10 02 03 01 02 03 ...geopoly...... +| 3584: 01 02 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 .....json1...... +| 3600: 01 02 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 .....load....... +| 3616: 02 03 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 ....max......... +| 3632: 02 05 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 ..emory......... +| 3648: 04 04 73 79 73 35 16 02 03 01 02 03 01 02 03 01 ..sys5.......... +| 3664: 06 6e 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 .nocase......... +| 3680: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3696: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 3712: 06 01 02 02 03 06 01 02 02 03 06 01 12 02 03 06 ................ +| 3728: 01 02 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 .....omit....... +| 3744: 02 02 01 05 72 74 72 65 65 19 02 03 01 02 03 01 ....rtree....... +| 3760: 02 03 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 ....im.......... +| 3776: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 3792: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3808: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3824: 02 02 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 ....threadsafe.. +| 3840: 02 01 02 02 01 02 02 01 04 76 74 61 62 07 02 04 .........vtab... +| 3856: 01 02 04 01 02 04 01 01 78 01 06 01 01 02 01 06 ........x....... +| 3872: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3888: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3904: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3920: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3936: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3952: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3968: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3984: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 4000: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 4016: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4032: 01 01 02 01 06 01 01 02 01 06 01 01 02 04 18 13 ................ +| 4048: 0c 44 10 12 11 0f 47 13 0f 0c 12 10 0f 0e 10 0f .D....G......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 57 24 24 0f D..@.......$W$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 1f 00 0f df 0f bd 0f 9b 0f 89 ....$........... +| 16: 0f 76 0f 63 0f 44 0f 24 0f 04 0e ec 0e d3 0e ba .v.c.D.$........ +| 32: 0e a2 0e 89 0e 70 e5 50 e3 90 e1 d0 e0 40 de a0 .....p.P.....@.. +| 48: dd 00 db 50 d9 90 d7 ea ca ea be d0 d6 40 d4 a0 ...P.........@.. +| 64: d3 00 d1 00 ce f0 cc e0 ca e0 c8 d0 c6 c0 c5 30 ...............0 +| 80: c3 90 c1 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3088: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 18 ................ +| 3104: 24 05 00 25 0f 19 54 48 52 45 41 44 53 41 46 45 $..%..THREADSAFE +| 3120: 3d 30 58 42 49 4e 41 52 59 18 23 05 00 25 0f 19 =0XBINARY.#..%.. +| 3136: 54 48 52 45 41 44 53 41 46 45 3d 30 58 4e 4f 43 THREADSAFE=0XNOC +| 3152: 41 53 45 17 22 05 00 25 0f 17 54 48 52 45 41 44 ASE....%..THREAD +| 3168: 53 41 46 45 3d 30 05 00 33 0f 19 4f 4d 49 54 20 SAFE=0..3..OMIT +| 3184: 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f 4e 58 42 LOAD EXTENSIONXB +| 3200: 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f 4d 49 54 INARY. ..3..OMIT +| 3216: 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f 4e 58 LOAD EXTENSIONX +| 3232: 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 4f 4d 49 NOCASE....3..OMI +| 3248: 54 20 4c f4 14 42 04 55 85 44 54 e5 34 94 f4 e5 T L..B.U.DT.4... +| 3264: 85 25 45 24 94 d1 f1 e0 50 03 30 f1 94 d4 15 82 .%E$....P.0..... +| 3280: 04 d4 54 d4 f5 25 93 d3 53 03 03 03 03 03 03 05 ..T..%..S....... +| 3296: 84 24 94 e4 15 25 91 f1 d0 50 03 30 f1 94 d4 15 .$...%...P.0.... +| 3312: 82 04 d4 54 d4 f5 25 93 d3 53 03 03 03 03 03 03 ...T..%..S...... +| 3328: 05 84 e4 f4 34 15 34 51 e1 c0 50 03 30 f1 74 d4 ....4.4Q..P.0.t. +| 3344: 15 82 04 d4 54 d4 f2 90 f1 74 54 e4 14 24 c4 52 ....T....tT..$.R +| 3360: 04 74 54 f5 04 f4 c5 95 85 25 45 24 94 d1 70 f0 .tT......%E$..p. +| 3376: 50 02 30 f1 94 54 e4 14 24 c4 52 04 65 45 33 55 P.0..T..$.R.eE3U +| 3392: 84 24 94 e4 15 25 91 70 e0 50 02 30 f1 94 54 e4 .$...%.p.P.0..T. +| 3408: 14 24 c4 52 04 65 45 33 55 84 e4 f4 34 15 34 51 .$.R.eE3U...4.4Q +| 3424: 60 d0 50 02 30 f1 74 54 e4 14 24 c4 52 04 65 45 `.P.0.tT..$.R.eE +| 3440: 33 55 85 25 45 24 94 d1 70 c0 50 02 30 f1 94 54 3U.%E$..p.P.0..T +| 3456: e4 14 24 c4 52 04 65 45 33 45 84 24 94 e4 15 25 ..$.R.eE3E.$...% +| 3472: 91 70 b0 50 02 30 f1 94 54 e4 14 24 c4 52 04 65 .p.P.0..T..$.R.e +| 3488: 45 33 45 84 e4 f4 34 15 34 51 60 a0 74 54 e4 14 E3E...4.4Q`.tT.. +| 3504: 24 c4 52 04 65 45 33 45 85 25 45 24 94 d1 e0 90 $.R.eE3E.%E$.... +| 3520: 50 03 10 f1 94 54 e4 14 24 c4 52 04 44 25 35 44 P....T..$.R.D%5D +| 3536: 15 42 05 65 44 14 25 84 24 94 e4 15 25 91 e0 80 .B.eD.%.$...%... +| 3552: 50 03 10 f1 94 54 e4 14 24 c4 52 04 44 25 35 44 P....T..$.R.D%5D +| 3568: 15 42 05 65 44 14 25 84 e4 f4 34 15 34 51 d0 70 .B.eD.%...4.4Q.p +| 3584: 50 03 10 f1 74 54 e4 14 24 c4 52 04 44 25 35 44 P...tT..$.R.D%5D +| 3600: 15 42 05 65 44 14 25 85 25 45 24 94 d1 10 60 50 .B.eD.%.%E$...`P +| 3616: 01 70 f1 94 44 54 25 54 75 84 24 94 e4 15 25 91 .p..DT%Tu.$...%. +| 3632: 10 50 50 01 70 f1 94 44 54 25 54 75 84 e4 f4 34 .PP.p..DT%Tu...4 +| 3648: 15 34 51 00 40 50 01 70 f1 74 44 54 25 54 75 85 .4Q.@P.p.tDT%Tu. +| 3664: 25 45 24 94 d2 00 30 50 03 50 f1 94 34 f4 d5 04 %E$...0P.P..4... +| 3680: 94 c4 55 23 d6 36 c6 16 e6 72 d3 62 e3 02 e3 05 ..U#.6...r.b.... +| 3696: 84 24 94 e4 15 25 92 00 20 50 03 50 f1 94 34 f4 .$...%.. P.P..4. +| 3712: d5 04 94 c4 55 23 d6 36 c6 16 e6 72 d3 62 e3 02 ....U#.6...r.b.. +| 3728: e3 05 84 e4 f4 34 15 34 51 f0 10 50 03 50 f1 74 .....4.4Q..P.P.t +| 3744: 34 f4 d5 04 94 c4 55 23 d6 36 c6 16 e6 72 d3 62 4.....U#.6...r.b +| 3760: e3 02 e3 05 85 25 45 24 94 d0 d0 00 00 02 40 ee .....%E$......@. +| 3776: 00 00 ff 80 ff 00 fe 80 fe 00 fd 80 fd 00 fc 80 ................ +| 3792: fc 00 fb 80 fb 00 fa 80 fa 00 f9 80 f9 00 f8 80 ................ +| 3808: f8 00 f7 80 f7 00 f6 80 f6 00 f5 80 f5 00 f4 80 ................ +| 3824: f4 00 f3 80 f3 00 f2 80 f2 00 f1 80 f1 00 f0 80 ................ +| 3840: f0 00 ef 80 ef 00 ee 80 ee 00 00 00 00 00 00 00 ................ +| page 6 offset 20480 +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 05 01 01 ................ +| 4080: 06 02 03 00 12 05 01 01 06 01 03 00 12 05 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-e114c036e13dde.db +}]} {} + +do_catchsql_test 29.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1','col'); +} {0 {}} +do_catchsql_test 29.2 { + SELECT rowid, quote(term), * FROM t3 WHERE term=='nocase'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 30.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-eef41e30b388a0.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 00 00 66 46 08 08 0f ef 00 14 2a 00 00 00 ....fF......*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 52 06 66 72 06 82 06 90 .a b cd R.fr.... +| page 5 offset 16384 +| 0: d0 00 00 00 30 fe 80 00 ff 80 ff 00 fe 00 00 00 ....0........... +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-eef41e30b388a0.db +}]} {} + +do_catchsql_test 30.1 { + SELECT fts5_decode(id, block) FROM t1_data; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 31.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 8192 pagesize 4096 filename crash-7629f35f11d48e.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 01 0f c7 00 0f c7 00 00 ................ +| 4032: 00 00 00 00 00 00 00 37 01 06 17 15 15 01 53 74 .......7......St +| 4048: 61 62 6c 65 64 75 61 6c 64 75 61 6c 02 43 52 45 abledualdual.CRE +| 4064: 41 54 45 20 54 41 42 4c 45 20 64 75 61 6c 28 64 ATE TABLE dual(d +| 4080: 75 6d 6d 79 20 76 61 72 28 31 29 29 0d 00 00 00 ummy var(1)).... +| page 2 offset 4096 +| 0: 01 0f fb 00 0f fb 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 00 03 01 02 0f 58 ...............X +| end crash-7629f35f11d48e.db +}]} {} + +do_execsql_test 31.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a,b,c); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<72) + INSERT INTO t1(a) SELECT randomblob(2829) FROM c; + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<10) + INSERT INTO t1(a) SELECT randomblob(3000) FROM c; +} + +do_catchsql_test 31.2 { + DELETE FROM t1 WHERE a MATCH X'6620e574f32a'; +} {0 {}} + +#------------------------------------------------------------------------- +reset_db +do_test 32.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-e2d47e0624a42c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 20 ....c.....g.... +| 4048: 10 16 80 10 60 10 20 30 10 16 90 10 60 10 20 40 ....`. 0....`. @ +| 4064: 40 60 60 60 80 80 70 10 30 01 40 30 90 00 90 00 @```..p.0.@0.... +| 4080: 00 01 12 40 00 00 00 00 10 10 10 00 10 10 10 10 ...@............ +| page 3 offset 8192 +| 0: a0 00 00 00 30 fe c0 00 ff a0 ff 30 fe 00 00 00 ....0......0.... +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 01 01 68 .d...e...f.....h +| 4048: 01 01 01 01 01 69 01 01 01 04 06 06 06 04 04 04 .....i.......... +| 4064: 06 06 07 01 03 00 14 03 09 09 09 0f 0a 03 00 24 ...............$ +| 4080: 00 00 00 00 01 01 01 00 01 01 01 01 0a 00 00 00 ................ +| page 8 offset 28672 +| 0: 01 0f fa 00 0f fa 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 1f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-e2d47e0624a42c.db +}]} {} + +do_catchsql_test 32.1 { + SELECT snippet(t1, -1, '.', '..', '[', ']'), + highlight(t1, 2, '[', ']') + FROM t1('g + h') + WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; +} {1 {database disk image is malformed}} + +do_catchsql_test 32.2 { + SELECT * FROM t3; +} {1 {database disk image is malformed}} + +do_catchsql_test 32.3 { + SELECT * FROM t4; +} {1 {database disk image is malformed}} + +do_catchsql_test 32.4 { + SELECT fts5_decode(id, block) FROM t1_data; +} {1 {database disk image is malformed}} + +do_catchsql_test 32.5 { + SELECT fts5_decode(id, block) FROM t2_data; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 33.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-fed6e90021ba5d.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 06 0d b6 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d b6 00 00 00 00 00 00 ...k............ +| 3504: 00 00 00 00 00 00 56 07 06 17 1f 1f 01 7d 74 61 ......V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 8a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsi.et1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c af 00 0f e6 0f ef 0c af 00 00 ................ +| 3232: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 86 ................ +| 3248: 2f 84 80 80 80 80 01 04 00 8c 62 00 00 03 12 02 /.........b..... +| 3264: 30 30 01 04 05 03 01 04 05 03 01 04 05 03 1f 02 00.............. +| 3280: 03 01 02 03 01 02 03 01 08 35 30 30 30 30 30 30 .........5000000 +| 3296: 30 1c 02 04 01 0e ee ca ec ea ea ab e4 f5 ca b1 0............... +| 3312: ac ee ec de ef 3e ee ca ee ec f2 f8 0f f0 0f e8 .....>.......... +| 3328: 0f e0 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 ................ +| 3344: 0f a0 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 ...........x.p.h +| 3360: 0f 60 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 .`.X.P.H.@.8.0.( +| 3376: 0f 20 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 . .............. +| 3392: 0e e0 02 03 01 02 03 01 02 03 02 07 6f 6d 70 69 ............ompi +| 3408: 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 62 ler...........db +| 3424: 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 65 stat...........e +| 3440: 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 6e bug...........en +| 3456: 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 02 able............ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3520: 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 04 .....xtension... +| 3536: 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 03 ........fts4.... +| 3552: 01 02 01 02 03 04 01 35 0d 02 03 01 02 03 01 02 .......5........ +| 3568: 03 01 07 67 65 6f 70 6f 6c 79 10 02 03 01 02 03 ...geopoly...... +| 3584: 01 02 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 .....json1...... +| 3600: 01 02 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 .....load....... +| 3616: 02 03 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 ....max......... +| 3632: 02 05 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 ..emory......... +| 3648: 04 04 73 79 73 35 16 02 03 01 02 03 01 02 03 01 ..sys5.......... +| 3664: 06 6e 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 .nocase......... +| 3680: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3696: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 3712: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3728: 01 02 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 .....omit....... +| 3744: 02 02 01 05 72 74 72 65 65 19 02 03 01 02 03 01 ....rtree....... +| 3760: 02 03 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 ....im.......... +| 3776: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 3792: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3808: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3824: 02 02 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 ....threadsafe.. +| 3840: 02 01 02 02 01 02 02 01 04 76 74 61 62 07 02 04 .........vtab... +| 3856: 01 02 04 01 02 04 01 01 78 01 06 01 01 02 01 06 ........x....... +| 3872: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3888: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3904: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3920: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3936: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3952: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3968: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3984: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 4000: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 4016: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4032: 01 01 02 01 06 01 01 02 01 06 01 01 02 04 18 13 ................ +| 4048: 0c 44 10 12 11 0f 47 13 0f 0c 12 10 0f 0e 10 0f .D....G......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 57 24 24 0f D..@.......$W$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 1f 00 0f df 0f bd 0f 9b 0f 89 ....$........... +| 16: 0f 76 0f 63 0f 44 0f 24 0f 04 0e ec 0e d3 0e ba .v.c.D.$........ +| 32: 0e a2 0e 89 0e 70 0e 55 0e 39 0e 1d 0e 04 0d ea .....p.U.9...... +| 48: 0d d0 0d b5 0d 99 0d 7d 0d 64 0d 4a 0d 30 0d 10 .........d.J.0.. +| 64: 0c ef 0c ce 0c ae 54 d5 35 95 33 55 84 24 94 e4 ......T.5.3U.$.. +| 80: 15 25 91 a1 70 50 02 90 f1 94 54 e4 14 24 c4 52 .%..pP....T..$.R +| 96: 04 d4 54 d5 35 95 33 55 84 e4 f4 34 15 34 51 91 ..T.5.3U...4.4Q. +| 112: 60 50 02 90 f1 74 54 e4 14 24 c4 52 04 d4 54 d5 `P...tT..$.R..T. +| 128: 35 95 33 55 85 25 45 24 94 d1 81 50 50 02 50 f1 5.3U.%E$...PP.P. +| 144: 94 54 e4 14 24 c4 52 04 a5 34 f4 e3 15 84 24 94 .T..$.R..4....$. +| 160: e4 15 25 91 81 40 50 02 50 f1 94 54 e4 14 24 c4 ..%..@P.P..T..$. +| 176: 52 04 a5 34 f4 e3 15 84 e4 f4 34 15 34 51 71 30 R..4......4.4Qq0 +| 192: 50 02 50 f1 74 54 e4 14 24 c4 52 04 ae 4f 41 33 P.P.tT..$.R..OA3 +| 208: 55 85 25 45 24 94 d1 a1 20 50 02 90 f1 94 54 e4 U.%E$... P....T. +| 224: 14 24 c4 52 04 74 54 f5 04 f4 c5 95 84 24 94 e4 .$.R.tT......$.. +| 240: 15 25 91 a1 10 50 02 90 f1 94 54 e4 14 24 c4 52 .%...P....T..$.R +| 256: 04 74 54 f5 04 f4 c5 95 84 e4 f4 34 15 34 51 91 .tT........4.4Q. +| 272: 00 50 02 90 f1 74 54 e4 14 24 c4 52 04 74 54 f5 .P...tT..$.R.tT. +| 288: 04 f4 c5 95 85 25 45 24 94 d1 70 f0 50 02 30 f1 .....%E$..p.P.0. +| 304: 94 54 e4 14 24 c4 52 04 65 45 33 55 84 24 94 e4 .T..$.R.eE3U.$.. +| 320: 15 25 91 70 e0 50 02 30 f1 94 54 e4 40 0f 38 0f .%.p.P.0..T.@.8. +| 336: 30 0f 28 0f 20 0f 18 0f 10 0f 08 0f 00 0e f8 0e 0.(. ........... +| 352: f0 0e e8 0e e0 00 00 00 00 00 00 00 00 00 00 00 ................ +| page 6 offset 20480 +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 05 01 01 ................ +| 4080: 06 02 03 00 12 05 01 01 06 01 03 00 12 05 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 52 59 11 05 05 00 ..........RY.... +| 16: 17 0f 19 44 45 42 55 47 58 4e 4f 43 41 53 45 10 ...DEBUGXNOCASE. +| 32: 04 05 00 17 0f 17 44 45 42 55 47 58 52 54 52 49 ......DEBUGXRTRI +| 48: 4d 20 03 05 00 35 0f 19 43 4f 4d 50 49 4c 45 52 M ...5..COMPILER +| 64: 3d 63 6c 61 6e 67 2d 36 2e 30 2e 30 58 42 49 4e =clang-6.0.0XBIN +| 80: 41 52 59 20 02 05 00 35 0f 19 43 4f 4d 50 49 4c ARY ...5..COMPIL +| 96: 45 52 3d 63 6c 61 6e 67 2d 36 2e 30 2e 30 58 4e ER=clang-6.0.0XN +| 112: 4f 43 41 53 45 1f 01 05 00 35 0f 17 43 4f 4d 50 OCASE....5..COMP +| 128: 49 4c 45 52 3d 63 6c 61 6e 67 2d 36 2e 30 2e 30 ILER=clang-6.0.0 +| 144: 58 52 54 52 49 4d 0d 00 00 00 24 0e e0 00 0f 6f XRTRIM....$....o +| 160: 6e 74 65 6e 74 05 43 52 45 41 54 45 20 54 41 42 ntent.CREATE TAB +| 176: 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 LE 't1_content'( +| 192: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 208: 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 RY KEY, c0, c1, +| 224: 63 32 29 69 04 07 17 19 19 01 81 2d 74 61 62 6c c2)i.......-tabl +| 240: 65 74 31 5f 69 64 78 74 31 5f 69 64 78 04 43 52 et1_idxt1_idx.CR +| 256: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 EATE TABLE 't1_i +| 272: 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c dx'(segid, term, +| 288: 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b pgno, PRIMARY K +| 304: 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 EY(segid, term)) +| 320: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 03 WITHOUT ROWIDU. +| 336: 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 .......tablet1_d +| 352: 61 74 61 74 31 5f 64 61 74 61 03 43 52 45 41 54 atat1_data.CREAT +| 368: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 E TABLE 't1_data +| 384: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 '(id INTEGER PRI +| 400: 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 MARY KEY, block +| 416: 42 4c 4f 42 29 38 02 06 17 11 11 08 5f 74 61 62 BLOB)8......_tab +| 432: 6c 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 let1t1CREATE VIR +| 448: 54 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 TUAL TABLE t1 US +| 464: 49 4e 47 20 66 74 73 35 28 61 2c 62 2c 63 29 00 ING fts5(a,b,c). +| 480: 00 00 39 00 00 00 00 00 00 00 00 00 00 00 00 00 ..9............. +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-fed6e90021ba5d.db +}]} {} + +do_catchsql_test 33.1 { + CREATE VIRTUAL TABLE t2 USING fts5vocab('t1','row'); + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1','col'); + CREATE VIRTUAL TABLE t4 USING fts5vocab('t1','instance'); +} {/*malformed database schema*/} + +do_catchsql_test 33.2 { + SELECT * FROM t2; +} {/*malformed database schema*/} + +do_catchsql_test 33.3 { + SELECT * FROM t2, t3, t4 WHERE t2.term=t3.term AND t3.term=t4.term; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 34.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-a60a9da4c8932f.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 20 68 20 69 0d 00 00 00 03 0f e8 ...t. h i....... +| 3904: 00 0f f8 0f f0 0f e8 00 00 00 00 00 00 00 00 00 ................ +| page 5 offset 16384 +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 00 ................ +| 4080: 60 20 30 01 20 30 00 30 60 10 30 01 20 30 00 30 ` 0. 0.0`.0. 0.0 +| page 6 offset 20480 +| 0: a0 00 00 00 10 ff 40 00 ff 00 00 00 00 00 00 00 ......@......... +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 00 00 00 00 00 00 00 00 00 11 87 89 06 26 ...............& +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 44 00 06 06 07 01 03 00 14 03 09 09 09 0f ..D............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-a60a9da4c8932f.db +}]} {} + +do_catchsql_test 34.1 { + SELECT fts5_decode(id, block) FROM t1_data; +} {1 {database disk image is malformed}} + +do_catchsql_test 34.2 { + SELECT fts5_decode(id, block) FROM t2_data; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 35.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename crash-ae135cb10977c7.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 07 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0e fc 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 01 2f 0d d5 ...t.[.@.$.../.. +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 8f DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 57 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000WBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 59 4e 4f 43 41 53 45 17 LE RTREEYNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4f 41 42 4c NARY....)..EOABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 51 53 45 E GEOPOLYXNOCQSE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 1d 05 E FTS5XNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 55 53 34 58 42 49 4e 41 52 59 17 0b LE FUS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end crash-ae135cb10977c7.db +}]} {} + +do_catchsql_test 35.1 { + SELECT * FROM t1 WHERE t1 MATCH 'e*'; +} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}} + +#------------------------------------------------------------------------- +reset_db +do_test 36.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 24576 pagesize 4096 filename crash-a6651222df1bd1.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 11 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 00 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 13 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 64 61 62 ............0dab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 14 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 0c e9 ................ +| end crash-a6651222df1bd1.db +}]} {} + +do_catchsql_test 36.1 { + INSERT INTO t1(b) VALUES( + x'78de3fa24af3733ca8769291a0fee3669f9fddefc5cba913e4225d4b6ce2b04f26b87fad3ee6f9b7d90a1ea62a169bf41e5d32707a6ca5c3d05e4bde05c9d89eaaa8c50e74333d2e9fcd7dfe95528a3a016aac1102d825c5cd70cf99d8a88e0ea7f798d4334386518b7ad359beb168b93aba059a2a3bd93112d65b44c12b9904ea786b204d80531cdf0504bf9b203dbe927061974caf7b9f30cbc3397b61f802e732012a6663d41c3607d6f1c0dbcfd489adac05ca500c0b04439d894cd93a840159225ef73b627e178b9f84b3ffe66cf22a963a8368813ff7961fc47f573211ccec95e0220dcbb3bf429f4a50ba54d7a53784ac51bfef346e6a'); +} {0 {}} + +#------------------------------------------------------------------------- +reset_db +do_test 37.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename null-memcmp-param-1..db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 66 74 34 74 34 43 .....utablft4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 64 44 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G dDs5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 01 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 5a 02 01 01 62 01 .t.....0a.Z...b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 30 69 .a b cd e fg h0i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end null-memcmp-param-1..db +}]} {} + +do_catchsql_test 37.1 { + SELECT * FROM t3; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 37a.0 { + CREATE VIRTUAL TABLE t1 USING fts5(b, c); + INSERT INTO t1 VALUES('a', 'b'); + SELECT quote(block) FROM t1_data WHERE rowid=10; +} {X'000000000101010001010101'} + +do_execsql_test 37a.1 { + UPDATE t1_data SET block = X'FFFFFFFF0101010001010101' WHERE rowid = 10; + SELECT rowid FROM t1('a'); +} {1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 38.0 { + CREATE VIRTUAL TABLE t1 USING fts5(b, c); + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('a', 'b'); + SELECT quote(block) FROM t1_data WHERE rowid=1; +} {X'020202'} + +do_execsql_test 38.1 { + SELECT * FROM t1('a b') ORDER BY rank; +} {a b a b} + +do_execsql_test 38.2 { + UPDATE t1_data SET block = X'000202' WHERE rowid=1; +} +do_catchsql_test 38.3 { + SELECT * FROM t1('a b') ORDER BY rank; +} {1 {database disk image is malformed}} + +db close +sqlite3 db test.db +do_catchsql_test 38.4 { + SELECT * FROM t1('a b') ORDER BY rank; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 38.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename crash-fd2a1313e5b5e9.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 b8 31 5f 63 6f 6e 74 !...table.1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 01 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 ec 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 01 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 60 35 30 30 30 30 MAX MEMORY`50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 44 4d 4f 52 59 3d 35 30 30 30 .MAX MDMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 16 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 05 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 10 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 05 f1 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end crash-fd2a1313e5b5e9.db +}]} {} + +do_catchsql_test 38.1 { + UPDATE t1 SET b=quote(zeroblob(200)) WHERE t1 MATCH 'thread*'; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 39.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-e650fe95502908.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 32 2c 32 2c 33 ,b,prefix=.2,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a ff 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 00 f1 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 81 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 61 80 80 80 11 03 00 3c 00 00 ......a......<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 71 .......6.....2tq +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 0b 89 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 f4 01 02 03 01 03 66 .....0eac......f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 b3 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 05 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 2d .....2th......2- +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 03 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end crash-e650fe95502908.db +}]} {} + +do_execsql_test 39.1 { + SELECT rowid FROM t1('t*'); +} {1 2 3} + +#------------------------------------------------------------------------- +reset_db +do_test 40.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash2.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 52 6c 65 (a)V.......taRle +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 73 73 !...tablet1_doss +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 75 73 34 03 02 02 01 04 6e .....4fus4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 05 62 03 04 0a 19 8c 80 80 ....tab.b....... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a ff 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 11 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 02 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 06 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 00 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 10 f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: bd 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 05 52 08 04 01 ...0n.......R... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 b3 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end crash2.txt.db +}]} {} + +do_catchsql_test 40.1 { + BEGIN; + INSERT INTO t1(b) VALUES(X'819192e578de3fa24af3733ca8769291a0fee3669f9fddefc5cba913e4225d4b6ce2b04f26b87fad3ee6f9b7d90a1ea62a169bf41e5d32707a6ca5c3d05e4bde05c9d89eaaa8c50e74333d2e9fcd7dfe95528a3a016aac1102d825c5cd70cf99d8a88e0ea7f798d4334386518b7ad359beb168b93aba059a2a3bd93112d65b44c12b9904ea786b204d80531cdf0504bf9b203dbe927061974caf7b9f30cbc3397b61f802e732012a6663d41c3607d6f1c0dbcfd489adac05ca500c0b04439d894cd93a840159225ef73b627e178b9f84b3ffe66cf22a963a8368813ff7961fc47f573211ccec95e0220dcbb3bf429f4a50ba54d7a53784ac51bf'); + INSERT INTO t1(b) VALUES(X'c8ae0d0e7c3175946e62ba2b449511d4eb504079984a20f77969f62206c9f3d7ea25358ab705e6978627290b6d48db9032f815a06a79a4f4b809841a0942eed12954ed166f666111812a508abc3bec87958846edaec0a6fe14564bc0a4b78f1c35ebcacca6bae29cc37ae9b59d8a2d7593af1e47dda0ece2268a98d20febafad037964f139851f9a57f48b3706b01721769071991412044cd6006f1d72eb6eb4aa5ad77e378176db8c15575fbeee47165e38a7c6c5a557ac2dfe11813976eaf6741cf593a9e457053a3c34cddfbe605a6e25419f993de8374fafcd3636509d8416a51dc7bcc14cfca322ae343078f47e23522431c17d0da0c033'); + INSERT INTO t1(b) VALUES(X'dc29a94e873a45a4243fce9b912aaefbadf1d0423e0345793874b356eeb500b92fb05284c1601fe9bad3143f72162f10242cec27c44ebf764c8fc9fb0824e32c4161472a4f914f579e0e8274f08ca1a02e59b9d8eec1f31061f9ccb9ed97a6f06534e991f7992c761489e6a7724f6e9c2b581e77487ded3a986d53c4419bbd3e9747cee300e670dd7294874c77e2ed48da68eaa6c3ec954a09ac410493d98e34d6686e54fbbe80696705f10e040c66093efb40746b33600685c94c664c7942835a9e954866121d5dcfb2cb12e92521ea3df175ee17072502dad9b9c1565f801b2179799011eb7418bfa00323e3157589e648ff7378be233c79b7'); +} {/*malformed database schema*/} + +do_catchsql_test 40.2 { + INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3, true ),(4,44); +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 41.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a,b,c); + REPLACE INTO t1_data VALUES(1,X'255a5824'); + REPLACE INTO t1_data VALUES(10,X'0a1000000102020002010101020101'); + INSERT INTO t1_data VALUES(137438953473,X'0000032b0030300102060102060102061f0203010203010203010832303136303630390102070102070102070101340102050102050102050101350102040102040102040207303030303030301c0204010204010204010662696e6272790306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020108636f6d70696c657201020201020201020201066462737461740702030102030102030424656275670402020102020102020106656e61626c6507020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020202087874656e73696f6e1f02040102040102040104667473340a02030102030102030401350d020301020301020301036763630102030102030102030206656f706f6c7910020301020301020301056a736f6e3113020301020301020301046c6f61641f020301020301020301036d61781c02020102020102020205656d6f72791c020301020301020304047379733516020301020301020301066e6f6361736502060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020301b10202030601020201046f6d69741f0202010202010202010572747265651902030102030102030402696d010601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202010a746872656164736166652202020102020102020104767461620702040102040102040101780106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020415130c0c124413110f47130f0c0e11100f0e100f440f1040150f'); + INSERT INTO t1_data VALUES(274877906945,X'00000183023030250601011d010331c2ba250601010d0101342506010121010135250601011e02036ec2ba250601012b0101382506010112010161250a0101021a1d02016f2506010111010162250601013201026377250601012f010166250801011f0c010167250601012701026863250601010f02026473250601013002016b2506010133020175250601010e010169250601012c0204386ec2be250601012001016a250601010401056bc2b2cebc250601010901016c25060101150203cebc71250601011301036dd18a250601010c01016f25060101260102706425060101240101712506010122010173250a010116040d02016f2506010134010175250801011b14020161250601010b010376c2aa25060101100202d7ac250601010601017725060101030201752506010114010179250a0101190e050202357a250601010701017a250601012e0102c2aa250801011c100201b3250601010a0202ba6225060101310203be656625060101080103c5a77425060101050102de8e250601011704080a08080a080a080809090809090808080b080c080a0a0809080a0809080a0908080a09080a08090a0a'); + INSERT INTO t1_idx VALUES(1,X'',2); + INSERT INTO t1_idx VALUES(2,X'',2); +} + +do_catchsql_test 41.1 { + INSERT INTO t1(t1) VALUES('optimize'); +} {1 {database disk image is malformed}} + +do_catchsql_test 41.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 42.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename 8cfba7fbb67e48de92c6.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 62 72 79 03 06 ........binbry.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 01 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0b 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4f 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 OARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 57 42 49 NABLE GEOPOLYWBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 42 41 53 45 E GEOPOLYXNOBASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 52 02 49 4e 41 52 59 27 20160609R.INARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 5d 69 7a 65 uild....opti]ize +| end 8cfba7fbb67e48de92c6.db +}]} {} + +do_catchsql_test 42.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 43.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 24576 pagesize 4096 filename 89028ffd2c29b679e250.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 12 6f 63 73 69 7a 65 04 43 52 izet1_.ocsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5e 64 61 74 61 02 et1_datat1^data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e8 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 00 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 94 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 04 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 00 90 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 07 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 71 01 02 05 04 08 18 84 80 80 ......q......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 91 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 05 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 02 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end 89028ffd2c29b679e250.db +}]} {} + +do_catchsql_test 43.1 { + INSERT INTO t1(t1) VALUES('optimize'); +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 44.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii"); + REPLACE INTO t1_data VALUES(1,X'03090009'); + REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101'); + INSERT INTO t1_data VALUES(137438953473,X'0000002e023061011202010162010203010163010204010167010601020201016801050102030101690106010204040606060808'); + INSERT INTO t1_data VALUES(274877906945,X'0000001f02306702080201020201016802080301020301016a420804010204040909'); + INSERT INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808'); + INSERT INTO t1_idx VALUES(1,X'',2); + INSERT INTO t1_idx VALUES(2,X'',2); + INSERT INTO t1_idx VALUES(3,X'',2); + INSERT INTO t1_content VALUES(1,'a b c','d e f','g h i'); + INSERT INTO t1_content VALUES(2,'g h i','a b c','g h i'); + INSERT INTO t1_content VALUES(3,'a b c','g h i','g h i'); + INSERT INTO t1_docsize VALUES(1,X'030003'); + INSERT INTO t1_docsize VALUES(2,X'030003'); + INSERT INTO t1_docsize VALUES(3,X'030003'); +} {} + +do_catchsql_test 44.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +do_catchsql_test 44.3 { + SELECT snippet(t1, -1, '.', '..', '', 2 ) FROM t1('g h') ORDER BY rank; +} {0 {{.g.. .h..} {.g.. h} {.g.. .h..}}} + +#-------------------------------------------------------------------------- +reset_db +do_test 45.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 24576 pagesize 4096 filename crash-0b162c9e69b999.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 52 6c 65 (a)V.......taRle +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 73 73 !...tablet1_doss +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 75 73 34 03 02 02 01 04 6e .....4fus4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 74 6d 03 06 01 ...<.....3ntm... +| 2528: 01 05 01 03 74 61 62 05 62 03 04 0a 19 8c 80 80 ....tab.b....... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a ff 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 11 03 00 16 00 00 01 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 02 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 06 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 00 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 10 f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: bd 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 2f 03 34 74 20 07 04 01 0e 01 03 34 1e .../.4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 02 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 05 52 08 04 01 ...0n.......R... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 b3 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +}]} {} + +do_catchsql_test 45.2 { + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); + INSERT INTO t1(t1, rank) VALUES('merge', 5); +} {/*malformed database schema*/} + +#-------------------------------------------------------------------------- +reset_db +do_test 46.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename crash-1ee8bd451dd1ad.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 60 t2(x)V.......t` +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 07 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 06 e2 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 31 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DS1FE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 40 42 4c 45 20 52 54 52 ...%..EN@BLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 59 42 ..ENABLE JSON1YB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 15 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 04 f0 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 22 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 a2 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end crash-1ee8bd451dd1ad.db +}]} {} + +do_catchsql_test 46.1 { + SELECT snippet(t1,'[','', '--',-1,10) FROM t1('*'); +} {/*malformed database schema*/} + +#-------------------------------------------------------------------------- +reset_db +do_test 47.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename 4b6fc659283f2735616c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 7d 42 5f 63 6f 6e 66 69 67 74 32 table.B_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6f 74 74 31 5f 63 6f 6e 74 1_conteott1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 46 52 20 50 52 49 4d 41 52 59 INTEGFR PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 01 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end 4b6fc659283f2735616c.db +}]} {} + +do_catchsql_test 47.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {/*malformed database schema*/} + +do_catchsql_test 47.2 { + SELECT count(*) FROM ( + SELECT snippet(t1, -1, '.', '..', '[', 50), + highlight(t1, 2, '[', ']') FROM t1('g h') + WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank + ) +} {/*malformed database schema*/} + +#-------------------------------------------------------------------------- +reset_db +do_test 48.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename crash-44a8305b4bd86f.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 62 72 79 03 06 ........binbry.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 01 03 16 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 04 71 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ..q.........comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 03 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 13 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0b 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4f 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 OARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 46 45 58 4e 4f 43 41 53 45 17 LE RTRFEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 57 42 49 NABLE GEOPOLYWBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 42 41 53 45 E GEOPOLYXNOBASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 66 54 41 42 58 52 54 52 49 4d 11 06 TAT fTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 62 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XbTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 52 02 49 4e 41 52 59 27 20160609R.INARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 5d 69 71 a5 uild....opti]iq. +| end crash-44a8305b4bd86f.db +}]} {} + +do_catchsql_test 48.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 49.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-fd87385402ecf5.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 20 68 20 69 0d 00 00 00 03 0f e8 ...t. h i....... +| 3904: 00 0f f8 0f f0 0f e8 00 00 00 00 00 00 00 00 00 ................ +| page 5 offset 16384 +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 00 ................ +| 4080: 60 20 30 d6 20 30 00 30 60 10 30 01 20 30 00 30 ` 0. 0.0`.0. 0.0 +| page 6 offset 20480 +| 0: a0 00 00 00 10 ff 40 00 ff 00 00 00 00 00 00 00 ......@......... +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 00 00 00 00 00 00 00 00 00 11 87 89 06 26 ...............& +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 44 00 06 06 07 01 03 00 14 03 09 09 09 0f ..D............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 9d 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-fd87385402ecf5.db +}]} {} + +do_catchsql_test 49.1 { + SELECT term FROM t4 WHERE term LIKE 'oase'; +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 50.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-695bce8a3e107c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 20 68 20 69 0d 00 00 00 03 0f e8 ...t. h i....... +| 3904: 00 0f f8 0f f0 0f e8 00 00 00 00 00 00 00 00 00 ................ +| page 5 offset 16384 +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 00 ................ +| 4080: 60 20 30 d6 20 30 00 30 60 10 30 01 20 30 00 30 ` 0. 0.0`.0. 0.0 +| page 6 offset 20480 +| 0: a0 00 00 00 10 ff 40 00 ff 00 00 00 00 00 00 00 ......@......... +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 00 00 00 00 00 00 00 00 00 11 87 89 06 26 ...............& +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 44 00 06 06 07 01 03 00 14 03 09 09 09 0f ..D............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 9d 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-695bce8a3e107c.db +}]} {} + +do_catchsql_test 50.1 { + SELECT term FROM t4 WHERE term LIKE '»as'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 51.0 { +BEGIN TRANSACTION; +PRAGMA writable_schema=ON; +CREATE VIRTUAL TABLE t1 USING fts5(a,b,c); +CREATE TABLE IF NOT EXISTS 't1_data'(id INTEGER PRIMARY KEY, block BLOB); +REPLACE INTO t1_data VALUES(1,X'2eb1182424'); +REPLACE INTO t1_data VALUES(10,X'000000000102080002010101020107'); +INSERT INTO t1_data VALUES(137438953473,X'0000032b0230300102060102060102061f0203010203010203010832303136303630390102070102070102070101340102050102050102050101350102040102040102040207303030303030301c023d010204010204010662696e6172790306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020108636f6d70696c657201020201020201020201066462737461740702030102030102030204656275670402020102020102020107656e61626c6507020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020202087874656e73696f6e1f02040102040102040104667473340a02030102030102030401350d020301020301020301036763630102030102030102030206656f706f6c7910020301020301020301056a736f6e3113020301020301020301046c6f61641f020301020301020301036d61781c02020102020102020205656d6f72791c020301020301020304047379733516020301020301020301066e6f6361736502060102020306010202030601020213060102020306010202030601020203060102020306010202030601020203060102020306010202030601020201046f6d69741f0202010202010202010572747265651902030102030102030402696d010601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202010a7468726561647361666522020201020201020201047674616207020401020401020401017801060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102ad060101020106010102010601010201060101020106010101010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020415130c0c124413110f47130efc0e11100f0e100f440f1040150f'); +INSERT INTO t1_data VALUES(274877906945,X'00000e96023030011a042319320d3b123d812b5a31120110446e581b66814a05010a4537814274010e8102815c810f3d0104846d01081581204401103741043c59416b44010a404655265301103f73811a11114213010a821235820f020135030484320201360104816a020162020484550302390301710a04824a020166030483690201670704837d0201690404822602016a0504825c02026b620504817502016f0904810d0303e79c88060482760201700a04826302017204048155020373c2be050481130201770204846202027962050482710202c2ba010482140203e58496070483330204e8b2b879010483710101310110545c0c814b0e3a6501082c815d5b011a2a0e2f0d765c3d686014061d0d0112810733112c2e82141101048313010e5c6f632e813e42010c811882370548010e19158146822f1f01104d364a708146135a010a237b0a55210201610904841703027678090481270201620304810002026374060484660202657a0704827602016601048351090483540301660704814b02016b03025f0304c582caba0204816602016c01025f0302cebc0904843e02016e0804821802016f0a04817503016f070483100201720304822c020484380201740404842e0102460201770104812f0204836c02027a6f040483110202cebc02048267040161020484650205d5bd62cebc0604845b0204f2a580880204842a0206f38184a179670502750204f696a3aa0a04814601013201063330390110812281378114600d010c03716c5e822d010e81226b542a814d010a72740f83000108813a1e0b010c5681046f812c010c07814a777328011664244219531b1a2f811e4a010c4d81557c7f1b0201300704810702013307048230010484050202367807048175020239710804832502016204026b0204814d020363306108048262020265650504817602026667070483150201690704832f0301360a04814d02016c08024702016d0304843e0303cfb2630204814002016e0804837503016203048416030370c2be0304821b02016f0604834b0201700504816b030175070210020273660604841c020676c2bac2b2640604830a02017704027d02017808048141010482700201790504811d0202c2ba0502470206ca8d73ecbab9010483340204f09e9ab504048367010133010c3e04814f82250114812e814b2e0411811305010c811337811e6e010c82085e2b0e5d010c61812054811c01148122451b0781050c813d010c17823762643e011e080c1720814a10364306143b0d33260112810f0c2a810c816b13010a8163810e470201370404811102056176c2aa36050481530202646e0904846202026a730404827402016b0a020c02036f616b0404830a0201750504820d02017605025e0201770a04820702027a73050482460202c2ba0604824a020483330203cba434040483200203cebc790304847e01013401124181442c1d091f81580108601d8336011081320a2b8125820001123b0b81158116811f070110078112817a817308010e6682410d810e2601122d0d6413378147351e01105081021d3525812d01128246510a622204054101105c1b620e81302b05020130020483480104822702013102027e020132030483270201350304844802023770030207020261710604823f0201640802570204830a020265770304831f020168070483210201690204825b02016f020481280704835402057037e18b8d0904810802017304048439030172020481440303c2bd6b0a02630201760202490804815e0201770304816e050483550201780704816902017a070483280207c2b2f093aabc780a0482240301b303024e0301bc0604837a0202cb800204834a0202cebc04048201040484410203d3ad770a04814f0202d5a508048371010135011630817e0f81040d2c041552010c813d3b7e8115010c40692182693a01121f810d810d0a32814701101d1d1f642281742e01068229240110811231810a387f4c01100f50810f8165810d0114811f26443152593c104a010e641e1a3357820e020132030481540201340204815402023778040207020163060481020204815b020164020483010201670a0481540304f0948f870904811a0201690704835502016d0604832203027b01022803017a0a025102016e020484260404816002026f690502680301720104834e02017208024f02021d020475e69c8e0504814c0202767602025f0302de870a04837b020178090483200104835003026a72090484400201790504837204022302017a0104836b0202c2aa0a0481070303bcc2b3090484370203c7866403027501013601087c158303011212814305813e7b0e090118141a1c49713a211e0c74630f010a59826d8113011203328166037781561a01101d7f1d2a1f822533010e820e070f7b40160110811f40292c813226010a2d20824a32010a81418158670201300304840a0201660404817d02016d0804826902016e080626817f0104820b02016f0404825a02017003023b020272750804840c0301760a04844d02017403025b020175080484200202766e060482360303c2ba6c030482220202c2aa0204810e0301b20204835703048421040484240301bc0802410202d2a1010482630204e1b18f3704048354010137010c08816337812f01101382211532424d39010881248123010e7724810267815f011081236029813e273301101b7b29812a5b813b01128150810324814b220b01060c8417010e165b6c81708117010a1782346f6c0201380804816803016b0604840102013908023f0201610204816c0201630302760201640304832604023e0204833902016502048203020266770804821a0201670a04830002026964080484500304f29e9eb70802250201700204811b0201710904832d0201730304826b05048403020174010481000205776a62c2b2050482630201790202260206c2b2eaaeb464050485020301b3080482480303bac7af0a0484550203c695650804822a0202ceb90a0481170202dbae05023f010138011819814a2703390a61090c6912011a21181304812523811b5f164e050114110f35128123423f810c010c817573817c03010e7182590c812b0401142503597e6e0e2f3a3759011252813a811a2b75091a010882162a31010e17450a81048279010858658208020231670a048205020361c2b906023b02016301048236020164050482520201650904833d0201670904811b0201690604825002026a7a0604837c02016e0204832002017002026d0302c789010481020201720504835e0201740604810002017502020a0702630302676306025f0303e4a0a70102640203786a75010484440201790104841402047ac2be72070481340207c2ba3766673576090482790301bd07048142010481600202c7b309027a0401740604823b0202d2bf0304830f0204e989a6300a02600204f4bd91b60702120101390106518369010e19254641823711010c258267288121010e817c810d2b17250110810a578133812f4c0110415681067b288121010e0881208119347101140b8131543c8100343d1101088203813e01100d742e3230820f3802013006048107020134080481440202356501021a02013808048147020162020482230201630304833c030162050483390206656cf093b5bd070484140202677303048502020769e3ad9669c2b90304847402016b0804836402016c0404841f02016d010481250904825202016e06025402016f0704842a0201700a04834a0201720a0483530201750304822e020676f097b18374030482140201780604833d0202c2b9010481550301be070483720204826f0202ca80060481630202d5a80504833f0101610114551047810e130a78660c011a364611206c0b13080705733d5501240f08070c090b0c20813d1471042e4351131e011204412d814f0913104201263036110d060b1f811a301b0f4e1a29092f181c012808071e221a2a81075b320503065a0f140c1a0a26011c07231d0e6f3715063b760c6b091501121111303e3a71566d6d010e0867814d816a0c01181e18240d41724d221b3f384b0201300204830b060483080302c2b30204837f0201310a026e020134050481560301730902690201360406827c0f0201370204825d06020a0201390904815b0304f3bfb2a70a04822c0201610404810e020262660604841a03017608025b02016309088112825f020164090482310201650a0484480302396f01025d02026774010482090302df9b06048321020168070661813303016f03048248020169030483610504814001048401010483460301300704824203016a0204824402016a0504813303021d03017209048412030277380804824502016c0104814c02036ec2ba0804835702016f0204811d030176020238020270360a04840d02017107048201030469ca99690602350201720304812d0104845e02017304022c02017409022b020175050484140302caaf0402410201760404831f020177010484180704845203026f6d04024b02017a030482660202c2aa0204810a0301b30a04817b0302bd76020483780302be6a0302440202c58207025e0202c69901027a0302ad77010483200206c993f099b183070484270203caa1660204841a0204f29788ac0804831d0204f4ba9f950504843c010162011c0e33810216341c2413042130780501184d373e53131f2f052907423e010c830e3781390e011c1320461f81041b811b041e15243d011e241b10816c310b130c3133033b0741011a11816d3139100c13140b395848011c580e411a06304306810a3138330d011a441707092c70140c1643813920010c73653581374f010c826c81210f0402013803024f07048172020161020481650204810c020162030483470301370404813602016307048379020483280201640304815501048176010481060201650104812a0104841f0201660504821f0201670302700201680804846403016b0a04831d0301700904845502016908025c02016a010483560904827602026b6306023402016c0404832602016d040484410204825702016e0504831603027831020482160302c2be0504827d02026f6a05048121020171030232020483220304845402023a0201720204845c0304846e03013607048224020174060484480201780704844303016f0604814f045807070a0707070707080709070709070808090a5c07080708080b07060a06080707070b0a0b0808070b0a0b0a55070b08080a0908080707060709070709070706080c060b07070c0a66070b08080609070607080c0909660b06070707080a0807070b0b0707080a0b07070d0607080c0908630707070b07070a070d060b0707090a07080b080a070809085f0707070c0706080706070809080f06080a5807070607060e070807080907070b070b060c0709090807690808070707070708070608070709070809070a0d0b07070809095b070707070707070c080d07070b06070707070c07080b0808811a0b08060706080a070a07080609070707080808071307070a0708070907060807090b06060707070b0707080708070707080c090a0a81080a0b07070b0f0b0706070707060b07070b07080808110b070707'); +INSERT INTO t1_data VALUES(274877906946,X'00000e880330627a020482240202c2aa0a04833f0301b30704844b0302b9650704824f0301ba0204845f0202c9820a0483640202d194060482300203e19cbd0904844b0203e691b4050483510205e78dadde9b0104821201016301142a6c033b8151085c094601140b813d49313f81110e1c011681163611221527257f5d38011c150f22811a0a3c12350631238117011c3e26420b402c1d81080c40150b2f01181c3143382640273d60132e070118663b1d162a1b0e2e8111393e0110821117310e52811c01141a2f49810181391f2b130112323c0305812a6f2e390201320204842702023334020481340201350202610201360304844603023362040484470203376a360a04826b02013808026203016f0704830d02013902025502016106088170827a0301320a04820403016c0404831502016204048327050484030201630a04814302016401048349020484760302430301640204845d02016504048249020367c2bd07020a0302cebc0902150201680a02500201690204846102016a06024c0301770504842b02016b0104830e0704811803023370040483580301710404845f02016c0504844f0204820d0204837f0302c2aa0104833702016d0104844c02016e0804834f03026c6a07025702046fecbd9a01023a020270330204830a0301740304837c020271350204811e0201720706833b310206736ef09289b70104832b020174070483290301320204827c02017608022802017806025b0302c2b30904835202027978080483040303c9b56f0904846a0202c2aa04048127050482120301b30504813a0301b901024204016a0704840c0202c5820704823c0202c999010482470202cebc0602400203db91670602730202dca7050482760203e1a3950304817e0203e786a702048273010164011a0612105b292b817c1211080d5a01147c1d420b35451c36811a011e0e168117081c0c2e051d474055192d011e02050a1c81180420250f815f300f21011c02316a37143321443a10042d54230112761428810e4750054101101805072b8215294e0116680f0f5381445a3e0b070901224e4a41210c361c281b101c43051325130f01185a1e19108106300f2e3f4538020130060481370202327305022002013405048168020335377a03021802013704027b02013804048260020161030882118101030263650904814d0201620604822502016303048419030135090483240201640602280301380404817e0201650404823c0304f097ac9f010482680201660a0267030566e2b6936f0104821c0201680704813302016b0604832002016c01026a0301610204843202016d0804845e03026d6d03023c020270730a04817f0304c2be797a0804832e020271710504835f02017203021b0201740204825a02017706027202017805048451010683572e03016e0a04814c0201790304811702017a080484450302c2b90204837b0202c2aa0604825b0303b273630104841c0301b9050485040301bc0a027b0303bd37680502670202c98b0204826b0203cfa1740504823a0203d199610202350203e3bf87040483570204f1baaba90504817301016501120b8104392d0d20180f011645213f292e4d0d082f8165011e0b400c07341b2329307f193338173a012055292409050c560a272a0f4403245718011a1c3a183f1c43264c3126060829012081208102043a044d0621650b180e150e011a066c030e513d7d265e1313130c0118171953040457347b114d191901261b1c060c26090d6f0d332a1519096e03101d1d012207342f1f2c7e2517251d0f310d2a17081e02013005020a02013308048247020135030483660201380704841b030132030482180201610102600604825f0304c2bdc2ba02023a02016209021f0201630604813002056663cebc610604841e0201670504816a0104842703037177310604833f03027872080482350201680604825e02016a040483320104840d02016b0304813f02026c6408020602016d070481590104837d03016c08020d02016e060484630301780404815c02016f0104826b0804825e02017008027b0302c2aa0504847c0201710404836d0201730402510302cebc08048338020375c7bf05048344020177010482660304822303026479070481630204786ec2ba0204814f0202796a0a04834f02017a01048407030484660604810603026561020483180204c2b278390504813d0301b304024f04026536010483110302b9330604813e0301be0304840b040484560202c4a702022d0206c6a5f7ada9990402350202c79f090481180202caa60502140204cebcc2bd080483320202db900a0481250205f4b5aa9079040484360204f7b985bd0204835e01016601128101285c096981190e01121f813f0d431a8135530114698102813228492f190a011260161881328101812601188155780d813257050c0b04060114161681340772811b5e25011c4505810e13290b253a0c0c0a1a4b011a3714133e1235812b136b062c0b011a6b591356810c3c240906250b1001148127810d413e0e81090d020231680504822c0302c2bc0204830c0201360104840c0303656a740902110302d1950504824f0201370902130201390a0482530303e0a9ae0904844c0201610804810d0201620a04810c03023039030481330302356902048268020163060483470201650504822d02016706048200030483560302713509025002026a79010484410804825902016c0504822002016e07020d0304843a02016f0204842d0201710304837604048361030482430201720904840f020173020482520804810b02017406068425320201760202420201780804845d02037979650704814802017a080481110301780804812c0202c2bd070484600202cebc0508813082410102770401610202250202ddb40302310205f19e9a937a030482410101670118365558195a0a062d0581260a011881068143330844041f0a1851011a2025141e1081204f550e077521011a193b1f58351912265681220821011812070528472f4e2f407f204a01124d1e1f811b810d7b4d01180f1d3481034a35580a12811f011c2303340d1470150778070c812331011620411939703c032915143f01104281116a3d323c6a0201350604842902013703023f0201390a04816e0201610604826402016202023f0201630a04843302016409048258020165020482620404814702016602061b833f0201670404827d03026369020227020169040483490301670604847602016a0604845d0404840902016c04024601088150822b010482350204830a0301690304844902016d0202710704820002016f03048509020482380304836603026e74010483580201700504817a020171020481080204826a0201720304837b0202410302160302c2b3080484550201730904816e020174040248010484280304834e020275730404836e020176060484720104815c02017709026a020178060275040483790201790504821501026c030170080483770304cfb269710704815102017a0204813303016f090483160202c2aa050481400301ba0804810b0401630804830d0301be0604844d0202c8a30a02110203cba0640204816a0202cebf060482420204e487856e030481080204ec97bd6d080484080205f09eb3a0770502260208f687999931ed878703048424010168011a12460e090c036e151b812e065501161708411982151738471f35011a2d1c0678340c1f04425c21200c010c2a087f255d4a011a0d0b6c33814a212c3a0a401b1e011c501f2381010a0481201c0c6012280118150b5228520e0a036c1c8123011a15810a060408030a81563f381601185b1b06212a143f332a60160e011a221b1e62411d2048090e0b0f5502033072350804826a020131020482530201320304823b0201330104814502033677380204813102026174020483540304dbbf6f620404835d0201620504846a0104831803017103048323020164060483740201660a02410201690102130104821402016a0504823f02016c0404832e02016d09022602026e640a04822702016f070482000301750402670201710304813106020d0202726304048220020173060484530201750504831e020483400302c2bd0704843a020177020483470203786371030483740201790904810002017a0302300202c2b9080483280301bc010481700303bd33720304825e0205c99973c2bd040483160203d5a6330a04842b0204e7b3b3300904813c0205f099a68f72090209010169011c21101d4b2d0e0e066b4253074c140118070a0910447556030833541d01163733816837402b3909122501183c5b1139102e2d430c662334011e27050f21621230323503332b6a0332011e1e07031843202e6e3c2850094d410c01163955220b16812d24521212011681250b0a3505460481176f011a2c09266b162968051c0a1481170116022e1e820c352037263a070201310104825d0602110303696869020484070201320204826e0201610304832203016f040232020162050642843f03048336020484540301370804833c03026c6105027a0203636165090483120201640502770304833e03017107022f0301780702470201650a04811e020167080238020168040481160102230404826b030170040483000201690304836f0302766c0304811402016b0504812e02026c6108027702016d0308827d81530604837302026e790904842602016f06048208020170060481680302320201710204812902017307048255020274320104822a02017506026803016e0a04821303017207025f0302c2b90504834a020177020483130201780604836b0402210301320604847302017a010483130202c2aa0804823c0301b9030482600301ba0104845304016c0504837e0202c3b8080484600204cf9d6379020483660202d3860704812e0203e3a4be0402560203e58784010481210204f09e95ac0102580204f5aea5890a023301016a01123428131a1f6c81445601141e227c1a7b5f1918810301182318812e17455605460d811c011a28820221311a6e12093f050a0c0120082c0b0f1362074457460c3b070d5132011c2143052a20133d160a358117591f01103136813b136e6247011c100e4c28060d16815a320a3e11070124462c03582e262d45110804113326040808070807080809090b81050708060708090607060907070b070e07070807060706070b08070f0807070709080708080c070706060808090c07060708080708080909811307070708060709080707070607070a060b070706070707080a080607060c070707080809070608080908090a812406070707070a0906070b0b0908070b07070b0607070b0608070608090b080a080f080a0608080b070b08070a080b0a810408080708080607090707080807070b070c070a070f070b080607090707080d06070b810b070607070607070b08070707070b14070a0f08070b0d08070e080b060a0a070a0707080707070709080a0a0a0e810f0907070709080a0b0707060a0707060807060a08070b08070907060807090b090a0a81140a09070706100707090a060607060e07070807070d08070a07070806070608070a070708070707080a0808090909'); +INSERT INTO t1_data VALUES(274877906947,X'00080e7f073c23110a1a18392f66090524183704276d6703306a320404824e030164080483520305c2bd7ac2bc0604815a0201360704833202016106021f020482400201630304822a0708817e8204030173040483500201640404824803016d0804824a03017709023002016606048367020268680a04815802016902088339811804027f0302656e0704834e0303d5a5370604816702036a3366090484470303c2ba660904826e02016b0904837c02016c07021403026c610604835802016d0204816802016e0104831202016f0104822f020270720602060201710704822202017206048174020273690204824602017409020c020175090482140201760a0482720301660404824403016a090484290201790404845703025d0203c2bc33040484620301bd0304824c020484540202c78607022403019a010482380202ca87070484390202d39d030485050203e184940404831b0203e6a881060483480203e8b18c0a04816d0203ee8d850104814801016b0110467257393c81272c011a053e815d3b190517064524521f011c3823590a8115372004313b1f3216011a5a20780b102d0804426916112c011a182f810781082d12137026161501221a180516811611051c131207811515173501180320112581062e05621c1407011c2d0e0617811522062208065a21520114582841621e6c203f1e2001161647411a272533815b1c2602013009048309020232630104835a0301720104817f0201330604836f0302ddb5080482560202347a07048102020135020483460104827b02043678ca800a04835f0201370404814b0104846002016103048246010482220301700204833f0201620404824d060481150201650304824f02016606088110834c0201670604821d0303c2be66010481790201680404843b030176050482270201690a04830e02016e0904844202016f040481010301630304822f020270640204822f03016e0704845802027177090482710206736ec2b2796a0104832e0306dab1d485377004048304020174050481700201750a0212020378627604048164030173080483190201790704833d0204823a02017a0506820e67030178070484530202c2b2060481500104823f020483030301b3020484310301bc04027e0402caaf0a026a0301be040482590204842e0202540202c79f0804824d0202cfa30804815a0204f29a92970204823301016c01140f63351a0a653b650d22011c09117a3e1538123537046a15043101141310082f49052f772b0c011c11121781583c2a5010133228241301287f3e0a2b1244080503060a100f413b4f0d070e2a01103e4e1f04814e7b1601183d0404052877111f230f811d01123a100f053e5c076910011a031732102381243d1b1727507301180e5d273e810803812e0f192a02013301048271070481330204821d020134080263020135060481280201610104830a0201640604826d020165060483050201670204841c0504841c0304841b0201680a04845602016a0104811c01024f030481080204813102016b0204837008024502016d0404836c04068207780301670704842302016f0404821203016d040484490301720404837e0201700104821d03048407030165050483050201720a04811602017307023502017407020503016f080484240302c2b90504821b020175090484090201770a088119822503026d6905048300020178030482680604812a0201790104830c0204833a0303d9a06806022002017a060482600203c2aa33030481560301b904020a0301bd0504820f0202d0b90904817c0202d3820202200202daa9080482030203de966e02024b0202df9d080484350204f098b0a20604845e01016d01220304456608322258060a031d4c38340f090112310c070e4238626e6601124a318109030513812f0118240d561e533742188113101b01160b24444b224d44814d4806011c05774e483410330d23541b28090401141f29062581131e221b6d011e81053a037a03320b0e4c24360d2310011a0e321d3c141825111d54637a1c0114093d3c2e58571a35293a0201350104840a03017701048330020136060217020138030483370201610a0482650201620504815e0201630704827701048201020164080483690201660804846703016904048113020167070483080201680504837d05022c0302cdb10a04815f0201690104833a0404824302026a360a04823b02016b0a04813502016d0504831a0204833803021a02026e360404825e02016f080484140201720304844b0404816603056ff09d899b0304823f020275390204816e0301780a04824202017604088308812703027902027770050482040201790104827e040482750204812902017a060483030304c2bdc2b30104836f0203c2aa62040484040301b903021c0302bd6b090484300301be0704814d0202c99402025603049a65656b090484020202ca92090482060203d19a730504844a0203d49f690804836e0202dfa8020482710204f09180860704822901016e011a0c0b8104243647521f43231f36011a2e1b33432c3d0b414905054d17011010573a6c0a816c1801160e063582340a5239050b06011a4481063d1b67250f2044200839012044591d1857291214135814101a1b361d011225067e8147111a4a4301166b13362e17195f3812186f01141c465b032b290406373301182a152a2281300f8107054e3f02023274080481770305c2bacf8168020481450201340604832f0201350704842e02013605020e0201380404841d0201610404810d020483750201620304812b020484230301610804834503026c6a0304816d0201630102380305613577337405048359020165040482720201660904826202066736f094b0af0a0482250201680104811f02016b0304847202016c0404822403016f0904822c0302c2b301025002016d0504817b01023f02016e020483090802040303e7bda10804832d02036f6b740404811402017005048419020484220202716506026303026b760904830a020172080482430304706c73620504825f02017308048413020174080481070201760104827f0204836e020477e7b89a0104840e02017a030483700206c2aa35657065050482740301b30804842b04046cc2be78090481040301b903020d0301bc010484260904813f0203c7a5620302330203c99f36050481010301a30704815b0202ca8b090483250202cdbb0604820a0202cebc0102170401380304842b0207eca2a6f29c87950904824001016f01221d17052b58101241060e3a201f1021633a0114816919811c142443100801280426080e2620042a812c531a490e121707131710011273432e493347811a340112195f671f46721c325e0118380c052b812822478107600b0116021c21821b2019263433040126021b05351b2a286b05181f071b5628111a330a012014533e073d0c0e5469141d1e2734050901220318051b44412803632e0642370e0a3a2b020131070481770201320a04812f0202346e04022b020136030483590304f09a81b60404834702016105048210020162030205030167010268020163010481540604820202048300020264310902420201650804834b02016703048247030365c6b602048205030573f098b890030481450201690204832802016a0a04826703016c0104825e02016b0604815e03016c03048334030677c2aa74c2bc09023d02016c0304823903027777060484540303df866c0104815b02016d0204811f0303796f7704020302016e0204814d07024a02016f0902680201700604840a0104831a0204835a020172070484440201730a023703026b660604830e030278790304815b0201750904822402017704025b020178030482350307f4b2a3896a343407026b0201790902720302633409020402017a020484590302dea004025f0202c2b20204816b050481200202de900402160204ee85a5770204822c0101700114143a0d391a60812d4e09011a2f313104201c372c3a3411321b011a268140144226334145050d1c4d01164e081f20671f088107237901186b123c1f6d07261e2b732e210116511116342a3d32376e083001106882257a0a17141101163039192b0c05812d735f3b01262a3e0841030b17181411051e0a18530e272b6d01182f322b260e24581d5381050f02013104048353020132050482370301690204843e02013302020d0201340104841f0201350a048139020137050482770201380204833a02016105025a0504832f0201620304836d020163050484100304832003016c030481290201640304837e020482490304822b010482290201660a04827002046964dc960204833102016c0704814502016d010482000201700104817e02037176760904821f020473eb91a708048152020174090482770201750404831e01063a825d03017a0904826a020276730a0254020177080260030277630104815c020178070481220202020301720804841202017a0204834c0202c2aa040484010301ba080482580202c6a3020481320203cdbf690502790202ce90070483140301bc030481470205d1a371cebc060481590203d2976a0404830c0203dfba6e0604814b01017101163732393b8120422f054b0e010e030b211d815d1c01165641757c080d81311d090e0112816581542d2313054301224e07121706516606080e39102d231c4b39010a2d81402d5e011a132527428114080d6e1111721c011a814a1a341538251023100d1c4c011e22182622623712411e38162a182d3b01142b67611981470f1f1f250201310a04824e0202336207048217020238730204815a0303cf886d06025a02033962620404833803016f0a04814e020161060483140302726d050483450301790904810c02036376690504811c02016403024d020165010483280802550301650904827f0304ebb8b561070482340201670302670301660804810e020168040482340201690704844d0302616404020f02016a030481060301700704827802016b070481240104814b0302c2bd0504816102016c0604837f03017a0404837902036dc2b90804810002016e0904821602016f0304812c03016401024b02017103048233060483600303e5848e04023a020172040481050305f3978aa06c070481151708070b070a0d0707070607080c080909090706080707070707070806070707070a090b070708080909090981140708070708080b0a0b0b070b070907090707070707070807080c0c070609070b0807100706070e08080a81110f06070707070f07120a0c070707070b0707060607080709080b0b080709060708070808080a810f0707060707070b070707070a080b08070e08070b0b08070c080f070a09060807070a080909080a810b080b070706070b0b0708060b07070c07070707070a0a09090b0708070a07070b0a070c070a060b080907080807070d8123070707070a0706060f070707090b07070707070b07080907080a060f070608080706070c060707070c070a810f07070706070707070a070b0713070a070707090a070c070706080a07070807080808070b0909810607080808090707080709060a070a060707070707070b080707090707060b0807'); +INSERT INTO t1_data VALUES(274877906948,X'00000e8a0330717304048359030134050481100203756371070482190201760704817b0301770804821d0201770204844d0201780204836c0404826103017504026a0202c2aa020482420301b305022c040267390602570301b9080481540301bc0102290206c99cf6b5aa80080481430202cebc02026e02048120010172011a1c2f15158108048125463f251d010811811539011412423105812181171549011847284a30234e5b33042632120118351e8113817d0f2b220d111901264f104a211004061d0a2a0b35121a0a2118341f011c81160a1b030d2a0610243e445f0c011c6f0c1e3b1768141e322717500b140110537f810169811625011a492847203e210f532c16480627020135020481780302caae0104811a020261330904846c0201620804812d0201630404814a0201650704837503017301048276020168050213020169010484540604842b0302796f0504833302016a0304831b02016b0604701302016d0604815302016e070483630204815202027071020481520201710104835b02037266700704843002027362040481490201740904817d020175020481040304f59e9c9407048218020176060484060204776dc2aa020483070201780504812503016601048159020279790704840502027a730904826c030178030482740202c2aa04023d010483540301bd06026b0203cab877010483290202cdbf060482410202cebc04025c0401690204827f04016c0904840e0202cf880104835d0203dfbe6c01025b0204f0aeb7b2030481680205f1a7b5bb390504826a010173011a22810d12415003071f81181839011a3220221511546d810012052b57011a0c4274300d154e81111f041e10011e293f4213051b2276560817312811170120092136122418370e4e782b3912080f3201262b3b340f222b0c09142a0822116a135c1c130c0114320c4e385a0d0415075f01163543340f06362381133c0c012224180981742048191d110e180e180d310f011a20632450281f043027114b034e0203316a61030482160201360104826e0201380a0484570201610104844d0504814a0201620904820f020164010481630304810403027761040258020165080481550302c9b70204827a0204677367690102660201680104831b020169020483760301720a04814502016a04020c02016b04023405023c030269630504840502016c08048463020170030482670202716d080482000201720304835c03016507024d020174030482520504821503016505048207020175040208030137060481710201760a04833d0201780404832302027a6f080481200202c2ba0204812c0301bd0404821e0205c3a66865730702540202c7890804816f0205cebcc7af730504815c0202d2930202540202db89020481160203e8b8a00304825d0205f0958db331070481620305989b8569780a04813d0204f69299a5020210010174011a7b3829100a4e511f1a281c17140114812626032c372634234c01140a520e815a810815200501123e4f3531042d57615b011a041f3e64070f1f1913274a20770114811d0f5d743e0634161c01162c2782130c1b810520280d01164a513110480b402b810d13011e522d08042c1146137012201e810512011a290903182c05301e5d811944290201390a04836d0301610104836e02016208027803036cc2bc07048261020263640a0481480301730704813602016408022b020165050484310301720702260301780402500203666d73070484470201670104825b02016a040481590304836702016c0304835601025e02016d060484110301340204813702016f04021802017006068336280201710304813507025902017201026508022903016d0604812f0201740304827e03016a010482440201750404834b02017604021a010484150504836a02057773c2b2380602520201780204823e0302cebc080484040303d2956403048171020179020484240204813c0202c2b303048307020484410301be0204816f030484250203c798680104843d03019c030482570204e19ea86a020482350302a0950a0482280204e5a4bc780304810e0101750126090a35030a03220a1731630f31252f0c4b1e31011e39200e3715282a03103b56090f6b1501121d4916246e6d460d6501162609380406361e816d203f011a22166008124f58202e182025150114390f3a25713f0e3f715c011a5a11191123466025710c313312011e3c191326811c1444055f1f5109051201143b106f1181000d068155012043381381020d81080d0603171824260a0201300404844a02013207024203026b390204833803066eeebabb35660604842102013308020302023f02013503048243020436716b66040481440201380a04843d020261690704836002016203048209030484670201630a04841d020264790104822b0201650502340302c2bd060484300201670202620201680704810d020169010484430104843402026b67040481540306eea3ad77c2aa0a04836702016c020258040482270104830a02016d0404824102036e716a0604843e02016f0104832a04020c0204836803036530650a04817402017005025b0301630604843d020171070275020273720404827002017504048133010484120301370102270201760104814f03026203016d0904844802017701048375080481220301660202330301700a04827f020378c2b2030484710202796806023c0301730704813d02017a0404815f0104817202048407010483390202c2b204026802048254040266640204831e0301b3090482230302b963060482010401750a0483290306ba35f2999dac09020f0203ca926e090483350203cdb4780302350202cebc0a020f0204d7a7696d010483100206f097bc996d71040481480101760114185b2258291610821c0e01160272173107154f5b813722011a81020c200e1826250d39811f07011a7911152a2a45131504422c81070120050d3f5b23342e3e4139032a3813042d0116592d1c15630c0c0a814649011c1a362f5c4a35511f0804033e372b01102981262a352e8205010e0b4b6282388106011e26810a2d125f361a12170d1721311e0201300204832b0201320104811b050483790201350204832202016404024c0202657a090483710202666307022703016c07048362030277750604842f0201670304843d0104844c0203686f7a07027202016a0102430204847502046ce0a2b20704810302016d0204827b02026e330804826c02016f0104835502017102027606026c020172060483490202736505048371020174030484130302387602025d02017502048345020376346b0904825c0201770904814b0303c2bd720804812b020178040483600201790804816402017a08026e0202c2b90704811c0301bd0504821e0204cdbcc7a108021a0202cfb8080481490204f09f96a50204842101017701180207232d37812d0c045c4a0a011a06163b3408171c52213a26592201206d08581605811a171e0c0a1347104914011282181324082b73320f01122f6e811d2c3d410a44011e551414206a092f133f333d150a3e0f011e235b170e37060627471b13373b3e27011a0e1e816b270c10102d53381045011a060e1e254d044932651234691e011a158138300a04810c0a8121071802013003027707048214020131080481370201320a04835502013304048271020538ceb369650802310201390a0481440202616c0502570302c7a104026c0201620804811c02016301048168020241020164040481560104820b020165080484460102400202686c060232020269670604827302016a0108810c826e0704824e02026b6c0604816902016c0304831c02016d0304811903016407022b02016e01022e0604845a030237780702040301710804815102016f0104842903017a0704826902017003048445020482080202713002024e0201720804822003016202022302017303048111010482790204812d03067479c39f66700a0210020174020229020175090484430301690604820402017606020202017a01021b0104843a0202c2b90604842b0301bd030484570202c69b0504815d0202c8a30a02240202ceb8090484690301bc0404832f0202df85020481230101780116812b0a16810e4b045a3b2a01205b1305811134092f62072343100f0f05011e5734152612030b4c4134123009361601121781653207780a6a0d01164a25210824138107738139011481341f088158060c8133010e5920193a4c2331011a0510358101231a1b3609702732011a2f07631610033436810256174c011a1342040a58110721378139101602033067750802720201320604834303017109048244020133090481700203366f79010483520201390404810902016301048411050483420201640a048432020265310704832b02026774040481000302d5b2010259020369756c0504832902016c07048365030233700904824102016d010482670404834c0504830c02016e0104831604048120030169090481750303eba6990104835f02016f0604834c030379c2ba0a0481560201700404815e06048256020174040482370201760a04820d0201770604811c02017a0404812e06020b0202c2b9030481660301bd0a04816c0301be030483620206c99b6d7777750304835b0206ceb0646b66610402490202d8bf030482250206e8bfbc626964080482510204f09c9a9e0404830f010179010e2081335661371c01220e4e2718124f0d0649812b0b0a063b040b011402741d1235810805211a011409161d732b8106325f6a01182e330325068107703728302b011e3723081c0d0a3f810c183e061b067f0106834a12011a044030185a1e810704220a0541011245602b0e421441817801144b03811a1a29614e224b02013003048139020132050485050304f09caba6010482230201350104822708048413020138020483430201610402230201620902440108812181070301300304815e020263710404831002016403048226010483660604823c020165010484510201670104816407048418030334c2b20a0481120201690404814c02016a010481500904810f02036b75610304836402016d01027902016f0304817b03056f6373cebc0a0483010201700104817d020171050482680104843b0302383203048128040807090707070b0608060707060c0b810e07080807070707060b080707070b0807090807070a070a07070808070b06090807070708080a0b81230907070b070b0707080907070706090807070807060b07060707070808070a080b0708090b0b09810a0707060908070607060609070b0a070706080a09070707070e0a0708090b0c0b09070a080a811a0706080c09070a07080b0708060806070b080c0e07090e09060706080b060a070b0607090707130b080708070b0908070a0c810d070b0706080707080b080a0a070807090708070707090709070706080709080a81170a0707070a070707070a0b0a07080d080707060a070707070b0707060f0b060707060a08070807080708810d0807070709070b070808070907080f0b070907090b0707070a0807070c0b080c0a810107070a0b07060c07080f070b09070b0906070b070b'); +INSERT INTO t1_data VALUES(274877906949,X'00000e5c033079720404826c0404833002021b03026f6b020482100201740904842e010484150303c2b36f0204840e0201760704826b02017708048273020578f48ba5b50a0481400201790904845902017a050483280304c2bac2bd0404841e0203c2aa680904843a0301bc070481100301be0304820304016a050481630202ca8103027b0202cb860604840d0204ceb56e370204832e0302bc740202520202cf8d07027e0205d8ae39c2aa0104813c0204e887b3770404816a0204f1bfb0970204832f01017a0118101e282f07045961813a193e011e69162f0d2b051c060f084460063053011e06810c20330d0733815c220515220c011210290a7e07810c3a18011a1f2f064a19155212472781047c010a123e45825501166c6062182718167131092f0112331b812c0b6e81470b01184f3a230d45261e271c36111701140a8128456b291248391a0201300104840d0204812502013401048318020137070483070201380904823b03016704026402013907023403016e060481380201610204812f0404824504045243020162050481150104824003016a0804827a020163070484590204817903017a02048209020164020484200203653077030482610302737601048424020166070481730201670304841e0504840e03016b0504825302016901048235010484400404841002016a010483680404845f02046bc2b97407023302026c7a0704812d02027161070209030378cebc0a021f020172040483750404815f0201730304813c030131020482040201740704824d0302793901027502017604048423020177040620813103048313020179030484540204833c04022503017a0904826302017a050483530203c2b2660404840f0307ba6272f397bd92010483070301be020482760202c6b9050482410203c9a3650604812c0202cbae020482180202d38c0704812f0204f096adb7070483490204f6a69c8b030484390102c2aa010c815e81147969010c811c827f0417010e81077a4c03815f010a2e8100820c010a810d148140010c0a7481201a13010c0b831525323d010a8206358129010c21637a33812701083c8340390301300504820a0304326939770404841c03016107048334030165030484150302696c0704810103016c0704812a03027178090481620301730504830e0301750504846303017802021c0301790304836803017a0604834e040135030482650302cdbf010483330304ceb23169070484090303e4849a0504817d0201b201088219744601082210832c010629846001121c8137182211816232010a81668122690108817281080110816a433581102f3e010e7481001b3481190106814e1b010e3646823a810e070301310504840b030161010481720301660604822f0301680804821003016a0704844803016b0104845b03016c0504823403026f700802340301710804826004026d3106024a03017509068458070301770a04836c04016b01021f0301790a0482790302c2b20a02270401b3040482130303ceae6e010484330302df9e040482100201b3010881656e750106820b50010e81434a27048153010a812b068122010843810c0401048211010a50817d812e010c811a8163810801061e832c010c811f81418101030163030481210301640504831102048104030482440301680a048202040271300404841303066b6576cebc6a0104843603026f7a08026d0303756a7808023e030176040483610301770704814403027973020484360302c2b906020c0401bc010484490401bd0804835c0302d38c0304846f0304eaae96750a0484020201b9010c3b824018322d010e0468315a5c817901067d583401060b810e010865118169010c0e07826b813a01066f8265010c3b8239633852010e61161e7030821b01068241210301350804843d0301620302480301690402610403e7a1910704812103016d080269030673c2bac2b36c090483070301760504821d02048454030278350702620302c2ba010483320304ca897a750804813b0401b5070482350303cebc6d050483120302d199020483020402ad660604840c0302d2a1010482550302daa30202340201ba01026d01085119826a010c2f5e82008110010882028221010481090108821b812d01068128460208810b8264010c810971812d440301310304820e0301320504825e05048221030333756d010482720301340502700301640604813d03016607020c0301690204823a03016b0104812203016c0704842103016d0304835f03016e0a04813c03016f0304834d030173060482660401710604810f0301740104843c0303d48174040482690201bc010482690108813c83290114090f816045242e148111010a810616822701048456010e2c81167a638115010a3a3b83204001067a8367010c8114127a2265010a824f7f5c230301300604833e0302616f020483560301640104827d090481280301650502150301660804841a03016804048221030169030481120301720104827803017606023d04016e020481700302c2aa0904836c0401ba090484100201bd01088240224f010843813674020a81185068620106822a44010e14154a8101825e010c19814a1b826c010c81221f81651b010a815a4d812c01082d7281160301300804843c030231300304836703013509048353030365c2bc0304814d0302667504027a0301670604831503016c09025d03016d03048178030172020483620301730104816b0402c2b30404823203017401027204016a08048451030175090237040177090481690301770302190301780a04823d03027a350a0481260302c2aa060481620401b3090481090304f098a1a30a04825d0201be010a0d730b816f010c821d81236c2b010c6f8208358119010681236801087d4e831e0108823c8235010667794b010683165e010e05812e3d3c820d010c81148123817403043531c2ba0204824c03083875c2bceba7957a03020803016302020804036dcebc0304814e0301640304843704033379790504823203016b08023603016d070481090304825804013707027a03016f080484530301710a04842203017809025e0304c2aa6135010481460401b90a04836b0401bc0704814f0306c6ab66e3afb9080482660103c39f350304821f0201b0050481110301680602030201b8040481310604810403016c0a024803017108024b0201be06048323030482650301300504812d0304f48990ae060482740103c491680a0483700301760a0481290201a7010484090202b177020482380201b3030482280102c5800602090301360104826a02018202048173040483780301370104814102018b090482730201930504810f0102c680020483790404830c020183020228020185030484400704820b02028d7a0a04821c0201920404835e02019507048437010484070301760204836102019a020214080483100303d795680504820602019e030481050201a30404820a05048436030177070484290201a50704843b0201a8060482690304816f03027735020483630202ab79050484430201b6060481050201b904024003016d070481130201bd090484350201bf06048361030163020481580102c781040484560201830604833a020286650404842a0301750a04821a020189070483370303e0b9b30904836602018c0104816c04020602021d030171050484570201960a04842d03016b0204821903016c04021e03016e0204845002019a0a04844402019d0704835103013103021702039f77750704827e0201a1090483340201a301022b0404831d0304616363750502520302cfb2040483720201a50a0481530202ad760902310201af0104811d0908816281390301640404843c0201b3080483240201bb030483480201bd040484290304840a0203bf646a0404843f03017a040482450102c89d05020f0201a1090483550201a3060483550201ab0904813203017a04024a0201ad030482210201b10204824302025d0303656577030483600202b46e010482240201b6090484410204bce39f9d020484570102c9870604813202018b0804843603017a06027102018d080481130204840c02018f0104836309026202029164020484430201930504816d0201970304843402039c6c73070483700201a0070483470201a203048262030364d7960a0484010201a80304832a060484390202a96a080481170303c69532090481440201ac040214020481180202ae770604832c0201b0090482300201b40804845a0201b5070482110302716f010482160201ba0504837b0201bb02048149030165050483250201bc07088105833002067f824d0302d19f0402330201bd04024d03017a0904814f0102ca800a048321020181040481580201820604821e020383693707048417020287630704840002018a0904834c0104835c02018b06022f02018c0704814702018d0704810a0104831a020490e7b38308048423020191020482510201920304832f030135030483710201950304833207026802019902026b02019b0904832a02019d09024c02019f0802210201a4010484080202a5710a0481680203a777670804831e0301790902100201ae0402700104834e0201af040484650202b673080484190201b705048221010483520201b808048465030167010484400104cb81cab90104834802018602048317020689777568cf920804842602018b0404817b020191050484200202a0720304833e0201a3040625827f0201a4030485080102cdb1070881408205020483700201b30404840e0202b735080482640201b80504815a0201b90a0482370206bb31cebc6c730304842f030133070483010103ce80370304826d0201900202770201ae0504844e0201b1050484080201b20904815f0201b3040483430201b40702600201b50804823b0201b90304846e0302c9a1070483300201ba0404844d0104847e0201bb050482780201bc010882378223010e811105814d8134010a8142823f2d0106811c52010a6e1814816b02088168824b010a438137812d011019060c6b812f811c010a81314e811b03033366660404825503013502048263030238620904820703016303048120040f080b0907070b07070a0907070707080a07070b0a0a81060b0707070606070f0b070b07070908070b070f0b090807080b07070707070c0e0707090d07080908080a0a50070a0707080708070706070707080a094d07070707070707070707080706070707090844070f07080c0708070708070707080a4707060609060c0b07080a07090808080737070b09060706070707070707070707094807080b0607070707060708074107080709070706070707080607060706070808070a460a0d06090709060b060707060a07070c0907060b06060b070a090707080707070b0707070c060b08070b070a09070b07070b08080706070707070807080707090d070707060707070609070a090807070d0707070b09070707070706070a0908070a0807060b0a080707090707090b08090a08070707080707070e07060708070709080b06070b0a0707070a06070606070809060a07080b07070a070c07070808070e070807070c07090607070707060707080b0743090708'); +INSERT INTO t1_data VALUES(274877906950,X'00040e1807020e0830cebc66f09f97b60604840703016b06021003016c0804835103016d0a0484050301710602560301730304822d030275770904834e0301770204824d0302797707022e0302c2b2070482640402bc6d0a025e0303c3a6370602530305cebc7768690a04823f0201be0504816f0301780a0482780201bf0a04833a0305f09e919a790a0483440103cf80340604827e0201810404823e0201830304843a0504837f0301780704834c02018407048139020186030881038372030178010231020187010481100201880204835b0204845a0104825a0201890204825c0604831802018a0704841502028c340304845e02018e0602180301390204833c0203936e770104820f0201940704823702039765630304847b020199080484110301710704821402029b650704827102019d0304845802019f0504845c0303e7ba880502690202a163090484160201a3060483020301680102470201a90a0482480201ab0104837f030482570201af050482190201b20a0481020201b30504842903026f64040483650203b873780704826d0201bb0704831c030177020481410102d0b0070482560202b17a050484460201b2040483400201b40402390202b671020484220202b777090483510201ba030481750202bb6301021e0202bc620a0484080201bd0204815d0201be060482470201bf040483340102d1800904825002028470050482290202876f06022202048872c5820104826202028b680504845d02018c0a027302018d0a0205020197090483210202986b0a02780201a3030481580201a70604835e020483670201a901048379060483780201ad0a04816b0201af010482310201b1080482020201b3060484610202b5640804814c03026e690304847c0201bb080482710201bd020482020201bf06048435010482590107d28b37c2ba61640104831702018d0704810b020393c5890204836e0201950602070305f6a5989a74040484450201970104813702029f790a04843f0202a57407027f0201a90504835b0202ab750502490202b9760a04827d0301780304826e0202bb6f060481780203bd7130050483630103d3866f09023602028a6c0604825302018c0304845302028e650404833a02028f6f0104821302039d356d090484320201a5020481130404813f0201ab0704835b0201b90a0482380201bd0904833c0201bf060484450302307604027f0102d491050483300201930a024603036bd291070483060201970704826702079b72c2b3356f730904844f02019d010482610201a10404814d0202a36b0502250203a5c7b3080483780201ab02023d0103d5a1640102260201a206026a0301660804813c0201a5030481180201a7050483360201a9040483510201aa060482020201ab0504811e0201bb090481680202bd740702160202be700304846d0102d6840a027d02068534f0988d910a0483730102d78f0404832804048462020390766601027802019202027405024b0201940a04834d020298770904815702039a6c6b0304846802049c73616e080483420201b0010481050202b565050481290206b67376c2aa640a0483280201b80704830e0102d89d0202150802720201a60602480202b073090481480201ba0204836f0102d9a50202670201b40a04822a030337c2b20a04820f0102da800604831c0201840604842802038636380804811e020188090484190201920504815402019a0304830c0201a301023d0201a6040481460201a8020484100201af0104842a0204825f0201b50304827c0201bb0804830f0105dbb878c2bc0502580107dc97e69eaf6764020259020299610104812902029d680502110202a0690a0483270203a56372030483540202a96a0204842c0205acf094abac0704821c0102dd92020483410203946d790304814f020395d7ac090228020296300404830302039b397809020a02029c770504833d0202a2700504821a0201ab010483400202b0760a04813a0201b2040483130203b3c7a10104833d0204b4e8a5a40604816f0201b90604830d0201ba090483630102de800904810a020281740a02380201880704826802018f030481670201900202790203916e710404833502019206048418020399ddba080481240202a3630a02250202b86e070481670204830d0201b90604811f0201bc070481530102df820102390201830304830102038e367901024f02039134620a0482080201920104814b020198030484790202a06b060481240201ba0802090201be08022f03023670010482170202bf6f01022d0103e0a1960304831d0302bb75060483700202a2900804817c0202a49e050485010202a6b2040482030202aa80040481720202abb30304843c0301ba0204833b0203aca677010481120206ae9f613566730504841e0202afa001020a0302a979080484060202b1a1010483730203b2a46a070482460301ba020484580203b39b610804834a0202b6a60304813b0202bfae0404837c0103e185930304845f02038889780a04826802028a85070484620301b90304826c020298a3070483270203af8962050484620202b5b7070481170202b8970602190202bd8f0a0482620103e2848f04021b0203afa861020484010301b70704824a0103e3828f05048214020284a50304812f0202909e0904824d020292b907026302049ba0336604020602029c8e0204840d0202ad860202750302a16a0802180202ae880602040202b480010483150202b7b1090483780202bdbf070482050103e480ab06020f0202889605024f020293b9030483310204968fdc93080483110202a295010484150202a69a0804844d0202a78e0a02080202aa9b0a04825f0203b2b8670402720203b5b764060482550204babacf87070481270202bd9b010481780206be9a6878696805021e0103e58bba0304821202049eab6a630a04827702029faf08024d0202a2b50a02400202a3af0704827c0203a4bd6f0504840c0202ad9d030485010202b1be040483780202b5b9060484500103e689b10404844602028abc0704811902038e98690802320203968b35090482680203979971010483060202b6a4030484050301be0a04833e0103e780a601048171020282be0304824d020286b105021b02028f960a048266020291be0804831c020299900204815502029ea70702400202a0ba040481410204a69e3870030481220202a9800602310202afb8020482170203b1af6d0802140203b5a06a0702740104e88c9a6c06048426020295bc05048326020296af0304846b02059cad72d4aa060484780205baa97364680404837b0103e983840104843902029a8a010481490203a1ac770504812c0301ba050483410202af8c0302620202b1af0302500103ea8fa60a023e020297b50904812e020298a90704825702039ea76b0304847f0202a59e080481460202a68e080482590204a9b1c6a8010481350203b38d75040482750203b89e6f0a0484030103eb828f0704824e020384837109048178020397b6690104821a02039e9e330704837f0203a68b73030484440203b980320804822c0103ec8fa80204814302039588760a04826902029bb80a024e0203afb6630802710203b4a268060483000103ed8cb50104811602029982060481550104ee8085760304834502028a870904845702028f8b04027602039da470030483040202a1a20504813c0202a8ae0504837f0202af9d0402570205b1b773d8b4080482120202bd880904832c0202be930104827b0104ef88ae680704835d0205a8ab63c2be06022e0203aaa16c080481090204adb779660804823f0202b8ae080484220203bb9165030481340106f08798b1636a0604824f0203908ea80404826203029f880402530303b4ad740a04833302039184b6090484650302acb3080483340302b1b703048272020a9281a23578f098b1893001048210030291a50404816803029d8f070483200302afa5080484440302b49f020481010303bd833603026a020393839a07024c0304a3ad6e300504842d0303abb16d0502660302adae0904830e02039481a406048164030386a16c070483530302b2a8050484320302bfb9080483170204958eb8720304832b0302b1ba090481200302babe05048209020496888b6708048358030293b4050483400304b4927a6f050483770204978b93320104810a03048fb9c7890704822803029ba702022a0302a39d060484490302a4be0a0484530302adaa0a026b0302af92040483700302b9bc0404821b02049884b56404048352030486b1716f0904827a030296b80904811d03049d9e797a060483390302a5a00a04844a0303b5856d0a04822e0302bcbd040483310303bd91740704845102039983a20a04816503039d8a330a02140303b9826b0404814e02069a8b8438687403026b0302a599010481740302a6ac0704840f0303b6896d0404831a0302b785010484370302bfac08027d02059b86b770610404845203028fbd020483550303a0be75090483620302a295040482390302b6970404813002039c97bb0a04841103059d8976c2be0604824e0302afbc05023c02039d87b40504825903028daf03048172030292be040482310302a0ab0502290303b2a16b03025d02039e929d050483040302a69f060484150302aa8607025502049f9f8b66020483160302a3be060481120203a8afad0904835e0204acb1af640204845b0204b282a47a040481610203baa68a010482490204bb9f8366060483130203bfa48f0704817c0104f18191920802220204828cad360a0482730203859c9a050481670204888eb23408025f020492ba9c3202026502049bb0bf7008021f02039f94a60104835c0203a6b496030482760203a98c960504824e0203acb7b8030483240203afa0b703023d0206b3a38c683972060483340203bbaaad0902070105f282848f790804833a020384b79f0a048170020590a98c616a0804843b02049a92be730904822f02039da18b080482530203a28f8f06048309070d06070706070807070807080b0707070b09070b07070906070f0b07080607090709070708070708080706070b070707080907070808070608080707080707070808070a0806060707070b0b07070707080807070b0d0709060b0708070707080708090808070808090b0707070707080609070d07070709060806070707070707070708070c0c08090708090a07080c070a0608070707090807090707070607070b07070a0c08070809080b08090808080808070807090a07070807070706090709070c070707070809070708060608070908080808080807090c07080809070908080909080708090807080809070908080709080707070808080807080a0808070808090a080b090a070708090808080908080909080709080708080807080a070808080a08080b0b0908090707070808080908080a0909090909090909090907080909080a0807090808070b08080a0a090a08090c090709090808100808080808080a0808090908080a08080a080a0a0a0708080708080a0a080a080908090908090b08080908070b08090808090b0709080807080908070a08090a0a090a09090a0909090909090909080c080b090b0a09'); +INSERT INTO t1_data VALUES(274877906951,X'000002020630f2abab907a060481130204b381af77040481790203bea6a6070484250108f380b5a33633737701048229020383be9f0404837102048495bf73070481060204938c9a6e0904823502039eb39201022302049fbb9f6c0a0481500204a3909e620404837d0204a9ac9a75020483110203bd8c850a048111030293ad0a048247030294b9040483550203bea4ac0702220105f483b5ab66030482440302bda306048141020584b2bb643804048142020586b1a8677a0204831d0205878c916c770804821f02038da1bf0704824302038f81bd0304825102039e91b20302420203b29b9403021f0203b3a1b10704825a0203b5afb4040481210204b98da0780a02060104f582b1b00404811002038f9fbc010482200302b8a406048419020394b7b80404834c02039bb5b005022b0203aa88b9060481390203aba48f0604842a0205af969438680602360203b88d8903026c0104f68386ad0904846702049aae8f79030483520203a8989a0604836c0203b18cb4070482260203b9a98d050483730203bebc870104824c0104f78283840804816703028db20302600303a8b46106020b020389b2a80104835002048e809339010481190205929193d0b409048253020393aebc0204816202059787b96f690704817202079b949dc78934620404817f02039db28b020483750205a288a7c7a3080482780203a5948d0204811a040b0a090e090a0a080a0a0a090808080b080b0b0b090908080909090a0908090809090a080a0a090909090a0708090a0b090b0d090b'); +CREATE TABLE IF NOT EXISTS 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID; +INSERT INTO t1_idx VALUES(2,X'',2); +INSERT INTO t1_idx VALUES(2,X'30627a',4); +INSERT INTO t1_idx VALUES(2,X'306a32',6); +INSERT INTO t1_idx VALUES(2,X'307173',8); +INSERT INTO t1_idx VALUES(2,X'307972',10); +INSERT INTO t1_idx VALUES(2,X'30cebc66',12); +INSERT INTO t1_idx VALUES(2,X'30f2ab',14); +CREATE TABLE IF NOT EXISTS 't1_content'(id INTEGER PRIMARY KEY, c0, c1,Öc2); +INSERT INTO t1_content VALUES(1,'COMPILER=gcc-5.4.0 20160609','X','RTRIM'); +INSERT INTO t1_content VALUES(2,'COMPILER=gcc-5.4.0 20160609','X','NOCASE'); +INSERT INTO t1_content VALUES(3,'COMPILER=gcc-5.4.0 20160609','X','BINARY'); +INSERT INTO t1_content VALUES(4,'DEBUG','X','RTRIM'); +INSERT INTO t1_content VALUES(5,'DEBUG','X','NOCASE'); +INSERT INTO t1_content VALUES(6,'DEBUG','X','BINARY'); +INSERT INTO t1_content VALUES(7,'ENABLE DBSTAT VTAB','X','RTRIM'); +INSERT INTO t1_content VALUES(8,'ENABLE DBSTAT VTAB','X','NOCASE'); +INSERT INTO t1_content VALUES(9,'ENABLE DBSTAT VTAB','X','BINARY'); +INSERT INTO t1_content VALUES(10,'ENABLE FTS4','X','RTRIM'); +INSERT INTO t1_content VALUES(11,'ENABLE FTS4','X','NOCASE'); +INSERT INTO t1_content VALUES(12,'ENABLE FUS4','X','BINARY'); +INSERT INTO t1_content VALUES(0,NULL,NULL,NULL); +INSERT INTO t1_content VALUES(33,'OMIT LOAD EXTENSION','X','BINARY'); +INSERT INTO t1_content VALUES(32,'OMIT LOAD EXTENSION','X','NOCASE'); +INSERT INTO t1_content VALUES(31,'OMYT LOAD EXTENSION','X','RTRIM'); +INSERT INTO t1_content VALUES(30,'MAX MEMORY=50000000','W','BINARY'); +INSERT INTO t1_content VALUES(29,'MAX MEMORY=50000000','X','NOCASE'); +INSERT INTO t1_content VALUES(28,'MAX MEMORY=50000000','X','RTRIM'); +INSERT INTO t1_content VALUES(27,'ENABLE RTREE','X','BINARY'); +INSERT INTO t1_content VALUES(26,'ENABLE RTREE','Y','NOCASE'); +INSERT INTO t1_content VALUES(25,'ENABLE RTREE','X','RTRIM'); +INSERT INTO t1_content VALUES(24,'ENABLE MEMSYS5','X','BINARY'); +INSERT INTO t1_content VALUES(23,'ENABLE MEMSYS5','X','NOCASE'); +INSERT INTO t1_content VALUES(22,'ENABLE MEMSYS5','X','RTRIM'); +INSERT INTO t1_content VALUES(21,'ENABLE JSON1','X','BINARY'); +INSERT INTO t1_content VALUES(20,'ENABLE JSON1','X','NOCASE'); +INSERT INTO t1_content VALUES(18,'ENABLE GEOPOLY','X','BINARY'); +INSERT INTO t1_content VALUES(17,'EOABLE GEOPOLY','X','NOCQSE'); +INSERT INTO t1_content VALUES(16,'ENABLE GEOPOLY','X','RTRIM'); +INSERT INTO t1_content VALUES(15,'ENABLE FTS5','X','BINARY'); +INSERT INTO t1_content VALUES(14,'ENABLE FTS5','X','NOCASE'); +CREATE TABLE IF NOT EXISTS 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB); +CREATE TABLE IF NOT EXISTS 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID; +CREATE TABLE t2(x); +INSERT INTO t2 VALUES('optimize'); +INSERT INTO t2 VALUES('rebuild'); +INSERT INTO t2 VALUES('integrity-check'); +PRAGMA writable_schema=OFF; +COMMIT; +} {} + +do_catchsql_test 51.1 { + SELECT max(rowid)==0 FROM t1('e*'); +} {1 {database disk image is malformed}} + +#-------------------------------------------------------------------------- +reset_db +do_test 52.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 40960 pagesize 4096 filename crash-2b92f77ddfe191.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 75 74 32 5f 63 6f 6e 66 69 67 74 32 tablut2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: e8 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 .csize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6f 74 74 31 5f 63 6f 6e 74 1_conteott1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 08 c1 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 14 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 03 ..h.......i..... +| 4064: f4 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 00 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 16: 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 04 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3968: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 3984: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4000: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4016: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4032: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4048: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4064: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| 4080: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| page 8 offset 28672 +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-2b92f77ddfe191.db +}]} {} + +do_catchsql_test 52.1 { + SELECT fts5_decode(id, block) FROM t1_data; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 53.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 24576 pagesize 4096 filename crash-dbe9b7614da103.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6b WIDU........tabk +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 10 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e 2a ...m.K.,.......* +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0e 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 89 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 16 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 70 .......e.......p +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 03 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 02 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 86 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 07 ....2.......1t.. +| 3744: f4 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 0f 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 00 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 33 44 65 ......1......3De +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 75 ......0t......0u +| 4080: 26 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 &.....0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 08 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f 00 00 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 0c e9 ................ +| end crash-dbe9b7614da103.db +}]} {} + +do_catchsql_test 53.1 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x<>1 FROM c WHERE x<10) + INSERT INTO t1(a) SELECT randomblob(3000) FROM c; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 54.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 24576 pagesize 4096 filename crash-03a1855566d9ae.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 2a 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_c*nfigt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 73 73 !...tablet1_doss +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 32 2c 32 2c 33 ,b,prefix=.2,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cd 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 05 62 03 04 0a 19 8c 80 80 ....tab.b....... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 39 a7 68 03 02 .....8.....9.h.. +| 2560: 04 10 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a ff 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c cc 03 02 03 07 1c 8c 80 80 80 .0tabl.......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 05 0c 00 00 00 16 01 01 02 04 ................ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 84 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 .h........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 12 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 11 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 08 f0 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 06 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 00 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 de 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 00 f2 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 05 52 08 04 01 ...0n.......R... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 b3 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 f4 09 11 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 e4 .....2th......2. +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 23 03 02 01 03 03 02 02 01 02 02 00 f2 09 ..#............. +| end crash-03a1855566d9ae.db +}]} {} + +do_catchsql_test 54.1 { + SELECT rowid==-1 FROM t1('t*'); +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 55.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 32768 pagesize 4096 filename crash-b366b5ac0d3887.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 05 90 00 00 09 00 00 00 04 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 43 4c 4f 42 29 5e 05 07 17 21 Y, sz CLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 32 5f 69 64 78 74 ...-tablet2_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 42 52 59 20 4b 45 59 EGER PRIMBRY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 f1 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 08 d6 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 00 00 00 00 .......x.W.>.... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 20 f5 00 33 0f 19 4f 4d 0XRTRIM. ..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4c 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 LAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 30 45 4e 41 42 INARY....%.0ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4f 41 42 4c 45 20 46 54 53 35 58 .#..EOABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 31 42 58 42 49 4e 41 52 59 1e TAT VT1BXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 15 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 42 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XBTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 1f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 07 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 05 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 01 17 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 02 f0 12 06 01 01 ................ +| 4080: 06 02 03 00 13 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 77 72 .........+intewr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 uild....optimize +| end crash-b366b5ac0d3887.db +}]} {} + +do_catchsql_test 55.1 { + SAVEPOINT one; + DELETE FROM t1 WHERE a MATCH 'ts'; +} {/*malformed database schema*/} + +do_execsql_test 55.2 { + ROLLBACK TO one; +} + +#------------------------------------------------------------------------- +reset_db +do_test 56.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-2acc487d09f033.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 4f 64 6f 63 73 69 7a 65 04 43 52 izet1Odocsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 32 2c 32 2c 33 ,b,prefix=.2,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 09 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 3b 01 01 06 04 09 19 8c 80 ....of.;........ +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a ff 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 00 f1 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 81 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 61 80 80 80 11 03 00 3c 00 00 ......a......<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 14 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 71 .......6.....2tq +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 0b 89 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 f4 01 02 03 01 03 66 .....0eac......f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 96: 0e 7c 0e 73 0e 6a 0e 60 0e 58 0e 4f 00 00 00 00 .|.s.j.`.X.O.... +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 b3 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 05 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 2d .....2th......2- +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f eb 00 0f f9 0f f2 0f eb 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 03 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end crash-2acc487d09f033.db +}]} {} + +do_test 56.1 { + set res [catchsql { + INSERT INTO t1(b) VALUES(randomblob(250)); + INSERT INTO t1(b) VALUES(randomblob(250)); + }] + + # For some permutations - those that use the page-cache - this test + # may return SQLITE_CONSTRAINT instead of SQLITE_CORRUPT. This is because + # the corrupt db in the test over-reads the page buffer slightly, with + # different results depending on whether or not the page-cache is in use. + if {$res=="1 {constraint failed}"} { + set res "1 {database disk image is malformed}" + } + set res +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 57.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename x.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 34 20 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..4 ...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 61 6b 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 aket1_configt1_c +| 3600: 6f 7e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 o~fig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 1d !.wtablet1_cont. +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 59 64 78 03 43 52 45 t1_idxt1_Ydx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 64 61 02 43 52 45 41 54 45 tat1_dada.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 0e b4 00 06 0e 35 00 0f e8 0e 35 0f bd 0f 4e ......5....5...N +| 16: 0e cb 0e 4f 00 00 00 00 00 00 00 00 00 00 00 00 ...O............ +| 3632: 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 01 04 .........6...... +| 3648: 04 00 04 01 01 01 02 01 01 03 01 01 04 01 01 5e ...............^ +| 3664: 90 80 80 80 80 01 04 00 81 40 00 00 00 51 06 30 .........@...Q.0 +| 3680: 61 62 61 63 6b 01 01 04 04 6e 64 6f 6e 01 01 02 aback....ndon... +| 3696: 04 63 69 76 65 01 01 02 04 6c 70 68 61 01 01 02 .cive....lpha... +| 3712: 03 74 6f 6d 01 01 01 06 62 61 63 6b 75 70 01 01 .tom....backup.. +| 3728: 02 05 6f 6f 6d 65 72 01 01 01 06 63 68 61 6e 6e ..oomer....chann +| 3744: 65 01 01 01 04 74 65 73 74 01 01 04 09 08 08 08 e....test....... +| 3760: 07 0a 09 0a 0f 3a 00 17 30 00 00 00 00 01 03 03 .....:..0....... +| 3776: 00 03 01 01 01 02 01 01 03 01 01 68 8c 80 80 80 ...........h.... +| 3792: 80 01 04 00 81 54 00 00 00 5b 06 30 61 62 61 63 .....T...[.0abac +| 3808: 6b 02 02 07 04 04 6e 64 6f 6e 02 02 05 02 04 63 k.....ndon.....c +| 3824: 69 76 65 02 02 0b 02 04 6c 70 68 61 02 04 02 0a ive.....lpha.... +| 3840: 02 03 74 6f 6d 02 02 09 01 06 62 61 63 6b 75 70 ..tom.....backup +| 3856: 02 02 04 02 05 6f 6f 6d 65 72 02 02 08 01 06 63 .....oomer.....c +| 3872: 68 61 6e 6e 65 02 02 03 01 04 74 65 73 74 02 02 hanne.....test.. +| 3888: 06 04 0a 09 09 0a 08 0b 0a 0b 0f ef 00 14 2a 00 ..............*. +| 3904: 00 00 00 01 02 02 00 02 01 01 01 02 01 01 68 88 ..............h. +| 3920: 80 80 80 80 01 04 00 81 54 00 00 00 5b 06 30 61 ........T...[.0a +| 3936: 62 61 63 6b 01 02 07 04 04 6e 64 6f 6e 01 02 05 back.....ndon... +| 3952: 02 04 63 69 76 65 01 02 0b 02 04 6c 70 68 61 01 ..cive.....lpha. +| 3968: 04 02 0a 02 03 74 6f 6d 01 02 09 01 06 62 61 63 .....tom.....bac +| 3984: 6b 75 70 01 02 04 02 05 6f 6f 6d 65 72 01 02 08 kup.....oomer... +| 4000: 01 06 63 68 61 6e 6e 65 01 02 03 01 04 74 65 73 ..channe.....tes +| 4016: 74 01 02 06 04 0a 09 09 0a 08 0b 0a 0b 24 84 80 t............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 05 42 66 74 02 02 02 04 04 6e 64 ck....Bft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 04 0d 00 on.............. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 04 0f e5 00 00 00 0f f3 0f ec 0f e5 ................ +| 4064: 00 00 00 00 00 06 04 01 0c 01 04 02 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0d 01 02 ................ +| page 4 offset 12288 +| 0: 0d 0e bc 00 04 0e 78 00 00 00 00 00 00 00 0e 78 ......x........x +| 16: 0e 78 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .x.............. +| 3696: 00 00 00 00 00 00 00 00 42 02 04 00 81 09 61 6c ........B.....al +| 3712: 70 68 61 20 63 68 61 6e 6e 65 20 62 61 63 6b 75 pha channe backu +| 3728: 70 20 61 62 61 6e 64 6f 6e 20 74 65 73 74 20 61 p abandon test a +| 3744: 62 61 63 6b 20 62 6f 6f 6d 65 72 20 61 74 6f 6d back boomer atom +| 3760: 20 61 6c 70 68 61 20 61 63 69 76 65 00 00 00 44 alpha acive...D +| 3776: 81 09 61 6c 70 68 61 20 63 68 61 6e 6e 65 20 62 ..alpha channe b +| 3792: 61 63 6b 75 70 20 61 62 61 6e 64 6f 6e 20 74 65 ackup abandon te +| 3808: 73 74 20 61 62 61 63 6b 20 62 6f 6f 6d 65 72 20 st aback boomer +| 3824: 61 74 6f 6d 20 61 6c 70 68 61 20 61 63 69 76 65 atom alpha acive +| 4064: 0a 03 03 00 1b 61 4e 61 6e 64 6f 6e 08 02 03 00 .....aNandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 71 63 6b .abaft.....abqck +| page 5 offset 16384 +| 0: 0d 0f e8 00 04 0f e2 00 00 00 00 00 00 00 0f e2 ................ +| 16: 0f e2 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 04 02 03 00 0e 0a 00 00 00 06 0e 0a 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 10 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end x.db +}]} {} + +do_catchsql_test 57.1 { + INSERT INTO t1(t1) VALUES('optimize') +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +reset_db +do_test 58.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-5a5acd0ab42d31.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 2a 6e 66 69 68 74 31 5f 63 6f 6e 66 t1_c*nfiht1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 73 73 !...tablet1_doss +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 32 2c 32 2c 33 ,b,prefix=.2,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ae ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 30 fd 15 0c f3 0c d3 0c b5 ...v.T.0........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 00 00 00 00 00 00 ...m.M.+........ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 9c 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 05 62 03 04 0a 19 8c 80 80 ....tab.b....... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 39 a7 68 03 02 .....8.....9.h.. +| 2560: 04 10 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a ff 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 01 f1 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 14 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c cc 03 02 03 07 1c 8c 80 80 80 .0tabl.......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 02 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 3e 10 00 00 11 02 01 .........>...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 05 0c 00 00 00 16 01 01 02 04 ................ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 01 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 bf 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 03 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 01 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 12 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 71 01 07 .......0the..q.. +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 08 f0 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 07 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 03 f8 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 00 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 10 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 00 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 de 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 00 f2 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f c1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d7 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 8e 0e 85 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 34 03 33 74 68 1c 08 04 01 10 01 03 ....4.3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 0f f1 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 30 fc .....1n.......0. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 05 52 08 04 01 ...0n.......R... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 b3 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 09 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 f4 09 11 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 e4 .....2th......2. +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 00 00 00 00 00 00 00 ......0t........ +| page 4 offset 12288 +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 4080: 00 00 23 03 02 01 03 03 02 00 00 00 00 00 00 00 ..#............. +| end crash-5a5acd0ab42d31.db +}]} {} + +do_catchsql_test 58.1 { + SELECT * FROM t1('t*'); +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +do_test 59.0 { + sqlite3 db {} + sqlite3_fts5_register_matchinfo db + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-96b136358d01ec.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 00 00 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c d6 63 32 29 69 04 07 17 19 c0, c1,.c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 93 ff 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 31 31 36 30 ...........21160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 33 f1 609...........3. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 03 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 01 f2 03 06 4e 02 02 03 06 01 ..........N..... +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 13 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 07 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 01 f1 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 25 0d 02 03 01 02 03 01 ........%....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 0f f2 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 00 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 13 06 00 f2 02 03 ocase........... +| 3680: 06 01 02 02 13 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 0a 22 74 72 65 65 19 02 03 01 02 03 01 02 03 ...tree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 11 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 01 f1 06 01 01 02 ad 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 01 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0e fc 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 8f DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 45 ed 0XRTRIM.!..3..E. +| 3168: 49 54 20 4c 4f 41 44 21 45 58 54 45 4e 53 49 4f IT LOAD!EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 5a 29 MIT LOAD EXTENZ) +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 59 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMYT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 56 a9 4d 1f 1e 05 00 33 0f 19 IONXRTV.M....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 57 42 49 4e 31 52 59 1f 1d 05 00 33 0f 000WBIN1RY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 32 0000XNOCASE....2 +| 3328: 0f 17 4e 41 58 20 4d 45 4d 4f 52 59 2d 35 30 30 ..NAX MEMORY-500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 59 4e 4f 43 41 53 45 17 LE RTREEYNOCASE. +| 3408: 19 66 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 .f.%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 76 35 58 52 54 52 49 4d 18 15 05 10 25 MSYv5XRTRIM....% +| 3520: 0f 19 45 4e 40 42 4c 45 20 4a 53 4f 4e 31 58 42 ..EN@BLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 32 58 4e 4f 43 41 53 45 17 LE JSON2XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 5f 81 42 4c NARY....)..E_.BL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 51 53 45 E GEOPOLYXNOCQSE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 1a 45 4e 41 42 4c 45 20 56 54 43 35 58 42 49 ..ENABLE VTC5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 48 4e 4f 43 41 53 45 16 1d 05 E FTS5HNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 55 53 34 58 42 49 4e 41 52 59 17 0b LE FUS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 57 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e WNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 05 52 54 52 49 4d 1e ABLE FTS4.RTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3f 87 ...C..COMPILER?. +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 45 0d 60 59 4c 45 52 3d 67 63 63 2d 35 2e 34 00 E.`YLER=gcc-5.4. +| page 6 offset 20480 +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 01 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 02 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 00 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 10 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 07 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 4048: 00 00 00 00 00 00 5d 03 02 2b 69 6e 74 00 00 00 ......]..+int... +| end crash-96b136358d01ec.db +}]} {} + +do_catchsql_test 59.1 { + SELECT (matchinfo(591,t1)) FROM t1 WHERE t1 MATCH 'e*eŸ' +} {0 {}} + +#------------------------------------------------------------------------- +do_test 60.0 { + sqlite3 db {} + sqlite3_fts5_register_matchinfo db + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-c77b90b929dc92.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 00 00 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c d6 63 32 29 69 04 07 17 19 c0, c1,.c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 93 ff 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 31 31 36 30 ...........21160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 33 f1 609...........3. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 03 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 01 f2 03 06 4e 02 02 03 06 01 ..........N..... +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 13 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 07 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 76 b4 65 6e 73 69 6f 6e 1f 02 ......v.ension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 25 0d 02 03 01 02 03 01 ........%....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 0f f2 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 00 03 6d 61 78 1c 02 0c 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 13 06 00 f2 02 03 ocase........... +| 3680: 06 01 12 02 13 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 11 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 01 f1 06 01 01 02 ad 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 01 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0e fc 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f 00 00 00 00 00 00 00 ....$........... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 8f DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 43 41 46 45 3d ..%..THREADCAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4b 4f 41 44 21 45 58 54 45 4e 53 49 4f IT KOAD!EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 59 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMYT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 56 a9 4d 1f 1e 05 00 33 0f 19 IONXRTV.M....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 57 42 49 4e 31 52 59 1f 1d 05 00 33 0f 000WBIN1RY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 32 0000XNOCASE....2 +| 3328: 0f 17 4e 41 58 20 4d 45 4d 4f 52 59 2d 35 30 30 ..NAX MEMORY-500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 59 4e 4f 43 41 53 45 17 LE RTREEYNOCASE. +| 3408: 19 66 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 .f.%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 10 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 40 42 4c 45 20 4a 53 4f 4e 31 58 42 ..EN@BLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 32 58 4e 4f 43 41 53 45 17 LE JSON2XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4f 81 42 4c NARY....)..EO.BL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 51 53 45 E GEOPOLYXNOCQSE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 1a 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 48 4e 4f 43 41 53 45 16 1d 05 E FTS5HNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 55 53 34 58 42 49 4e 41 52 59 17 0b LE FUS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 57 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e WNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 05 52 54 52 49 4d 1e ABLE FTS4.RTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 8a 4e 41 52 .....DEBUGXB.NAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3f 87 ...C..COMPILER?. +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 45 0d 60 59 4c 45 52 3d 67 63 63 2d 35 2e 34 2e E.`YLER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 68 52 54 52 49 4d 0 20160609hRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e 00 00 00 00 00 00 00 00 00 00 ....$........... +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 01 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 02 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 00 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 4048: 00 00 00 00 00 00 5d 03 00 00 00 00 00 00 00 00 ......]......... +| end crash-c77b90b929dc92.db +}]} {} + + +do_catchsql_test 60.2 { + SELECT (matchinfo(t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ' +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +do_test 61.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-e5fa281edabddf.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 0f c7 00 06 0d b6 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 00 00 00 00 00 00 00 00 ...k............ +| 3504: 00 00 00 00 00 00 56 07 06 17 1f 1f 01 7d 74 61 ......V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 51 62 6c 65 74 31 5f 64 ..!!...tQblet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 63 6f 63 73 69 7a 65 ocsizet1_cocsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 ea 74 31 43 52 ...._tablet.t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 13 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 11 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 01 13 05 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 01 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 0e 16 01 01 02 01 06 01 01 02 01 06 01 02 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 07 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 e2 05 00 25 0f 19 54 48 52 45 41 NARY....%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 40 52 59 1f 20 05 00 33 0f 19 4f NXBIN@RY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4e 45 4d 4f 52 59 3d 35 30 30 30 30 MAX NEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 45 30 30 30 .MAX MEMORY=E000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 20 54 52 ...%..ENABLE TR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 e5 54 53 35 58 42 49 ..ENABLE .TS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4d NARY....#..ENABM +| 3712: 45 b5 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E.FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 b7 4e 41 52 59 17 0b LE FTS4XB.NARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 53 9XNOCASE&...C..S +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 2f 31 00 00 00 00 00 00 00 00 00 00 00 0 2/1........... +| page 6 offset 20480 +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 10 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 01 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-e5fa281edabddf.db +}]} {} + +do_catchsql_test 61.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1'(),'col' ); +} {/*malformed database schema*/} + +do_catchsql_test 61.2 { + SELECT * FROM t3 ORDER BY rowid; +} {/*malformed database schema*/} + +#------------------------------------------------------------------------- +do_test 62.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-44942694542e1e.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 00 00 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 79 67 74 31 5f 63 blet1_confygt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 52 4c 4f 42 29 5e 05 07 17 21 Y, sz RLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 35 ff 63 6f 6e 74 65 6e 74 05 43 52 entt5.content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 42 29 69 04 07 17 19 c0, c1, cB)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 79 64 78 04 43 52 45 41 54 45 20 54 41 42 1_ydx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 74 61 ablet1_datat1_ta +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 2f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 /..........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 c7 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 04 16 01 02 02 03 06 01 02 02 02 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 02 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 00 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 74 67 04 02 02 01 02 02 01 02 02 01 06 65 ebtg...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 01 f1 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 45 02 02 01 02 02 01 02 02 01 02 .....E.......... +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 09 c1 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 00 35 0d 02 03 01 02 04 01 ........5....... +| 3552: 02 03 01 0f d7 63 63 01 02 03 01 02 03 01 02 03 .....cc......... +| 3568: 02 06 65 6f 70 6f 6b 79 10 02 03 01 02 03 01 02 ..eopoky........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 14 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 12 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 09 f6 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 11 02 ................ +| 3728: 02 01 04 6f 7d 69 74 1f 02 02 01 02 02 01 02 02 ...o.it......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 11 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 00 fa 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 03 04 01 40 .......vtab....@ +| 3856: 04 01 02 04 11 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 00 02 01 06 01 01 02 01 03 91 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 76 01 01 02 01 06 01 01 02 5c ......v......... +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 11 06 ................ +| 4000: 01 02 02 01 06 08 11 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 05 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 ca 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 14 24 0f D..@.......$Z.$. +| 4080: 0a 03 00 24 ff ff ff ff 01 01 02 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fb 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 09 00 00 00 00 00 00 00 00 00 ....$........... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 47 17 22 DSAFE=0XNOCASG.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 3f 41 44 20 45 58 54 45 4e 53 49 4f IT L?AD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 64 20 4c 4f 41 44 20 45 58 54 45 d9 53 49 MId LOAD EXTE.SI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 39 54 20 4c 4f 41 44 20 45 58 55 45 4e 53 OM9T LOAD EXUENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4c 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 LAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 af 4f 43 41 53 45 1e 1c 05 00 33 0000X.OCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 ab 30 30 58 62 54 52 49 4d 18 1b 05 00 25 00.00XbTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1b 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 63 35 58 42 49 NABLE MEMSYc5XBI +| 3456: 4e 41 52 59 1a 17 04 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 3d 45 ....)..ENABLE =E +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 46 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LF JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 46 45 46 50 4f 4c 59 57 42 49 NABLE FEFPOLYWBI +| 3616: 4e 41 52 59 18 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 5f 43 41 53 45 E GEOPOLYXN_CASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 42 ....)..ENABLE GB +| 3664: 2f 50 4f 4c 59 58 51 54 52 49 4d 17 0f 05 00 23 /POLYXQTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 1c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 16 0b LE FTS4XBINARY.. +| 3776: 05 00 22 0f e9 45 4e 41 42 4c 35 20 46 54 53 34 .....ENABL5 FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 00 47 45 4e XNOCASE....#.GEN +| 3808: 41 42 4c 45 20 46 54 53 34 57 52 54 52 49 4d 1e ABLE FTS4WRTRIM. +| 3824: 60 05 00 31 0f 19 45 4e 41 42 4c 55 20 43 42 53 `..1..ENABLU CBS +| 3840: 54 41 54 20 56 54 42 42 58 42 49 4e 41 52 59 1e TAT VTBBXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 40 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d T@T VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 55 20 44 42 53 ...1..ENABLU DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 12 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 21 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y!......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 18 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 42 54 52 49 4d 27 11 05 00 43 0f 19 43 4f 4d XBTRIM'...C..COM +| 3984: 50 49 48 f5 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PIH.R=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 22 32 30 31 36 30 36 30 cc-5.4.0.2016060 +| 4048: 39 c2 3e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9.>OCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 30 32 30 31 26 30 36 30 39 58 52 54 52 49 4d 00201&0609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 00 00 00 00 00 00 00 00 ....$........... +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 00 f6 17 03 00 19 e2 f9 01 ................ +| 3920: 06 16 03 00 12 02 05 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 10 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 00 f1 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 05 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 02 ff 84 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 07 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-44942694542e1e.db +}]} {} + +do_catchsql_test 62.1 { + WITH c(x) AS (VALUES(false) UNION ALL SELECT x+1 FROM c WHERE x<72) + INSERT INTO t1(a) SELECT randomblob(2829) FROM c; +} {/*malformed database schema*/} + +#--------------------------------------------------------------------------- +do_test 63.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-8230e6c3b368f5.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 7c 65 62 63 62 62 ......1tab|ebcbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 9d EATE TABLE 't1_. +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 64 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 ed1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 10 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a ad ...~.H.......... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 ec 09 ca 09 a8 ...m.M.+........ +| 112: 09 86 09 63 0f f1 00 00 00 00 00 00 00 00 00 00 ...c............ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 13 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 75 61 62 03 02 03 04 0a 19 8c 80 80 ....uab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 ec 68 03 02 .....8.....2.h.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 17 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 64 02 08 05 0a 1b 88 .......thd...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 43 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .C...........6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 01 00 02 30 21 02 ..0there.....0!. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 51 .............<.Q +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 51 62 6c 01 06 01 01 05 02 03 65 ...4tQbl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 17 63 68 01 02 03 01 04 70 .....4e.ch.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 03 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 12 02 31 65 01 02 02 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 01 f4 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 00 00 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 07 ....2.......1t.. +| 3744: f4 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 73 22 07 04 01 0e 01 02 34 20 08 04 ...4s.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 00 00 00 00 00 00 00 .......0t....... +| page 4 offset 12288 +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 02 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 08 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| end crash-8230e6c3b368f5.db +}]} {} + +do_catchsql_test 63.1 { + SELECT * FROM t1 WHERE b MATCH 'thead*thead*theSt*'; +} {/*malformed database schema*/} + +do_catchsql_test 63.2 { + INSERT INTO t1(t1) VALUES('optimize'); +} {/*malformed database schema*/} + +do_catchsql_test 63.3 { + SELECT * FROM t1 WHERE b MATCH 'thead*thead*theSt*'; +} {/*malformed database schema*/} + +#--------------------------------------------------------------------------- +do_test 64.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-4470f0b94422f7.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0d e2 00 0f c4 0f 6a ...............j +| 112: 0e fc 0e 9d 0e 3d 0d e2 00 00 00 00 00 01 00 00 .....=.......... +| 3552: 00 00 59 06 06 17 21 21 01 7f 74 61 62 6c 65 74 ..Y...!!..tablet +| 3568: 74 74 5f 63 6f 6e 66 69 67 74 74 74 5f 63 6f 6e tt_configttt_con +| 3584: 66 69 67 06 43 52 45 41 54 45 20 54 41 42 4c 45 fig.CREATE TABLE +| 3600: 20 27 74 74 74 5f 63 6f 6e 66 69 67 27 28 6b 20 'ttt_config'(k +| 3616: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 PRIMARY KEY, v) +| 3632: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5e 05 07 WITHOUT ROWID^.. +| 3648: 17 23 23 01 81 03 74 61 62 6c 65 74 74 74 5f 64 .##...tablettt_d +| 3664: 6f 63 73 69 7a 65 74 74 74 5f 64 6f 63 73 69 7a ocsizettt_docsiz +| 3680: 65 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 e.CREATE TABLE ' +| 3696: 74 74 74 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 ttt_docsize'(id +| 3712: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3728: 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 29 5d 04 07 KEY, sz BLOB)].. +| 3744: 17 23 23 01 81 01 74 61 62 6c 65 74 74 74 5f 63 .##...tablettt_c +| 3760: 6f 6e 74 65 6e 74 74 74 74 5f 63 6f 6e 74 65 6e ontentttt_conten +| 3776: 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 t.CREATE TABLE ' +| 3792: 74 74 74 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 ttt_content'(id +| 3808: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3824: 4b 45 59 2c 20 63 30 2c 20 63 31 29 6c 03 07 17 KEY, c0, c1)l... +| 3840: 1b 1b 01 81 2f 74 61 62 6c 65 74 74 74 5f 69 64 ..../tablettt_id +| 3856: 78 74 74 74 5f 69 64 78 03 43 52 45 41 54 45 20 xttt_idx.CREATE +| 3872: 54 41 42 4c 45 20 27 74 74 74 5f 69 64 78 27 28 TABLE 'ttt_idx'( +| 3888: 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 6e segid, term, pgn +| 3904: 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 73 o, PRIMARY KEY(s +| 3920: 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 54 egid, term)) WIT +| 3936: 48 4f 55 54 20 52 4f 57 49 44 58 02 07 17 1d 1d HOUT ROWIDX..... +| 3952: 01 81 03 74 61 62 6c 65 74 74 74 5f 64 61 74 61 ...tablettt_data +| 3968: 74 74 74 5f 64 61 74 61 02 43 52 45 41 54 45 20 ttt_data.CREATE +| 3984: 54 41 42 4c 45 20 27 74 74 74 5f 64 61 74 61 27 TABLE 'ttt_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 13 13 08 5f 74 61 62 6c LOB):......_tabl +| 4048: 65 74 74 74 74 74 74 43 52 45 41 54 45 20 56 49 ettttttCREATE VI +| 4064: 52 54 55 41 4c 20 54 41 42 4c 45 20 74 74 74 20 RTUAL TABLE ttt +| 4080: 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 29 USING fts5(a, b) +| page 2 offset 4096 +| 0: 0d 0f 44 00 05 0e 81 00 0f 1a 0e 81 0f af 0f 58 ..D............X +| 16: 0e 98 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3712: 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 03 01 .....0.......... +| 3728: 01 01 02 01 01 03 01 01 81 24 8c 80 80 80 80 01 .........$...... +| 3744: 04 00 82 4c 00 00 00 9b 02 30 65 03 1a 02 05 05 ...L.....0e..... +| 3760: 07 05 01 01 04 03 03 08 03 03 01 2e 02 05 05 07 ................ +| 3776: 05 07 05 07 05 01 01 04 03 03 08 03 03 08 03 03 ................ +| 3792: 07 f3 03 02 01 65 03 1e 03 05 05 04 05 05 01 00 .....e.......... +| 3808: 03 06 04 04 06 04 03 01 36 03 05 05 04 06 05 04 ........6....... +| 3824: 06 05 04 05 05 01 01 03 06 04 04 06 04 04 06 04 ................ +| 3840: 04 06 04 03 03 01 65 03 14 04 05 06 f5 05 01 01 ......e......... +| 3856: 02 08 09 01 20 04 05 07 05 07 05 07 05 05 01 00 .... ........... +| 3872: 02 08 0a 0a 0a 04 01 65 03 02 0a 01 06 0a 0a 0a .......e........ +| 3888: 05 01 65 03 06 01 01 0a 01 0a 01 01 0a 0a 0a 04 ..e............. +| 3904: 2b 31 21 0b 0f ef 00 14 2a 00 00 00 00 01 02 02 +1!.....*....... +| 3920: 00 02 01 01 01 02 01 01 50 88 80 80 80 80 01 04 ........P....... +| 3936: 00 81 24 00 00 00 47 02 30 65 02 1a 02 05 05 07 ..$...G.0e...... +| 3952: 05 01 01 04 03 03 08 03 03 02 01 65 02 1e 03 05 ...........e.... +| 3968: 05 04 05 05 01 01 03 06 04 04 06 04 03 03 01 65 ...............e +| 3984: 02 14 04 05 07 05 05 01 01 02 08 0a 04 01 65 02 ..............e. +| 4000: 02 0a 05 01 65 02 06 01 01 0a 04 12 14 0f 06 31 ....e..........1 +| 4016: 84 80 80 80 80 01 03 00 68 00 00 00 2b 02 30 65 ........h...+.0e +| 4032: 01 10 02 05 05 01 01 04 03 03 02 01 65 01 12 03 ............e... +| 4048: 05 05 01 01 03 06 04 03 03 01 65 01 0e 04 05 05 ..........e..... +| 4064: 01 01 02 08 04 0d 0e 06 01 03 00 12 04 4c 4c 00 .............LL. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 04 0e 1a 00 0f c7 0f 5b 0e ef 0e 1a ...........[.... +| 3600: 00 00 00 00 00 00 00 00 00 00 81 52 04 06 00 81 ...........R.... +| 3616: 5d 81 55 65 20 65 65 20 65 65 65 20 65 20 65 65 ].Ue ee eee e ee +| 3632: 20 65 65 65 20 65 20 65 65 20 65 65 65 66 20 65 eee e ee eeef e +| 3648: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 3664: 20 65 65 20 65 65 65 65 20 65 65 20 65 65 65 20 ee eeee ee eee +| 3680: 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 e ee eee e ee ee +| 3696: 65 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 ee ee eee e ee e +| 3712: 65 65 20 65 20 65 65 20 65 65 65 65 65 65 20 65 ee e ee eeeeee e +| 3728: 65 20 65 20 65 20 65 20 65 65 20 65 65 65 20 65 e e e e ee eee e +| 3744: 65 20 65 65 65 65 65 20 65 65 20 65 20 65 1f 65 e eeeee ee e e.e +| 3760: 20 65 65 20 65 65 65 20 65 65 20 65 65 65 65 65 ee eee ee eeeee +| 3776: 20 65 65 20 65 20 65 20 65 20 65 65 20 65 65 65 ee e e e ee eee +| 3792: 20 65 65 20 65 65 65 65 65 20 65 65 20 65 20 65 ee eeeee ee e e +| 3808: 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 6a e ee eee ee eej +| 3824: 03 03 ff 75 71 65 20 65 65 1f 65 65 65 20 65 20 ...uqe ee.eee e +| 3840: 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 65 ee eee e ee eeee +| 3856: 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 ee eee e ee eee +| 3872: 20 65 20 65 65 20 65 65 65 65 65 65 20 65 65 20 e ee eeeeee ee +| 3888: 65 20 65 20 65 20 65 65 20 65 65 65 20 65 65 20 e e e ee eee ee +| 3904: 65 65 65 65 65 20 65 65 20 65 20 65 20 65 20 65 eeeee ee e e e e +| 3920: 65 20 65 65 65 20 65 65 20 65 65 6a 02 04 00 75 e eee ee eej...u +| 3936: 40 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 @e ee eee e ee e +| 3952: 65 65 20 65 20 65 65 20 65 65 65 65 20 65 65 20 ee e ee eeee ee +| 3968: 65 65 65 20 65 20 65 65 20 65 65 65 20 65 20 65 eee e ee eee e e +| 3984: 65 20 65 65 65 65 65 65 20 65 65 20 65 20 65 20 e eeeeee ee e e +| 4000: 65 20 65 65 20 65 65 65 20 65 65 20 65 65 65 65 e ee eee ee eeee +| 4016: 65 20 65 65 20 65 20 65 20 65 20 65 65 20 65 65 e ee e e e ee ee +| 4032: 65 20 65 65 20 65 65 37 01 04 00 41 3f 65 20 65 e ee ee7...A?e e +| 4048: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 4064: 20 65 65 20 65 65 65 65 65 65 20 65 65 20 65 20 ee eeeeee ee e +| 4080: 65 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 e e ee eee ee ee +| page 5 offset 16384 +| 0: 0d 00 00 00 04 0f e4 00 0f f9 0f f2 0f eb 0f e4 ................ +| 4064: 00 00 00 00 05 04 03 00 10 21 21 05 03 03 00 10 .........!!..... +| 4080: 11 11 05 02 03 00 10 11 11 05 01 03 00 10 09 09 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-4470f0b94422f7.db +}]} {} + +do_catchsql_test 64.1 { + SELECT * FROM ttt('e*'); +} {1 {database disk image is malformed}} + +#--------------------------------------------------------------------------- +do_test 65.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-3aef66940ace0c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 00 00 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c d6 63 32 29 69 04 07 17 19 c0, c1,.c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 93 ff 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 31 31 36 30 ...........21160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 33 f1 609...........3. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 03 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 01 f2 03 06 4e 02 02 03 06 01 ..........N..... +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 13 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 07 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 01 f1 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 76 b4 65 6e 73 69 6f 6e 1f 02 ......v.ension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 25 0d 02 03 01 02 03 01 ........%....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 0f f2 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 00 03 6d 61 78 1c 02 0c 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 13 06 00 f2 02 03 ocase........... +| 3680: 06 01 12 02 13 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 8e 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 11 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 01 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 01 f1 06 01 01 02 ad 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 01 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0e fc 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 00 00 00 00 00 00 00 00 ....$........... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 8f DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 43 41 46 45 3d ..%..THREADCAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4b 4f 41 44 21 45 58 54 45 4e 53 49 4f IT KOAD!EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 59 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMYT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 56 a9 4d 1f 1e 05 00 33 0f 19 IONXRTV.M....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 57 42 49 4e 31 52 59 1f 1d 05 00 33 0f 000WBIN1RY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 32 0000XNOCASE....2 +| 3328: 0f 17 4e 41 58 20 4d 45 4d 4f 52 59 2d 35 30 30 ..NAX MEMORY-500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 59 4e 4f 43 41 53 45 17 LE RTREEYNOCASE. +| 3408: 19 66 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 .f.%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 10 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 40 42 4c 45 20 4a 53 4f 4e 31 58 42 ..EN@BLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 32 58 4e 4f 43 41 53 45 17 LE JSON2XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4d 41 42 4c 45 20 4a 53 4f ...%..EMABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4f 81 42 4c NARY....)..EO.BL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 51 53 45 E GEOPOLYXNOCQSE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 1a 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 48 4e 4f 43 41 53 45 16 1d 05 E FTS5HNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 55 53 34 58 42 49 4e 41 52 59 17 0b LE FUS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 57 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e WNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 05 52 54 52 49 4d 1e ABLE FTS4.RTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 8a 4e 41 52 .....DEBUGXB.NAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3f 87 ...C..COMPILER?. +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 45 0d 60 59 4c 45 52 3d 67 63 63 2d 35 2e 34 2d E.`YLER=gcc-5.4- +| 4080: 30 20 32 30 31 36 30 36 30 39 00 00 00 00 00 00 0 20160609...... +| page 6 offset 20480 +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 01 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 02 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 00 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-3aef66940ace0c.db +}]} {} + +do_catchsql_test 65.1 { + SELECT ( MATCH (t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ' +} {1 {malformed database schema (t2) - invalid rootpage}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 66.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-37cecb4e784e9f.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 49 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARI KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 01 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 00 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 01 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 09 01 52 1b 72 65 62 75 69 6c ........R.rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end crash-37cecb4e784e9f.db +}]} {} + +do_catchsql_test 66.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 67.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-43ed0ad79c0194.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 06 0d e2 00 0f c4 0f 6a ...............j +| 112: 0e fc 0e 9d 0e 3d 0d e2 01 00 00 00 00 00 00 00 .....=.......... +| 3552: 00 00 59 06 06 17 21 21 01 7f 74 61 62 6c 65 74 ..Y...!!..tablet +| 3568: 74 74 5f 63 6f 6e 66 69 67 74 74 74 5f 63 6f 6e tt_configttt_con +| 3584: 66 69 67 06 43 52 45 41 54 45 20 54 41 42 4c 45 fig.CREATE TABLE +| 3600: 20 27 74 74 74 5f 63 6f 6e 66 69 67 27 28 6b 20 'ttt_config'(k +| 3616: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 PRIMARY KEY, v) +| 3632: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5e 05 07 WITHOUT ROWID^.. +| 3648: 17 23 23 01 81 03 74 61 62 6c 65 74 74 74 5f 64 .##...tablettt_d +| 3664: 6f 63 73 69 7a 65 74 74 74 5f 64 6f 63 73 69 7a ocsizettt_docsiz +| 3680: 65 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 e.CREATE TABLE ' +| 3696: 74 74 74 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 ttt_docsize'(id +| 3712: 49 4e 54 45 47 45 52 20 51 52 49 4d 41 52 59 20 INTEGER QRIMARY +| 3728: 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 29 5d 04 07 KEY, sz BLOB)].. +| 3744: 17 23 23 01 81 01 74 61 62 6c 65 74 74 74 5f 63 .##...tablettt_c +| 3760: 6f 6e 74 65 6e 74 74 74 74 5f 63 6f 6e 74 65 6e ontentttt_conten +| 3776: 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 t.CREATE TABLE ' +| 3792: 74 74 74 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 ttt_content'(id +| 3808: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 f1 59 20 INTEGER PRIMA.Y +| 3824: 4b 45 59 2c 20 63 30 2c 20 63 31 29 6c 03 07 17 KEY, c0, c1)l... +| 3840: 1b 1b 01 81 2f 74 61 62 6c 65 74 74 74 5f 69 64 ..../tablettt_id +| 3856: 78 74 74 74 5f 69 64 78 03 43 52 45 41 54 45 20 xttt_idx.CREATE +| 3872: 54 41 42 4c 45 20 27 74 74 74 5f 69 64 78 27 28 TABLE 'ttt_idx'( +| 3888: 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 6e segid, term, pgn +| 3904: 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 73 o, PRIMARY KEY(s +| 3920: 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 54 egid, term)) WIT +| 3936: 48 4f 55 54 20 52 4f 57 49 44 58 02 07 17 1d 1d HOUT ROWIDX..... +| 3952: 01 81 03 74 61 62 6c 65 74 74 74 5f 64 61 74 61 ...tablettt_data +| 3968: 74 74 74 5f 64 61 74 61 02 43 52 45 41 54 45 20 ttt_data.CREATE +| 3984: 54 41 42 4c 45 20 27 74 74 74 5f 64 61 74 61 27 TABLE 'ttt_data' +| 4000: 28 69 64 20 49 4e 54 45 47 55 52 20 50 52 49 4d (id INTEGUR PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 50 42 29 3a 02 06 17 13 13 08 5f 74 61 62 6c LPB):......_tabl +| 4048: 65 74 74 74 74 74 74 43 52 45 41 54 45 20 56 49 ettttttCREATE VI +| 4064: 52 54 55 41 4c 20 54 41 42 4c 45 20 74 74 74 20 RTUAL TABLE ttt +| 4080: 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 29 USING fts5(a, b) +| page 2 offset 4096 +| 0: 0d 0f 44 00 05 0e 71 00 0f e7 0e 81 0f af 0f 58 ..D...q........X +| 16: 0e 98 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3712: 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 03 01 .....0.......... +| 3728: 01 01 02 01 01 03 01 01 81 24 8c 80 80 80 80 01 .........$...... +| 3744: 04 00 82 4c 00 00 00 9b 02 30 65 03 1a 12 05 05 ...L.....0e..... +| 3760: 07 05 01 01 04 03 03 08 04 03 01 2e 02 05 f7 07 ................ +| 3776: 01 e6 f5 07 05 01 01 04 03 03 01 22 03 18 03 03 ................ +| 3792: 08 03 03 02 01 65 03 1e 03 05 05 04 05 05 01 01 .....e.......... +| 3808: 03 06 03 f4 06 04 03 00 36 03 ff 05 04 05 05 04 ........6....... +| 3824: 05 05 04 05 04 f1 01 03 06 04 04 06 04 04 06 04 ................ +| 3840: 04 07 04 03 03 01 65 03 14 04 05 07 05 05 01 01 ......e......... +| 3856: 02 08 a5 01 20 04 05 01 94 f7 05 07 05 05 01 01 .... ........... +| 3872: 02 08 0a 0a 0a 04 01 65 03 02 0a 00 06 0a 0a 0a .......e........ +| 3888: 05 01 65 03 06 a7 01 0a 01 0a 01 01 0a 0a 0a 04 ..e............. +| 3904: 2b 31 21 0b 0f ef 00 14 2a 00 00 00 00 01 02 02 +1!.....*....... +| 3920: 00 02 01 01 01 02 11 01 50 88 80 80 80 80 01 04 ........P....... +| 3936: 00 81 24 00 00 00 47 02 30 65 02 1a 02 05 05 07 ..$...G.0e...... +| 3952: 05 e6 01 07 aa e3 08 03 03 02 01 65 02 1e 03 05 ...........e.... +| 3968: 05 05 04 f5 01 01 03 06 04 04 06 04 13 03 01 65 ...............e +| 3984: 02 14 04 05 07 05 05 01 f7 f2 08 0a 04 01 65 02 ..............e. +| 4000: 02 0a 05 01 65 02 06 00 f1 0a 04 12 14 0f 06 31 ....e..........1 +| 4016: 84 80 80 80 80 01 03 00 68 00 00 00 2b 02 30 65 ........h...+.0e +| 4032: 01 10 02 05 05 00 01 04 03 03 02 01 65 01 12 03 ............e... +| 4048: 05 05 01 01 03 06 04 03 03 01 65 01 0e 04 05 04 ..........e..... +| 4064: 01 01 02 08 04 0d 0e 06 01 03 00 12 04 4c 4c 00 .............LL. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 02 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| page 4 offset 12288 +| 3600: 00 00 00 00 00 00 00 00 00 00 81 52 04 06 00 81 ...........R.... +| 3616: 5d 81 55 65 20 65 65 20 65 65 65 20 65 20 65 65 ].Ue ee eee e ee +| 3632: 20 65 65 65 28 15 20 65 65 20 65 65 65 65 20 65 eee(. ee eeee e +| 3648: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 3664: 20 65 65 20 65 65 65 65 20 65 66 20 65 65 55 20 ee eeee ef eeU +| 3680: 65 20 65 55 20 65 65 65 20 65 20 65 65 20 65 65 e eU eee e ee ee +| 3696: 65 64 20 65 61 c0 65 65 65 20 65 20 65 65 20 65 ed ea.eee e ee e +| 3712: 65 65 20 79 20 65 65 20 65 65 65 65 65 65 20 65 ee y ee eeeeee e +| 3728: 65 1f 65 20 65 20 65 20 65 65 20 65 65 65 20 65 e.e e e ee eee e +| 3744: 65 20 65 65 65 65 65 20 65 65 20 65 20 65 20 65 e eeeee ee e e e +| 3760: 20 65 65 20 65 65 65 20 6b 85 20 65 65 65 66 65 ee eee k. eeefe +| 3776: 20 65 65 10 65 20 65 20 65 20 65 65 20 65 65 65 ee.e e e ee eee +| 3792: 20 65 65 20 65 65 65 65 65 20 65 65 20 65 20 65 ee eeeee ee e e +| 3808: 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 6a e ee eee ee eej +| 3824: 03 04 00 75 71 65 20 65 65 20 65 65 65 20 65 30 ...uqe ee eee e0 +| 3840: 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 65 ee eee e ee eeee +| 3856: 20 65 65 20 65 65 65 20 65 1f 65 65 20 65 65 65 ee eee e.ee eee +| 3872: 20 65 20 65 65 20 65 65 65 65 65 66 20 65 65 20 e ee eeeeef ee +| 3888: 65 21 27 20 65 20 55 65 20 66 65 64 20 65 65 00 e!' e Ue fed ee. +| page 5 offset 16384 +| 4064: 00 00 00 00 05 04 03 00 10 11 20 05 03 03 00 10 .......... ..... +| 4080: 11 11 05 02 03 00 00 11 11 05 01 03 00 10 09 09 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 01 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-43ed0ad79c0194.db +}]} {} + +do_catchsql_test 67.1 { + SELECT snippet(ttt, null,null, + EXISTS(SELECT 1 FROM ttt('e NuOT ee*e*ÏNuOY ee*') ) , '', + (SELECT 1 FROM ttt('eu NuOT ee*e* NuOY ee*')) + ), * FROM ttt('e') +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 68.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-41234e232809e7.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 16: 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 00 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 03 f1 06 62 69 6e 62 72 79 03 06 ........binbry.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 01 03 16 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 04 71 02 02 03 06 11 02 02 01 08 63 6f 6d 70 ..q.........comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 03 02 ................ +| 3472: 00 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 66 63 63 01 02 03 01 02 03 01 02 03 ....fcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 5e 31 13 02 03 01 02 03 01 02 ...jso^1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 13 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 12 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 05 f1 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 5b 02 01 06 01 01 02 01 06 .......[........ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0b 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4f 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 OARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 46 45 58 4e 4f 43 41 53 45 17 LE RTRFEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 49 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 IABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 57 42 49 NABLE GEOPOLYWBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 42 41 53 45 E GEOPOLYXNOBASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 b7 4e 41 42 4c 45 20 44 42 53 ...1...NABLE DBS +| 3904: 54 41 54 20 66 54 41 42 58 52 54 52 49 4d 11 06 TAT fTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 62 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XbTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 52 02 4a 4e 41 52 59 27 20160609R.JNARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 04 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 5d 69 71 a5 uild....opti]iq. +| end crash-41234e232809e7.db +.testctrl prng_seed 1 db +}]} {} + +do_catchsql_test 68.1 { + PRAGMA reverse_unordered_selects=ON; + INSERT INTO t1(t1) SELECT x FROM t2; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 69.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-31c462b8b665d0.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 00 00 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 39 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c9, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 00 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 01 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 1a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 aa 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 02 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 01 ...omit......... +| 3744: ff ff ff ff ff ff ff ff f0 00 00 00 00 00 01 02 ................ +| 3760: 58 81 96 4d 01 06 01 02 02 03 06 01 02 02 03 06 X..M............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 1e 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 00 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0b 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 00 00 00 00 00 .......h.O...... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 d3 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 28 2c 4f 41 44 b2 04 55 85 44 54 e5 34 MIT(,OAD..U.DT.4 +| 3216: 94 f4 e5 84 e4 f4 34 15 34 51 e1 f0 50 03 30 f1 ......4.4Q..P.0. +| 3232: 74 f4 d4 95 42 04 c4 f4 14 42 04 55 85 44 54 e5 t...B....B.U.DT. +| 3248: 34 94 f4 e5 85 25 45 24 94 d1 f1 e0 50 03 30 f1 4....%E$....P.0. +| 3264: 94 d4 15 82 04 d4 54 d4 f5 25 93 d3 53 03 03 03 ......T..%..S... +| 3280: 03 03 03 05 84 24 94 e4 15 25 91 f1 d0 50 03 30 .....$...%...P.0 +| 3296: f1 94 d4 15 82 04 d4 54 d4 f5 25 93 d3 53 03 03 .......T..%..S.. +| 3312: 03 03 03 03 05 84 e4 f4 34 15 34 51 e1 c0 50 03 ........4.4Q..P. +| 3328: 30 f1 74 d4 15 82 04 d4 54 d4 f5 25 93 d3 53 03 0.t.....T..%..S. +| 3344: 03 03 03 03 03 05 85 25 45 24 94 d1 81 b0 50 02 .......%E$....P. +| 3360: 50 f1 94 54 e4 14 24 c4 52 05 25 45 24 54 55 84 P..T..$.R.%E$TU. +| 3376: 24 94 e4 15 25 91 81 a0 50 02 50 f1 94 54 e4 14 $...%...P.P..T.. +| 3392: 24 c4 52 05 25 45 24 54 55 84 e4 f4 34 15 34 51 $.R.%E$TU...4.4Q +| 3408: 71 90 50 02 50 f1 74 54 e4 14 24 c4 52 05 25 45 q.P.P.tT..$.R.%E +| 3424: 24 54 55 85 25 45 24 94 d1 a1 80 50 02 90 f1 94 $TU.%E$....P.... +| 3440: 54 e4 14 24 c4 52 04 d4 54 d5 35 95 33 55 84 24 T..$.R..T.5.3U.$ +| 3456: 94 e4 15 25 91 a1 70 50 02 90 f1 94 54 e4 14 24 ...%..pP....T..$ +| 3472: c4 52 04 d4 54 d5 35 95 33 55 84 e4 f4 34 15 34 .R..T.5.3U...4.4 +| 3488: 51 91 60 50 02 90 f1 74 54 e4 14 24 c4 52 04 d4 Q.`P...tT..$.R.. +| 3504: 54 d5 35 95 33 55 85 25 45 24 94 d1 81 50 50 02 T.5.3U.%E$...PP. +| 3520: 50 f1 94 54 e4 14 24 c4 52 04 a5 34 f4 e3 15 84 P..T..$.R..4.... +| 3536: 24 94 e4 15 25 91 81 40 50 02 50 f1 94 54 e4 14 $...%..@P.P..T.. +| 3552: 24 c4 52 04 a5 34 f4 e3 15 84 e4 f4 34 15 34 51 $.R..4......4.4Q +| 3568: 71 30 50 02 4f f1 74 54 e4 14 24 c4 52 04 a5 34 q0P.O.tT..$.R..4 +| 3584: f4 e3 15 85 25 45 24 94 d1 a1 20 50 02 90 f1 94 ....%E$... P.... +| 3600: 54 e4 14 24 c4 52 04 74 54 f5 04 f4 c5 95 84 24 T..$.R.tT......$ +| 3616: 94 e4 15 25 91 a1 10 50 02 90 f1 94 54 e4 14 24 ...%...P....T..$ +| 3632: c4 52 04 74 54 f5 04 f4 c5 95 84 e4 f4 34 15 34 .R.tT........4.4 +| 3648: 51 91 00 50 02 90 f1 74 54 e4 14 24 c4 51 f4 74 Q..P...tT..$.Q.t +| 3664: 54 f5 04 f4 c5 95 85 25 45 24 94 d1 70 f0 50 02 T......%E$..p.P. +| 3680: 30 f1 94 54 e4 14 24 c5 20 46 54 53 35 58 42 49 0..T..$. FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4f 4f 43 41 53 45 16 0d 05 E FTS5XOOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 97 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 3e 5f 19 45 4e 41 42 4c 45 20 44 42 53 ...>_.ENABLE DBS +| 3840: 44 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e DAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 4d e3 45 1d TAT VTABXNOCM.E. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 02 02 50 08 5f 17 44 45 42 55 47 CASE...P._.DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 44 0f 19 43 4f 4d XRTRIM'...D..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 c9 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 47 02 3d 67 63 63 2d 35 2e 34 2e OMPILG.=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 00 00 00 00 00 00 .X.P.H.@.8...... +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 1f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 00 00 00 00 00 00 ity-check....... +| end crash-31c462b8b665d0.db +}]} {} + + +do_catchsql_test 69.2 { + SELECT * FROM t1 WHERE a MATCH 'fx*' +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 70.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename sql022250.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 06 0d e2 00 0f c4 0f 6a ...............j +| 112: 0e fc 0e 9d 0e 3d 0d e2 00 00 00 00 00 00 00 00 .....=.......... +| 3552: 00 00 59 06 06 17 21 21 01 7f 74 61 62 6c 65 74 ..Y...!!..tablet +| 3568: 74 74 5f 63 6f 6e 66 69 67 74 74 74 5f 63 6f 6e tt_configttt_con +| 3584: 66 69 67 06 43 52 45 41 54 45 20 54 41 42 4c 45 fig.CREATE TABLE +| 3600: 20 27 74 74 74 5f 63 6f 6e 66 69 67 27 28 6b 20 'ttt_config'(k +| 3616: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 PRIMARY KEY, v) +| 3632: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5e 15 07 WITHOUT ROWID^.. +| 3648: 17 23 23 01 81 03 74 61 62 6c 65 74 74 74 5f 64 .##...tablettt_d +| 3664: 6f 63 73 69 7a 65 74 74 74 5f 64 6f 63 73 69 7a ocsizettt_docsiz +| 3680: 65 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 e.CREATE TABLE ' +| 3696: 74 74 74 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 ttt_docsize'(id +| 3712: 49 4d 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 IMTEGER PRIMARY +| 3728: 4b 45 59 2c 20 73 7a 20 42 4c 4f 20 29 5d 04 07 KEY, sz BLO )].. +| 3744: 17 23 23 01 81 01 74 61 62 6c 65 74 74 74 5f 63 .##...tablettt_c +| 3760: 6f 6e 74 65 6e 74 74 74 74 5f 63 6f 6e 74 65 6e ontentttt_conten +| 3776: 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 t.CREATE TABLE ' +| 3792: 74 74 74 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 ttt_content'(id +| 3808: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3824: 4b 45 59 2c 20 63 30 2c 20 63 31 29 6c 03 07 17 KEY, c0, c1)l... +| 3840: 1b 1b 01 81 2f 74 61 62 6c 65 74 74 74 5f 69 64 ..../tablettt_id +| 3856: 78 74 74 74 5f 69 64 78 03 43 52 45 41 54 45 20 xttt_idx.CREATE +| 3872: 54 41 42 4c 45 20 27 74 74 74 5f 69 64 78 27 28 TABLE 'ttt_idx'( +| 3888: 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 6e segid, term, pgn +| 3904: 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 73 o, PRIMARY KEY(s +| 3920: 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 54 egid, term)) WIT +| 3936: 48 4f 55 54 20 52 4f 57 49 44 58 02 07 17 1d 1d HOUT ROWIDX..... +| 3952: 01 81 03 74 61 62 6c 65 74 74 74 5f 64 61 74 61 ...tablettt_data +| 3968: 74 74 74 5f 64 61 74 61 02 43 52 45 41 54 45 20 ttt_data.CREATE +| 3984: 54 41 42 4c 45 20 27 74 74 74 5f 64 61 74 61 27 TABLE 'ttt_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 13 13 08 5f 74 61 62 6c LOB):......_tabl +| 4048: 65 74 74 74 74 74 74 43 52 45 41 54 45 20 56 49 ettttttCREATE VI +| 4064: 52 54 55 41 4c 20 54 41 42 4c 45 20 74 74 74 20 RTUAL TABLE ttt +| 4080: 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 29 USING fts5(a, b) +| page 2 offset 4096 +| 0: 0d 0f 54 00 05 0e 81 00 0f e7 0e 81 0f af 0f 58 ..T............X +| 16: 0e 98 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3712: 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 03 01 .....0.......... +| 3728: 01 01 02 01 01 03 01 01 81 24 8c 80 80 80 80 01 .........$...... +| 3744: 04 00 82 4c 00 00 00 9b 02 30 65 03 1a 02 05 05 ...L.....0e..... +| 3760: 07 05 01 01 04 03 03 08 03 03 01 2e 02 05 05 07 ................ +| 3776: 05 07 05 07 05 01 01 14 03 03 08 03 03 08 03 bf ................ +| 3792: 07 f2 f3 02 01 65 03 1e 03 05 05 04 05 05 01 01 .....e.......... +| 3808: 03 06 04 04 06 04 03 01 36 03 05 05 04 05 05 04 ........6....... +| 3824: 05 15 04 05 05 01 01 03 06 04 04 06 04 04 a1 04 ................ +| 3840: 04 06 04 03 03 01 65 03 14 04 05 07 05 05 01 01 ......e......... +| 3856: 02 09 0a 01 20 04 05 07 05 07 05 07 05 05 01 01 .... ........... +| 3872: 02 0f da 0a 0a 04 01 64 f3 02 0a 01 06 0a 0a 0a .......d........ +| 3888: 05 01 65 03 06 01 01 0a 01 0a 01 01 0a 0a 0a 04 ..e............. +| 3904: 2b 31 21 0b 0f ef 00 14 2a 00 00 00 00 01 02 02 +1!.....*....... +| 3920: 00 02 01 01 01 02 01 01 50 88 80 80 80 80 01 04 ........P....... +| 3936: 00 81 24 00 00 00 47 01 30 65 02 1a 02 05 05 07 ..$...G.0e...... +| 3952: 05 01 01 04 03 03 08 03 03 02 01 65 02 1d f3 05 ...........e.... +| 3968: 05 04 05 05 01 01 04 06 04 04 06 04 03 03 01 65 ...............e +| 3984: 02 14 04 05 07 05 05 01 01 02 08 0a 04 01 65 02 ..............e. +| 4000: 02 0a 05 01 65 02 06 01 01 0a 04 12 14 0f 06 31 ....e..........1 +| 4016: 84 80 80 80 80 01 03 00 68 00 00 00 2b 02 30 65 ........h...+.0e +| 4032: 01 10 02 05 05 01 01 04 03 03 02 01 66 01 12 03 ............f... +| 4048: 05 05 01 01 03 06 04 03 03 01 65 01 0e 04 05 05 ..........e..... +| 4064: 01 01 02 08 04 0d 0e 06 01 03 00 12 04 4c 4c 00 .............LL. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 2f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ./.............. +| page 4 offset 12288 +| 3344: 00 00 00 00 00 00 00 00 00 00 81 52 04 06 10 81 ...........R.... +| 3360: 5d 81 55 65 20 65 65 20 65 65 65 20 65 20 65 65 ].Ue ee eee e ee +| 3376: 20 65 65 65 20 65 20 65 65 20 65 65 65 65 20 65 eee e ee eeee e +| 3392: 65 20 65 65 65 24 a5 20 65 65 20 65 65 65 20 65 e eee$. ee eee e +| 3408: 24 05 65 20 65 65 65 65 20 65 65 20 65 65 65 20 $.e eeee ee eee +| 3424: 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 e ee eee e ee ee +| 3440: 65 65 20 65 65 20 65 65 65 20 65 20 65 65 10 65 ee ee eee e ee.e +| 3456: 65 65 20 65 20 65 65 20 65 65 65 65 65 65 20 65 ee e ee eeeeee e +| 3472: 65 20 65 20 65 20 65 20 65 65 20 65 65 65 20 65 e e e e ee eee e +| 3488: 1f 20 65 65 66 65 65 20 65 65 20 65 20 65 20 2d . eefee ee e e - +| page 5 offset 16384 +| 4064: 00 00 00 00 05 04 d0 00 10 21 21 05 03 03 00 10 .........!!..... +| 4080: 11 11 05 02 03 00 10 11 10 05 01 03 00 10 09 09 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end sql022250.txt.db +}]} {} + +do_catchsql_test 70.1 { + SELECT snippet(ttt, -1, '', '','','>')FROM ttt('e* NOT ee*e* NOT ee*'); +} {1 {database disk image is malformed}} + +do_catchsql_test 70.2 { + SELECT snippet(ttt, -1, '', '','',13)FROM ttt('e* NOT ee*e* NOT ee*') +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 71.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename sql025294.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 00 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 58 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARX KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 60 eb 01 02 02 04 01 66 74 02 02 02 04 04 6e 64 `......ft.....nd +| 4064: 6f 6e 03 01 f2 04 1a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 4b 00 01 01 01 01 ...$......K..... +| page 3 offset 8192 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e1 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end sql025294.txt.db +}]} {} + +do_catchsql_test 71.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 72.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-77b86d070d0ac6.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0d e2 00 0f c4 0f 6a ...............j +| 112: 0e fc 0e 9d 0e 3d 0d e2 00 00 00 00 00 00 00 00 .....=.......... +| 3552: 00 00 59 06 06 17 21 21 01 7f 74 61 62 6c 65 74 ..Y...!!..tablet +| 3568: 74 74 5f 63 6f 6e 66 69 67 74 74 74 5f 63 6f 6e tt_configttt_con +| 3584: 66 69 67 06 43 52 45 41 54 45 20 54 41 42 4c 45 fig.CREATE TABLE +| 3600: 20 27 74 74 74 5f 63 6f 6e 66 69 67 27 28 6b 20 'ttt_config'(k +| 3616: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 PRIMARY KEY, v) +| 3632: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5e 05 07 WITHOUT ROWID^.. +| 3648: 17 23 23 01 81 03 74 61 62 6c 65 74 74 74 5f 64 .##...tablettt_d +| 3664: 6f 63 73 69 7a 65 74 74 74 5f 64 6f 63 73 69 7a ocsizettt_docsiz +| 3680: 65 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 e.CREATE TABLE ' +| 3696: 74 74 74 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 ttt_docsize'(id +| 3712: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3728: 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 29 5d 04 07 KEY, sz BLOB)].. +| 3744: 17 23 23 01 81 01 74 61 62 6c 65 74 74 74 5f 63 .##...tablettt_c +| 3760: 6f 6e 74 65 6e 74 74 74 74 5f 63 6f 6e 74 65 6e ontentttt_conten +| 3776: 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 t.CREATE TABLE ' +| 3792: 74 74 74 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 ttt_content'(id +| 3808: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3824: 4b 45 59 2c 20 63 30 2c 20 63 31 29 6c 03 07 17 KEY, c0, c1)l... +| 3840: 1b 1b 01 81 2f 74 61 62 6c 65 74 74 74 5f 69 64 ..../tablettt_id +| 3856: 78 74 74 74 5f 69 64 78 03 43 52 45 41 54 45 20 xttt_idx.CREATE +| 3872: 54 41 42 4c 45 20 27 74 74 74 5f 69 64 78 27 28 TABLE 'ttt_idx'( +| 3888: 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 6e segid, term, pgn +| 3904: 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 73 o, PRIMARY KEY(s +| 3920: 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 54 egid, term)) WIT +| 3936: 48 4f 55 54 20 52 4f 57 49 44 58 02 07 17 1d 1d HOUT ROWIDX..... +| 3952: 01 81 03 74 61 62 6c 65 74 74 74 5f 64 61 74 61 ...tablettt_data +| 3968: 74 74 74 5f 64 61 74 61 02 43 52 45 41 54 45 20 ttt_data.CREATE +| 3984: 54 41 42 4c 45 20 27 74 74 74 5f 64 61 74 61 27 TABLE 'ttt_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 13 13 08 5f 74 61 62 6c LOB):......_tabl +| 4048: 65 74 74 74 74 74 74 43 52 45 41 54 45 20 56 49 ettttttCREATE VI +| 4064: 52 54 55 41 4c 20 54 41 42 4c 45 20 74 74 74 20 RTUAL TABLE ttt +| 4080: 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 29 USING fts5(a, b) +| page 2 offset 4096 +| 0: 0d 0f 44 00 05 0e 81 00 0f e7 0e 81 0f af 0f 58 ..D............X +| 16: 0e 98 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3712: 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 03 01 .....0.......... +| 3728: 01 01 02 01 01 03 01 01 81 24 8c 80 80 80 80 01 .........$...... +| 3744: 04 00 82 4c 00 00 00 9b 02 30 65 03 1a 02 05 05 ...L.....0e..... +| 3760: 07 05 01 01 04 03 03 08 03 03 01 2e 02 05 05 07 ................ +| 3776: 05 07 05 07 05 01 01 04 03 03 08 03 03 08 03 03 ................ +| 3792: 08 03 03 02 01 65 03 1e 03 05 05 04 05 05 01 01 .....e.......... +| 3808: 03 06 04 04 06 04 03 01 36 03 05 05 04 05 05 04 ........6....... +| 3824: 05 05 04 05 05 01 01 03 06 04 04 06 04 04 06 04 ................ +| 3840: 04 06 04 03 03 01 65 03 14 04 05 07 05 05 01 01 ......e......... +| 3856: 02 08 0a 01 20 04 05 07 05 07 05 07 05 05 01 01 .... ........... +| 3872: 02 08 0a 09 fa 04 01 65 03 02 0a 01 06 0a 1a 0a .......e........ +| 3888: 05 01 65 03 06 01 01 0a 01 0a 01 01 0a 0a 0a 04 ..e............. +| 3904: 2b 31 21 0b 0f ef 00 14 2a 00 00 00 00 01 02 02 +1!.....*....... +| 3920: 00 02 01 01 01 02 01 01 50 88 80 80 80 80 01 04 ........P....... +| 3936: 00 81 24 00 00 00 47 02 30 65 02 1a 02 05 05 07 ..$...G.0e...... +| 3952: 05 01 01 04 03 03 08 03 03 02 01 65 02 1e 03 05 ...........e.... +| 3968: 05 04 05 05 01 01 03 06 04 04 06 04 03 03 01 65 ...............e +| 3984: 02 14 04 05 07 05 05 01 01 02 08 0a 04 01 65 02 ..............e. +| 4000: 02 0a 05 01 65 02 06 01 01 0a 04 12 14 0f 06 31 ....e..........1 +| 4016: 84 80 80 80 80 01 03 00 68 00 00 00 2b 02 30 65 ........h...+.0e +| 4032: 01 10 02 05 05 01 01 04 03 9f 02 01 65 01 12 03 ............e... +| 4048: 05 05 01 01 03 06 04 03 03 01 65 01 0e 14 05 05 ..........e..... +| 4064: 01 01 02 08 04 0d 0e 06 01 03 00 12 04 4c 4c 00 .............LL. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3600: 00 00 00 00 00 00 00 00 00 00 81 52 04 06 00 81 ...........R.... +| 3616: 5d 81 55 65 20 65 65 20 65 65 65 20 65 20 65 65 ].Ue ee eee e ee +| 3632: 20 65 65 65 20 65 20 65 65 20 65 65 65 65 20 65 eee e ee eeee e +| 3648: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 3664: 20 65 65 20 65 65 55 65 20 65 65 20 65 65 65 20 ee eeUe ee eee +| 3680: 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 e ee eee e ee ee +| 3696: 65 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 ee ee eee e ee e +| 3712: 65 65 20 65 20 65 65 20 65 65 65 65 65 65 20 65 ee e ee eeeeee e +| 3728: 65 20 65 20 65 20 65 20 65 65 20 65 85 65 20 65 e e e e ee e.e e +| 3744: 65 20 65 65 65 65 65 20 65 65 20 65 20 65 20 65 e eeeee ee e e e +| 3760: 20 65 65 20 65 65 65 20 65 65 20 65 65 65 65 65 ee eee ee eeeee +| 3776: 20 65 65 20 65 20 65 20 65 20 65 65 20 65 65 65 ee e e e ee eee +| 3792: 20 65 65 20 65 65 65 65 65 20 65 65 20 65 20 65 ee eeeee ee e e +| 3808: 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 6a e ee eee ee eej +| 3824: 03 04 00 75 71 65 20 65 65 20 65 65 65 20 65 20 ...uqe ee eee e +| 3840: 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 65 ee eee e ee eeee +| 3856: 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 ee eee e ee eee +| 3872: 20 65 20 65 65 20 65 65 65 65 65 65 20 65 65 20 e ee eeeeee ee +| 3888: 65 20 65 20 65 20 65 64 20 65 65 65 20 65 65 20 e e e ed eee ee +| 3904: 65 65 65 65 65 20 65 65 20 65 20 65 20 65 10 65 eeeee ee e e e.e +| 3920: 65 20 65 65 65 10 65 65 20 65 65 6a 02 04 00 75 e eee.ee eej...u +| 3936: 71 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 qe ee eee e ee e +| 3952: 65 65 20 65 20 65 65 20 65 65 65 65 20 65 65 20 ee e ee eeee ee +| 3968: 65 65 65 20 65 20 65 65 20 65 65 65 20 65 20 65 eee e ee eee e e +| 3984: 65 20 65 65 65 66 65 65 20 65 65 20 65 20 65 20 e eeefee ee e e +| 4000: 65 88 65 65 20 65 65 65 30 65 65 20 65 65 65 65 e.ee eee0ee eeee +| 4016: 65 20 65 65 20 65 20 65 20 65 20 65 65 20 65 65 e ee e e e ee ee +| 4032: 65 20 65 65 20 65 65 37 01 04 00 41 3f 65 20 65 e ee ee7...A?e e +| 4048: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 4064: 20 65 65 20 65 65 65 65 65 65 20 65 65 20 65 20 ee eeeeee ee e +| 4080: 65 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 e e ee eee ee ee +| page 5 offset 16384 +| 0: 0d 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 05 04 03 00 10 21 21 05 03 03 00 10 .........!!..... +| 4080: 11 11 05 02 03 00 10 11 11 05 01 03 00 10 09 09 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-77b86d070d0ac6.db +}]} {} + +do_catchsql_test 72.1 { + INSERT INTO ttt(ttt) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +do_catchsql_test 72.1 { + SELECT 1 FROM ttt('e* NOT ee*'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 73.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-b02ca2cc4d7dda.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0d e2 00 0f c4 0f 6a ...............j +| 112: 0e fc 0e 9d 0e 3d 0d e2 00 00 00 00 00 00 00 00 .....=.......... +| 3552: 00 00 59 06 06 17 21 21 01 7f 74 61 62 6c 65 74 ..Y...!!..tablet +| 3568: 74 74 5f 63 6f 6e 66 69 67 74 74 74 5f 63 6f 6e tt_configttt_con +| 3584: 66 69 67 06 43 52 45 41 54 45 20 54 41 42 4c 45 fig.CREATE TABLE +| 3600: 20 27 74 74 74 5f 63 6f 6e 66 69 67 27 28 6b 20 'ttt_config'(k +| 3616: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 PRIMARY KEY, v) +| 3632: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5e 05 07 WITHOUT ROWID^.. +| 3648: 17 23 23 01 81 03 74 61 62 6c 65 74 74 74 5f 64 .##...tablettt_d +| 3664: 6f 63 73 69 7a 65 74 74 74 5f 64 6f 63 73 69 7a ocsizettt_docsiz +| 3680: 65 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 e.CREATE TABLE ' +| 3696: 74 74 74 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 ttt_docsize'(id +| 3712: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3728: 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 29 5d 04 07 KEY, sz BLOB)].. +| 3744: 17 23 23 01 81 01 74 61 62 6c 65 74 74 74 5f 63 .##...tablettt_c +| 3760: 6f 6e 74 65 6e 74 74 74 74 5f 63 6f 6e 74 65 6e ontentttt_conten +| 3776: 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 t.CREATE TABLE ' +| 3792: 74 74 74 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 ttt_content'(id +| 3808: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3824: 4b 45 59 2c 20 63 30 2c 20 63 31 29 6c 03 07 17 KEY, c0, c1)l... +| 3840: 1b 1b 01 81 2f 74 61 62 6c 65 74 74 74 5f 69 64 ..../tablettt_id +| 3856: 78 74 74 74 5f 69 64 78 03 43 52 45 41 54 45 20 xttt_idx.CREATE +| 3872: 54 41 42 4c 45 20 27 74 74 74 5f 69 64 78 27 28 TABLE 'ttt_idx'( +| 3888: 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 6e segid, term, pgn +| 3904: 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 73 o, PRIMARY KEY(s +| 3920: 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 54 egid, term)) WIT +| 3936: 48 4f 55 54 20 52 4f 57 49 44 58 02 07 17 1d 1d HOUT ROWIDX..... +| 3952: 01 81 03 74 61 62 6c 65 74 74 74 5f 64 61 74 61 ...tablettt_data +| 3968: 74 74 74 5f 64 61 74 61 02 43 52 45 41 54 45 20 ttt_data.CREATE +| 3984: 54 41 42 4c 45 20 27 74 74 74 5f 64 61 74 61 27 TABLE 'ttt_data' +| 4000: 28 69 65 20 49 4e 54 45 47 45 52 20 50 52 49 4d (ie INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 13 13 08 5f 74 61 62 6c LOB):......_tabl +| 4048: 65 74 74 74 74 74 74 43 52 45 41 54 45 20 56 49 ettttttCREATE VI +| 4064: 52 54 55 41 4c 20 54 41 42 4c 45 20 74 74 74 20 RTUAL TABLE ttt +| 4080: 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 29 USING fts5(a, b) +| page 2 offset 4096 +| 0: 0d 0f 44 00 05 0e 81 00 0f e7 0e 81 0f af 0f 58 ..D............X +| 16: 0e 98 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3712: 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 03 01 .....0.......... +| 3728: 01 01 02 01 01 03 01 01 81 24 8c 80 80 80 80 01 .........$...... +| 3744: 04 00 82 4c 00 00 00 9b 02 30 65 03 1a 02 05 05 ...L.....0e..... +| 3760: 07 05 01 01 04 03 03 08 03 03 01 2e 02 05 05 07 ................ +| 3776: 05 07 05 07 05 01 01 04 03 03 08 03 03 08 03 03 ................ +| 3792: 08 03 03 02 01 65 03 1e 03 05 05 04 05 05 01 01 .....e.......... +| 3808: 03 06 04 04 06 04 03 01 36 03 05 05 04 05 05 04 ........6....... +| 3824: 05 04 f4 04 05 01 01 03 06 04 04 06 04 04 06 04 ................ +| 3840: 04 06 04 db 03 01 65 03 14 04 05 07 05 05 01 01 ......e......... +| 3856: 02 08 0a 01 20 04 05 07 05 07 05 07 05 05 01 01 .... ........... +| 3872: 02 08 0a 0a 0a 04 01 65 03 02 0a 01 06 0a 0a 0a .......e........ +| 3888: 05 01 65 03 06 01 01 0a 01 0a 01 01 0a 0a 0a 04 ..e............. +| 3904: 2b 31 21 0b 0f ef 00 14 2a 00 00 00 00 01 02 02 +1!.....*....... +| 3920: 00 02 01 01 01 02 01 01 50 88 80 80 80 80 01 04 ........P....... +| 3936: 00 81 24 00 00 00 47 02 30 65 02 1a 02 05 05 07 ..$...G.0e...... +| 3952: 05 01 01 04 03 03 08 03 03 02 01 65 02 1e 03 05 ...........e.... +| 3968: 05 04 05 05 01 01 03 06 09 14 06 04 03 03 01 65 ...............e +| 3984: 02 14 04 05 07 05 05 01 01 02 08 ed 04 01 65 02 ..............e. +| 4000: 02 0a 05 01 65 02 06 01 01 0a 04 12 14 0f 06 31 ....e..........1 +| 4016: 84 80 80 80 80 01 03 00 68 00 00 00 2b 02 30 65 ........h...+.0e +| 4032: 01 10 02 05 05 01 01 04 03 03 02 01 55 01 12 03 ............U... +| 4048: 05 05 01 01 03 06 05 03 03 01 65 01 0e 04 05 05 ..........e..... +| 4064: 01 01 02 08 04 0d 0e 06 01 03 00 12 04 4c 4c 00 .............LL. +| 4080: 00 00 11 24 00 00 00 00 01 0f c1 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3600: 00 00 00 00 00 00 00 00 00 00 81 52 04 06 00 81 ...........R.... +| 3616: 5d 81 55 65 20 65 65 20 65 65 65 20 65 20 65 65 ].Ue ee eee e ee +| 3632: 20 65 65 65 20 65 20 65 65 20 65 65 65 65 20 65 eee e ee eeee e +| 3648: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 3664: 20 65 65 20 65 65 65 65 20 65 65 20 65 65 65 20 ee eeee ee eee +| 3680: 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 e ee eee e ee ee +| 3696: 65 65 20 65 65 20 65 65 65 20 65 20 65 65 20 65 ee ee eee e ee e +| 3712: 65 65 20 65 20 65 65 20 65 65 65 65 65 65 20 65 ee e ee eeeeee e +| 3728: 65 20 65 20 65 20 65 20 65 65 20 65 65 65 20 65 e e e e ee eee e +| 3744: 65 20 65 65 65 65 65 20 65 65 20 65 20 65 20 65 e eeeee ee e e e +| 3760: 20 65 65 20 65 62 d5 20 65 65 20 65 65 65 65 65 ee eb. ee eeeee +| 3776: 20 65 65 20 65 20 65 20 65 20 65 65 20 65 65 65 ee e e e ee eee +| 3792: 20 65 65 20 65 65 65 65 65 20 65 65 20 65 21 65 ee eeeee ee e!e +| 3808: 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 6a e ee eee ee eej +| 3824: 03 04 00 75 71 65 20 65 65 10 65 65 65 20 65 20 ...uqe ee.eee e +| 3840: 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 65 ee eee e ee eeee +| 3856: 20 65 65 20 65 65 65 20 65 20 65 65 20 65 65 65 ee eee e ee eee +| 3872: 20 65 20 65 65 20 65 65 65 65 65 65 20 65 65 20 e ee eeeeee ee +| 3888: 65 20 65 20 65 20 65 65 20 65 65 65 20 62 55 20 e e e ee eee bU +| 3904: 65 65 65 65 65 20 65 65 20 65 20 65 20 65 20 65 eeeee ee e e e e +| 3920: 65 20 65 65 65 20 55 65 20 65 65 6a 02 04 00 75 e eee Ue eej...u +| 3936: 71 65 20 65 65 20 65 65 65 20 65 10 65 65 20 65 qe ee eee e.ee e +| 3952: 65 65 20 65 20 65 65 20 65 65 65 65 20 65 65 20 ee e ee eeee ee +| 3968: 65 65 65 20 65 20 65 65 20 65 65 65 20 65 20 65 eee e ee eee e e +| 3984: 65 20 65 65 65 65 65 65 20 65 65 20 65 20 65 20 e eeeeee ee e e +| 4000: 65 20 65 65 20 65 65 65 20 65 65 20 65 65 65 65 e ee eee ee eeee +| 4016: 65 20 65 65 20 65 20 65 20 65 20 65 65 20 65 65 e ee e e e ee ee +| 4032: 65 20 65 65 21 65 65 37 0a 04 00 41 3f 65 20 65 e ee!ee7...A?e e +| 4048: 65 20 65 65 65 20 65 20 65 65 20 65 65 65 20 65 e eee e ee eee e +| 4064: 20 65 65 20 65 65 65 65 65 65 20 65 65 20 65 20 ee eeeeee ee e +| 4080: 65 20 65 20 65 65 20 65 65 65 20 65 65 20 65 65 e e ee eee ee ee +| page 5 offset 16384 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 05 04 03 00 10 21 21 05 13 03 00 10 .........!!..... +| 4080: 11 11 05 02 03 00 10 11 11 05 01 03 00 10 09 09 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-b02ca2cc4d7dda.db +}]} {} + +do_catchsql_test 73.1 { + SELECT snippet(ttt,ttt, NOT 54 ), + * FROM ttt('e* NOT ee*e* NOT ee* NOT ee*e* NOT e*') ; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 74.0 { + sqlite3 db {} + sqlite3_fts5_register_matchinfo db + db deserialize [decode_hexdb { +| size 106496 pagesize 4096 filename x.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 1a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 4f 78 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ..Ox...........6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c d6 63 32 29 69 04 07 17 19 c0, c1,.c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 2 offset 4096 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 01 2f 0d d5 ...t.[.@.$.../.. +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 00 00 00 00 00 .......x.W...... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 8f DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 59 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMYT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 57 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000WBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 59 4e 4f 43 41 53 45 17 LE RTREEYNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4f 41 42 4c NARY....)..EOABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 51 53 45 E GEOPOLYXNOCQSE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 1d 05 E FTS5XNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 55 53 34 58 42 49 4e 41 52 59 17 0b LE FUS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 53 49 4d 0 20160609XRTSIM +| page 3 offset 8192 +| 0: 05 00 00 00 07 0f ba 00 00 00 00 1a 0f f6 0f ec ................ +| 16: 0f e2 0f d8 0f ce 0f c4 0f ba 00 00 00 00 00 00 ................ +| 3200: 00 00 00 00 00 00 00 00 00 00 08 01 03 00 16 2e ................ +| 3216: b1 7d 24 24 86 4a 84 80 80 80 80 01 04 00 8d 18 ..$$.J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 00 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 07 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 d3 02 01 02 nable........... +| 3456: 02 01 02 02 02 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 54 01 02 03 01 02 03 ....gcc..T...... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 13 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 01 f3 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 08 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 ad 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 01 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 00 00 00 19 88 80 ................ +| 4032: 80 80 80 06 00 00 00 18 88 80 80 80 80 05 00 00 ................ +| 4048: 00 17 88 80 80 80 80 04 00 00 00 16 88 80 80 80 ................ +| 4064: 80 03 00 00 00 15 88 80 80 80 80 02 00 00 00 14 ................ +| 4080: 88 80 80 80 80 01 00 00 00 13 84 80 80 80 80 01 ................ +| page 4 offset 12288 +| 0: 0a 00 00 00 08 0e bd 00 00 00 0e f9 0e ef 0e e5 ................ +| 16: 0e db 0e d1 0e c7 0e bd 00 00 00 00 00 00 00 00 ................ +| 3760: 00 00 00 00 00 00 00 00 00 00 00 00 00 09 04 01 ................ +| 3776: 12 01 02 30 f4 a3 0e 09 04 01 12 01 02 30 cf 8c ...0.........0.. +| 3792: 0c 09 04 01 12 01 02 30 7a 34 0a 09 04 01 12 01 .......0z4...... +| 3808: 02 30 72 64 08 09 04 01 12 01 02 30 6b 30 06 09 .0rd.......0k0.. +| 3824: 04 01 12 01 02 30 63 33 04 06 04 01 0c 01 02 02 .....0c3........ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 05 00 00 00 0a 0f ce 00 00 00 00 12 0f fb 0f f6 ................ +| 16: 0f f1 0f ec 0f e7 0f e2 0f dd 0f d8 0f d3 0f ce ................ +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 01 2f 0d d5 ...t.[.@.$.../.. +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 00 00 00 00 00 .......x.W...... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 8f DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 59 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMYT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 57 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000WBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 59 4e 4f 43 41 53 45 17 LE RTREEYNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 53 59 53 35 58 42 49 NABLE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4f 41 42 4c NARY....)..EOABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 51 53 45 E GEOPOLYXNOCQSE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 1d 05 E FTS5XNOCASE... +| 3728: 00 23 0f a4 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 55 53 34 58 42 49 4e 41 52 59 17 0b LE FUS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 00 00 cc-5.4.0 20160.. +| 4048: 00 11 09 00 00 00 10 08 00 00 00 0f 07 00 00 00 ................ +| 4064: 0e 06 00 00 00 0d 05 00 00 00 0c 04 00 00 00 0b ................ +| 4080: 03 00 00 00 0a 02 00 00 00 09 01 00 00 00 02 00 ................ +| page 6 offset 20480 +| 0: 0d 0f b0 00 25 0e bc 03 0e d7 0e ce 0f f0 0e c5 ....%........... +| 16: 0f e7 0f de 0f d5 0f cc 0f c3 0e bc 0f b7 0f a8 ................ +| 32: 0f a0 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 ...........x.p.h +| 48: 0f 60 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 .`.X.P.H.@.8.0.( +| 64: 0f 20 0f 18 0f 10 0f 08 0f 00 0e f8 0e 00 00 00 . .............. +| 3760: 00 00 00 00 00 00 00 00 00 00 00 00 07 09 03 00 ................ +| 3776: 14 84 6b 00 00 07 03 03 00 14 84 5b 00 00 07 02 ..k........[.... +| 3792: 03 00 14 84 63 00 00 07 01 03 00 14 85 07 00 00 ....c........... +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 00 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 0f f8 00 07 12 02 01 07 0a 03 00 14 85 05 00 00 ................ +| 4032: 0f f8 00 07 08 03 00 14 84 73 00 00 07 07 03 00 .........s...... +| 4048: 14 85 0b 00 00 07 06 03 00 14 85 02 00 00 07 05 ................ +| 4064: 03 00 14 84 70 00 00 07 04 03 00 14 84 7e 00 00 ....p........~.. +| 4080: 06 de 03 00 12 06 01 01 00 00 00 08 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 01 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 00 00 00 00 uild....opti.... +| page 9 offset 32768 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 01 06 00 ae 7c 00 00 ee a4 ad af cf 26 bc c7 >....|.......&.. +| 1104: a8 a1 59 d7 43 c2 93 4f ad 82 19 2e 25 e1 97 f9 ..Y.C..O....%... +| 1120: 21 59 34 ce e8 b2 e2 3c 4b f2 b4 ee 6b 35 84 c3 !Y4.....@..r... +| 1792: 8d 30 50 e6 54 9d 62 0d 11 b0 80 ad 64 44 7f 82 .0P.T.b.....dD.. +| 1808: 9a 61 c7 d5 f1 6b 34 2b ec bf 1a ee 5f bf 10 fe .a...k4+...._... +| 1824: 0f 9e 42 3d 34 1e ff 00 82 db 52 e0 65 d1 d7 0d ..B=4.....R.e... +| 1840: 77 fb 84 5c ec c3 b7 72 cf 21 26 40 85 20 f4 78 w......r.!&@. .x +| 1856: 6a f2 f4 52 8a c8 cd d9 99 73 8f 65 f5 f5 a8 af j..R.....s.e.... +| 1872: 75 bd a0 c6 ff 1a bf 64 53 01 4e ed be 22 95 1a u......dS.N..... +| 1888: 57 06 21 1d f3 f3 97 4f c7 de 83 ab 80 40 c4 02 W.!....O.....@.. +| 1904: 12 65 0d d5 55 87 6f 33 a1 b9 45 86 56 aa df fc .e..U.o3..E.V... +| 1920: 24 e6 70 37 c3 e8 bc b6 b6 e1 02 29 dc ba 7e 16 $.p7.......)..~. +| 1936: e9 52 3b 9d e7 f8 d9 d1 5c b2 db 4c 18 29 56 80 .R;........L.)V. +| 1952: f0 f1 b4 07 34 30 2c 2f ee f6 a5 80 f7 a0 b6 7c ....40,/.......| +| 1968: 43 e7 6f e2 a9 87 21 a2 96 82 d7 17 55 4d 33 ff C.o...!.....UM3. +| 1984: 78 75 40 fe db de 63 c3 b5 45 c8 37 40 97 f8 68 xu@...c..E.7@..h +| 2000: b3 03 46 2d 50 1e 2b b0 90 23 09 47 f5 05 db 9c ..F-P.+..#.G.... +| 2016: 23 71 b1 12 e4 b2 ff 41 b4 1f e3 bb 3c 50 d2 ab #q.....A...... +| 2096: 24 2d b9 ba ec 3d e0 88 60 b2 9d 02 07 86 97 13 $-...=..`....... +| 2112: e9 c5 64 c8 0f 03 e1 1b 12 0b ff 39 e1 49 95 fa ..d........9.I.. +| 2128: 6f c3 63 4a 7c 86 38 66 95 b3 57 0b 95 96 c5 e5 o.cJ|.8f..W..... +| 2144: 55 3b 4b 58 d2 59 89 fa 70 40 b5 59 a1 b9 9e 46 U;KX.Y..p@.Y...F +| 2160: ec 3a 5a c8 6d fb ac da d0 62 f6 fb 8c 26 ff 3e .:Z.m....b...&.> +| 2176: fb 54 69 ca 27 4b b5 a3 d1 27 f4 f5 2e d4 26 5a .Ti.'K...'....&Z +| 2192: 0b 3e d6 f4 11 b9 ae fd 42 65 08 6f c1 83 51 1a .>......Be.o..Q. +| 2208: 98 bc ad a9 77 8c da ef 70 dd 9b 6e e6 89 a0 75 ....w...p..n...u +| 2224: 5b c5 14 70 9c 3c 8d 99 b5 83 59 1e 77 0f 16 5e [..p.<....Y.w..^ +| 2240: 38 44 b9 da 9c c0 f0 61 23 5a 67 0e 43 48 81 ff 8D.....a#Zg.CH.. +| 2256: 50 9b 2a 36 d1 18 7e bb d2 4e 83 45 07 f8 35 5f P.*6..~..N.E..5_ +| 2272: a4 8a af 12 ab 91 3e 4f 4f 40 32 3f 7e fc 06 6a ......>OO@2?~..j +| 2288: c4 de 5e aa 2a a5 21 ba 14 51 28 65 c9 9e ef 7c ..^.*.!..Q(e...| +| 2304: 6a 65 67 fc 11 23 1d 76 e3 be e3 10 7a 34 b8 ef jeg..#.v....z4.. +| 2320: 37 b8 ba 9c 1a 13 c7 e1 91 c9 06 ad 98 3c 4b ea 7................ +| 2576: 93 1b 7c 28 fb 53 bc b2 13 2d 8e 22 50 97 4c cf ..|(.S...-..P.L. +| 2592: 06 f1 ac 55 9d c8 ce cd 59 74 9c af 7b bd 6f 7c ...U....Yt....o| +| 2608: a5 b3 a4 87 6a 67 22 f1 82 5b 3e 9e 76 b0 2f d6 ....jg...[>.v./. +| 2624: f6 6d 7a c0 f5 8f 06 c2 5b 09 0f b5 df b9 d7 c2 .mz.....[....... +| 2640: be 51 e0 5e 5a 0a 79 62 3a 3c c8 a1 50 0d a3 36 .Q.^Z.yb:<..P..6 +| 2656: e1 7d 61 4a 1b af f0 42 20 dc 10 a9 13 3d 2b 46 ..aJ...B ....=+F +| 2672: 16 98 d1 24 a8 a1 1a c8 88 0d 63 23 2b 23 b3 8d ...$......c#+#.. +| 2688: 19 13 f9 4c 76 0b 48 40 bb 02 db 1a 54 e8 ea f4 ...Lv.H@....T... +| 2704: f4 57 25 0d 50 1d 31 af 74 00 d8 f4 22 d9 53 e8 .W%.P.1.t.....S. +| 2720: 35 ae 1c c6 82 18 06 4f b6 f3 e8 2c 15 1c 38 1c 5......O...,..8. +| 2736: 5c 47 24 f4 49 44 ef a6 cc de 85 0d 61 aa f6 4f .G$.ID......a..O +| 2752: 18 4b 23 7e ca dd 04 7a 7f 6c d0 70 59 05 0f 31 .K#~...z.l.pY..1 +| 2768: de 19 71 96 7a 1b 93 a2 20 18 d6 a6 f2 8d 28 1d ..q.z... .....(. +| 2784: c0 f3 a9 20 87 f7 dc 10 eb bf bc 1e cf 7a 54 a2 ... .........zT. +| 2800: 7f 96 0d 32 a9 30 30 5c 6d 31 3c 76 4d 65 1f d0 ...2.00.m1..=......USJ +| 2928: 53 80 b1 9e 23 83 49 1a e0 22 9f 5c f3 df 87 d8 S...#.I......... +| 2944: 6f 45 ff 9b 8a 69 80 95 f8 ca bc 5c 0c b1 46 04 oE...i........F. +| 2960: 7e d7 61 72 d8 bf a1 b6 bc b3 48 c7 c2 b3 9d 7c ~.ar......H....| +| 2976: de fb a4 0a 04 2f 97 99 eb ca 8c b5 39 fc 2b 45 ...../......9.+E +| 2992: 69 7f a9 70 c6 73 a1 22 71 b0 0d 53 0c f4 c7 68 i..p.s..q..S...h +| 3008: 85 ec 11 44 0b f9 b7 3b ff b7 91 1b fb bd bf e1 ...D...;........ +| 3024: 01 90 4c 74 35 f3 ac 5a 70 bd e1 4e bb fd a8 dc ..Lt5..Zp..N.... +| 3040: 4d 38 c4 68 a8 e1 8f d1 69 b8 7a 20 b6 5c ed 2b M8.h....i.z ...+ +| 3056: e8 20 dc 0f 7e 29 3e 5f 83 76 e2 d9 b1 c0 07 66 . ..~)>_.v.....f +| 3072: cd a2 d4 57 bc 27 ff 1d 16 7a 11 d6 3d df 04 89 ...W.'...z..=... +| 3088: 45 91 e5 37 62 51 5a e6 0f 0d b8 e9 1a 43 e4 c3 E..7bQZ......C.. +| 3104: 47 94 bf 16 fb 8c e2 f7 b8 c8 7f 7f 34 3b a3 fa G...........4;.. +| 3120: 4f 39 b3 89 ee 7a 1b 80 a6 04 46 24 7f 0c d2 48 O9...z....F$...H +| 3136: b4 cb a2 df 5c af 55 11 a5 c6 f6 de 6a a4 0f dc ......U.....j... +| 3152: ae 12 98 11 9e 95 e2 b1 a6 c2 67 bb 37 ea e8 8e ..........g.7... +| 3168: d1 6f 6c 7a 3a 13 5e 1c 31 92 7e 24 72 b0 f5 b6 .olz:.^.1.~$r... +| 3184: f5 db 3e b9 3b 2a 18 da 93 29 da 2c be 4a de c6 ..>.;*...).,.J.. +| 3200: 6c 55 a0 3e 47 25 76 5c 73 08 99 17 87 e9 30 0e lU.>G%v.s.....0. +| 3216: 91 a8 cd da 9a cd 90 8b 2d 5f 0e 88 7c a7 00 42 ........-_..|..B +| 3232: 84 bd 59 1b ce fa 76 27 33 78 c1 a4 0d 29 98 45 ..Y...v'3x...).E +| 3248: d1 7f b6 7d 56 f6 67 fe 78 ae 83 03 39 66 ce 5a ....V.g.x...9f.Z +| 3264: 62 a1 e3 c5 fd 29 53 06 6e cd ff 0e 5a 95 ca 91 b....)S.n...Z... +| 3280: 1b 24 0d 42 ec e2 24 8a 01 ff 12 0b bf b1 18 74 .$.B..$........t +| 3296: 77 2d f5 9e 0a 74 e6 d4 7d 1c c1 53 d9 f5 65 9c w-...t.....S..e. +| 3312: 40 6d 8f a9 f6 7b b0 96 37 71 c2 96 8c 90 f8 29 @m......7q.....) +| 3328: 07 10 c7 2d f5 1d 80 dc 96 b7 25 65 a6 a2 ff ba ...-......%e.... +| 3344: 5d 1e c1 0d ed b1 6a 83 20 6d 06 28 6d 54 8c 88 ].....j. m.(mT.. +| 3360: 08 02 3d cf f6 79 81 f1 36 3b f0 6e e6 80 39 43 ..=..y..6;.n..9C +| 3376: 64 d6 4a 24 8b 3d 21 41 a9 48 d2 36 65 2f 5a 71 d.J$.=!A.H.6e/Zq +| 3392: eb 6f 2b 47 78 2d 8c 28 91 60 25 3c 35 81 5b 1d .o+Gx-.(.`%<5.[. +| 3408: b7 36 34 71 4c 38 f2 29 7e f5 a8 45 71 95 78 19 .64qL8.)~..Eq.x. +| 3424: 00 e8 87 4d da 50 78 5e f7 dc aa 2d 15 92 49 d8 ...M.Px^...-..I. +| 3440: 4e af 77 30 bd ad 22 1b 6a 84 ff 78 6d 37 cf 1b N.w0....j..xm7.. +| 3456: 8a d9 81 dd 34 15 a7 3a c0 53 d6 ab 5a 38 ec 69 ....4..:.S..Z8.i +| 3472: 6a 88 64 8c a6 ce 50 12 45 a1 7f a2 aa 3a a8 cf j.d...P.E....:.. +| 3488: d6 a0 80 4e d6 7a b6 50 90 64 c0 52 30 51 04 6f ...N.z.P.d.R0Q.o +| 3504: 89 25 02 b3 54 0b fb 24 89 cf f7 98 bd 8e fc 9f .%..T..$........ +| 3520: 62 0c cd fd 57 fd ac 64 b9 2a 94 62 94 38 c6 01 b...W..d.*.b.8.. +| 3536: 0c f1 b9 75 f4 3c 5d 0e d4 1f 96 b3 74 3f 96 03 ...u.<].....t?.. +| 3552: 13 b6 76 32 07 e0 1f 82 d8 27 f3 e7 2e f4 60 d0 ..v2.....'....`. +| 3568: 56 a5 8f 04 37 bd 5c 17 1e 33 94 75 d8 30 59 0d V...7....3.u.0Y. +| 3584: e5 90 f5 09 ee 5c 01 88 14 ca 69 27 08 fa e7 3c ..........i'...< +| 3600: a2 69 df e2 be 35 44 96 b5 06 69 5c 01 3f 52 67 .i...5D...i..?Rg +| 3616: 18 d2 c9 64 a7 ba 0b 59 d8 b8 53 21 74 de 2b 21 ...d...Y..S!t.+! +| 3632: 8a 53 3d 97 14 92 77 ed 51 21 4b f0 2d 69 93 09 .S=...w.Q!K.-i.. +| 3648: 57 3e 92 9f 3e 20 6c 4d bf 8b fd 4f 75 4b 19 5d W>..> lM...OuK.] +| 3664: 48 ef 23 1e 53 11 ee 76 b7 04 08 5a c4 9a 1f 6c H.#.S..v...Z...l +| 3680: 24 cb 15 7f 0b f7 86 8e 60 a4 8d 3c 2a fe 14 13 $.......`..<*... +| 3696: 03 28 80 fa 6b d7 1b 02 a7 0d 9e 88 4d 1f b2 a4 .(..k.......M... +| 3712: 63 c7 65 56 14 df 51 7e d3 d4 3b e3 45 e1 7a 49 c.eV..Q~..;.E.zI +| 3728: 1e 71 40 fe b7 ae 65 10 b1 27 3a 02 31 21 47 11 .q@...e..':.1!G. +| 3744: d9 fc 9c 32 e5 c8 40 0d b6 4b 02 ed bc da 4c 98 ...2..@..K....L. +| 3760: 35 2c d5 9e 6f b3 42 c7 8e 0a c7 fa ae ff 36 5b 5,..o.B.......6[ +| 3776: 76 08 69 3e 3c cd 4d eb 6f 0c a0 f6 23 93 a6 bb v.i><.M.o...#... +| 3792: 2f ed 44 64 22 df e8 6b 21 68 5b 35 d6 8f 68 c5 /.Dd...k!h[5..h. +| 3808: 15 1f 46 fd 12 bc b5 b5 3e a7 e4 9b b2 83 f4 12 ..F.....>....... +| 3824: ea bb 50 84 f4 40 96 c4 64 30 d8 fe 74 5b f2 ba ..P..@..d0..t[.. +| 3840: 9a 64 23 67 4a 3d 7e 54 da 8f 39 18 df 31 88 23 .d#gJ=~T..9..1.# +| 3856: d6 80 5f e9 10 9f 37 22 6f 4a 21 13 20 13 fc 66 .._...7.oJ!. ..f +| 3872: fc 4b db a8 d9 aa 55 01 48 3e 8c ac bf 16 fc 62 .K....U.H>.....b +| 3888: 95 2c 44 1f 27 bf 7b 45 7d 28 55 14 1f ed 56 ed .,D.'..E.(U...V. +| 3904: 24 5b 11 ff be a0 7a 20 3b 3e 9c 2c e6 d6 b0 ef $[....z ;>.,.... +| 3920: b4 df 16 73 f2 d3 a9 90 2b 54 c7 7a fa 25 e7 ee ...s....+T.z.%.. +| 3936: da 99 8d d7 b5 7d 0f 72 c7 61 75 d1 d7 23 dd 41 .......r.au..#.A +| 3952: 1e 46 ee ef 41 86 00 9f 1c 47 36 75 95 f6 1d 89 .F..A....G6u.... +| 3968: 13 c0 75 f8 cc 5e 08 93 e4 de a8 ee d2 ce c2 32 ..u..^.........2 +| 3984: e4 16 b0 c8 82 c1 2a 74 ed 5c 5f c1 99 f7 07 a7 ......*t.._..... +| 4000: b3 50 21 87 a1 43 dc 17 4f 2d 47 e0 be 53 ad 17 .P!..C..O-G..S.. +| 4016: f9 09 67 d1 4f 1f 72 17 62 b7 03 fa cd de 3e a7 ..g.O.r.b.....>. +| 4032: 25 a9 e7 a0 e2 3d a3 6b 2b 34 3f 55 46 18 df ef %....=.k+4?UF... +| 4048: 16 0a ce c8 67 58 eb 64 eb 7e a3 5b 4e 85 49 64 ....gX.d.~.[N.Id +| 4064: d7 f9 ec 0d 4d b6 1d 49 bb 93 e9 79 3d e1 f9 ad ....M..I...y=... +| 4080: 6d c0 45 9c 46 26 21 41 10 dd 4a 12 fc 28 f4 cc m.E.F&!A..J..(.. +| page 10 offset 36864 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 02 06 00 ae 7c 00 00 9a 84 85 31 87 79 cc e4 >....|.....1.y.. +| 1104: 12 98 6f 2c 46 b6 5d 4e d9 5c ba 21 2f cf 51 9d ..o,F.]N...!/.Q. +| 1120: 9f 8c 48 7d 5a 15 ae 1c 54 d7 f0 7f a2 e6 cc fd ..H.Z...T....... +| 1136: ad e6 3c d5 e5 98 5d f2 64 82 5b 9e 6c d6 05 8f ..<...].d.[.l... +| 1152: de fb 33 3a 37 b3 1d 58 04 47 13 cd d9 39 17 a9 ..3:7..X.G...9.. +| 1168: a7 14 f1 fe c8 3b 21 40 a8 22 f5 3b 64 74 14 dc .....;!@...;dt.. +| 1184: c5 9e 99 8b f3 d4 5f 1a e8 1d 93 98 9e 00 7d ee ......_......... +| 1200: 9f 0f 49 b5 af 12 7e 47 8e fe 5f 70 a1 9c 32 ae ..I...~G.._p..2. +| 1216: d1 23 33 d1 c0 65 b6 8e 5a 4a 03 2f 74 27 0f 3d .#3..e..ZJ./t'.= +| 1232: 3f 42 0d 03 5f 6c ef 2c 54 6a bd ec 88 eb cd 1f ?B.._l.,Tj...... +| 1248: dd 03 9c 06 18 8e a4 c1 a5 3c ec 9d 68 da d9 d3 .........<..h... +| 1264: d8 8f ba 16 95 fd 94 ed 19 2b 2f 5e 0e a5 fa de .........+/^.... +| 1280: dc c5 a0 a2 d4 31 0f dc f5 8b 4a 28 4f 6f 9f ff .....1....J(Oo.. +| 1296: 7f e6 d4 d0 08 7e 5c 76 9a 3a ea 0e ea b4 9d 45 .....~.v.:.....E +| 1312: 4b db 96 ed 27 ad a8 09 63 3c d0 1d d9 b1 dc 94 K...'...c<...... +| 1328: 6a 8e e7 6a 9b 6e d1 8b f7 c8 60 1c 85 e9 a2 3e j..j.n....`....> +| 1344: 0c b6 2f 40 b0 2e b6 53 8f 94 74 6b 39 13 fd c9 ../@...S..tk9... +| 1360: 44 77 64 7c 62 e0 51 2e 04 0f 6a 99 5e 68 6f 72 Dwd|b.Q...j.^hor +| 1376: 08 92 02 bf 36 78 0c 98 c3 3f 3d e3 ac a4 2d 3b ....6x...?=...-; +| 1392: d7 51 cf f9 25 72 59 36 b0 ea 51 a1 9c bf 13 f1 .Q..%rY6..Q..... +| 1408: dc f5 36 f0 d9 83 fe 0a ff b5 00 ab 5b 4e 0b 33 ..6.........[N.3 +| 1424: 0a a1 fa c1 02 c8 7a af 00 30 54 d3 6e a8 37 b4 ......z..0T.n.7. +| 1440: 02 88 16 41 95 86 1f 36 e0 98 43 d5 55 57 c6 5e ...A...6..C.UW.^ +| 1456: 0d 10 40 fd 3d 85 a1 5f 19 3f 18 87 11 d9 6a ca ..@.=.._.?....j. +| 1472: 8b 5c 14 cc 64 48 b9 42 71 61 30 ca 0d 2c a5 67 ....dH.Bqa0..,.g +| 1488: 6b 4f 5b 34 1b f3 7b a1 1f ed e4 3b ef 68 27 00 kO[4.......;.h'. +| 1504: f3 8a 27 ff de 2e 07 ab 03 b1 5b b9 c3 84 53 f1 ..'.......[...S. +| 1520: d0 c5 8c 65 50 4b 7a 35 06 0c d5 2a ce 4c b9 02 ...ePKz5...*.L.. +| 1536: 7f cf 2a d9 0d 2a 1c c1 5f 94 cb 5c 05 4d 3a 7a ..*..*.._....M:z +| 1552: aa c5 3e 4e 26 93 8a 4d 1d fc 75 23 9f b5 27 87 ..>N&..M..u#..'. +| 1568: fc f7 3d aa a6 3e 5d c3 55 63 e2 a8 2a 7b 2e 26 ..=..>].Uc..*..& +| 1584: e9 64 59 1f 2a e7 ff 7c d4 69 a6 34 bb d9 23 81 .dY.*..|.i.4..#. +| 1600: de 4b ba f3 91 cd 9a 8b 98 56 f0 b8 73 7b 83 f1 .K.......V..s... +| 1616: fc e9 5c 01 7e 74 66 d0 0c 01 79 c0 b2 49 0f 78 ....~tf...y..I.x +| 1632: 79 ef 20 96 ec cb 63 67 fe 43 a7 ea 5d af 68 12 y. ...cg.C..].h. +| 1648: 84 dc 0c 5c 31 6a 1d d6 a1 1e d4 25 a3 f0 7c 19 ....1j.....%..|. +| 1664: dc 84 4f 7b e4 5a e5 40 23 07 b3 5b 92 92 4b 3a ..O..Z.@#..[..K: +| 1680: 0e 11 9c b4 ba d4 d1 ff 22 af cd 7d e8 86 c3 0a ................ +| 1696: 14 eb 01 13 f8 99 56 8c c6 26 55 8d a7 cc cc 00 ......V..&U..... +| 1712: 4c 16 7a 07 de 6c 69 02 e2 a1 e6 e1 d5 ff 12 f5 L.z..li......... +| 1728: ee d7 8f 90 e1 78 99 09 de 27 b8 b3 e1 7a df 8a .....x...'...z.. +| 1744: 08 cb c0 67 ff d4 48 bd 0b 4c 65 56 5a ed 02 47 ...g..H..LeVZ..G +| 1760: 1c de b7 58 7c cf 68 a4 7f e6 db 74 26 55 9c 14 ...X|.h....t&U.. +| 1776: 76 5a bf f3 e1 79 23 5e 33 f8 65 13 bb 36 cc ed vZ...y#^3.e..6.. +| 1792: 9e 12 63 b2 c2 14 3f 6a 4d e3 a4 63 bf 30 0f cb ..c...?jM..c.0.. +| 1808: f2 f0 d1 81 a7 26 d7 c0 92 9f 06 79 bd a7 a0 8d .....&.....y.... +| 1824: 74 21 7b c6 46 49 5c fd 01 a6 92 56 3b eb e6 d5 t!..FI.....V;... +| 1840: b4 4d 76 2a e8 00 98 3b a3 67 a9 7d 02 25 96 3b .Mv*...;.g...%.; +| 1856: 45 19 a0 d4 b5 67 88 cf 48 eb 1a b6 0e f3 2d 62 E....g..H.....-b +| 1872: 3e 0c fa b3 e7 a4 f1 76 d4 d7 f5 19 6f b2 9e e8 >......v....o... +| 1888: e1 a1 b7 be 3a ff 69 db eb 75 1c 0c 91 7f 02 4f ....:.i..u.....O +| 1904: df 15 af 09 48 2c d0 86 bd b0 80 82 e7 7b 1d a6 ....H,.......... +| 1920: 38 f1 24 79 d0 8e 4c 07 9f ed 9f fb 90 a1 7e fd 8.$y..L.......~. +| 1936: 50 c6 fe d3 61 04 a1 03 34 30 bd 98 db 5c 18 e3 P...a...40...... +| 1952: 94 d9 ba 8f 64 f8 6b c1 21 3b 22 b7 3e 71 5a 66 ....d.k.!;..>qZf +| 1968: cf 9f 51 c4 a2 36 c8 ba c3 2a 95 2e 67 e4 87 3e ..Q..6...*..g..> +| 1984: 07 ae 66 ad ec af c6 17 62 11 be 5f 15 fc 61 53 ..f.....b.._..aS +| 2000: 76 eb f5 78 da 32 8c fa 4e d3 cc 27 19 dc 89 fc v..x.2..N..'.... +| 2016: 37 4f 74 61 f7 5e 97 0f fe fc af aa 12 ee ea f2 7Ota.^.......... +| 2032: 68 36 36 cd 7e 57 41 75 48 be cd 46 e3 cd 3a 99 h66.~WAuH..F..:. +| 2048: 31 33 9a 84 8a 83 a2 fc 85 85 3c bc cf 07 b4 6d 13........<....m +| 2064: 57 d2 c9 63 a2 9d 42 07 4e cd 65 2e 65 0a af 03 W..c..B.N.e.e... +| 2080: dc a9 98 57 b6 7f da 1d b7 3c e0 ef aa 18 eb 3f ...W.....<.....? +| 2096: 78 f1 34 e6 bf c7 28 34 11 a7 bc b3 34 79 f2 85 x.4...(4....4y.. +| 2112: ed 7d fe 38 a5 48 b3 b4 8a 0d 75 12 65 04 f8 88 ...8.H....u.e... +| 2128: 71 b9 d4 86 48 c7 ea 2f af 4d 50 b9 50 a7 17 f6 q...H../.MP.P... +| 2144: 1f a2 e1 b8 aa ed 6d 85 3a e2 91 be 94 c8 fc db ......m.:....... +| 2160: 93 50 0e 50 7c cf 52 f7 55 81 3e 1a 59 4d a8 36 .P.P|.R.U.>.YM.6 +| 2176: ee 07 f1 9e 26 4a 1e d3 0b 7d 52 e3 bc 7c 91 78 ....&J....R..|.x +| 2192: 02 48 7e b5 4c 32 1e a3 ba db 93 61 94 3b d7 22 .H~.L2.....a.;.. +| 2208: 7b cb 46 5b c0 a5 1e e6 5a ed c9 82 07 48 17 d6 ..F[....Z....H.. +| 2224: de 85 ca 47 e1 16 5c c8 dc 30 5e 27 65 60 a5 41 ...G.....0^'e`.A +| 2240: 46 0a 12 0e 97 63 3f 05 5f 62 83 e0 cb 72 b6 61 F....c?._b...r.a +| 2256: 6a 4c 1d 7b 09 28 75 54 1c cd 29 bc f8 71 34 56 jL...(uT..)..q4V +| 2272: d6 ac 77 34 44 0d 5f c3 0e f1 d7 b3 dc 0d 1f 85 ..w4D._......... +| 2288: e3 db 45 88 fd db ab bf a6 ff bc 08 f3 3c 00 f0 ..E..........<.. +| 2304: 9f 3b 07 36 cc 37 f0 6d 14 a8 6b 69 d6 e4 3a ab .;.6.7.m..ki..:. +| 2320: 4f 5a b5 ad 3a e5 e3 d6 1c e1 6b 15 97 69 b0 41 OZ..:.....k..i.A +| 2336: 91 09 50 02 6a 5c c1 9e fe e7 38 7e 19 8a 36 44 ..P.j.....8~..6D +| 2352: 51 04 8d 61 d9 a3 12 34 00 b2 b1 60 f3 35 eb 8a Q..a...4...`.5.. +| 2368: dc bb a1 67 48 b6 95 81 91 fe 20 dd 03 74 bb 3b ...gH..... ..t.; +| 2384: 8c 33 58 b2 d8 55 7e 04 ea 42 6c 02 e8 26 18 19 .3X..U~..Bl..&.. +| 2400: 23 15 21 a5 73 ca 08 7b dd db fb b2 12 df 6a 5a #.!.s.........jZ +| 2416: d7 ac e4 57 61 ac 3f 81 77 df f6 a3 97 5c 69 47 ...Wa.?.w.....iG +| 2432: 5a b7 75 23 6c 60 be 97 ee b5 5d a0 c3 60 15 4a Z.u#l`....]..`.J +| 2448: 79 eb 81 9f 2a 74 22 13 7e ca 4d 3b 19 13 62 58 y...*t..~.M;..bX +| 2464: 78 25 ca 21 c3 10 1e 96 34 82 37 6b 08 b5 9b f6 x%.!....4.7k.... +| 2480: 7b 87 97 cd bf 64 67 b4 10 f6 84 16 62 74 a5 b0 .....dg.....bt.. +| 2496: 1f d4 c7 1d 8c 2d b5 10 36 5b fb 06 80 fe 97 59 .....-..6[.....Y +| 2512: b6 5a 08 0f 1e ff 0f 5f bf 28 46 4b d8 84 6c ad .Z....._.(FK..l. +| 2528: 05 c0 25 89 a9 cd 6a be a3 59 84 f0 17 1c 37 8a ..%...j..Y....7. +| 2544: 8d 09 17 bf 7d fe 47 b4 d6 d6 56 1f d8 04 66 59 ......G...V...fY +| 2560: 2c c4 b4 91 a9 ff da 4a 9c b1 29 ff 92 db 6f 19 ,......J..)...o. +| 2576: ef eb 99 ba 6e 65 2f 6a 7f 1a cf ad a3 96 8c 1d ....ne/j........ +| 2592: 62 86 42 3e a3 64 fc e0 40 4c 7c 60 77 b5 42 68 b.B>.d..@L|`w.Bh +| 2608: 3f 09 37 68 02 75 2c 22 83 d5 04 17 eb a7 e2 71 ?.7h.u,........q +| 2624: 29 36 b7 1b c5 1f 11 ce 8d 91 5a 25 39 50 16 2b )6........Z%9P.+ +| 2640: 60 29 50 9f 17 55 b0 9b a5 92 92 f8 1b e3 9c a3 `)P..U.......... +| 2656: e2 a4 cd 90 1e 21 ac 30 ac 35 de 25 30 88 6c 2c .....!.0.5.%0.l, +| 2672: 79 ea b5 0d 58 a5 37 2b ac af 7d 1f af 32 ca 58 y...X.7+.....2.X +| 2688: 27 17 68 f2 a3 ca a6 cc b2 be 12 c1 a0 43 1e 7d '.h..........C.. +| 2704: 11 ec 8e 23 22 0a ca cd 70 d1 05 fd c0 68 92 e9 ...#....p....h.. +| 2720: 1c 55 85 48 10 37 7c 02 69 bf 3f 86 cf d6 40 38 .U.H.7|.i.?...@8 +| 2736: a3 9c 83 40 b8 4b 83 51 50 0d d2 b9 c3 32 09 f8 ...@.K.QP....2.. +| 2752: bc 7b 9b b8 d7 2e 4f a0 96 48 a7 5a 1b c9 71 fb ......O..H.Z..q. +| 2768: f4 2b ff 05 54 89 26 b9 6f 25 4a b9 e2 2b e8 86 .+..T.&.o%J..+.. +| 2784: 43 22 f6 20 28 a9 39 d9 09 a2 dc 60 14 09 d6 0d C.. (.9....`.... +| 2800: 61 7c 15 5a 8f 3f cc 00 12 f5 e0 45 fe 14 1a cc a|.Z.?.....E.... +| 2816: 98 c4 de 48 75 12 02 2b 79 a1 4a 33 a5 7c 3d cd ...Hu..+y.J3.|=. +| 2832: b0 5c dd 77 15 5f d9 24 b7 6b 62 80 cb 35 5c e6 ...w._.$.kb..5.. +| 2848: a6 57 2e e9 00 9e 20 a9 c6 f0 63 a2 0e eb 9d f3 .W.... ...c..... +| 2864: bb 2c 56 03 68 35 53 5a fb 4f 44 8e 0f a4 9c 9a .,V.h5SZ.OD..... +| 2880: 0d b7 2c a9 03 14 8c 51 23 21 fb fd 46 07 68 a7 ..,....Q#!..F.h. +| 2896: f3 09 25 e4 98 55 24 da 72 ee 50 00 95 04 7c 74 ..%..U$.r.P...|t +| 2912: d0 07 8b 92 f9 27 11 3e 41 b4 3e 6c aa 56 ed 54 .....'.>A.>l.V.T +| 2928: e3 40 4d 67 8b 7b 63 cd 62 37 ec e2 1b b4 f9 eb .@Mg..c.b7...... +| 2944: ca c7 6e 8a d3 7e f5 e9 e1 33 84 31 05 cb f8 e4 ..n..~...3.1.... +| 2960: 02 76 c2 2c b9 00 32 5f be b5 f1 c8 78 e8 cf 22 .v.,..2_....x... +| 2976: 65 d8 2b 43 2c 2e 5d fb 2e 58 92 d2 3e 9b 7e 67 e.+C,.]..X..>.~g +| 2992: a7 3b 59 18 84 5d 61 92 ec 57 97 b6 bd e3 7b f2 .;Y..]a..W...... +| 3008: ff 3b 9f 48 39 ad 10 81 51 85 7c 6b 5c b0 53 10 .;.H9...Q.|k..S. +| 3024: 60 9b 4e 0d 86 c0 31 b9 a1 95 05 f4 54 13 52 6a `.N...1.....T.Rj +| 3040: 96 50 1b e5 f5 54 98 63 8f 07 28 cf b4 ba 9c 3b .P...T.c..(....; +| 3056: dd 85 3c a6 15 a6 31 f6 aa 1c 3b 31 3e d9 6e c3 ..<...1...;1>.n. +| 3072: 09 74 b9 8e 59 a1 50 10 e0 f0 48 a9 f1 dd a1 ab .t..Y.P...H..... +| 3088: b6 fc 53 9f bb 38 69 78 07 50 8c 77 cb d1 89 e2 ..S..8ix.P.w.... +| 3104: cb b9 51 dc c7 2e 8a 56 47 14 67 80 eb 7a 16 f8 ..Q....VG.g..z.. +| 3120: 87 89 58 d1 55 58 c5 bd 62 24 4d ef 46 9f 68 94 ..X.UX..b$M.F.h. +| 3136: 61 9a 08 81 dd dd f1 51 c6 40 63 9b a1 3c 3e 5a a......Q.@c..<>Z +| 3152: 78 b2 10 7c a2 47 45 57 ef 98 85 82 7e 21 3b 77 x..|.GEW....~!;w +| 3168: 1e d2 fd 0d bf a5 41 6f f5 ee 4c ed 82 f3 15 ea ......Ao..L..... +| 3184: 2e 57 5e 78 7c 22 2e a4 a6 9c 3f 50 e9 4b 02 11 .W^x|.....?P.K.. +| 3200: a0 7c 5a 19 82 9b 34 13 39 32 07 f1 46 4c ad fa .|Z...4.92..FL.. +| 3216: a7 5a fa 22 5c 84 4d 8d 07 1f 17 1f 1a 49 5f fa .Z....M......I_. +| 3232: df 22 56 ac ce 27 4d 00 a9 b0 c0 77 d8 10 56 cb ..V..'M....w..V. +| 3248: 76 e0 c8 4a 8a d1 37 c1 cc b8 74 d3 e9 61 1b 0c v..J..7...t..a.. +| 3264: 81 7b be 3d e3 17 43 75 5c ba 6f 61 74 d4 a4 e9 ...=..Cu..oat... +| 3280: d7 0f 2f d2 bc 7d 0b 6e 32 ae fc 20 db 08 39 b5 ../....n2.. ..9. +| 3296: 8f 78 1f d6 4e 84 46 05 a3 a1 15 e8 9d 36 b1 96 .x..N.F......6.. +| 3312: 3e e3 12 1b 72 28 28 e8 a5 0f e1 16 a2 31 22 48 >...r((......1.H +| 3328: 4b 77 c6 e5 3f b8 f5 bb 29 13 e1 c1 da 79 81 72 Kw..?...)....y.r +| 3344: fa 12 ab 62 da 63 ac 95 2a 1b 4a a4 4f 96 95 c0 ...b.c..*.J.O... +| 3360: 30 38 3b af 2d c5 cd aa 56 ef e4 35 1d 97 9c 3f 08;.-...V..5...? +| 3376: 57 30 2a 7a 1a 0f 4d 9e be 82 a6 fc 5e a7 9d 0d W0*z..M.....^... +| 3392: a8 11 12 39 b5 e7 5f 8d 18 1d bb a0 2b 02 10 d3 ...9.._.....+... +| 3408: 08 3a 47 de a1 93 f3 7b b7 07 21 c2 62 22 f2 b3 .:G.......!.b... +| 3424: dd 67 9c 8c 87 59 7d 87 75 46 0c 85 64 f3 09 7f .g...Y..uF..d... +| 3440: e2 7f ee ca c8 d2 2e 96 14 50 30 89 86 f6 ed 07 .........P0..... +| 3456: ee 9d 78 0c 84 c8 80 ee 2f d4 59 22 12 92 e9 80 ..x...../.Y..... +| 3472: 04 f5 ed aa 7a f5 cd 53 d7 de 45 a5 c3 e1 4e 9d ....z..S..E...N. +| 3488: fb 78 98 4d cb 5b 0b bf 1b 6a bc 50 ba 45 e5 ff .x.M.[...j.P.E.. +| 3504: d7 d0 7b 39 3d 5b eb 92 46 7f bb 21 7f 81 8b da ...9=[..F..!.... +| 3520: e7 c1 e2 12 41 82 19 7a bd 78 de bc e0 51 cf a7 ....A..z.x...Q.. +| 3536: 0e 20 0b 74 cb c8 fe e9 26 64 6e c4 e1 cc 3f b8 . .t....&dn...?. +| 3552: 62 a1 9b 79 bd 9f 05 21 b1 d6 93 66 6f 38 e5 df b..y...!...fo8.. +| 3568: 56 0c 57 c3 3b 57 04 79 f7 f4 b5 20 16 65 18 dc V.W.;W.y... .e.. +| 3584: a8 88 ce 1c bd 7f 40 c8 05 46 17 d7 99 a7 8e 07 ......@..F...... +| 3600: b7 02 2f 8c a6 49 5a 1f ce 63 3a 66 31 c2 3b 78 ../..IZ..c:f1.;x +| 3616: 84 00 80 8e 81 43 07 3a 43 ef e4 df 33 4b 18 33 .....C.:C...3K.3 +| 3632: 45 27 cc 2e e8 cf e8 be 1a 90 97 ed 99 9b b2 6f E'.............o +| 3648: 65 ff 53 ad df f8 58 47 d9 5d 71 6a 38 e4 e8 17 e.S...XG.]qj8... +| 3664: e1 1d 4c 03 cc c1 33 18 96 3e 9c 11 98 55 8e 62 ..L...3..>...U.b +| 3680: 1a af a0 33 36 f7 0c 87 0a 0c f0 43 ff e4 71 19 ...36......C..q. +| 3696: 5a 38 f8 9d 9a 53 d4 48 ff e3 40 89 e2 18 d5 3c Z8...S.H..@....< +| 3712: fb 2c 67 2c b9 f3 e5 d7 5c 97 e9 8f fb 3e 3e a2 .,g,.........>>. +| 3728: 22 63 47 fb d0 65 ed 87 b0 93 e4 28 e5 85 87 68 .cG..e.....(...h +| 3744: fc 0b 64 4c 5b 3c 5f 9f 40 96 d7 34 5b cb d0 a9 ..dL[<_.@..4[... +| 3760: 63 f5 f7 80 f4 67 49 1d 20 0d 84 f9 39 85 28 8b c....gI. ...9.(. +| 3776: 78 52 fc 86 fb f8 33 cd d6 ef ef ca 32 51 98 2c xR....3.....2Q., +| 3792: 54 d4 2e b9 78 0c 2a 5e c0 e6 2c 58 83 60 85 ea T...x.*^..,X.`.. +| 3808: 0b 33 de 33 40 a4 05 80 0d 21 b5 0e 4f 87 a1 a2 .3.3@....!..O... +| 3824: 3a 20 d6 ee 63 7a 31 93 5c 64 f4 f4 57 43 8a 65 : ..cz1..d..WC.e +| 3840: 53 e0 6e 84 c0 ee f6 44 ce 46 ea 97 fc af ec 4b S.n....D.F.....K +| 3856: ff af da 33 b8 69 e5 08 bd 36 17 aa be 1b 99 5f ...3.i...6....._ +| 3872: 80 c2 8c 27 56 1c f4 99 fa c3 1c 54 01 b7 3e 96 ...'V......T..>. +| 3888: 67 b7 f0 31 f7 20 63 51 17 61 f1 b7 06 23 24 35 g..1. cQ.a...#$5 +| 3904: f3 84 c9 01 ec 59 ba 00 83 b8 6c f9 4b 94 01 53 .....Y....l.K..S +| 3920: d4 06 ef b7 a9 08 3c 29 7d ab 8a 88 12 e2 3f a6 ......<)......?. +| 3936: a1 27 be 2e 09 aa d4 15 bc 43 2a 10 ff 2a 2d 89 .'.......C*..*-. +| 3952: fd a0 73 e7 91 14 a3 e2 2c 00 e8 c8 a7 bf 7c c3 ..s.....,.....|. +| 3968: 4c a0 89 bb 70 da 99 1a 39 5f 81 77 af 17 8d af L...p...9_.w.... +| 3984: e8 b0 e0 27 fb f0 b2 39 45 4a 82 97 93 00 a5 54 ...'...9EJ.....T +| 4000: 44 07 b7 1d ce 52 5f bd cd 07 6f ba 6f 8d 6d d7 D....R_...o.o.m. +| 4016: 2b 4f f5 4b 7b 23 45 74 d9 d8 aa e0 15 14 f6 be +O.K.#Et........ +| 4032: 74 f2 b8 38 f9 ba 64 58 93 b2 b6 0c e7 62 10 81 t..8..dX.....b.. +| 4048: ad ce 6f 23 0e cd 1c 3c 1e 30 62 64 95 d6 48 55 ..o#...<.0bd..HU +| 4064: fc 63 67 73 85 40 16 2b ab 0b 96 c4 90 8e ea 04 .cgs.@.+........ +| 4080: 64 13 bb 60 63 d8 6c d3 73 ed a9 10 03 bf 20 04 d..`c.l.s..... . +| page 11 offset 40960 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 03 06 00 ae 7c 00 00 68 95 7f 98 0f 98 e0 e6 >....|..h....... +| 1104: fd 92 02 0f 0f 29 aa bd bc 40 56 6c 5e fa 3f 54 .....)...@Vl^.?T +| 1120: 10 24 70 48 b1 c5 fe e5 0b 21 21 65 1e a4 f1 2a .$pH.....!!e...* +| 1136: fa 3e 9a a1 b8 67 92 ab cd 27 a2 0f f3 b1 77 9b .>...g...'....w. +| 1152: 97 e6 c0 f2 c4 db a4 5c d8 f1 01 bd 3c f4 fd 0f ............<... +| 1168: 00 ca 6d 75 11 fd 56 e2 12 96 e1 f7 18 99 f3 78 ..mu..V........x +| 1184: 9b b7 ef 8c 50 1f dc c5 fa f5 92 fb 91 70 cc d3 ....P........p.. +| 1200: ca 0a c6 9b 42 d4 2f cc ce 98 48 a6 ab cd d1 d8 ....B./...H..... +| 1216: 4c 8a 98 2f 2d dc 48 d9 84 52 8f 86 57 04 b8 14 L../-.H..R..W... +| 1232: ec 33 d4 5e 69 da c5 45 8e 2c 4f d6 fc a3 ee f1 .3.^i..E.,O..... +| 1248: 7c 47 c1 5c 8d 48 1b c2 be f1 ca b8 f4 91 d8 bd |G...H.......... +| 1264: d6 41 f6 40 ee 41 90 51 14 09 a8 c5 cf 38 16 21 .A.@.A.Q.....8.! +| 1280: b5 84 48 6c 8e eb e7 b0 62 69 dd 04 c4 4a 75 f6 ..Hl....bi...Ju. +| 1296: 6f 92 af 16 33 94 81 eb 26 00 9f dd d4 01 73 32 o...3...&.....s2 +| 1312: ec ee 9d 23 bf 82 8f e0 c8 eb 8f aa 4f 69 4b 68 ...#........OiKh +| 1328: a8 4e 3d 6e d8 2c 99 49 fb 96 d1 77 f6 a0 c3 97 .N=n.,.I...w.... +| 1344: 7e c6 7d ff 90 fc 12 97 e7 71 7b 20 ea dd ba 47 ~........q. ...G +| 1360: 3f bc bf 89 2e 26 82 91 71 97 6c 5e 2e ba 8d b9 ?....&..q.l^.... +| 1376: c6 ff 51 b3 17 7d e6 ca 99 75 53 73 99 9f 5a b8 ..Q......uSs..Z. +| 1392: 11 db 5a fb b3 73 55 bd 61 0c 2c e5 03 b2 d5 1c ..Z..sU.a.,..... +| 1408: ce ea ef b4 2e 2c dd d4 55 df cc 7b 13 c9 ba 87 .....,..U....... +| 1424: 25 38 2d d7 fc 7b 47 c1 da 56 42 fe 50 8e 12 a8 %8-...G..VB.P... +| 1440: fd 9d 90 e3 ae e1 f2 5d fa 1e c5 e2 d9 74 b9 55 .......].....t.U +| 1456: d0 5b 90 1d 31 f0 fb 36 d6 e5 df 91 72 4b 69 41 .[..1..6....rKiA +| 1472: 06 33 4e 77 cc 1c a7 d1 3e 49 7f 76 8d d3 73 88 .3Nw....>I.v..s. +| 1488: 27 53 13 95 87 9a 99 cc b2 54 46 34 7f 96 98 1e 'S.......TF4.... +| 1504: fb 60 06 85 93 37 0a 90 30 4b 8a ca 7f e8 a6 6a .`...7..0K.....j +| 1520: a1 94 da ac 1f 47 80 1a 81 8f dd 19 9d 70 88 73 .....G.......p.s +| 1536: b7 4c 22 f7 fe 8b c1 05 5a ea e0 20 5d 01 55 c4 .L......Z.. ].U. +| 1552: 7c 3d 99 f7 8a 1a 15 23 1a a6 5a 02 52 35 2f 14 |=.....#..Z.R5/. +| 1568: ea 01 d4 2b 97 3f 7c e3 4f 7b 58 ea 83 2f 1f 03 ...+.?|.O.X../.. +| 1584: c3 64 4f e8 81 f2 ec 19 92 5f c7 b1 27 5a 74 12 .dO......_..'Zt. +| 1600: fd f3 d1 48 0c eb fd 57 9d 28 1e 44 28 3d 69 aa ...H...W.(.D(=i. +| 1616: 94 66 23 55 11 3b 01 52 76 0b 3b e0 fe 67 e0 fe .f#U.;.Rv.;..g.. +| 1632: f1 e5 18 2b 19 ec 22 a0 33 94 e2 33 3f 59 fc 04 ...+....3..3?Y.. +| 1648: 15 c3 ee a0 f8 18 7d 07 39 04 04 e5 df ee 0c 38 ........9......8 +| 1664: 31 94 76 b0 ec 42 89 fe 3a d9 d2 1a 4d ee b0 67 1.v..B..:...M..g +| 1680: 10 f4 e6 d1 dd 96 9c d8 ec 10 f0 4b 8c 02 37 54 ...........K..7T +| 1696: 5b b8 e6 d6 d2 d3 ad cd cc 84 02 bd 1f bf 36 de [.............6. +| 1712: fe d3 8f 68 94 5d c7 19 18 d0 91 e5 be c2 e6 40 ...h.].........@ +| 1728: b6 21 ec 29 08 49 08 96 97 a1 14 3a 32 cb 0b 24 .!.).I.....:2..$ +| 1744: d4 8c 77 f9 f8 4b 78 8c cf de 90 27 b8 a1 5b dd ..w..Kx....'..[. +| 1760: 84 c5 88 21 39 fe 1e 88 c0 1c 9b d7 46 44 4f a7 ...!9.......FDO. +| 1776: 13 49 c0 49 d6 c4 c2 06 ab 24 9b 8d 80 83 32 32 .I.I.....$....22 +| 1792: b0 ff e5 47 db 08 c0 17 c5 98 67 12 fa 2f 86 39 ...G......g../.9 +| 1808: a5 f6 13 03 4d 25 af dc 2c 52 e2 5e f0 36 b3 a2 ....M%..,R.^.6.. +| 1824: 13 5b d4 47 14 5e 26 66 f5 a8 48 b3 1a 9e c5 03 .[.G.^&f..H..... +| 1840: 39 66 f3 34 fc c2 ff a8 0f a5 66 a7 7a 6b ad dc 9f.4......f.zk.. +| 1856: 77 e2 c7 fb 10 c6 fe a6 2a 64 fc e3 84 8c 12 0e w.......*d...... +| 1872: 5d 96 80 c8 d2 0d d9 e4 f1 bd 96 51 a0 ad a2 cb ]..........Q.... +| 1888: 99 d9 13 4b de c3 f6 ab 20 d8 ad 61 23 fd be 39 ...K.... ..a#..9 +| 1904: 97 0b 90 22 49 08 e2 38 3e 43 e6 91 9a ef e5 d7 ....I..8>C...... +| 1920: 9e d3 ff 82 51 99 6d 12 f5 74 0d 84 6e f8 ed 63 ....Q.m..t..n..c +| 1936: b8 ff 4d dd ea ae a3 0c 55 13 a1 03 4c 8b 3f 0e ..M.....U...L.?. +| 1952: 54 b7 02 3e 01 5a 77 ad 6f fc 92 bd b3 6f 3e 89 T..>.Zw.o....o>. +| 1968: 8e fe e1 2e 1c d4 56 c8 5f 4c 57 73 99 95 d6 be ......V._LWs.... +| 1984: f8 f5 17 22 5f 3f 13 5d 98 c9 b5 74 b2 17 7c c8 ...._?.]...t..|. +| 2000: dd d0 a8 d1 fa da 22 5a e8 34 f9 83 93 1b 7f e7 .......Z.4...... +| 2016: ba 48 ab e4 cd 3d 54 ec a2 9b 4b ca cf 84 0a d3 .H...=T...K..... +| 2032: 4d 8c bc 0d 73 1a 29 05 0c 60 0a 4a 6e 54 d3 0f M...s.)..`.JnT.. +| 2048: 84 00 df c0 f3 0b 73 2b 3b f0 60 68 91 ae cd 60 ......s+;.`h...` +| 2064: 59 0b ee b7 dc 7c eb b1 cc 70 f3 bb 6a 27 3b bb Y....|...p..j';. +| 2080: 20 41 3c 84 9d 3d 06 94 0c 53 eb 9c 31 e3 8a a0 A<..=...S..1... +| 2096: 1e c1 65 ef a6 78 92 ae 2e f9 64 49 58 b7 c0 23 ..e..x....dIX..# +| 2112: 2b 4c ab 93 2c 78 c2 86 32 09 d0 8e bf 34 b4 9e +L..,x..2....4.. +| 2128: 59 5c 6f 69 bb 85 5d a6 02 b2 01 85 89 23 40 7f Y.oi..]......#@. +| 2144: 23 3f c7 67 da 35 cf 2e d0 36 1e 71 fa 78 da c5 #?.g.5...6.q.x.. +| 2160: 41 db 9a 14 b1 48 d4 02 36 2f ed 6a 85 4a f4 f6 A....H..6/.j.J.. +| 2176: f4 3f 46 81 2d fa 92 47 21 16 14 84 f8 c9 18 86 .?F.-..G!....... +| 2192: 74 45 16 8d b3 cd 58 93 40 62 9b 24 6b af d3 44 tE....X.@b.$k..D +| 2208: 67 f9 0a 7f e5 25 01 b9 76 3f 8e 4d 82 b8 04 ae g....%..v?.M.... +| 2224: ef ed 12 c3 9f c6 a9 54 03 8a 78 0d f4 7e bf 7d .......T..x..~.. +| 2240: c1 f5 be 24 33 54 77 e3 7f c4 c9 fd 5c 79 6d 54 ...$3Tw......ymT +| 2256: 67 2f 83 a3 04 8b 16 09 71 ff 47 67 14 6a 84 71 g/......q.Gg.j.q +| 2272: 39 54 18 0b 7d 7e ac ef 62 0f 4b dd 9c d0 0d 20 9T...~..b.K.... +| 2288: 9e 62 98 79 11 53 de 73 a9 9d 44 5e f5 5a 23 62 .b.y.S.s..D^.Z#b +| 2304: 08 01 fb de 39 57 24 ac c6 b5 5f e2 6b 07 18 2e ....9W$..._.k... +| 2320: 27 24 42 96 d8 31 68 d3 0e bb 65 9c 01 f8 93 ba '$B..1h...e..... +| 2336: 4a 2a 60 3a b3 c2 9c 17 66 1d 34 4c 0f 90 38 5e J*`:....f.4L..8^ +| 2352: a4 9c 72 9c 16 d7 c4 98 33 ed c1 95 a2 d7 cc a1 ..r.....3....... +| 2368: 3b 36 28 1c 44 1c 8f c9 f7 bc eb ed 5a d3 2d 69 ;6(.D.......Z.-i +| 2384: 6f 86 9a 2f 15 91 aa 63 0d 19 f9 bb 07 6b 87 f5 o../...c.....k.. +| 2400: 0a 48 7e db b6 9a c3 97 01 d1 91 44 37 7c ce 5c .H~........D7|.. +| 2416: 63 43 c0 92 20 31 b4 5c 36 98 01 50 05 ec 1d ac cC.. 1..6..P.... +| 2432: 73 10 66 97 48 60 c2 2b 46 3e 2b fc 52 1d 42 87 s.f.H`.+F>+.R.B. +| 2448: d3 af 2c 42 7f c4 a2 0b 2b d6 09 64 0a f9 9d f5 ..,B....+..d.... +| 2464: 54 b8 1d 96 23 1f 22 0e 56 de dc 0a f8 97 55 41 T...#...V.....UA +| 2480: 56 ff 72 20 73 2b 50 fe 0d 79 c2 d3 61 73 22 da V.r s+P..y..as.. +| 2496: 68 8e 77 23 19 ca f9 7c a6 9c 91 27 ed eb f8 9c h.w#...|...'.... +| 2512: b6 bb 72 dc 3a 62 23 17 fb 04 98 bb ce 1a fb 1f ..r.:b#......... +| 2528: 17 ab aa 46 46 5a 8b 98 c6 7e 0e 18 74 8a 62 9e ...FFZ...~..t.b. +| 2544: 1f 7f f4 dd 8c 1d 68 8d 54 3d 0c 06 95 f9 9f 06 ......h.T=...... +| 2560: 3f 97 71 00 21 d5 e9 c0 44 7b 98 27 16 ba d2 fa ?.q.!...D..'.... +| 2576: c2 33 1d 4a 34 ec ae 4d a7 6c 68 77 9b f6 7b 2c .3.J4..M.lhw..., +| 2592: 6e d3 f6 1b b6 35 6f 47 69 be 5f 96 66 13 c3 8e n....5oGi._.f... +| 2608: 2f f1 4a 5e cb 26 00 73 33 a9 05 12 7b 6d 5b 96 /.J^.&.s3....m[. +| 2624: b7 3d 7e bc 62 aa c7 fe d2 fb 11 d5 7c a3 bf 90 .=~.b.......|... +| 2640: 09 fa ba 2b 2e 8d 65 c6 3f 21 41 fa 3d 71 f8 8d ...+..e.?!A.=q.. +| 2656: e5 77 34 b2 ea 4d 28 d1 48 fc 4c 1f 2e db 9c 58 .w4..M(.H.L....X +| 2672: b1 04 54 ce 4b 68 a3 7b b6 28 16 19 22 d7 fe d3 ..T.Kh...(...... +| 2688: f9 88 dd ca f6 26 43 88 28 bb 0f 62 3b d5 d4 cf .....&C.(..b;... +| 2704: af 90 27 ca 8e 40 62 f6 50 a8 2b 23 d4 3e 6d 32 ..'..@b.P.+#.>m2 +| 2720: e0 62 79 28 29 ab fe 77 21 ad 98 62 11 8a d2 90 .by()..w!..b.... +| 2736: 9a 83 73 c5 44 45 cb dd 71 7e 45 b7 79 d8 ab 3f ..s.DE..q~E.y..? +| 2752: ea 53 89 0b 8c 18 b3 95 1b 45 d5 dd 45 5b 79 b0 .S.......E..E[y. +| 2768: e8 c2 a9 58 77 cf c2 5b 43 a2 81 0e 43 9c c2 6d ...Xw..[C...C..m +| 2784: 5b a3 7a ef ed e1 24 56 54 a4 e1 ef 07 e7 9b 0e [.z...$VT....... +| 2800: 1f 32 78 3d a4 0f c2 52 ee 7d 4e 4d 80 4d ff 00 .2x=...R..NM.M.. +| 2816: 6c 8f da b7 ff aa fd a6 05 c1 31 e1 03 5c a4 e1 l.........1..... +| 2832: 92 39 d9 be 7a 16 9a c2 4b c7 01 eb ef 54 f8 2a .9..z...K....T.* +| 2848: 60 82 bb f6 4c 86 d5 8b 23 ce 88 52 a5 35 a7 ba `...L...#..R.5.. +| 2864: b6 31 e5 ec fe 30 f9 06 98 e7 bd eb 83 08 33 e5 .1...0........3. +| 2880: c5 a2 12 68 ca cb 97 77 db 60 94 4a 43 fb c1 04 ...h...w.`.JC... +| 2896: f1 8d 16 af 2e 8d bf 61 2a ff b5 4b 80 65 8a 07 .......a*..K.e.. +| 2912: 91 17 7f b7 ee c0 57 ce 40 82 c7 c8 57 89 62 61 ......W.@...W.ba +| 2928: b8 63 fe eb c2 97 30 ed 71 84 23 b3 48 e9 d9 ba .c....0.q.#.H... +| 2944: 71 c3 66 f5 6a 66 5a ee e6 bf 72 fe 80 a1 40 24 q.f.jfZ...r...@$ +| 2960: 75 0e 01 f9 1d 18 c3 fd 73 1c 21 92 5c 2b 07 a0 u.......s.!..+.. +| 2976: 83 80 5c 0f 20 fe e9 55 d6 f4 4c 96 d8 ab 7a 5c .... ..U..L...z. +| 2992: d6 d1 1e 44 60 ee 0a 30 7e 92 cf af a3 41 20 d5 ...D`..0~....A . +| 3008: 0b de fb 9c f4 81 76 5c d6 58 5d 86 1e 14 10 c5 ......v..X]..... +| 3024: 12 b5 f1 2d 73 09 0b ad 33 2c 49 5f 59 a7 08 80 ...-s...3,I_Y... +| 3040: 30 bf 50 61 b4 0b b2 3c 53 f3 de 2f e0 87 59 58 0.Pa...MC....u. +| 3200: 36 53 ad 19 cc d1 3c 57 45 64 fc e4 19 89 42 81 6S........... +| 3424: 65 16 ad c6 c4 a3 75 78 94 b2 ce b5 9c 50 df 02 e.....ux.....P.. +| 3440: f3 c2 91 11 5a 81 de eb 31 24 16 50 16 d8 5f c5 ....Z...1$.P.._. +| 3456: df 62 1b 24 37 bb cd 5d a8 a7 93 d0 6a 5e d8 55 .b.$7..]....j^.U +| 3472: 74 cf 69 a2 4a 05 07 2e 61 09 0e fa 1e 12 ab 62 t.i.J...a......b +| 3488: d9 ca ff ab 2b b0 3b 16 16 b0 95 15 27 26 89 46 ....+.;.....'&.F +| 3504: d3 67 fb 0f 33 00 18 44 c3 a1 92 de 6e 6d b0 67 .g..3..D....nm.g +| 3520: b0 65 97 fa b9 10 1d 39 15 10 95 f1 b3 cc a3 d4 .e.....9........ +| 3536: 28 06 9f b7 00 be a1 06 8f 61 34 66 92 fa 58 0b (........a4f..X. +| 3552: 4f 91 6d 93 31 32 cb a1 97 a5 18 b2 f1 bc fc 70 O.m.12.........p +| 3568: c0 86 62 24 4b 82 ce b3 71 30 53 ac 42 c7 32 2d ..b$K...q0S.B.2- +| 3584: 3a 0e 3f 8a 91 89 0b 46 55 f3 5d 73 b1 74 a2 9e :.?....FU.]s.t.. +| 3600: 97 6e 85 b6 f0 3e 56 6d 31 50 96 87 3d 18 cf 5f .n...>Vm1P..=.._ +| 3616: 74 57 1a 65 b4 d2 4e 96 1c 6e 3f 5b a9 a0 47 2d tW.e..N..n?[..G- +| 3632: 22 5b 16 ab 27 f4 86 36 f7 8e a1 f2 9c 69 7f 0c .[..'..6.....i.. +| 3648: 85 4d 3c 08 3b fa 96 ee 92 af f9 8d c2 ca a9 f1 .M<.;........... +| 3664: 2b fa 90 4b a3 fb 35 c4 c6 05 c3 0f ad 5e bb f6 +..K..5......^.. +| 3680: 55 66 73 bd db 98 2f 70 7d 25 3f 05 3d 24 c6 0d Ufs.../p.%?.=$.. +| 3696: 8a e2 e1 3e 19 59 51 25 a9 e8 4a 0a 68 69 05 c9 ...>.YQ%..J.hi.. +| 3712: ce 1e 5f 96 c4 a1 98 2a 7b b9 e9 99 ce 84 af 0b .._....*........ +| 3728: 82 61 5b 97 b1 8c 2e ea 81 98 44 39 79 e0 d5 9c .a[.......D9y... +| 3744: 6a b9 14 0f 36 98 ea c8 cd 6b 66 fa d1 23 cf cb j...6....kf..#.. +| 3760: c6 1b 25 6c 42 13 b2 67 f9 68 0a 62 ff 37 ab ca ..%lB..g.h.b.7.. +| 3776: 22 60 77 c1 30 e4 59 5f 17 b4 47 c3 ea c5 c4 2b .`w.0.Y_..G....+ +| 3792: 62 44 94 08 87 03 1a a5 10 ae b1 53 cd 2d 40 5a bD.........S.-@Z +| 3808: 80 ef dd 56 6b 88 92 d7 3c d0 7a a0 2f 63 22 02 ...Vk...<.z./c.. +| 3824: 61 f1 ab 36 f9 16 f0 14 ba 67 7d 9c 7f 90 cc d9 a..6.....g...... +| 3840: b1 ac d3 1d 73 1e e8 bf 88 6a 6b 15 32 b3 b5 9c ....s....jk.2... +| 3856: 4a ad bd 4f f8 f8 b8 b8 2e 48 0f e2 e9 d6 d9 72 J..O.....H.....r +| 3872: 22 9a 28 8d 24 e2 f0 33 23 27 ff 37 5e 36 55 37 ..(.$..3#'.7^6U7 +| 3888: e1 92 78 19 a4 9d ff b9 22 e5 47 79 b8 de b4 9f ..x.......Gy.... +| 3904: 4e f0 65 1d 89 5e 55 86 bf 25 ff 6f 7e 27 f2 c1 N.e..^U..%.o~'.. +| 3920: 9d 26 69 ab 5b 1a 16 1d a1 b7 0a f0 60 06 12 4b .&i.[.......`..K +| 3936: ce 6b 17 d3 e7 66 ef 6d 83 10 30 96 49 8b 8d 52 .k...f.m..0.I..R +| 3952: 98 65 b1 48 a8 0d 96 ae 15 cd 00 5b f5 3e 4c f6 .e.H.......[.>L. +| 3968: 16 b3 15 5b cf 3e ba 15 86 7e 9b 92 be cb 8a de ...[.>...~...... +| 3984: d0 de c8 0e ac cb 79 cc 8f ad f6 3a 40 33 9f 91 ......y....:@3.. +| 4000: 48 7c 3f d1 33 c1 d3 51 ba 44 47 0e 41 8d fa 2d H|?.3..Q.DG.A..- +| 4016: 55 99 7f 4d 44 4a 85 dc 6b b1 9f 8a 4c 55 70 b7 U..MDJ..k...LUp. +| 4032: 2d 14 4d 72 ea 76 c8 0c 9f 9c 99 0a c2 7e 95 94 -.Mr.v.......~.. +| 4048: 61 84 76 8c 20 1f a3 2d 5f 54 ab 5b 8c fc 04 0d a.v. ..-_T.[.... +| 4064: 25 33 9f d1 f3 f7 38 1d 64 4c c1 0d 38 0e 4d b8 %3....8.dL..8.M. +| 4080: 61 16 f8 ea 3d 01 e9 f7 7b 6d 10 ed 08 07 86 f3 a...=....m...... +| page 12 offset 45056 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 04 06 00 ae 7c 00 00 72 d0 44 9f 95 5b ad 14 >....|..r.D..[.. +| 1104: 76 bf ee ef 64 b3 05 0c b8 3c fa 37 1b ab 5a f0 v...d....<.7..Z. +| 1120: ae 4c 14 79 98 2a 95 49 a0 1e 3e ff 44 91 47 ad .L.y.*.I..>.D.G. +| 1136: 40 a0 7b 5e 0c 5f 86 45 28 c7 d9 03 81 2b 26 13 @..^._.E(....+&. +| 1152: 0c 38 01 db 73 e2 42 53 3c 8c d1 6c fd 51 a6 5a .8..s.BS<..l.Q.Z +| 1168: fa 65 81 fc 71 89 ea 74 84 df da ea e0 d8 40 ce .e..q..t......@. +| 1184: 56 a4 cb 65 0b 6b 79 7c 81 a9 55 79 8c 3e 02 dd V..e.ky|..Uy.>.. +| 1200: 17 7f 47 b6 84 c0 c2 a3 5e 79 66 83 e4 f2 73 6a ..G.....^yf...sj +| 1216: 46 9f 20 24 f3 4e 2b 6e a9 2c 9b 51 39 96 42 82 F. $.N+n.,.Q9.B. +| 1232: ec c0 c9 78 ec ec 90 e3 57 af fb 1f 51 41 a3 37 ...x....W...QA.7 +| 1248: b7 68 5d 84 5e 6a 9e 17 13 e1 87 7d 93 f5 92 8e .h].^j.......... +| 1264: 7a 8d 3b d4 40 35 df 0e 8f 80 13 f3 54 9c 57 a8 z.;.@5......T.W. +| 1280: 86 88 69 32 7d a2 22 13 ab 14 db e6 8a 1c 84 ed ..i2............ +| 1296: 54 4e d3 0a ba 5f 9e 66 88 2a 64 d8 04 86 3c 5e TN..._.f.*d...<^ +| 1312: 41 b7 2d c2 c1 a0 73 61 d8 80 3b 9d 9d 1e 11 ab A.-...sa..;..... +| 1328: ae 20 63 ac 31 80 2e 66 d5 a7 c8 8e 0a 55 78 53 . c.1..f.....UxS +| 1344: 66 8d 5d 1e cb 04 13 d9 a8 d1 09 4b 31 3c e9 ba f.]........K1<.. +| 1360: 83 69 76 4c 12 e8 95 84 e1 fe c0 64 74 c5 74 8d .ivL.......dt.t. +| 1376: 1f 75 28 75 1f 51 23 b7 df ef 45 66 61 1b 18 2e .u(u.Q#...Efa... +| 1392: c8 56 69 8f fa 2a 08 41 21 2d 6a c7 79 46 8f 95 .Vi..*.A!-j.yF.. +| 1408: dc b1 b2 d7 39 b9 a6 41 af 06 3e 53 85 c2 db c3 ....9..A..>S.... +| 1424: 42 fb b7 4f e9 e0 11 c1 a6 9a 48 fb d6 c9 26 31 B..O......H...&1 +| 1440: 3b c4 7a e2 bd 53 d7 3e d7 56 60 8f 6a 80 60 19 ;.z..S.>.V`.j.`. +| 1456: db fd 5a 7a 8c c7 43 c7 29 57 8b 6f 31 ea 13 af ..Zz..C.)W.o1... +| 1472: de 2e 32 75 5f 63 b7 bb 45 07 9e da 69 54 a0 fd ..2u_c..E...iT.. +| 1488: 0e dc 0d f5 42 17 eb 64 60 0d d8 56 4b 61 6d 8c ....B..d`..VKam. +| 1504: 82 09 f1 80 a2 25 5d 8b e5 67 76 13 9d 46 4a 6f .....%]..gv..FJo +| 1520: 30 f6 eb e0 7b 0e 01 e1 a8 b2 38 1b 9b 21 48 ed 0.........8..!H. +| 1536: c8 7f 7d 6b 0c 25 ce a8 0a 78 a8 48 45 2d b5 e7 ...k.%...x.HE-.. +| 1552: fd ca b2 a1 2b 2a b9 72 61 76 76 0c ae 9c b3 17 ....+*.ravv..... +| 1568: fa 83 07 24 24 62 18 18 d6 12 2d ad 58 0b b3 5e ...$$b....-.X..^ +| 1584: 80 da 40 b5 d0 e7 aa 28 52 bc 7f 68 7a 1d 5a 1f ..@....(R..hz.Z. +| 1600: 79 65 10 9e 4d fe be 32 c7 0f aa 7d a5 0a 43 0f ye..M..2......C. +| 1616: 3d 17 0e 88 a3 7f 3f ff 04 bc 79 56 62 0e 9b a0 =.....?...yVb... +| 1632: 51 1f 49 04 ff 40 7a c2 51 e8 36 44 85 63 df a4 Q.I..@z.Q.6D.c.. +| 1648: d5 63 a4 eb a3 71 44 b4 02 51 a1 2b 00 73 69 4e .c...qD..Q.+.siN +| 1664: 07 76 42 84 bc 87 1a e6 15 13 f7 ac 7a e8 bf c0 .vB.........z... +| 1680: a4 8a 64 fe 9c 17 a7 20 cb 67 cd e6 da f1 95 26 ..d.... .g.....& +| 1696: b6 79 d4 93 e2 52 fd 6f 44 29 db 1e 34 02 46 a4 .y...R.oD)..4.F. +| 1712: f1 c5 f0 b5 c3 19 1d 0e 14 d4 ba 08 1c 01 bf 39 ...............9 +| 1728: 54 02 e2 f5 d3 cf fb d1 d6 76 65 91 ba 7b c8 22 T........ve..... +| 1744: 85 f8 fa c7 ae ae 21 85 b9 d2 09 5f f4 3b 27 c5 ......!...._.;'. +| 1760: 13 39 39 7e 79 ae 07 37 26 72 07 22 01 40 2c 10 .99~y..7&r...@,. +| 1776: 5b 2c 9b bf 30 92 2e f5 12 9f 19 f9 2d 69 2a c6 [,..0.......-i*. +| 1792: 9a 26 1b de 18 b8 05 65 a8 b4 b3 3e 2c 04 5b bc .&.....e...>,.[. +| 1808: 63 79 56 7a ba 24 ae 69 ee d6 d5 7c 3d 5f 2f 7b cyVz.$.i...|=_/. +| 1824: e9 e5 55 f2 88 58 72 49 89 3e d5 91 be ae ea 9e ..U..XrI.>...... +| 1840: 58 20 45 2c da c7 b8 f9 db 97 68 59 c7 f4 d3 4b X E,......hY...K +| 1856: 95 b6 c7 73 1c 9b 96 d2 1c 56 17 eb 18 a1 fa 17 ...s.....V...... +| 1872: 05 72 44 05 4f c3 2b 8c 89 d2 3c a2 25 d8 16 74 .rD.O.+...<.%..t +| 1888: 48 b7 78 36 7a 86 88 ff cc 47 ac bd 73 28 1f 3f H.x6z....G..s(.? +| 1904: 13 3d b9 d2 df 9a 93 43 a6 9f b9 fc 7c b2 d4 84 .=.....C....|... +| 1920: c4 7e d0 14 60 01 63 fb 09 de a3 2f 1c ae 2e 6b .~..`.c..../...k +| 1936: 9f 8d c8 f1 2d f0 c9 a5 1f 48 c1 7c 53 c5 63 8a ....-....H.|S.c. +| 1952: 9b 0b fd f3 7f 3a 63 eb 2c 4f df 3a 57 8c 20 3e .....:c.,O.:W. > +| 1968: 0b d1 00 0f ce e3 ab a8 77 31 63 4a aa 35 5f 3f ........w1cJ.5_? +| 1984: 77 ba d3 38 7b a3 53 94 e6 9d 34 ec 5a 28 09 6e w..8..S...4.Z(.n +| 2000: 5a 85 bc 72 18 bb 15 8a 20 01 f3 88 19 74 65 1b Z..r.... ....te. +| 2016: 51 bc 8a 4d 32 a6 00 80 fc 06 f8 aa 1c ee 0e 84 Q..M2........... +| 2032: b5 70 fe 09 02 78 d4 ae 3a bb 02 ed 5a 90 d2 a9 .p...x..:...Z... +| 2048: 3e 58 8a 08 2d 4c 79 7c f2 94 89 ac 04 59 d8 17 >X..-Ly|.....Y.. +| 2064: 09 e4 c3 e0 78 6b 77 58 64 4f 2f 77 15 16 c2 41 ....xkwXdO/w...A +| 2080: 96 61 8c 23 59 73 b9 56 c3 4e da 79 34 94 e5 31 .a.#Ys.V.N.y4..1 +| 2096: 9b a6 92 ca 23 64 e4 78 32 e2 9d 2e ad 18 11 b7 ....#d.x2....... +| 2112: 12 0d 1b 40 44 4f 92 cf 78 80 86 fb af c9 30 20 ...@DO..x.....0 +| 2128: 52 9b 22 8f f4 ca 65 e5 f2 e7 a3 c3 e7 00 ee a1 R.....e......... +| 2144: ca ab 3d 71 40 bd cf c3 2f 7b 59 e7 9d 18 6a 65 ..=q@.../.Y...je +| 2160: a1 c9 84 cf 0f f5 36 d5 22 06 d4 6a b6 ad 38 03 ......6....j..8. +| 2176: b4 ac 9e c4 7a b0 71 24 a9 01 f0 5c 5b e3 c9 bd ....z.q$....[... +| 2192: d6 e7 ba 71 fd 44 22 14 af f7 18 ef 29 1d f2 0f ...q.D......)... +| 2208: 5d e1 3a 71 3b 47 44 da 21 f6 73 8b 6a a1 ff 93 ].:q;GD.!.s.j... +| 2224: d5 eb 5d d0 61 23 41 53 99 7f 23 0f c0 a3 18 3c ..].a#AS..#....< +| 2240: bd 30 e0 bf 16 41 27 71 80 67 49 61 e9 64 71 e0 .0...A'q.gIa.dq. +| 2256: 9d 42 e6 48 9b 0e 40 82 f9 e2 a5 b4 51 37 7d 6b .B.H..@.....Q7.k +| 2272: 74 54 03 bb a2 93 ac 47 f8 a5 75 86 73 0f 12 47 tT.....G..u.s..G +| 2288: 27 65 16 16 a2 ab 99 08 14 0b 7e 1b 2c 7e 9d 20 'e........~.,~. +| 2304: 58 23 14 70 bd f5 7c 2b 74 ce d1 46 b3 29 03 8f X#.p..|+t..F.).. +| 2320: 76 f3 20 ca 9e a3 fc 1a 17 f2 8e 78 97 4e 0b 5e v. ........x.N.^ +| 2336: 61 2b e1 3d f1 14 ae 9a 1f e8 a8 3c 99 02 a5 c0 a+.=.......<.... +| 2352: a6 ee bd 47 a4 3c 6d db 6c 20 dd c4 60 09 17 e7 ...G.h$^0....z..... +| 2464: 89 df 3b c6 e1 09 1d 16 69 40 e9 d7 f7 41 d3 2b ..;.....i@...A.+ +| 2480: 52 6a 9a d9 65 e2 44 de 45 63 be 85 41 b8 d4 4a Rj..e.D.Ec..A..J +| 2496: 49 b9 b8 bb 4d a6 6e 60 46 13 6f 7f b1 57 9c 5b I...M.n`F.o..W.[ +| 2512: 78 69 87 3d 42 e5 c2 0a 46 9c 38 9d e2 44 80 fe xi.=B...F.8..D.. +| 2528: c8 81 f4 fe bb ef 61 46 88 c7 6f 1a f8 00 0b 63 ......aF..o....c +| 2544: d0 ac 8e 5e 88 8d 2d 56 15 79 d2 12 d9 94 ba 2b ...^..-V.y.....+ +| 2560: ae 71 63 be 62 08 75 b9 97 fc 1e 17 89 83 3b 30 .qc.b.u.......;0 +| 2576: 07 6e f0 db 44 17 c0 49 7f ed 0a 0a 43 13 6a 72 .n..D..I....C.jr +| 2592: 78 9c 96 52 2c 02 ac da a0 90 b5 66 34 b7 a2 17 x..R,......f4... +| 2608: e7 71 11 8d 8f 22 0e 40 90 9b 0f 99 f5 10 c0 64 .q.....@.......d +| 2624: ac e3 24 0d 49 1a f4 2f 0b 13 fc 94 a7 18 b1 5f ..$.I../......._ +| 2640: a6 64 49 0b c2 35 99 b6 05 81 bd a8 f8 88 56 83 .dI..5........V. +| 2656: 17 56 cc f8 91 db 5b 18 7c 42 46 3f 3d 9a 2d b6 .V....[.|BF?=.-. +| 2672: 63 8c 62 bf 78 53 a0 23 53 40 c0 32 06 f1 c8 3a c.b.xS.#S@.2...: +| 2688: f8 17 34 a7 29 1e ab 92 2d 2d 68 c5 83 e0 0a 2e ..4.)...--h..... +| 2704: 7b f8 9c b7 32 86 c0 1e f5 29 44 a6 24 e1 d7 66 ....2....)D.$..f +| 2720: 0d 48 2b 0e 49 f8 e4 52 4a 7d bd 1c c1 44 27 f0 .H+.I..RJ....D'. +| 2736: c9 db 87 93 13 62 82 ef ad 2c ea 8f d1 3d a4 a6 .....b...,...=.. +| 2752: b8 80 87 e2 0e 27 27 b3 3d 56 66 39 de e8 21 5f .....''.=Vf9..!_ +| 2768: 95 25 d9 68 f1 57 50 0e 15 54 0b a6 44 27 e8 d9 .%.h.WP..T..D'.. +| 2784: f2 dc 5e 79 f0 ec 2b a3 39 77 8f 3d 53 70 8a d3 ..^y..+.9w.=Sp.. +| 2800: e5 aa 14 cb 7b dc 31 72 f6 90 5e 8b 3a 8c f3 77 ......1r..^.:..w +| 2816: 4d 00 a3 1d 3a 63 47 c0 2a a2 32 98 6e 5c bc 21 M...:cG.*.2.n..! +| 2832: f0 6a 34 6c 89 a5 bc 04 f6 3b 8c 96 b0 eb 0d 70 .j4l.....;.....p +| 2848: 9f 18 d5 64 ec 2e df 19 7d 1d a4 61 48 e7 0b eb ...d.......aH... +| 2864: f3 94 16 f5 7f 4c 9e 0c 78 aa 4b 61 14 13 eb e2 .....L..x.Ka.... +| 2880: 72 14 56 27 41 70 8f 7f cb e6 a1 c3 37 c4 78 32 r.V'Ap......7.x2 +| 2896: 85 ea e5 af 96 46 4b c3 93 af 8f 26 0c ea 08 b6 .....FK....&.... +| 2912: b4 a6 a4 78 6d 82 51 ab fb e5 a9 e9 89 c3 a1 be ...xm.Q......... +| 2928: 3e b9 e3 8c 61 cf 42 1d de 25 45 9e f0 ff 8b 75 >...a.B..%E....u +| 2944: 63 9b 3d d6 92 c3 ad ca c5 4c 79 9d 72 37 fd 3a c.=......Ly.r7.: +| 2960: 21 f2 8d 34 37 b9 eb a0 86 d8 1a f2 9d aa 6a 53 !..47.........jS +| 2976: f6 c9 29 d0 39 2c 66 24 c6 8e 68 9c 70 cc 9e cc ..).9,f$..h.p... +| 2992: 25 a8 dd 5e d8 e6 87 1b dc 3a 26 20 e3 1c 3b ba %..^.....:& ..;. +| 3008: f8 c4 80 83 79 49 f1 2a f0 e8 70 31 e7 b2 a5 e3 ....yI.*..p1.... +| 3024: 9f 5b 49 25 2c 33 b6 ea 38 ab a8 0a 9a a2 0e fd .[I%,3..8....... +| 3040: 3a a8 9c 0f ab 93 45 a7 54 d2 18 2f 2e 94 34 cc :.....E.T../..4. +| 3056: fa 14 b6 8e 01 8d af 80 94 e7 80 31 dc 2c b9 1e ...........1.,.. +| 3072: dd 35 91 29 28 a0 29 65 bc e3 a2 52 b0 e5 56 5a .5.)(.)e...R..VZ +| 3088: 12 21 06 fd f3 04 77 31 6d f2 e0 66 0a a3 77 f3 .!....w1m..f..w. +| 3104: 5a 17 37 59 65 d9 32 68 82 2c 72 05 16 e9 9a 89 Z.7Ye.2h.,r..... +| 3120: 10 94 53 e9 be 63 2e 82 27 28 30 1d 2a bf ad 00 ..S..c..'(0.*... +| 3136: 49 9a a7 e3 50 d0 61 b4 bd 73 3e 99 a3 a7 69 7f I...P.a..s>...i. +| 3152: a8 4c 4d 12 7f c0 55 b7 de 0c 93 5e 27 cf 4a 53 .LM...U....^'.JS +| 3168: 78 08 ee da 22 37 3f 94 2f dd a6 b2 28 84 0f 62 x....7?./...(..b +| 3184: bc 7f be 3c af ad 84 82 70 b2 98 d5 9a d5 07 34 ...<....p......4 +| 3200: 41 6f 9c 0a 17 54 cf fc 43 1e 7c ca 48 6f 23 c5 Ao...T..C.|.Ho#. +| 3216: 50 7b b6 ec e5 3a 27 0f 65 29 97 ad 37 b1 9f e6 P....:'.e)..7... +| 3232: 6b d1 d4 41 ed 5e 0e 20 22 00 ca 41 58 d2 b9 73 k..A.^. ...AX..s +| 3248: 08 2d 01 83 db d1 57 a2 e3 d2 34 c6 73 fd 2e b2 .-....W...4.s... +| 3264: 9b f2 a4 a4 f1 1d 9f 7a cc d8 93 37 b3 95 2c 99 .......z...7..,. +| 3280: f5 92 0c 91 d5 e2 2e b2 bb 49 76 19 c9 58 16 28 .........Iv..X.( +| 3296: 1e 76 60 ba d9 a8 04 e9 91 5e 7f e6 b1 dc de 9f .v`......^...... +| 3312: 6f e6 52 f2 61 53 7b 09 f4 df c4 96 15 23 99 67 o.R.aS.......#.g +| 3328: 6b b8 e8 04 05 62 0d 5f 05 d1 8a 6c 49 e8 2b 71 k....b._...lI.+q +| 3344: 10 44 8e 93 13 bc 27 14 32 52 22 2d 11 cf cd 43 .D....'.2R.-...C +| 3360: 77 a1 60 21 70 44 60 cb 64 9e b4 c6 23 aa f2 0d w.`!pD`.d...#... +| 3376: c0 6d 50 0b d9 73 d0 eb 6d d5 03 fc 74 e1 fb 49 .mP..s..m...t..I +| 3392: a1 18 4f bb a2 2f 09 54 28 80 8c 04 86 3d e1 fd ..O../.T(....=.. +| 3408: ff 55 c9 7a a0 35 ef 64 b8 87 1b 10 8c fc 3d 27 .U.z.5.d......=' +| 3424: e3 6c 30 3c 05 4d b3 37 b8 29 ba a2 46 9b 76 5d .l0<.M.7.)..F.v] +| 3440: ff 1b 30 65 70 57 f2 89 c3 17 3f 21 5d 26 5f 58 ..0epW....?!]&_X +| 3456: b1 50 6e 75 c3 dc be a9 0e 43 3e 34 4e 40 0e a1 .Pnu.....C>4N@.. +| 3472: f4 69 6d b1 56 a7 0f 09 7e 25 74 08 83 bb 9d 67 .im.V...~%t....g +| 3488: a6 f6 68 87 e9 d9 ba 16 97 42 ca 66 29 e6 df cd ..h......B.f)... +| 3504: e8 e4 ef ef c9 1f f7 d8 9c 8a a3 24 c9 43 60 4d ...........$.C`M +| 3520: 9a 73 cb 73 9f b2 b1 fe 51 9d 55 9f 37 a0 d2 3c .s.s....Q.U.7..< +| 3536: 7e be e4 35 1e 8e 81 f5 67 29 29 ac 95 0a 99 28 ~..5....g))....( +| 3552: ec 4c f2 c6 03 6b 01 76 f5 87 77 58 51 51 40 d4 .L...k.v..wXQQ@. +| 3568: 81 c7 d7 7b 40 09 8f f5 c8 a7 49 00 c9 ee 99 c9 ....@.....I..... +| 3584: 6f 4a c5 6f e7 51 8e 4c 95 a3 de 6d 53 45 7b 97 oJ.o.Q.L...mSE.. +| 3600: 7b 7f 8b b4 db f7 52 97 08 2e 4f 36 41 24 79 bd ......R...O6A$y. +| 3616: 54 bb b8 2c 32 6c 1f 54 e6 82 47 27 53 7f 57 ea T..,2l.T..G'S.W. +| 3632: 4c 10 71 9b b2 0d 9a 80 fb d2 a6 ec 8d 6e 60 64 L.q..........n`d +| 3648: 1a 73 43 13 b9 cb b1 bd da 84 82 8d 17 ae 75 4d .sC...........uM +| 3664: 1a af 28 9d 01 6d c7 f7 25 8c b5 ef 1b 5e 33 86 ..(..m..%....^3. +| 3680: e7 ce 59 c0 5b 3d e2 46 ab 85 3b b2 c0 9e 7a 75 ..Y.[=.F..;...zu +| 3696: 26 15 cd 6e 6d c2 19 fc cb 3e 5c 5e 95 94 ec 29 &..nm....>.^...) +| 3712: 94 8e d4 e6 0a 38 8c ef fc eb cb 3a 06 97 20 f3 .....8.....:.. . +| 3728: 5f 53 b5 5f 40 33 9e 41 9c db a8 f3 7a d8 aa 0e _S._@3.A....z... +| 3744: a3 37 00 87 ff 13 78 1a 98 c2 b2 aa bf 2d 10 24 .7....x......-.$ +| 3760: 8e e5 9f 9f 3c 4f e4 90 e0 6a 90 44 32 9f ca 6a ....9=J.4.... +| 3888: 9c 65 c7 29 02 bf a6 97 61 3c 6f 46 48 0a e5 a6 .e.)....a7~. +| 3952: 43 f8 b7 ae 2e af d4 9d 67 8d fa c0 24 b2 1c 55 C.......g...$..U +| 3968: ea aa 33 67 22 d4 f3 e7 87 a5 9d fe a3 04 55 ae ..3g..........U. +| 3984: 56 87 2d 21 a9 02 c2 4d a3 88 a3 3f 48 18 83 a7 V.-!...M...?H... +| 4000: 01 b9 6d 60 d4 0f 79 79 8a 4b 66 c3 49 c7 51 0b ..m`..yy.Kf.I.Q. +| 4016: 15 73 df f2 53 eb 79 03 da 55 2f 5d 9f 8f 01 a7 .s..S.y..U/].... +| 4032: c7 5d cf 61 f0 c2 78 52 56 b8 45 7d f2 d8 56 ca .].a..xRV.E...V. +| 4048: 80 84 6e ee 2a 00 ba 00 74 46 0f 76 23 76 79 55 ..n.*...tF.v#vyU +| 4064: 4f 97 57 f2 63 07 11 1f a3 46 07 14 3b 50 43 c0 O.W.c....F..;PC. +| 4080: 72 b2 89 5a 44 15 dc c8 9b 2e f8 ad 69 80 3e d7 r..ZD.......i.>. +| page 13 offset 49152 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 05 06 00 ae 7c 00 00 40 9e 86 c7 f4 d7 a0 78 >....|..@......x +| 1104: b9 63 d2 14 9c 0c 6a 77 07 99 fe 03 10 f7 fd 07 .c....jw........ +| 1120: ea be 72 a1 24 46 49 15 05 16 1f 1c 80 72 1b 98 ..r.$FI......r.. +| 1136: e5 f9 d0 65 50 cb 29 09 d4 2b 17 e2 73 7e 45 4f ...eP.)..+..s~EO +| 1152: 05 c8 b5 bb 1a 81 34 52 32 17 53 96 c8 0d 54 19 ......4R2.S...T. +| 1168: bd d8 c3 af 35 79 a8 75 ae 2d 5d c4 44 eb b2 1f ....5y.u.-].D... +| 1184: ab 9d f6 d8 42 7b 3b dd 10 f4 34 95 57 ff ec cc ....B.;...4.W... +| 1200: 63 3f d3 93 e5 16 6d b5 3d e3 23 7b 32 88 b7 ce c?....m.=.#.2... +| 1216: 7b 9a 8c 46 26 6f ed 2f a7 83 fb fd a7 b7 1b 48 ...F&o./.......H +| 1232: 88 40 20 a5 9b de 7b d9 32 e5 72 1c 85 8a a2 63 .@ .....2.r....c +| 1248: e2 8d 7b 25 a2 13 51 ec fd 0a ed 1b 76 8f 9d 62 ...%..Q.....v..b +| 1264: 2e 9e cf 0f df 58 30 83 e5 90 40 df d2 9f c9 65 .....X0...@....e +| 1280: 3f ae c6 4a 11 a3 16 5b 6f 24 74 95 5e 21 6a c8 ?..J...[o$t.^!j. +| 1296: da 7a dc 82 c2 ab a0 93 c8 dd a8 f1 1d 78 5e 9b .z...........x^. +| 1312: c8 17 c7 e0 4f 53 1d e1 5a 5f 5c bf 23 25 fb f3 ....OS..Z_..#%.. +| 1328: a0 d4 f5 f7 16 78 f8 00 2e 49 1f 7b ff 41 5c 11 .....x...I...A.. +| 1344: 83 9e c7 f4 f5 c7 85 d3 97 dd 67 5a 1e 47 d0 41 ..........gZ.G.A +| 1360: ae cb 4c 47 4f 5f fc 9f bf 0b 20 04 94 08 11 8b ..LGO_.... ..... +| 1376: a8 98 f2 5e 86 19 3b f1 0f 9d 49 c8 02 14 68 24 ...^..;...I...h$ +| 1392: 20 71 28 49 3c 60 5a 92 ea d3 2c cd 8a 8a d7 cb q(I<`Z...,..... +| 1408: fd 14 db 41 3c d0 24 89 46 6a c4 52 8d f0 c0 dd ...A<.$.Fj.R.... +| 1424: b4 a6 0f bd c2 5a fd d1 53 bb a9 82 88 64 4a 34 .....Z..S....dJ4 +| 1440: 0f 8f ed 49 49 81 03 29 c3 c3 01 00 6a 83 21 dd ...II..)....j.!. +| 1456: bc 98 29 fb 2b 5e 78 95 4a 57 4f a7 d3 ec 44 a8 ..).+^x.JWO...D. +| 1472: 97 6c 7f 39 11 6f aa a0 60 b1 71 43 a6 45 ea 9b .l.9.o..`.qC.E.. +| 1488: e1 6f 67 2b f9 e4 8a 55 60 5b 69 f3 54 bf 5c 78 .og+...U`[i.T..x +| 1504: 32 d4 25 22 bf 9b 7a a6 a8 a3 70 53 7d e8 50 1c 2.%...z...pS..P. +| 1520: cd 2a 4a 92 a7 f1 e4 84 39 8e 72 a4 cf 3a c8 b2 .*J.....9.r..:.. +| 1536: 3e f1 06 f2 83 80 0c 1c 10 91 7b 41 92 f5 df 0f >..........A.... +| 1552: 7a dc 04 78 3c b4 cd 96 b6 15 f6 7f 85 33 db 09 z..x<........3.. +| 1568: 92 de 83 ef fc 2c 75 43 49 0c 12 61 48 14 38 d3 .....,uCI..aH.8. +| 1584: b4 fb 43 33 74 3a 0a 5b c5 03 0a d8 f2 2c 1d cf ..C3t:.[.....,.. +| 1600: e9 a0 7f be 25 24 90 77 75 2f 15 a7 ff 62 83 ef ....%$.wu/...b.. +| 1616: 01 04 b1 7c c8 5d 03 5a 28 1d a3 24 e9 7f 32 73 ...|.].Z(..$..2s +| 1632: 19 59 8c 07 30 10 be 58 c2 48 f6 d1 c6 3c b3 09 .Y..0..X.H...<.. +| 1648: e0 bf 5c c3 47 81 c8 04 71 ab 55 cb a5 7c 7a a8 ....G...q.U..|z. +| 1664: 45 28 03 75 e9 82 51 e8 bd 41 d4 1e cc b1 fe 43 E(.u..Q..A.....C +| 1680: 4d 5c 66 25 6f fa da a6 fa ea d3 de 0a 2e 5a 3d M.f%o.........Z= +| 1696: 68 65 10 b6 ee 3c 0b f6 13 bf 3e 6e 57 f3 30 43 he...<....>nW.0C +| 1712: f6 f4 e0 34 e5 47 88 73 9d 9c 36 f0 90 98 0e 5a ...4.G.s..6....Z +| 1728: 3b ab 0c 7f 8b ab ea 9c 8c 5e 34 26 d7 53 bb cb ;........^4&.S.. +| 1744: 18 58 a7 6f bf 78 43 7b 47 a6 3b 09 f7 9f 5f 81 .X.o.xC.G.;..._. +| 1760: 87 c0 17 02 8f ad 78 32 8c dd 96 0e d8 b8 66 66 ......x2......ff +| 1776: 23 91 e3 e3 37 5c 63 f4 16 c1 58 4f b7 63 de 7c #...7.c...XO.c.| +| 1792: 36 c3 67 fc 32 a3 d8 84 20 fe e6 0f 2e 2d ee 11 6.g.2... ....-.. +| 1808: 3b 97 1a 05 47 ea a1 71 c4 11 08 b1 a9 52 b7 54 ;...G..q.....R.T +| 1824: bf 4c 06 dd f5 a3 ec c2 ce eb c3 82 e0 5f 5c c1 .L..........._.. +| 1840: 51 9c e9 d7 d1 45 a6 ea 62 dd 3b 25 79 0a 5e ed Q....E..b.;%y.^. +| 1856: 5a 9d 81 8e 7c a8 fd 60 31 88 80 f3 df de 60 23 Z...|..`1.....`# +| 1872: 4e 07 7a 02 dd 89 7d 2c bd ce fa 45 ff c6 05 f4 N.z....,...E.... +| 1888: ff 2d 18 f5 13 37 c8 d8 be e9 6d 81 da 8f 03 e4 .-...7....m..... +| 1904: 82 0a c3 3b 19 10 de 4a d8 12 37 37 ce 51 4d 41 ...;...J..77.QMA +| 1920: 0a ad 6a 1f c2 80 49 84 15 7d d4 f0 53 5d d1 b1 ..j...I.....S].. +| 1936: b3 c7 27 34 2f bd 05 af df 5c 5f 86 90 55 99 e5 ..'4/....._..U.. +| 1952: 4a 9f 86 34 30 b1 fe 0d 14 2c 1a 85 28 0e 09 5e J..40....,..(..^ +| 1968: d0 31 99 e7 8c ff 6e f9 e6 a5 cb fd 85 9b f7 31 .1....n........1 +| 1984: a0 07 f4 cc ca 5d c2 37 f9 a3 6e d2 5b 2e 04 32 .....].7..n.[..2 +| 2000: 19 03 7a ee fa 9c 5c 68 89 f7 94 b9 19 fe 05 d2 ..z....h........ +| 2016: 19 f8 3b fd cf dd 6f 23 3f c3 28 15 8a 8c 86 4b ..;...o#?.(....K +| 2032: 8a ff ed ca b7 6a 69 4a fc 66 94 a0 ea b7 fa f0 .....jiJ.f...... +| 2048: b1 3b 67 85 2b 82 e4 7f 3b 8d da 02 9e 27 fb 69 .;g.+...;....'.i +| 2064: 7b 20 c6 f6 fb c2 18 fa d5 6d 60 22 be 01 93 28 . .......m`....( +| 2080: 08 71 9f b2 b0 e6 79 94 91 b6 b8 b0 ff a7 3f d8 .q....y.......?. +| 2096: e3 eb b8 8f c7 02 5b f1 35 b7 15 db 01 cd c9 f6 ......[.5....... +| 2112: d9 83 36 2f d0 8b 43 66 e2 3a 14 34 70 1e 68 e3 ..6/..Cf.:.4p.h. +| 2128: e3 a7 fd 7f 51 cb ec 60 ca fb 13 fc 11 56 97 46 ....Q..`.....V.F +| 2144: 23 53 0a aa 8d 15 de d4 cd 2b 1b de 3f d5 57 9c #S.......+..?.W. +| 2160: e5 91 3b 15 98 3e 09 d9 4e d9 25 14 01 32 fc 23 ..;..>..N.%..2.# +| 2176: 38 60 b2 8f 00 f3 21 8b 20 be 8f 6e 02 9a 58 3c 8`....!. ..n..X< +| 2192: a7 a7 6f 3e 24 cc 1f 87 ab e6 12 0f bd 0a 8b 38 ..o>$..........8 +| 2208: 17 b2 ee 80 80 bb 50 df e0 cb 72 0a 98 07 40 bc ......P...r...@. +| 2224: 8d 77 86 27 e8 07 57 4b 50 84 4c b4 73 b4 32 3d .w.'..WKP.L.s.2= +| 2240: 89 61 e8 b8 dd bf 22 c8 06 07 e6 8f 6d 54 c7 e3 .a..........mT.. +| 2256: 12 f9 1f 61 cb e2 40 e1 14 34 9a 2b 16 a7 1f ec ...a..@..4.+.... +| 2272: 85 b8 76 8e 25 59 51 5a 19 82 d9 54 7f 3d 11 05 ..v.%YQZ...T.=.. +| 2288: a6 4c 43 28 9c 65 44 5e 57 fe 04 84 ae 7f 26 5b .LC(.eD^W.....&[ +| 2304: ed 26 e2 1c 07 1f fa f1 99 7b 69 f0 03 16 bf 7c .&........i....| +| 2320: 13 16 75 80 d1 14 17 fa c9 f4 3b e5 9a 06 c0 d4 ..u.......;..... +| 2336: c2 55 53 0a 78 32 86 3c 23 a4 f8 b6 5b d4 9a 51 .US.x2.<#...[..Q +| 2352: 7c c3 1b 56 b1 e7 cd 83 02 75 8c 5e e3 6b b3 d3 |..V.....u.^.k.. +| 2368: 03 23 21 75 3f 78 a7 0f 66 87 2e 85 37 6b bc 0e .#!u?x..f...7k.. +| 2384: 2f 27 cd da 3c 1e 1a 35 2b bc 2d 97 ec 55 df 22 /'..<..5+.-..U.. +| 2400: fd 19 04 fa be 12 2a 1f 4b ed e3 0d 87 f0 61 db ......*.K.....a. +| 2416: 06 1f 86 2f 9d 80 32 97 df 70 ff b4 78 df 95 7d .../..2..p..x... +| 2432: a3 db a5 d0 db a0 b9 ed 4f a7 63 14 bf 8b 99 fd ........O.c..... +| 2448: 6e 1b d0 02 07 93 e1 be 56 4e d5 c1 41 a6 f0 ea n.......VN..A... +| 2464: 37 aa 21 7c 51 2f 5b c4 a0 ae ba e8 a7 1d c6 a5 7.!|Q/[......... +| 2480: 1e 8b 3b 86 7b f2 ea 57 b7 2f 88 37 5e 7a 89 e6 ..;....W./.7^z.. +| 2496: 09 c0 e4 3b aa bf 78 41 7c 00 14 cc 37 f1 1a f6 ...;..xA|...7... +| 2512: a5 79 49 f0 5a e0 1e 5d ba 33 ef fe 89 6e 20 01 .yI.Z..].3...n . +| 2528: 96 70 7d 58 0a 98 e0 c4 3e 20 b2 46 83 b8 1f c9 .p.X....> .F.... +| 2544: fe 78 8a 41 a9 ca 3f 2c c1 d4 35 a3 39 85 4d a1 .x.A..?,..5.9.M. +| 2560: 23 5e c7 b5 79 75 00 e7 d5 fb 30 99 00 c4 d0 31 #^..yu....0....1 +| 2576: 80 68 f2 80 1b 5e c8 30 f6 36 e1 a1 b5 87 92 35 .h...^.0.6.....5 +| 2592: 6b 9c ac 39 39 3f 54 e9 4d b4 8b bd 09 4c 6f 9a k..99?T.M....Lo. +| 2608: 9a 6d 65 1a 7a 2f 98 0f 33 32 3a 4c bd d6 c3 d5 .me.z/..32:L.... +| 2624: 2b 3f 7f 0a d7 f7 f0 05 88 5e 36 30 4d a2 0a 65 +?.......^60M..e +| 2640: f6 a9 5b 2d 5e 3f 0e 13 c9 45 1d c9 28 19 6d ba ..[-^?...E..(.m. +| 2656: ff 9d af 10 17 b2 49 34 d8 a1 c8 ce d2 b7 99 3e ......I4.......> +| 2672: 7c c6 f5 d6 74 84 05 84 2a bb 2d 25 27 67 04 f0 |...t...*.-%'g.. +| 2688: b3 6d 0a a9 43 f6 c4 51 71 1d 48 8d b3 a4 47 ed .m..C..Qq.H...G. +| 2704: d6 14 c3 27 3e 6a e9 64 ef d7 af fb b0 eb ec 3c ...'>j.d.......< +| 2720: bb 92 77 32 5e 4f 20 6d f7 a4 94 ce d9 9b c5 78 ..w2^O m.......x +| 2736: 84 f0 c3 1a ca d8 e2 ce 76 0a 5e f8 73 9d 5a 82 ........v.^.s.Z. +| 2752: 1e f5 0a 43 41 77 b9 d1 17 ef 47 0f 04 7b 5c b9 ...CAw....G..... +| 2768: 3d 96 ec 7c b6 45 7d f9 fe 46 cb d2 50 a3 7f 99 =..|.E...F..P... +| 2784: 16 ef 8d 2d d5 55 30 d8 27 18 bb f9 fa 76 80 55 ...-.U0.'....v.U +| 2800: 13 1e f3 c2 dc 11 ef 8b 5d 67 d2 cd 70 74 b0 93 ........]g..pt.. +| 2816: f9 51 ec 35 24 cc 01 55 9f 92 a5 10 7f 66 07 e9 .Q.5$..U.....f.. +| 2832: f9 0a 32 06 db fd a6 ed 3e c6 b4 55 41 45 6c d3 ..2.....>..UAEl. +| 2848: da a9 57 98 b1 79 3e 3a 6a e3 c4 37 3c 81 ba 16 ..W..y>:j..7<... +| 2864: 3b 6a 3b 29 93 80 50 6d 94 05 dd 6b 37 10 f7 5c ;j;)..Pm...k7... +| 2880: 5a 23 ef 13 1b 49 a5 fb 7b e8 49 a8 2a f4 e2 f4 Z#...I....I.*... +| 2896: 0b a6 28 03 5b 3e ce c0 f6 b4 d3 c3 b5 ed 2f 87 ..(.[>......../. +| 2912: a3 66 13 93 45 50 d5 3d 59 e4 42 fa f7 00 d8 b8 .f..EP.=Y.B..... +| 2928: 7a 3a cc ce fc d2 8d a5 e9 1b 63 f8 21 6e 4f a3 z:........c.!nO. +| 2944: 0e 17 92 56 d6 2e c9 a4 b8 07 2f 25 9e 3b ff a7 ...V....../%.;.. +| 2960: 9d 31 56 43 85 d6 ea 91 ab 29 8c 1e bd 29 f2 9c .1VC.....)...).. +| 2976: 9c 48 81 a2 a9 da 80 c8 b6 8a 67 48 1d c0 02 4a .H........gH...J +| 2992: fa a1 e3 69 15 83 c5 ca 9d 96 43 06 ad 97 c3 5c ...i......C..... +| 3008: 09 e0 de b9 0d 4e f5 99 75 e1 c0 98 20 91 c3 5d .....N..u... ..] +| 3024: 93 81 0f bf 2b 96 96 c0 9d 30 d3 84 c0 9e 60 56 ....+....0....`V +| 3040: 08 0a 3d 48 be 6b 3d 32 c4 5b e8 87 02 88 f6 f3 ..=H.k=2.[...... +| 3056: 71 71 07 f8 66 42 8d df 4e aa 44 6f ec 3a b8 7e qq..fB..N.Do.:.~ +| 3072: 4d b3 83 74 b4 bf 34 d2 2c 95 3c 44 9b c1 cc 9a M..t..4.,.+......6..#. +| 3120: 1b f5 ce 42 b0 0c 11 d4 69 a4 bf fa 4b 30 34 b4 ...B....i...K04. +| 3136: 52 48 84 cc 31 08 03 a9 93 d9 e5 c8 44 cd cf fe RH..1.......D... +| 3152: e4 2e c5 3a c2 b2 d9 bc 15 4d 7f 7a 81 52 9c 53 ...:.....M.z.R.S +| 3168: 3e 1a 74 78 c8 38 4a 02 ce 6b 96 2d 9c 3b 74 41 >.tx.8J..k.-.;tA +| 3184: 50 06 d9 6f fa c8 98 05 f5 70 c3 7f ca 0a 78 f4 P..o.....p....x. +| 3200: 01 f1 a2 f5 46 8a 02 b8 c3 37 7e cc a0 87 b0 fc ....F....7~..... +| 3216: 32 e5 86 6d 95 22 67 fd 10 56 c4 78 52 c5 12 e5 2..m..g..V.xR... +| 3232: e9 f2 b2 fa f6 78 2e a3 82 a6 73 1b 9f 96 a7 e6 .....x....s..... +| 3248: 81 49 e9 17 f1 0a e6 99 21 ce ec ff be aa 43 e6 .I......!.....C. +| 3264: ea 6a ac 2b a4 b9 0b 48 6c 81 1a 26 bf 93 69 36 .j.+...Hl..&..i6 +| 3280: 53 1e 46 2a 43 af cc ef 96 4e 0b 32 38 93 10 d5 S.F*C....N.28... +| 3296: d8 6f 88 e7 f9 92 f8 37 28 51 55 98 fa 1a 89 80 .o.....7(QU..... +| 3312: 35 d5 d1 90 f1 91 8f 4a 5a 81 2f 79 a7 e4 80 3c 5......JZ./y...< +| 3328: a2 5e e5 b3 f8 d3 56 ec cd 02 73 0e af 8d d8 81 .^....V...s..... +| 3344: 26 7b 4d 96 f1 b3 75 7a de c3 b8 a6 b6 75 f2 5c &.M...uz.....u.. +| 3360: 97 08 a3 50 9b d8 cf 43 c0 90 f7 52 da 87 8d 12 ...P...C...R.... +| 3376: 1c d4 0a de 56 26 3d af d1 d5 03 ff d8 62 a1 c5 ....V&=......b.. +| 3392: f2 7c ea 1d 39 3f f0 e1 5f cb 3d db 1a 8a a1 b7 .|..9?.._.=..... +| 3408: 10 01 f1 c6 17 53 bc 83 eb 2a c4 cf 3a 04 b1 8a .....S...*..:... +| 3424: 6d 80 20 be 7b 72 6c c9 be 4e eb f4 cd c8 60 d2 m. ..rl..N....`. +| 3440: 1c 46 b1 8a e5 9b 66 22 4f 05 bd c6 37 34 67 2d .F....f.O...74g- +| 3456: b1 c5 bb 60 de 4b c4 3a 29 14 dc de 38 9c 0d 09 ...`.K.:)...8... +| 3472: bd 78 2f f6 e4 34 18 9b 51 41 a7 57 f9 f2 ad 0d .x/..4..QA.W.... +| 3488: 82 70 5f b4 fa 84 19 5c 72 29 8e 5f b2 8b fa 2a .p_.....r)._...* +| 3504: e1 d1 55 bf b8 45 c3 79 e0 04 62 10 1f 76 79 de ..U..E.y..b..vy. +| 3520: 84 93 a8 df 92 8f 41 0d 38 14 70 b3 dd 9d c6 36 ......A.8.p....6 +| 3536: f6 12 72 ee 23 e6 ba ed 97 1f cd 36 55 ac 32 98 ..r.#......6U.2. +| 3552: 78 12 55 23 d5 64 ad ed cf 32 d3 2d 7b 51 d7 7f x.U#.d...2.-.Q.. +| 3568: 3c 6f 0f fb c7 06 75 96 ef 64 ce c4 09 15 4f b8 ....|....c/.... +| 1104: f2 f2 37 ff aa 18 0f 69 94 cc 49 85 cd f7 bb 7b ..7....i..I..... +| 1120: e5 c0 09 d1 a2 78 f4 c0 1b 23 e7 d5 ac 2d 85 27 .....x...#...-.' +| 1136: c2 48 c2 80 ad 93 a0 10 44 98 5a 6e 9e f1 92 a9 .H......D.Zn.... +| 1152: 98 f9 0d 00 2a 3a 22 36 c5 21 63 fb 95 fb c7 7e ....*:.6.!c....~ +| 1168: 6c 07 7d d1 a5 f1 a9 8f f2 d2 e2 0f 3d 02 56 94 l...........=.V. +| 1184: ea 44 2a 17 ee 99 82 71 9c 81 c7 9c 8f cd 2e 4d .D*....q.......M +| 1200: 8d 0a 1e 71 18 3d 4d 9d a6 ae 35 36 48 08 24 e8 ...q.=M...56H.$. +| 1216: 46 78 8e 66 f2 8a f1 3d c3 72 ad 65 18 5f ed 2d Fx.f...=.r.e._.- +| 1232: 7e 92 18 9e 4c 23 7c f8 8d 83 20 43 50 63 db 75 ~...L#|... CPc.u +| 1248: 12 7e 52 0e 71 13 e4 b6 31 c3 5f 88 b7 d0 76 80 .~R.q...1._...v. +| 1264: d5 9f 7e dc 8c b4 be 11 da 2c e1 a9 28 5e c8 46 ..~......,..(^.F +| 1280: 3c 0d cd 7a 2f b9 9f 53 9c 11 01 d1 09 31 5f 36 <..z/..S.....1_6 +| 1296: bb a4 f3 a0 ad ba b7 c4 ce d1 78 4e ad e8 cd f9 ..........xN.... +| 1312: 92 ed 51 05 69 ab 12 a3 89 4c 6d f5 7a c4 c5 54 ..Q.i....Lm.z..T +| 1328: 1e 26 e7 f8 22 1c 42 cb 9e ac 7a 53 79 09 3c 04 .&....B...zSy.<. +| 1344: 14 1d 49 15 c7 13 00 84 3c 7f 15 2d 5b 62 da d8 ..I.....<..-[b.. +| 1360: 8f 88 53 7a 78 f7 c4 7b be 67 25 cd 97 1f bf 19 ..Szx....g%..... +| 1376: b4 8f 4a 10 63 89 2f cf 56 c1 63 79 92 cc b1 75 ..J.c./.V.cy...u +| 1392: 45 ee 6a 9d c5 de 04 e0 60 49 c5 0e 09 a0 65 46 E.j.....`I....eF +| 1408: 57 f3 6c 5f 0f 07 99 fa 2b 2c 2c 5b bd b3 9b 50 W.l_....+,,[...P +| 1424: 38 43 94 4b 7b d2 b0 78 3b 4c e8 55 1d d3 6b 6a 8C.K...x;L.U..kj +| 1440: f7 dc 27 ef b5 b1 3a 52 9f 5a e9 7d 87 bb 61 5b ..'...:R.Z....a[ +| 1456: 39 a3 eb 5e eb f1 80 78 85 d8 86 16 76 3c b3 b0 9..^...x....v<.. +| 1472: 44 72 d7 0e 91 95 5e 7d d5 93 0a 4d 60 46 ad a8 Dr....^....M`F.. +| 1488: b7 d5 30 e2 39 4c d8 0a d4 ea 5e 46 95 39 1a 1e ..0.9L....^F.9.. +| 1504: 9d 81 27 20 bc 01 1e f5 2d 4a 19 4a a5 2d 94 91 ..' ....-J.J.-.. +| 1520: c2 a3 31 65 aa 13 f4 92 ce b2 c1 61 25 13 7d 93 ..1e.......a%... +| 1536: f0 c9 51 e5 ce 7c 0e 2d e4 f2 d4 b5 09 73 fc 96 ..Q..|.-.....s.. +| 1552: e0 a8 a7 85 b6 10 0f 95 a5 83 c5 4b ca 33 9f 98 ...........K.3.. +| 1568: 38 d9 50 29 38 08 cc b9 89 40 0e d7 1d 92 cc 4a 8.P)8....@.....J +| 1584: 7f a2 53 dd 19 d3 13 63 04 29 04 18 c2 79 97 36 ..S....c.)...y.6 +| 1600: 85 e8 95 4a 2c 34 7c 39 78 a5 fe 8b bb 22 93 4c ...J,4|9x......L +| 1616: 69 b6 86 ca 0c 61 39 b3 85 c1 df ee 28 b0 3d 73 i....a9.....(.=s +| 1632: 2f 78 84 69 f4 55 ed 6c b8 98 61 0c f5 64 15 7e /x.i.U.l..a..d.~ +| 1648: 84 bd 4f de ae 6c 1b db c1 60 c9 0a b9 22 f0 7d ..O..l...`...... +| 1664: 03 23 0f e3 f5 77 3a 3b 4b 7f 33 d9 e6 7c ee 92 .#...w:;K.3..|.. +| 1680: 0c 95 f6 9d 98 53 e3 7b 92 3b 15 a2 de 84 e6 75 .....S...;.....u +| 1696: 64 e1 44 b5 9d 16 77 10 78 65 e0 72 66 a9 13 db d.D...w.xe.rf... +| 1712: 27 cf 9f 0d 33 5d ee 89 37 cf 2f b6 fc 28 b1 93 '...3]..7./..(.. +| 1728: c3 6d 0f 80 d8 5a 16 33 ea 06 d7 81 d3 45 59 94 .m...Z.3.....EY. +| 1744: 07 73 bf 47 f6 fd 6e 78 13 5a ec 30 62 a4 c4 12 .s.G..nx.Z.0b... +| 1760: 2a 56 8f 29 03 37 0c 98 80 8e e9 50 4f 3a 37 a6 *V.).7.....PO:7. +| 1776: f6 29 3a 6f 17 54 45 8e df 98 62 1e 23 d0 ec 20 .):o.TE...b.#.. +| 1792: 4b c8 f6 cb ce ef f5 8b f2 fd a9 a8 51 5c 0f d0 K...........Q... +| 1808: e4 4d 7c a1 a1 2a 88 f1 24 78 9e cb 7c 81 6b 0a .M|..*..$x..|.k. +| 1824: 51 73 1e bf 48 21 fb b3 7a 5c 48 0f 53 04 00 5f Qs..H!..z.H.S.._ +| 1840: 7e 66 7a 2e c0 30 9f bf 2f e8 95 b5 fe ea 9c f8 ~fz..0../....... +| 1856: 0e 6d 1f 54 81 da e9 bc 28 ea 42 de 5b e0 7f 93 .m.T....(.B.[... +| 1872: 79 6d 6d 8e f3 3c ba 13 67 3a 08 31 03 43 da eb ymm..<..g:.1.C.. +| 1888: 34 40 04 3e a4 21 44 92 96 75 2c 9f 8f 03 89 98 4@.>.!D..u,..... +| 1904: f0 27 dd 27 49 3d e3 9b 37 4e 66 ef 98 bd 1b ed .'.'I=..7Nf..... +| 1920: 3f cc a4 10 85 a4 39 90 cc a3 80 3d fd e5 44 4f ?.....9....=..DO +| 1936: 45 79 64 37 d2 70 ba 69 99 0f 4d 33 5a 68 bd 53 Eyd7.p.i..M3Zh.S +| 1952: 92 24 0d 3f f4 e2 4b 72 35 cb 52 ea 3d d4 85 ee .$.?..Kr5.R.=... +| 1968: e3 12 b3 87 3a 9d 70 b8 5c bb c7 c4 98 03 84 25 ....:.p........% +| 1984: 5d 39 ef 24 bc ab b3 da 13 7b 2c bc 17 b1 76 4c ]9.$......,...vL +| 2000: 5d 5a 1c 62 14 8d 5d 20 32 dc 16 64 79 7e 13 95 ]Z.b..] 2..dy~.. +| 2016: 27 f6 99 dd f2 32 c2 43 c6 77 b4 f4 11 88 6d 3f '....2.C.w....m? +| 2032: 94 0e cb db c8 c3 9e 40 b1 7d d4 16 e6 95 f4 d9 .......@........ +| 2048: 3d 9b 5d df d5 a3 39 06 c1 eb a1 30 cb 85 77 d6 =.]...9....0..w. +| 2064: 9b d9 12 7a 6a 82 9e 3a bf 5a ff 5c b8 6f 4a bb ...zj..:.Z...oJ. +| 2080: 02 36 94 1d 5a 0c 95 da 85 d1 9c 22 5a 36 fd 28 .6..Z.......Z6.( +| 2096: 51 cf 48 60 84 a1 ce ad a9 5f 4c 51 43 71 7f d5 Q.H`....._LQCq.. +| 2112: 1a e1 6f b0 01 a5 6b f1 57 91 84 78 b0 d5 06 7c ..o...k.W..x...| +| 2128: f4 96 3d 9b e4 6d 04 b7 26 a0 5b 61 53 39 ba 49 ..=..m..&.[aS9.I +| 2144: 2c ea d8 18 e5 aa b3 ef 3e 8e 5d 91 ba 93 15 2b ,.......>.]....+ +| 2160: 4d e6 5c e7 47 ee 7c 9e d5 ca f4 82 91 84 f4 33 M...G.|........3 +| 2176: 02 46 9d dc 34 8d f1 5b 1c 15 5d db 2f c9 d0 09 .F..4..[..]./... +| 2192: fb a4 cf d9 0d 30 9f 16 3b 5e 80 ff 26 ac 19 b7 .....0..;^..&... +| 2208: b2 56 1a bb 72 7f b9 2f e7 41 6a e3 fc e5 cc e0 .V..r../.Aj..... +| 2224: 27 2c 78 86 1c 11 ee f4 51 2d 22 20 73 3b ed d6 ',x.....Q-. s;.. +| 2240: 42 c2 de 23 de c6 7d 43 8f 16 de 66 57 64 94 32 B..#...C...fWd.2 +| 2256: 13 da a6 91 b7 d5 8f 3a d6 c4 a7 e0 93 ff f8 9c .......:........ +| 2272: e3 70 a9 e9 e9 7a 50 37 3c 38 d3 fd 68 ff 17 5f .p...zP7<8..h.._ +| 2288: 96 b7 b4 77 c5 ce 1d 37 49 35 91 6c 2e 37 ef ac ...w...7I5.l.7.. +| 2304: 18 ee 9f 7a e5 28 88 a6 33 39 05 f7 c8 1f 1a 15 ...z.(..39...... +| 2320: f6 ad 55 5e 24 ea ce 21 c8 e2 1e 63 bd 7a 19 60 ..U^$..!...c.z.` +| 2336: c0 aa e4 42 2f 0f 53 f4 2c 04 99 a4 9a 13 86 6f ...B/.S.,......o +| 2352: 24 7b 42 ad c9 a9 c9 df 82 90 23 0a b6 e8 2a e4 $.B.......#...*. +| 2368: 36 c5 04 1a e4 dd 87 06 ca 6e 04 7b be 37 b4 ef 6........n...7.. +| 2384: 05 1d 6a ab 36 c8 58 52 3d ad 26 21 45 58 f4 66 ..j.6.XR=.&!EX.f +| 2400: 05 a2 95 89 9c 31 1c 3c 14 9b 59 90 29 28 f9 9d .....1.<..Y.)(.. +| 2416: f9 96 74 ab d7 30 91 81 17 ba b8 08 51 38 75 77 ..t..0......Q8uw +| 2432: 06 ca 81 1d e3 9b 67 3e 0f d9 fe 98 69 e0 d8 f1 ......g>....i... +| 2448: 78 dc 96 0e 88 50 9c b5 c1 1d 25 61 87 70 bf a5 x....P....%a.p.. +| 2464: 93 2f f9 c7 7b ab fd 2c 98 10 ef 0f ce a4 bd fa ./.....,........ +| 2480: 52 7f ff 29 53 5f 9c b5 92 2e 8c f5 36 a0 7c 88 R..)S_......6.|. +| 2496: 84 18 80 59 66 d2 c6 58 55 c0 5d 07 9f 69 d2 72 ...Yf..XU.]..i.r +| 2512: 88 0d 1d 8f 08 7c d5 b7 9c 15 4c 6c c6 1d f4 76 .....|....Ll...v +| 2528: 0a 82 32 6c 72 4e 3e cc ff 8b 62 c5 37 4e 0c 0d ..2lrN>...b.7N.. +| 2544: 09 89 8a 95 1b 63 d3 85 c6 4a 41 7c 49 46 60 95 .....c...JA|IF`. +| 2560: eb 37 60 82 84 55 ea fe 22 97 16 58 ad 33 06 ae .7`..U.....X.3.. +| 2576: d0 f5 5d 06 b3 2e b9 0a 6d d0 14 c8 d5 0b 8a cf ..].....m....... +| 2592: 51 ff 64 db a8 81 e1 8b 39 1a ef 85 18 fd a2 48 Q.d.....9......H +| 2608: 4f 47 b3 1e 5a 57 21 f0 6b 10 f0 ec 0a 4a 35 56 OG..ZW!.k....J5V +| 2624: f5 79 c5 de 62 d1 4a c2 e5 bc 69 89 9d 9b b9 4b .y..b.J...i....K +| 2640: ba 21 99 5a 58 c6 23 ab 67 45 13 4a 1d 3e da 68 .!.ZX.#.gE.J.>.h +| 2656: cf a6 4b 20 58 b0 4e 2f 0d f2 42 83 9d 96 43 6e ..K X.N/..B...Cn +| 2672: df 45 61 c5 bd ca 86 b3 6a ad 26 a2 8d c7 c4 03 .Ea.....j.&..... +| 2688: 63 d8 b8 02 0a f6 96 f9 15 46 2c bd 04 68 aa 1b c........F,..h.. +| 2704: 00 46 7e a0 bb 47 fe 5f 2b 7b b4 59 54 14 e3 25 .F~..G._+..YT..% +| 2720: cd 10 8b ff 05 6c 6d ab 9c 90 cb d4 d8 0d bd 58 .....lm........X +| 2736: be bc d5 c0 67 bd 0f d3 9d 22 71 c4 65 e6 2f bc ....g.....q.e./. +| 2752: 5e 3a 54 39 7a ec ab b1 46 80 25 5e 33 19 de 3b ^:T9z...F.%^3..; +| 2768: 4a 84 28 04 80 f6 86 aa 32 75 8e 5b 51 66 b0 64 J.(.....2u.[Qf.d +| 2784: 29 94 f6 56 6d 1c b1 2e 2f 4c 55 3b e5 82 01 75 )..Vm.../LU;...u +| 2800: 3f 58 d3 a0 b9 0b 6b 25 24 94 9e a5 ce a4 1c db ?X....k%$....... +| 2816: 51 f9 bb 53 71 19 0a 5a e3 2e 48 e1 06 24 7a b9 Q..Sq..Z..H..$z. +| 2832: 88 fb dc 4c 2a 23 b9 b9 7e 26 a4 3f 7f c7 b5 f1 ...L*#..~&.?.... +| 2848: cd cb 82 17 34 89 70 b4 10 ad be b8 4b 0d aa 07 ....4.p.....K... +| 2864: 61 6d b3 6f 56 ac a4 b2 e4 2f be 28 e6 24 3a ab am.oV..../.(.$:. +| 2880: cc 1a 8a 7f 54 c7 a3 c8 49 50 91 76 f9 75 d8 3a ....T...IP.v.u.: +| 2896: 76 1b ec 5b 51 c2 5f 6f c9 c7 60 24 0a 61 f4 ed v..[Q._o..`$.a.. +| 2912: d3 30 23 e7 a0 83 c7 5e f4 b8 1b 39 65 43 2a 8a .0#....^...9eC*. +| 2928: 1d a8 64 4d c5 4a a6 ab e1 f4 66 90 f7 2b 1c d6 ..dM.J....f..+.. +| 2944: ba 76 96 43 2d a1 6b 26 ee f1 3b 8b 2f e4 78 7e .v.C-.k&..;./.x~ +| 2960: 9b bd fa 2e 80 ff ec 26 74 5b 56 11 92 88 51 8c .......&t[V...Q. +| 2976: 6d 42 4f f5 3c 53 8b 42 3e b6 57 37 fe ad 66 59 mBO..W7..fY +| 2992: 4a f4 0e a1 39 a2 32 c0 23 78 e0 e7 db f9 f9 44 J...9.2.#x.....D +| 3008: 34 12 00 a0 d2 1f 45 25 4f 4a 72 74 9e b2 d5 e4 4.....E%OJrt.... +| 3024: 25 4e d5 0f ba 1b 87 7f 23 f2 7f dd 96 6c 1c 44 %N......#....l.D +| 3040: 0d 11 9a 22 6a f8 83 bf 37 e6 c9 5d 2e d3 dc d3 ....j...7..].... +| 3056: aa 03 03 7e bf a5 cc a1 d8 55 6f 05 1a 70 5e 89 ...~.....Uo..p^. +| 3072: d5 8e a5 08 f4 fc 90 7c d2 8a 67 53 82 25 82 02 .......|..gS.%.. +| 3088: 98 e0 ad 57 bb d4 7e fc 18 8f 25 9d d7 11 57 ab ...W..~...%...W. +| 3104: cf de 4b a5 1b 90 6b 38 a3 ef 93 ed 44 d7 9d f1 ..K...k8....D... +| 3120: 54 b1 0b 9b 2d e9 e7 d3 a6 6c 78 fd 40 d9 ea 7f T...-....lx.@... +| 3136: ec 10 60 2f ed 4c df f4 ef fb 4a cd fd 1a ea 94 ..`/.L....J..... +| 3152: 7b ac 69 70 be d1 2c ca e3 88 a7 b4 7f 95 83 eb ..ip..,......... +| 3168: 2e 82 0e 3e 60 27 b9 0c 62 56 3d b2 c4 7d 97 2c ...>`'..bV=...., +| 3184: ac cd 0b 05 b3 41 ab 7c 2e eb 4e a7 3e 8f db 53 .....A.|..N.>..S +| 3200: 88 59 f3 d6 f4 bf 5e 80 2e 5b f2 08 f8 dc 68 28 .Y....^..[....h( +| 3216: a7 a3 e3 4e ef 8f 3f 4e a6 c0 4b 04 d9 59 3e 53 ...N..?N..K..Y>S +| 3232: 07 78 2c b2 58 a6 78 a4 0a 6b 9e 6b 8d f2 3e 70 .x,.X.x..k.k..>p +| 3248: 45 07 82 f9 f0 d4 0f f7 81 e9 ab 09 c6 7a 3a ff E............z:. +| 3264: 98 6c bf 1f 43 c3 d5 ba c1 ad fe 9c a6 5e d5 f7 .l..C........^.. +| 3280: 93 ec 20 dd 33 ad 35 0b 74 47 80 1e 5b 8b c3 7c .. .3.5.tG..[..| +| 3296: 5c 81 66 82 dd 18 70 19 b0 54 18 2f 90 c2 22 bf ..f...p..T./.... +| 3312: 75 e8 b6 8e b7 3e 7c 2d b4 e9 7c b4 08 4e e9 c3 u....>|-..|..N.. +| 3328: e1 80 88 63 f2 90 ee ae 3d 9e 55 15 62 7f ee df ...c....=.U.b... +| 3344: 57 69 03 da 76 e9 f3 6b 90 9a a7 b9 bb 4b b7 2b Wi..v..k.....K.+ +| 3360: d0 a0 3b 70 b9 96 70 17 c8 0c cb ae 6e 3e e1 33 ..;p..p.....n>.3 +| 3376: 02 98 53 d6 af c6 d8 55 15 7a ea f1 8f 36 6b d4 ..S....U.z...6k. +| 3392: f2 8c fd db a5 81 50 fe c8 a8 cb 7f e6 9a a9 9f ......P......... +| 3408: 34 82 6d a5 11 6b e6 79 58 a8 54 ed 55 cd 19 46 4.m..k.yX.T.U..F +| 3424: e9 7f 92 f8 0b 3e ab b5 8d d3 9f 33 5c e7 f6 1e .....>.....3.... +| 3440: 28 f7 5f 2f 38 36 25 ed 7f 36 93 19 f1 a7 9c e7 (._/86%..6...... +| 3456: ad 4e 11 33 17 93 82 cf b4 a7 93 36 97 d9 e0 3d .N.3.......6...= +| 3472: 7e 54 b9 96 37 85 16 db e8 29 6f b0 b1 83 16 63 ~T..7....)o....c +| 3488: 9b 19 30 85 c6 17 2c b7 bc f2 2f 87 07 56 97 b3 ..0...,.../..V.. +| 3504: c4 a6 47 d5 8b 7a 68 a4 d0 11 ce ff 6c 4c 28 93 ..G..zh.....lL(. +| 3520: c3 37 ef 1e 7a 51 65 a9 6b 8a a0 a4 b0 b9 ef 17 .7..zQe.k....... +| 3536: dc 44 99 34 7b 33 f4 b9 3f d3 20 36 54 5c 10 6d .D.4.3..?. 6T..m +| 3552: 18 25 33 6e fc fa a0 16 c7 b6 78 f8 4b 0c 5e 7e .%3n......x.K.^~ +| 3568: c3 cb 95 54 90 88 57 20 d9 17 9b 82 dc e8 7f c6 ...T..W ........ +| 3584: bd d8 dc 13 ca 66 06 64 8e 46 2e 48 13 eb 4d 07 .....f.d.F.H..M. +| 3600: 24 7f 8e 35 f3 bf fb 54 80 c7 6e 93 da ce 7f 2b $..5...T..n....+ +| 3616: 60 3e 53 82 33 38 fe 69 7b f6 fa 2f d8 40 bd c8 `>S.38.i.../.@.. +| 3632: 95 75 6b a2 e5 53 7f 5c f1 f3 23 07 8f 42 04 8f .uk..S....#..B.. +| 3648: 64 43 a2 25 c1 40 07 e9 be b6 3c d6 1b 86 7f 91 dC.%.@....<..... +| 3664: f4 73 0f 6b ca 6b cb 8f 7c a8 5b 60 ee 20 a6 e6 .s.k.k..|.[`. .. +| 3680: dc 96 6b 8f 8d 62 1e 42 e0 1b 85 49 85 34 b4 c9 ..k..b.B...I.4.. +| 3696: 85 06 ec c4 b7 b5 89 47 af fa db d1 5b 9f ac 6f .......G....[..o +| 3712: 8a 27 53 48 4d ad a1 30 6b 3e 45 b1 b8 b9 50 17 .'SHM..0k>E...P. +| 3728: 7b b7 29 a8 b3 d5 9a 42 a9 2a e4 01 79 34 c5 21 ..)....B.*..y4.! +| 3744: a7 8b 21 69 ab c4 9b ec eb 0d 9f 34 b5 ff 46 2d ..!i.......4..F- +| 3760: 68 55 9b 63 9a ac ac 80 4c 45 8d eb dd b2 26 4a hU.c....LE....&J +| 3776: 91 8f ee f7 50 6f 84 81 c3 79 2e 6d 56 1c ee 01 ....Po...y.mV... +| 3792: 20 2b e5 ca 3a 51 ba 40 b6 44 15 a4 4a 3d d0 33 +..:Q.@.D..J=.3 +| 3808: 7d 2d 3c ca e3 f8 e9 1d bc b2 1e 59 1e 57 10 2f .-<........Y.W./ +| 3824: 92 4b b7 50 41 98 6f 07 0b 38 ba 31 08 3e d4 34 .K.PA.o..8.1.>.4 +| 3840: 4f f4 99 32 2f e5 7f 8f 14 8e 48 11 88 ad 18 d6 O..2/.....H..... +| 3856: db d0 e5 cc 17 4c b6 f7 45 f9 1e 9a d9 86 5d 56 .....L..E.....]V +| 3872: fa 76 52 49 cd f1 76 ea 84 b1 55 03 54 e2 ff 91 .vRI..v...U.T... +| 3888: 23 fb 22 b0 8a b8 da 6e 7e 54 25 03 f3 fb eb 9a #......n~T%..... +| 3904: ce 64 cb 70 38 c9 aa 11 e8 36 45 16 8c 82 0e 18 .d.p8....6E..... +| 3920: f3 cf 81 4c 7c ef 53 6a 5c 43 85 e3 01 c9 d5 97 ...L|.Sj.C...... +| 3936: 8b 03 78 90 35 c9 7c 3a 62 3d 66 d9 ff 6c 1f 3f ..x.5.|:b=f..l.? +| 3952: 7f a1 85 ad ec 87 65 3b 48 3c 0b 0c 94 c0 05 85 ......e;H<...... +| 3968: f9 a4 4a 79 c1 dc 7b 9f 87 c0 4f c9 7b a7 ce 5c ..Jy......O..... +| 3984: cd 11 68 d5 79 d2 e7 b5 8c 13 af d6 e9 81 ef 42 ..h.y..........B +| 4000: 4c 11 2f c0 55 33 4c f4 44 c1 75 67 ae 43 95 68 L./.U3L.D.ug.C.h +| 4016: da 90 26 d5 42 75 b6 ef 99 22 ce b8 2d 52 5f 43 ..&.Bu......-R_C +| 4032: a4 f7 23 72 17 c6 66 36 84 e6 d3 a7 da a9 ec 82 ..#r..f6........ +| 4048: b2 fd 87 90 51 2e 78 71 3f 56 49 96 d2 da 6b 67 ....Q.xq?VI...kg +| 4064: 2a f8 b1 eb ae 5d d0 e7 59 31 ed ed 24 b6 2c 32 *....]..Y1..$.,2 +| 4080: 7a b7 97 c5 e0 f1 a4 05 d3 8e fa 0f 10 ee 19 4d z..............M +| page 15 offset 57344 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 07 06 00 ae 7c 00 00 07 2c f5 e5 48 a9 82 86 >....|...,..H... +| 1104: 40 71 a9 80 b9 af 28 61 b5 17 5a cb c3 67 db aa @q....(a..Z..g.. +| 1120: fa 64 ec 63 b4 34 d5 37 ee 52 d8 d5 38 6a 25 d0 .d.c.4.7.R..8j%. +| 1136: 1f 05 80 72 bd c3 b9 1a d1 ea f7 f0 73 0e 97 29 ...r........s..) +| 1152: 41 19 09 d1 53 df c8 c5 02 ab a2 e8 76 b8 09 3d A...S.......v..= +| 1168: 8b 07 21 05 5f 80 3d 88 67 fd fe ab e6 4b 7f 97 ..!._.=.g....K.. +| 1184: a6 17 42 02 f5 ad f3 74 6b bb 6e 2a fd af 88 3b ..B....tk.n*...; +| 1200: d9 b2 3c 0d 33 a7 93 69 c0 97 c5 c4 c0 e9 29 d5 ..<.3..i......). +| 1216: c3 78 96 13 55 f3 41 e6 87 eb 7a 2f b9 e2 28 f3 .x..U.A...z/..(. +| 1232: e1 ed b8 c8 5d bd 80 80 a6 52 50 ff 0d db 2b f2 ....]....RP...+. +| 1248: 7e 9e 91 32 fd 08 46 ba db 2b 41 00 70 6f cf 80 ~..2..F..+A.po.. +| 1264: 46 1b 49 cf ab 6e 1d 14 21 f2 3f 55 0a 4a ed 19 F.I..n..!.?U.J.. +| 1280: 11 9b 9e c0 f2 c4 55 86 2c 05 4e a2 6f 8c b2 9c ......U.,.N.o... +| 1296: 11 fd 85 0c 5b d5 ee ef 9c f2 4b 1e 74 72 bd a4 ....[.....K.tr.. +| 1312: 29 8d 0d 92 67 97 b5 b5 03 b4 a7 68 15 50 c7 7e )...g......h.P.~ +| 1328: 91 22 a8 39 3e e2 18 bd 68 20 74 c1 e5 f2 8a e5 ...9>...h t..... +| 1344: 1e 72 0c 2a e3 1a b2 60 57 ad 2b ee 1f 59 57 ce .r.*...`W.+..YW. +| 1360: 6f b2 e0 15 96 73 f6 2e 5b 82 b7 ca 6b bf ed 78 o....s..[...k..x +| 1376: e5 af 82 82 73 16 6a ce cb 09 b9 5f b3 2d c0 0c ....s.j...._.-.. +| 1392: 1d 46 5f 7e be ee 0c b3 54 e7 ed e7 cc cf f9 9d .F_~....T....... +| 1408: 42 90 94 e2 e9 c2 76 2b 02 b9 a8 73 20 0a e3 6f B.....v+...s ..o +| 1424: 23 a0 c0 59 f2 af 9d c4 78 90 ba 8e 0c ef b4 d6 #..Y....x....... +| 1440: 37 03 5e f4 7f 54 a6 d7 87 bc 20 3f e1 7a 84 d9 7.^..T.... ?.z.. +| 1456: 1b 33 4a fc bd 1f c1 8b b3 db 8f 0c fa 49 7f d7 .3J..........I.. +| 1472: 9a d0 48 8e c6 3b 79 22 d6 0c fa 93 4d 43 f3 de ..H..;y.....MC.. +| 1488: bb 10 3e 81 d2 6b b4 9f 4c 91 05 d7 87 b5 35 98 ..>..k..L.....5. +| 1504: eb 84 9d 2c 0e 4e 2b de 1b 23 0d f5 bd aa 4a 9c ...,.N+..#....J. +| 1520: 6d 7e 48 23 18 f9 72 5b ef 9e da a9 ef 1b 29 08 m~H#..r[......). +| 1536: 03 34 e2 86 97 ff c9 2d c6 04 34 65 84 20 5b 7c .4.....-..4e. [| +| 1552: df e3 2c bc 73 04 22 70 f5 03 f2 8f fe fb 72 b5 ..,.s..p......r. +| 1568: a1 93 91 a4 3e a5 20 ac 21 19 8a 60 60 48 63 a0 ....>. .!..``Hc. +| 1584: 47 80 a8 ac 9f 68 a9 ab b7 0f 71 d5 44 1a fb 4a G....h....q.D..J +| 1600: 99 06 2b fc 96 ad ac 5e 37 37 84 6e 4e de 66 ba ..+....^77.nN.f. +| 1616: c6 10 fc e9 a0 26 cf 48 f6 f6 96 b7 69 35 30 06 .....&.H....i50. +| 1632: f2 7a db 70 53 4d f9 5b 9b 5f 51 b8 f6 af fb 1c .z.pSM.[._Q..... +| 1648: f5 2a d7 01 80 d2 0e b3 fe cc 99 ad 74 ad fa df .*..........t... +| 1664: 26 86 c7 29 b2 b3 36 98 b8 b7 2c de 9c 78 41 99 &..)..6...,..xA. +| 1680: ff 46 40 76 64 64 0c f5 46 d5 bd 72 a3 46 b0 b3 .F@vdd..F..r.F.. +| 1696: 75 45 03 d1 06 4b c9 ca a6 4f 90 f7 9b 60 4a 46 uE...K...O...`JF +| 1712: ed af a1 d5 68 30 db a7 5e ca 76 bf 66 64 e7 fa ....h0..^.v.fd.. +| 1728: 50 5c 31 5f 86 94 3a 5c f4 98 3d d5 52 80 26 1e P.1_..:...=.R.&. +| 1744: 55 07 14 d5 f5 2a 17 46 8b 6d 71 d2 da 4b e7 c8 U....*.F.mq..K.. +| 1760: 3d 8f 97 b7 97 51 c5 fb aa 4f 3f 3b 85 c1 aa e8 =....Q...O?;.... +| 1776: bf 3a d9 56 c0 d3 23 31 28 46 bd 45 2e 4b b5 e7 .:.V..#1(F.E.K.. +| 1792: f6 8e fa e2 28 65 6c 42 f2 68 8f 37 cb 4d e9 26 ....(elB.h.7.M.& +| 1808: f7 63 58 8c 99 09 71 7c 14 3a d7 92 c4 cf 14 51 .cX...q|.:.....Q +| 1824: af fe 6e 6b 50 e9 3a 70 0f ac 53 9d 57 01 4f 7a ..nkP.:p..S.W.Oz +| 1840: cc b2 c5 b9 2a 3e 02 6a d4 15 6f 2e d3 20 99 ed ....*>.j..o.. .. +| 1856: d7 1f 71 01 2f b4 e4 08 fe 8c 38 02 2c 14 e0 fd ..q./.....8.,... +| 1872: ec 3e 61 a8 b7 fa de f0 81 56 6a e2 d1 3d 2f de .>a......Vj..=/. +| 1888: f8 a3 75 5d 35 1b a4 a3 8e 6e 8c 9d b2 56 56 ba ..u]5....n...VV. +| 1904: 89 ab 42 86 70 59 5e 70 51 27 fc 87 b4 5c e5 89 ..B.pY^pQ'...... +| 1920: 20 b5 bd 40 6a 68 cb 1a 70 49 b5 2f c4 02 34 66 ..@jh..pI./..4f +| 1936: be cc 14 12 e0 f9 c7 4c 84 ff 37 a6 a5 ae 5c 36 .......L..7....6 +| 1952: 03 58 8e 44 17 e1 ca 30 3e 7f c9 54 94 f4 ff 86 .X.D...0>..T.... +| 1968: 57 8b 5e 2c aa 0b 28 30 ac 5a 1a 14 39 70 8a 84 W.^,..(0.Z..9p.. +| 1984: c2 4c 64 28 b1 87 6a 34 cf 50 b7 37 2e 96 ca ba .Ld(..j4.P.7.... +| 2000: c9 b1 37 d5 56 e7 39 b6 70 b1 16 2b 67 8a fb 0f ..7.V.9.p..+g... +| 2016: b3 ec 53 b4 27 ba 3e 60 16 96 86 b0 02 ba c9 d7 ..S.'.>`........ +| 2032: e1 d9 17 0f d4 12 aa 5d 65 53 29 47 04 47 2f 6f .......]eS)G.G/o +| 2048: 76 be c9 87 4b 41 27 63 28 f9 b0 48 a9 91 36 32 v...KA'c(..H..62 +| 2064: b7 e1 1f 84 52 c5 3f 81 98 6d de e0 31 27 eb 49 ....R.?..m..1'.I +| 2080: 0c f6 0c 00 5c db 30 f4 67 4e 90 f8 32 10 87 0f ......0.gN..2... +| 2096: 27 0c 19 71 b1 b8 e6 ea 40 e9 e0 fa 30 44 f0 d0 '..q....@...0D.. +| 2112: ae 89 05 c6 71 0d 18 36 f5 75 4d 6e b9 cf 60 2a ....q..6.uMn..`* +| 2128: e9 3e 54 d3 86 34 3b 69 87 47 64 9f 0d 99 d5 d7 .>T..4;i.Gd..... +| 2144: 42 bf c6 dd 24 51 32 5d e8 70 cf dc 67 55 58 bd B...$Q2].p..gUX. +| 2160: a0 93 c4 f2 05 73 91 bc 46 d1 4a ac 03 9d b1 ab .....s..F.J..... +| 2176: e9 6a de 28 8e d4 2a 75 e6 0b f5 ed a0 a6 45 e2 .j.(..*u......E. +| 2192: ea 86 1e a9 9d ad d9 8c 82 72 26 9e 7b ca a5 13 .........r&..... +| 2208: 3e ab 9f ed bf 81 9d 51 f0 f4 47 be 89 e1 0b a4 >......Q..G..... +| 2224: ec 5f 1c 5d b2 64 d0 7f 31 59 2e 68 2c 54 d2 69 ._.].d..1Y.h,T.i +| 2240: 91 34 56 e5 b9 83 0d 3a 00 9d e3 3d 78 6d 47 1d .4V....:...=xmG. +| 2256: 09 7d 67 de d8 a2 92 e6 b1 36 4d 89 c0 dd ce 76 ..g......6M....v +| 2272: df 73 aa 1d 88 38 c5 63 43 ac af 28 35 f8 48 5c .s...8.cC..(5.H. +| 2288: 9c 4d 50 ab f4 f3 c8 3c 6a 82 99 68 f9 25 7e 86 .MP....>..tJU..O() +| 2608: f8 00 70 ee 67 18 5e 7d 4d 1a fd b5 92 84 69 4e ..p.g.^.M.....iN +| 2624: c4 de 4d 45 90 3e 70 22 d3 b6 b0 74 b3 b0 21 86 ..ME.>p....t..!. +| 2640: 2d 10 d1 d3 2f d8 89 d5 b5 bd e0 92 66 a0 99 30 -.../.......f..0 +| 2656: 0f 6d de e8 db 9a 2d 6c 20 89 43 3c e0 61 32 67 .m....-l .C<.a2g +| 2672: ee f6 be 79 20 b4 06 11 63 b3 89 0f df 28 56 df ...y ...c....(V. +| 2688: 94 3c f2 31 b0 b8 5f 11 6c 6b 9b d3 43 cc f0 38 .<.1.._.lk..C..8 +| 2704: 6a 92 3d ee a6 92 95 9c 6c 02 94 31 7b ae 7a 0c j.=.....l..1..z. +| 2720: 0b 49 0f 82 54 ae c4 b8 58 4d 57 75 d6 e9 20 5a .I..T...XMWu.. Z +| 2736: bf 5a 2e ae 2d 18 4f 60 62 d9 1f 91 1d cd da f4 .Z..-.O`b....... +| 2752: 00 e2 d8 fd f0 20 d3 70 2b 4d 6f a1 80 a2 25 16 ..... .p+Mo...%. +| 2768: 7e 75 5e 7b 45 8d 72 62 11 ba 36 7e 4f 89 ba 4d ~u^.E.rb..6~O..M +| 2784: e2 0c e3 f4 0d da ac 14 89 36 e5 c8 4f 67 a5 33 .........6..Og.3 +| 2800: ff c2 91 ac 81 25 39 21 c4 79 5c 97 6d 97 45 3a .....%9!.y..m.E: +| 2816: 6e 00 60 1e c0 e4 ee f3 a6 98 16 43 ee 56 62 32 n.`........C.Vb2 +| 2832: 85 db 7a 83 af 26 a6 3f be 66 c6 5b ff 51 d7 f2 ..z..&.?.f.[.Q.. +| 2848: 8f 2e 13 c4 b2 67 72 ac c4 4d a3 3a 69 cb b5 bc .....gr..M.:i... +| 2864: 01 74 09 53 16 3a fa 7d 0e 24 8a c8 93 14 4d ca .t.S.:...$....M. +| 2880: 95 2f 30 dc d1 6c 42 d6 78 04 c8 91 4e 86 2a a0 ./0..lB.x...N.*. +| 2896: 0f 71 da 74 20 b0 a9 a5 a5 7e 96 44 e1 86 d1 af .q.t ....~.D.... +| 2912: bc 1d 24 19 11 54 3a e1 a6 db 38 f8 64 59 29 0f ..$..T:...8.dY). +| 2928: 3d 7b ce 0d c5 38 bd cd c2 85 13 f8 62 09 83 96 =....8......b... +| 2944: 20 2c 3d 81 6b 9d 48 24 4e d6 eb 88 7e f4 6f f4 ,=.k.H$N...~.o. +| 2960: fc 55 32 3d ab 74 bd f1 4f 75 66 f5 53 a5 a6 7f .U2=.t..Ouf.S... +| 2976: f5 89 da df 35 a7 44 0b 84 8d 85 f3 08 9b 63 14 ....5.D.......c. +| 2992: 8b ea 66 43 6f 07 a3 e5 33 eb fb 8b 11 40 10 7d ..fCo...3....@.. +| 3008: 11 cd 65 d7 38 4a 50 d7 02 73 ed a6 15 ac 82 77 ..e.8JP..s.....w +| 3024: 0d 8c f3 7a ea a2 36 32 11 52 71 d3 f4 24 13 5a ...z..62.Rq..$.Z +| 3040: 22 e6 75 14 28 7f b7 0c a8 71 16 20 3c 79 f5 ce ..u.(....q. ..L..... +| 3552: 7f e9 e1 a6 3b 68 b0 1e a0 81 5a 54 f7 3c a2 7a ....;h....ZT.<.z +| 3568: 24 ee 8e 43 a7 df 14 22 da b2 8c 80 6a 40 4c bf $..C........j@L. +| 3584: f8 5a 4e d1 34 af 19 3f dc 4f 93 29 83 d1 af 7a .ZN.4..?.O.)...z +| 3600: 33 fe 25 1a 1c d5 c1 e5 98 4e 30 4d 2b 2c 6e 55 3.%......N0M+,nU +| 3616: 8d d3 0f d5 f5 a3 a1 44 10 ba d6 2d 05 71 5d 4d .......D...-.q]M +| 3632: 15 e6 06 21 00 56 c1 45 21 63 f8 61 bf c5 75 79 ...!.V.E!c.a..uy +| 3648: f5 5f 83 53 ba cd a0 e6 12 45 3b 8b 33 b6 4d 29 ._.S.....E;.3.M) +| 3664: 41 c9 ad 09 a3 f1 a7 a6 6d 8d ec 16 68 b4 a3 fb A.......m...h... +| 3680: 01 1a c3 ee 78 17 d9 de cb 2e 42 d4 a3 56 8f f4 ....x.....B..V.. +| 3696: 3c 75 b1 92 d0 b5 75 10 69 cc c0 50 a4 bb d0 4c r. +| 3776: 8d e5 f5 f6 d4 49 8d b4 65 50 38 8b b3 3e 20 c1 .....I..eP8..> . +| 3792: 07 cd ad 7d e3 d1 8e 5f 6e 76 7b fe 01 ba 08 74 ......._nv.....t +| 3808: 16 a5 1b 5e 95 a2 ff 91 bb 9f 64 3f a4 e5 95 be ...^......d?.... +| 3824: d2 59 02 aa c5 06 66 9e fe 56 6e d1 1d 31 6f b3 .Y....f..Vn..1o. +| 3840: da 6d b4 3a 17 86 e0 e6 c1 37 21 68 6c 33 d3 58 .m.:.....7!hl3.X +| 3856: e8 67 fa 04 8d 59 ae e4 fb 1f 8c 48 17 7e bb 8c .g...Y.....H.~.. +| 3872: 17 61 b4 a7 e4 6b 87 c0 6a 8b 5f d1 76 16 5c f7 .a...k..j._.v... +| 3888: 81 b4 2d 66 ad ba 46 6f 95 bf a8 19 e8 82 05 7f ..-f..Fo........ +| 3904: 43 47 8b 63 b4 14 c6 2a de 90 8c 6d 04 3d 90 dc CG.c...*...m.=.. +| 3920: c4 6b d0 4c 33 9f 6c a7 38 6b fd b8 53 de 17 5e .k.L3.l.8k..S..^ +| 3936: ef e4 17 19 25 32 eb 12 86 03 3c 61 2e b4 06 ca ....%2........|..~v3.]... +| 1104: a8 31 d7 92 4a d2 68 29 8d e4 3c 76 6f d3 a0 f2 .1..J.h)..8...j..:. +| 1152: 2a 01 f9 21 05 ec 07 3f 6f 7d ba 17 5d ba 3a 04 *..!...?o...].:. +| 1168: a9 cc 88 35 4f 43 4c dc ab 1d 1d b0 24 1f a2 83 ...5OCL.....$... +| 1184: 69 4a 89 1e a8 a3 de bc 17 15 be 3d 07 ae 42 02 iJ.........=..B. +| 1200: f6 60 ff 36 08 cf 63 1f b5 be b1 1c 45 14 b0 56 .`.6..c.....E..V +| 1216: 0c 2e b9 49 49 69 47 17 da 41 4e 91 0b 5a bd 76 ...IIiG..AN..Z.v +| 1232: be 67 65 82 27 35 0a 23 a4 21 f4 ba 78 af 0a 7c .ge.'5.#.!..x..| +| 1248: 0e f0 ca 90 c6 32 d9 dc f1 dc 25 33 da 76 c0 f5 .....2....%3.v.. +| 1264: c2 4d ed 55 d7 f8 1f 44 82 37 c3 d3 90 12 23 52 .M.U...D.7....#R +| 1280: 8d 3c 3d f0 9e bf 7b 0e 49 8a 32 5d a1 64 53 26 .<=.....I.2].dS& +| 1296: cb be bc 26 dd bf f8 a6 ad 57 e5 35 68 c1 6c b9 ...&.....W.5h.l. +| 1312: 02 d7 0d 70 f4 d9 10 8f 75 35 8b 60 21 46 62 60 ...p....u5.`!Fb` +| 1328: e4 e7 70 35 0f a9 70 0d 50 da 5c c2 17 9b c6 3b ..p5..p.P......; +| 1344: 61 83 9f 3f 1d f4 28 dc c6 32 a6 12 ff 56 dd 0e a..?..(..2...V.. +| 1360: 57 08 0e 0d 5b f8 ea 32 33 20 a7 a1 a4 9b b3 77 W...[..23 .....w +| 1376: 4f db 85 06 dd 0b 52 b0 45 ff a4 e5 23 5e ea c0 O.....R.E...#^.. +| 1392: d6 54 d6 84 cd c4 fc 2e 95 ec 78 13 cf 9a fa e3 .T........x..... +| 1408: a1 5a b3 60 1a ab 1a 97 21 67 9c cd 44 6f ee a9 .Z.`....!g..Do.. +| 1424: bf 1f 07 52 d0 1a a4 ae 15 1e b5 01 0d ac 31 32 ...R..........12 +| 1440: 11 7f 67 b4 9c 20 78 28 8b 4a 79 32 44 c8 aa 1d ..g.. x(.Jy2D... +| 1456: 05 4b 94 3e e2 f0 d4 f2 16 3c 8e b7 67 13 98 47 .K.>.....<..g..G +| 1472: 2b 6a 1a 98 c9 82 7d a7 a7 7d ee 62 dd a8 29 58 +j.........b..)X +| 1488: 98 5b 2e 5b 62 1b 1b 21 37 4d eb e7 85 ac a3 8d .[.[b..!7M...... +| 1504: fe 69 ca 05 34 83 84 34 47 8d 5b b0 8d 71 f1 22 .i..4..4G.[..q.. +| 1520: 07 b5 a9 be 42 88 58 27 84 3e 37 1c 0f c7 1f 77 ....B.X'.>7....w +| 1536: 72 5e 2c fc 80 43 03 71 00 22 7f fe 5d fe ea a7 r^,..C.q....]... +| 1552: 33 3a c6 ae db e7 1d ba 8b 61 8c b9 b4 b2 ab b4 3:.......a...... +| 1568: 2f ec 9d b1 9c bb 13 7b 6e 3a b9 aa 43 b2 14 6f /.......n:..C..o +| 1584: f0 27 00 31 f6 5b 9c e9 96 40 3a 13 2f fc 6b ae .'.1.[...@:./.k. +| 1600: 0a 55 bf b3 cd 83 23 25 f1 15 e2 2e bc 7c 6b 29 .U....#%.....|k) +| 1616: 90 8f 85 ff 5b 5a 18 52 03 84 4d f9 f9 fa e1 a0 ....[Z.R..M..... +| 1632: e8 32 29 9c 5b 9e ed 39 b3 17 f8 ef 7f 55 b4 3f .2).[..9.....U.? +| 1648: c3 f3 66 f1 bc 15 1b 78 2c 9b ab cf e1 10 d0 46 ..f....x,......F +| 1664: 3d 21 2b eb 4f 8f 1d eb 8a 5e 87 23 72 50 04 ea =!+.O....^.#rP.. +| 1680: 8f 53 1b 3a 3f cf d5 92 5a 06 c1 18 25 33 41 84 .S.:?...Z...%3A. +| 1696: 77 56 bf a5 8a d1 ff 97 51 34 d3 24 88 12 dc 33 wV......Q4.$...3 +| 1712: 09 09 fc 68 b5 2a 4a cf 7c 73 d8 ff 93 29 40 1c ...h.*J.|s...)@. +| 1728: 8e ea 7d e3 7f 25 57 f5 bf 2b 19 80 4d fa 23 35 .....%W..+..M.#5 +| 1744: 18 8b ad af 46 85 3b 51 34 49 dd cb 39 d8 50 50 ....F.;Q4I..9.PP +| 1760: 7a a5 58 af 72 66 5d 21 c2 e2 03 1f ee bb d2 b4 z.X.rf]!........ +| 1776: 62 d2 f4 c0 fb 04 12 b2 35 0c 0f 0d b9 e2 a5 28 b.......5......( +| 1792: 87 5b 76 c7 39 4d 18 8d 3f 61 ab a0 84 4a 11 22 .[v.9M..?a...J.. +| 1808: e1 a9 69 55 2b 03 41 34 73 83 0d 0b ed da a6 d8 ..iU+.A4s....... +| 1824: f8 ff 9b d2 62 1b ca 1a 40 4f 0a 86 ad e7 92 af ....b...@O...... +| 1840: ce 69 19 8e 35 5d cd 50 4e 53 2d 90 46 90 ae 8a .i..5].PNS-.F... +| 1856: 43 ac 8b 30 7f 3b b3 05 78 63 b3 b7 0b 3d 2a 13 C..0.;..xc...=*. +| 1872: 55 83 ed a0 61 6b 12 30 5c 46 f3 1f 18 1f bd 89 U...ak.0.F...... +| 1888: af 86 9a 82 ed 89 35 0e 29 06 2c b8 97 c5 ef 46 ......5.).,....F +| 1904: 90 ce 6c 83 f8 7d 08 75 76 b4 07 6b 48 15 bb bd ..l....uv..kH... +| 1920: c2 fd 79 a8 7f 54 e4 d5 93 c2 17 09 3f bb 58 84 ..y..T......?.X. +| 1936: dd 78 7a 81 c4 13 70 e5 23 73 d4 60 25 0f 91 bc .xz...p.#s.`%... +| 1952: a9 8e 54 64 46 e1 8f 11 66 7c 1e 31 5b 9e 10 d9 ..TdF...f|.1[... +| 1968: 7e 09 e8 bc 2f 73 d4 f6 27 0c a0 62 5d ce 65 40 ~.../s..'..b].e@ +| 1984: f7 24 b0 bf f0 26 c0 17 1a dd d3 5d 16 35 22 10 .$...&.....].5.. +| 2000: 4d c4 cf 5b 22 2f 12 b2 5b f2 87 04 34 4d 5c 9d M..[./..[...4M.. +| 2016: 50 48 71 c7 8e bb 7d ce 89 f3 e4 4a c1 f2 92 4f PHq........J...O +| 2032: 90 4d 58 ac 27 60 a7 e9 31 41 ca b1 c0 38 ce 2d .MX.'`..1A...8.- +| 2048: df 40 e9 cd db 2f f3 28 09 ef 14 79 99 ef 66 1b .@.../.(...y..f. +| 2064: e6 c0 7f 4c cc 61 96 b6 f9 95 9d 1b 90 16 55 08 ...L.a........U. +| 2080: 71 ad 92 88 98 01 7f b9 8e 9c a2 f4 d4 24 33 a0 q............$3. +| 2096: 53 f2 01 df 47 3f 8d 15 a8 d1 70 a9 c2 70 f8 75 S...G?....p..p.u +| 2112: b4 ed b6 7d 82 95 63 5e 86 c1 3a 7c de b1 46 fd ......c^..:|..F. +| 2128: b8 30 42 36 83 35 e2 4a 78 c2 b5 b9 1e 57 f2 19 .0B6.5.Jx....W.. +| 2144: 91 6d ff 4f 87 97 55 98 a3 86 a2 54 1f 2f 0c 8e .m.O..U....T./.. +| 2160: 72 d6 a0 37 f2 bf 84 a3 a5 b3 87 12 a3 ef 0a 00 r..7............ +| 2176: 31 69 49 83 8c 6e 82 3a 84 62 a8 ce ee ce 91 61 1iI..n.:.b.....a +| 2192: b7 26 c1 fd bf 77 15 81 23 e6 d5 6f bb c4 ab eb .&...w..#..o.... +| 2208: 8b a7 fc b6 4e cb c1 61 ab 59 a2 dc ca ca f3 42 ....N..a.Y.....B +| 2224: 90 16 d8 d0 03 43 c1 e0 d9 25 50 b0 17 65 64 d0 .....C...%P..ed. +| 2240: 11 be 52 a8 e3 bd dd 7f 39 1e 0b 85 ae 92 c0 1c ..R.....9....... +| 2256: 0c 3b 06 2a bb be b6 c7 94 d6 83 6b b0 17 6d 24 .;.*.......k..m$ +| 2272: b1 b5 2d 22 5e 7f b9 db 47 65 e6 21 37 a2 2f d3 ..-.^...Ge.!7./. +| 2288: 03 8a 91 c0 5a de 52 27 dc 4b 9a 92 f1 fc f0 93 ....Z.R'.K...... +| 2304: d1 de 9e 04 73 13 c5 ec 25 36 54 15 94 92 b9 de ....s...%6T..... +| 2320: bd e2 39 b3 5c 88 8c e6 11 48 28 b8 0d 50 5a 6f ..9......H(..PZo +| 2336: d9 8a a6 17 4f a0 82 ff 94 70 28 10 0c 41 80 c4 ....O....p(..A.. +| 2352: d9 42 a9 fe d2 f7 1c 70 45 74 fa 65 d2 cf 49 00 .B.....pEt.e..I. +| 2368: b8 24 83 06 97 f5 1c 48 a7 dd 82 2f f0 77 f5 e6 .$.....H.../.w.. +| 2384: 5c 03 11 fa 65 46 57 90 c0 6f 1f 86 58 de 34 21 ....eFW..o..X.4! +| 2400: 5f 76 d7 1e 1a 16 6a e1 ad 26 ae 6a 32 53 30 8b _v....j..&.j2S0. +| 2416: db d9 05 93 22 87 58 e8 91 d8 26 80 85 f0 01 93 ......X...&..... +| 2432: 77 0e 88 91 bc bc ce e9 5e 6e e8 b8 aa 4e ad fa w.......^n...N.. +| 2448: a8 a2 5a 17 b8 88 56 f5 71 8a 70 fe 83 4f 5c 8c ..Z...V.q.p..O.. +| 2464: 07 1a 45 cf a9 89 05 c6 81 79 90 a5 d2 53 a4 3e ..E......y...S.> +| 2480: ac be 52 ae aa 9d 30 66 c5 b7 1f 7a c8 8e 6a 3b ..R...0f...z..j; +| 2496: 82 54 6a 62 aa 6e 4a c4 02 11 5b 69 12 6f 84 23 .Tjb.nJ...[i.o.# +| 2512: 17 f6 3d 81 1f 29 60 28 7c e2 95 b4 ae 39 e6 6b ..=..)`(|....9.k +| 2528: 5e c5 df 82 66 82 57 d4 84 cd 2b 1e f2 a0 31 82 ^...f.W...+...1. +| 2544: 8b 9f 0e 0c 72 76 6b 6c 5b cb 0c 5c 2a 77 22 df ....rvkl[...*w.. +| 2560: 1e 96 44 f9 4e 22 dd 22 ff fc 14 a2 cc 36 77 02 ..D.N........6w. +| 2576: 81 8f 22 1e 46 ea 11 e1 85 41 8a ee 69 64 e6 27 ....F....A..id.' +| 2592: 8b 46 0b 4a 47 35 f4 72 71 62 a1 0c 4d 55 be a0 .F.JG5.rqb..MU.. +| 2608: 1f ae ae 8b a6 2d de 54 04 24 05 98 06 43 04 57 .....-.T.$...C.W +| 2624: 09 8e ff 81 ff 9c 9a 18 02 bf c3 66 8f 65 fb e3 ...........f.e.. +| 2640: 66 0c 20 bb 50 f5 0c a8 a8 e9 6f 53 65 b5 3a e4 f. .P.....oSe.:. +| 2656: 48 d2 1c 86 e2 a6 35 c0 91 d4 72 b6 67 21 49 fb H.....5...r.g!I. +| 2672: 0c f9 91 b8 64 46 c2 75 28 df ac c0 bc 4f 61 d9 ....dF.u(....Oa. +| 2688: 92 06 6b 48 d8 29 ba 4f e6 40 a8 c8 35 8b 83 e6 ..kH.).O.@..5... +| 2704: 79 ec a1 d3 c1 73 82 64 42 13 3c 7b 73 3e 7a 14 y....s.dB.<.s>z. +| 2720: 2d db ac 00 76 00 81 ae fb c1 30 7b dc 57 39 f5 -...v.....0..W9. +| 2736: 27 6b 1a d5 ed c2 4b 21 12 3e 4d e0 8e 58 69 9b 'k....K!.>M..Xi. +| 2752: f5 cb 26 7a 62 e5 7b 3a fd 1a 8a e6 7d e6 f1 60 ..&zb..:.......` +| 2768: cc 4f 30 39 76 b7 3d 49 3f b7 59 a3 d0 a6 c8 8e .O09v.=I?.Y..... +| 2784: 3e a8 e1 a0 1b 7d 82 39 8b c6 a5 87 3e 2c da 16 >......9....>,.. +| 2800: f0 71 cc 5e 02 17 49 6d 48 24 8c 19 9e c4 d1 97 .q.^..ImH$...... +| 2816: a6 cc 28 28 c9 3e c2 e3 19 6a 05 f2 bd 4c ef 44 ..((.>...j...L.D +| 2832: df 84 da 8c 3f 41 16 2f 87 b8 88 bd 6d 8b 6e dc ....?A./....m.n. +| 2848: fa b0 44 08 ee ef 22 84 bf c0 5c a4 2f 2c 7e a2 ..D........./,~. +| 2864: 50 8b 84 cd 60 08 d8 53 4b 2d 4f 1e 3b 14 b4 62 P...`..SK-O.;..b +| 2880: 97 a1 66 49 2a 6f 3b 36 85 c2 42 58 98 5a db 3e ..fI*o;6..BX.Z.> +| 2896: 64 fd ad 32 f2 e0 f9 5a 13 dd e1 68 6c 35 34 a5 d..2...Z...hl54. +| 2912: 3c 9a b1 8a 51 78 49 5b c6 ef 57 7f 6e de fe 2d <...QxI[..W.n..- +| 2928: 0a 2c 1b 74 3f 19 bf 8e 6d 2f 0b 22 91 36 95 5d .,.t?...m/...6.] +| 2944: a3 65 b6 3b 3a e4 de 70 9e 29 f4 c6 0a 23 27 d6 .e.;:..p.)...#'. +| 2960: d1 4a f6 2e 2b a0 ee 8d cd fc 42 7a c8 1a a6 36 .J..+.....Bz...6 +| 2976: 17 aa f4 03 b5 cc e2 54 7a c4 fb 27 e5 90 62 20 .......Tz..'..b +| 2992: 03 77 4d fc 35 7c 59 87 01 49 86 ae 82 6b 8b a7 .wM.5|Y..I...k.. +| 3008: bc b0 b0 dd f2 ef f7 ef 0a 36 a6 25 f1 70 ba 89 .........6.%.p.. +| 3024: e8 00 f4 fc c0 98 73 f2 b0 9a a2 ed d5 1e 17 d1 ......s......... +| 3040: 79 82 18 84 b0 f8 2a 5d f2 a1 03 d6 45 b0 01 26 y.....*]....E..& +| 3056: 19 01 6d b3 0e 5f 55 6c 2b 21 72 33 84 a9 ab fb ..m.._Ul+!r3.... +| 3072: 64 4d bc f0 1d 16 ae aa 09 c1 29 60 e2 63 e1 d5 dM........)`.c.. +| 3088: 84 41 6e 5c 12 08 9a 04 dd 27 b8 fe 2f fb ca 83 .An......'../... +| 3104: 2a 7b eb 05 3b 77 fd 42 31 42 84 98 89 24 3c cc *...;w.B1B...$<. +| 3120: 47 f6 bc 13 37 d7 97 98 c4 61 24 50 0b 9e e5 53 G...7....a$P...S +| 3136: 46 05 49 5e a5 51 d5 48 26 fa 31 eb 0e 76 14 16 F.I^.Q.H&.1..v.. +| 3152: e9 60 f6 05 d5 bb 47 85 e2 da f6 5a 0a c0 14 38 .`....G....Z...8 +| 3168: bb 70 4d be eb d8 6d 10 61 d6 9b c6 c4 f4 56 7f .pM...m.a.....V. +| 3184: ff b8 fb 1f 92 a3 f5 74 78 7f c0 8d 9d f4 b1 a2 .......tx....... +| 3200: f4 0c 33 da 98 9e a2 61 4c c7 41 9c ea 0f 33 54 ..3....aL.A...3T +| 3216: 40 54 31 c3 04 fa d1 4b 67 80 e2 a7 6a 77 c6 ca @T1....Kg...jw.. +| 3232: 04 fc 71 ea fa 0f 92 8e d3 40 e8 0e 1c 48 a4 55 ..q......@...H.U +| 3248: 7f 74 67 ea c9 29 67 73 b9 b9 73 b1 00 1a d4 0b .tg..)gs..s..... +| 3264: 21 95 d6 1c b4 68 2b c5 e1 18 40 7e 8e 09 6f 28 !....h+...@~..o( +| 3280: 88 2d 6f 24 d3 73 7b 89 7a a6 aa df ad ae 7b 14 .-o$.s..z....... +| 3296: d9 f0 ff 20 ba fd bf a7 d7 04 6c 35 5c 76 4e f5 ... ......l5.vN. +| 3312: d3 5b a9 2b 8f 3a 11 fa 35 26 eb 78 45 da cb 00 .[.+.:..5&.xE... +| 3328: 78 97 b1 49 82 4e c1 4d b1 aa b7 80 75 fb 20 75 x..I.N.M....u. u +| 3344: cb 3f ba 05 95 33 cd e9 b3 bd b2 84 c4 4f df af .?...3.......O.. +| 3360: 77 c2 44 24 57 01 f9 9d c1 ef b6 ce 01 6f a6 5d w.D$W........o.] +| 3376: 3d 4b 12 e9 8f c2 a6 d5 1c 3b e4 05 83 48 aa 78 =K.......;...H.x +| 3392: 4b 3a 1c 1b ad 8e bc 49 c5 ee 91 68 28 8d 74 74 K:.....I...h(.tt +| 3408: a0 e1 20 ba 3d 62 97 0a 40 58 6b 1a c9 be 53 00 .. .=b..@Xk...S. +| 3424: 5d 9f e7 8e b6 05 7a d2 d1 89 ac fd 7f 9b ec 19 ].....z......... +| 3440: d0 95 35 e6 41 32 eb 68 85 eb 06 8c 53 f2 25 f3 ..5.A2.h....S.%. +| 3456: 19 d9 1c e7 a8 c2 c3 1c cd 78 50 4d 73 2d 5c 15 .........xPMs-.. +| 3472: 7c 3e b5 f0 e8 64 f5 72 a1 4a 9d f3 87 1d 12 0b |>...d.r.J...... +| 3488: d6 50 21 18 ca 10 f9 29 32 53 14 54 08 ef 51 6d .P!....)2S.T..Qm +| 3504: 6e 20 ae f1 3c b5 6b 8b 4c 4d 68 9f 73 99 2f d2 n ..<.k.LMh.s./. +| 3520: f2 ea ba bd 72 e4 d5 de 71 c2 15 a2 57 f7 69 c0 ....r...q...W.i. +| 3536: c4 6b cb c1 72 8e a8 fe f8 79 d2 6d 97 82 bf a1 .k..r....y.m.... +| 3552: 11 35 dc 30 36 e5 32 b5 81 f7 2b 58 06 c3 28 3a .5.06.2...+X..(: +| 3568: 43 c0 bd 16 a4 b8 f7 37 2b 1f 5f 8a 2a 09 21 4e C......7+._.*.!N +| 3584: d7 14 35 31 e6 36 8d 6f 2b 2a e2 63 1f 59 8d 62 ..51.6.o+*.c.Y.b +| 3600: 16 cd 16 d5 5a 20 b0 4b 9c 44 4d bb 0e 8a 01 6d ....Z .K.DM....m +| 3616: 6c 2e 09 48 6c 32 f5 96 45 0b df 3a a4 09 a4 1c l..Hl2..E..:.... +| 3632: 44 81 72 60 8a ed 10 29 c0 62 da ba 51 4f a0 7d D.r`...).b..QO.. +| 3648: e0 9b ed 31 6a 0e f8 b5 f4 69 b2 15 d4 01 ed c5 ...1j....i...... +| 3664: e2 09 df cc 97 13 70 57 48 1b bd 4a ad 0b ad 8a ......pWH..J.... +| 3680: 80 3d c1 c0 c7 f2 2d d9 a8 b7 3f b8 e5 aa 0f 5c .=....-...?..... +| 3696: e6 95 2a c6 80 83 69 ca 0e dd f9 7e 04 48 7d d3 ..*...i....~.H.. +| 3712: d5 29 fd 9d 7f 59 d7 1d 9b cd c3 06 e7 51 13 bc .)...Y.......Q.. +| 3728: 60 d5 c3 d5 a9 0b 78 ab 01 3e 34 06 4f 8a 47 4a `.....x..>4.O.GJ +| 3744: 0d a9 c1 b5 29 c4 26 c4 5d cb bd 80 f1 d4 eb b9 ....).&.]....... +| 3760: 6b 7d 2f c3 95 7c 2c e3 86 c9 5d 41 ee 76 89 9f k./..|,...]A.v.. +| 3776: 3a f3 e2 c8 6e f5 37 a7 68 01 06 26 47 9c 1b aa :...n.7.h..&G... +| 3792: 5a dc 55 89 af 53 89 9f f2 30 ee 04 90 cc 30 de Z.U..S...0....0. +| 3808: f4 2a c9 23 59 9d cf 65 7d 2c 30 7e ae 7d 08 43 .*.#Y..e.,0~...C +| 3824: 1f a9 b8 14 3b e8 3d 1e 3e 4b 46 93 18 4b 08 ee ....;.=.>KF..K.. +| 3840: 53 f4 07 0d 4e 69 87 a4 61 9f b0 75 a7 fc 64 34 S...Ni..a..u..d4 +| 3856: 86 aa a2 82 21 40 d7 7a 03 30 d0 f4 47 d2 4a ff ....!@.z.0..G.J. +| 3872: dd 27 02 69 99 d2 c9 61 d0 fa 46 dc 07 1d f7 0b .'.i...a..F..... +| 3888: 39 0b a3 2d 7f 9f 1b ce ca 79 69 62 bc 50 1a 07 9..-.....yib.P.. +| 3904: 2c 1b 38 37 9f 93 65 dc 45 3a 1e 2b ff 3f d0 bb ,.87..e.E:.+.?.. +| 3920: 05 f2 9b 36 b5 79 9c ca 4f 7d 8a 2f 6e 62 1d 20 ...6.y..O../nb. +| 3936: ab 6e 5f 75 19 67 e9 4a e1 c4 6a 2e 8a 06 b0 36 .n_u.g.J..j....6 +| 3952: 37 47 5f a1 27 93 d4 36 80 85 f9 29 94 db 83 13 7G_.'..6...).... +| 3968: f5 16 1e 0e 6b 6c b1 c1 f2 66 0a 8e 63 ea ca 6b ....kl...f..c..k +| 3984: f3 74 91 9e 73 62 d4 09 bd a9 64 54 e1 64 b0 e4 .t..sb....dT.d.. +| 4000: b6 47 e0 ff 92 09 88 39 e1 44 f8 50 20 b2 3c 6b .G.....9.D.P .....|..g.>@.E.y +| 1104: ad 46 a6 a7 42 e7 7c 05 62 a4 28 61 03 fc 1c e4 .F..B.|.b.(a.... +| 1120: 48 2e f1 ef d9 09 23 04 f4 d7 c2 ad 73 c4 d8 69 H.....#.....s..i +| 1136: cf 15 ef a5 1a ba db 63 7c d8 e9 4c 70 cf 08 d8 .......c|..Lp... +| 1152: 62 d3 ec 60 a0 93 06 fe 50 87 b5 22 48 37 b8 46 b..`....P...H7.F +| 1168: 30 26 ed 8d 2c f3 ed c1 51 62 e8 ce 13 45 b4 4a 0&..,...Qb...E.J +| 1184: 9a 3c 76 c4 fd a6 e2 de 26 04 93 69 30 3c 26 4d .1..3. .. +| 1248: 2f 8f 00 d5 47 ae 22 01 bf 83 40 da c4 74 43 56 /...G.....@..tCV +| 1264: 32 35 6e c9 3a 82 44 0e 86 32 5a 8e 46 5c 16 23 25n.:.D..2Z.F..# +| 1280: e5 dc 7e 38 26 fd ba d1 d3 d0 7e 4e 88 6e 2a 85 ..~8&.....~N.n*. +| 1296: d1 25 70 b4 fc 55 2b 5a 5b 4e 01 1b 20 21 d2 2b .%p..U+Z[N.. !.+ +| 1312: c1 5b e9 3e f1 fe 3f 65 18 07 a6 10 3a 88 e5 9b .[.>..?e....:... +| 1328: 4d 83 91 f2 1f b5 03 2a cb 57 21 e2 20 fc 35 99 M......*.W!. .5. +| 1344: b1 49 4a 0c b9 04 35 b9 b0 43 52 14 e7 2d 03 30 .IJ...5..CR..-.0 +| 1360: 73 cc 40 39 c3 d6 58 1b 90 85 df 21 6b e4 03 87 s.@9..X....!k... +| 1376: e1 de 4c 36 2c f3 c1 0b 75 92 a9 11 6e 9b de ac ..L6,...u...n... +| 1392: cb 26 b9 8d 48 47 f3 63 44 30 b0 e6 84 31 ec aa .&..HG.cD0...1.. +| 1408: f9 64 3b b2 12 b4 13 46 be 97 6f 36 37 3a ea 4b .d;....F..o67:.K +| 1424: ef 0d d6 04 44 21 e3 25 72 6c 70 d5 58 87 7d 16 ....D!.%rlp.X... +| 1440: 37 39 d3 0a 7b fe e2 81 d1 50 6a 62 6c a8 86 ca 79.......Pjbl... +| 1456: b9 f4 4e b5 04 c3 97 1b 4e 65 49 ee f4 cd 9c e4 ..N.....NeI..... +| 1472: 89 7d 7e c8 29 9b 3e 88 82 b7 68 c4 9b ea 75 48 ..~.).>...h...uH +| 1488: 1b 5b 93 0d 12 86 df 35 44 28 95 6c f2 c4 e5 3b .[.....5D(.l...; +| 1504: 89 a7 bf 11 ce 79 76 99 bb 0d 01 2e 58 e3 e1 0d .....yv.....X... +| 1520: 06 19 5e 28 bc 06 32 4e 76 bd e5 01 a0 20 5c 3f ..^(..2Nv.... .? +| 1536: 15 ad e8 4e 64 5a 67 a4 77 3f 23 c2 be 0d 96 b5 ...NdZg.w?#..... +| 1552: 28 e9 c3 60 ec 5d 09 73 e9 dd 40 a8 7d a0 ac 4f (..`.].s..@....O +| 1568: ef 82 ac 4d 25 19 a8 1b 22 71 c2 fa 97 c3 27 da ...M%....q....'. +| 1584: 44 14 f0 41 26 20 fd 57 4f 4e 9d f5 b3 d5 7c 72 D..A& .WON....|r +| 1600: f4 e5 56 0c 8e e8 c0 39 1c 3c 0e bb ce 8f 08 cb ..V....9.<...... +| 1616: 7f dd 0c fc 11 17 4b 41 03 b6 43 17 25 92 84 8b ......KA..C.%... +| 1632: f8 51 57 92 93 14 25 11 83 ce da bf 04 0d ed 32 .QW...%........2 +| 1648: 07 dc 6d b5 e1 c5 74 42 07 06 2a eb 2c a0 78 a7 ..m...tB..*.,.x. +| 1664: de 21 66 0b ea 49 fc e9 26 87 63 5c 8f 66 59 95 .!f..I..&.c..fY. +| 1680: c1 cd e9 00 79 05 0f 57 d0 d5 a9 1e 48 f6 7a 9d ....y..W....H.z. +| 1696: 80 6d 5c 74 55 3d fd 67 0e ff 74 26 70 bf c3 9b .m.tU=.g..t&p... +| 1712: 5f af 8b 43 69 e0 49 31 63 2b e4 9f 59 09 0e 5a _..Ci.I1c+..Y..Z +| 1728: 18 84 b3 f4 19 17 18 16 5b fd ed 78 5f 68 ec 2c ........[..x_h., +| 1744: d7 66 05 6e 47 eb 9b a1 52 90 0f 34 99 4f a0 fd .f.nG...R..4.O.. +| 1760: 21 f7 4d da f6 ae e4 57 51 63 59 e1 ee b8 32 6a !.M....WQcY...2j +| 1776: 03 ac c4 c4 69 73 10 32 d5 94 25 7e 6b d0 ec 62 ....is.2..%~k..b +| 1792: 0d 17 f7 16 24 b5 df 11 7b 29 01 03 d8 ff 1a 2f ....$....)...../ +| 1808: 7f 51 ab 9e 41 5b aa 0c a5 5d e9 69 ce 3b 85 4e .Q..A[...].i.;.N +| 1824: 7d ff 5a 37 b5 74 18 78 85 95 4d 44 8a d9 8c ea ..Z7.t.x..MD.... +| 1840: 2e 10 6e e6 cb a1 3d ef 57 38 f0 9b f0 8b 0b 68 ..n...=.W8.....h +| 1856: f9 f8 9f ac 58 da f3 9c 2b 2a 90 c7 4d 00 e4 4c ....X...+*..M..L +| 1872: 46 59 01 1c 0c 6b 08 6d 33 07 41 d0 5d 23 20 3b FY...k.m3.A.]# ; +| 1888: 00 e0 6b 32 2e 76 3a 7e 09 3e c9 66 90 4f d5 9b ..k2.v:~.>.f.O.. +| 1904: 8a bd b1 6d f9 33 d9 5a b3 bb a1 2a 45 31 0b c6 ...m.3.Z...*E1.. +| 1920: ce 44 34 5d 88 61 89 d1 cc 7c 14 86 a3 d7 6b 6b .D4].a...|....kk +| 1936: 22 d2 98 ab 62 7c 43 2e a8 92 12 de 43 d4 0a 9d ....b|C.....C... +| 1952: 94 f0 41 2d 14 86 69 fe 98 71 1a a2 2b db 5b ec ..A-..i..q..+.[. +| 1968: 23 00 e3 a3 03 1d 31 97 94 e4 85 1e 5f 53 44 66 #.....1....._SDf +| 1984: 0a 21 df ec ca 6a a6 a3 f4 c8 00 40 8e 0f 19 3f .!...j.....@...? +| 2000: 36 69 a3 7c de fc 73 16 d8 d3 f9 ff 03 ac 94 dd 6i.|..s......... +| 2016: 38 d5 d9 20 2e d8 97 2a 22 9d bb a5 cc 04 07 8f 8.. ...*........ +| 2032: be 18 b4 a8 ae e5 b4 45 27 7f e2 7e 3f 91 67 98 .......E'..~?.g. +| 2048: 86 3d 3c 05 3a 1e 71 bf 40 f8 a6 ca 48 7a ca c7 .=<.:.q.@...Hz.. +| 2064: b7 67 10 bd 99 79 53 32 70 b0 74 fc d9 93 60 c8 .g...yS2p.t...`. +| 2080: 3e 39 5c fa 3c 97 34 70 57 a3 0f f8 ab 4e 3f a6 >9..<.4pW....N?. +| 2096: a5 25 26 ea 0c 60 fe 32 f3 7a 68 d0 9f 50 a4 45 .%&..`.2.zh..P.E +| 2112: 13 2e f8 41 3f e9 d1 9f 78 4a d0 a7 d3 57 3c af ...A?...xJ...W<. +| 2128: a6 69 6f 2b 4e f0 86 b5 96 2d b2 de 06 5b 68 b2 .io+N....-...[h. +| 2144: 9a b9 de e7 7d 54 0a ff b1 26 17 ea bc a7 37 ff .....T...&....7. +| 2160: 46 7b f7 00 90 ec 71 92 cb 5a b7 c8 7f 45 60 ec F.....q..Z...E`. +| 2176: 9e 30 ca 80 69 4c 07 4d c8 c9 53 c4 77 3c 93 15 .0..iL.M..S.w<.. +| 2192: 17 63 15 c0 3a 1d a2 02 82 6d 4b ed 50 4c 5d 93 .c..:....mK.PL]. +| 2208: 2d ef bf 1e 9a ff 04 26 1a f6 7b da c9 21 7b 50 -......&.....!.P +| 2224: 4d 2f 53 c4 1e b4 dc 9f 5f 33 26 80 4a 8c ef 54 M/S....._3&.J..T +| 2240: fb 58 95 55 3c ec c0 a7 c5 78 a0 91 08 b4 d6 ce .X.U<....x...... +| 2256: d8 25 27 2b 37 e9 63 72 94 c3 89 5a 58 85 f4 95 .%'+7.cr...ZX... +| 2272: 09 fe db 6c 9c 71 54 af 1a 0c eb 2d e6 a9 db a0 ...l.qT....-.... +| 2288: 04 2c 29 70 12 b2 7e 66 ae 25 ce b4 b9 5c a2 12 .,)p..~f.%...... +| 2304: 1c 75 10 d5 54 f8 04 c5 8d be 38 29 64 f8 29 00 .u..T.....8)d.). +| 2320: 07 35 e4 e7 6e bc 64 db 39 d8 98 ee 72 28 a8 8b .5..n.d.9...r(.. +| 2336: 0f 1b 87 26 6e d4 73 1b ef b6 d4 db 05 12 1b c7 ...&n.s......... +| 2352: c6 1e 02 b1 ab bb e2 29 81 9a 9e b6 80 22 6e b4 .......)......n. +| 2368: 3c 30 ad e7 8b a9 60 7e de 01 a3 74 0b 76 0b d6 <0....`~...t.v.. +| 2384: e8 a0 91 65 ce bb 6a d7 96 cc fa 96 d0 c0 39 d0 ...e..j.......9. +| 2400: 54 80 1d 8d 79 17 11 86 c5 8f a7 7f 57 92 8d b2 T...y.......W... +| 2416: 41 5b 86 61 94 23 99 77 b0 5d 5d f9 29 86 36 c8 A[.a.#.w.]].).6. +| 2432: ea f8 2f b5 7d 42 66 fe c7 0d 19 17 e6 6f c5 79 ../..Bf......o.y +| 2448: e0 22 24 70 eb 79 23 ec 45 77 d3 44 8f 74 15 bc ..$p.y#.Ew.D.t.. +| 2464: cf 46 cc ff 5b 05 1e 03 7d ea c5 82 14 11 86 4b .F..[..........K +| 2480: cb 23 5b e0 1d 96 ca ea 62 d8 71 56 c4 1f d6 ed .#[.....b.qV.... +| 2496: 9e 8f bd b0 4a fe 53 87 c8 16 1a 31 cb 25 ba 18 ....J.S....1.%.. +| 2512: 66 c3 b2 a7 91 68 ff b9 2b bc c4 f5 50 36 79 e5 f....h..+...P6y. +| 2528: 33 c3 d4 98 e1 fd f9 3a c7 a1 ef 79 29 c1 bf 7e 3......:...y)..~ +| 2544: db 7b 66 3b 95 37 02 c7 ee ca 7f 9f 50 8f 47 5d ..f;.7......P.G] +| 2560: af 45 29 e3 f5 7e 07 93 73 a0 42 ef 05 a6 2b 74 .E)..~..s.B...+t +| 2576: 92 ab 8a 9a 19 be 2f 21 cf 6a 90 02 a3 1f f4 5c ....../!.j...... +| 2592: 72 b3 31 00 85 9b 20 d8 f3 c5 00 a7 15 30 56 6c r.1... ......0Vl +| 2608: 0f 7c 84 be 55 5b 8b aa df d6 bd 9e d9 55 54 93 .|..U[.......UT. +| 2624: 44 7d 9a 9b b8 38 46 b7 1e 1c 75 17 bb 29 05 5f D....8F...u..)._ +| 2640: da 39 7e fe 9e 40 e3 a5 7c a2 ba 5d 7e 9d 14 57 .9~..@..|..]~..W +| 2656: 19 10 2a d1 85 d3 06 0f d4 d8 ac f0 03 8a bd 61 ..*............a +| 2672: 26 89 4e c6 92 0a 4b 0b a2 3e 1d 5e 88 8c ce 33 &.N...K..>.^...3 +| 2688: 47 e7 4e 69 0c e3 09 ef 3a 9f af 19 ae 83 68 21 G.Ni....:.....h! +| 2704: 75 c3 54 7b 00 a5 f3 d2 32 41 69 ac fc 31 2e 0c u.T.....2Ai..1.. +| 2720: 91 dd 90 78 2c ab 3f 0c 37 a4 8c 69 a9 ae 41 8e ...x,.?.7..i..A. +| 2736: 6b b5 b5 ee bf df bc 1c 61 bb c9 51 37 a5 d5 96 k.......a..Q7... +| 2752: 0c 26 aa 97 67 72 73 cc c1 92 d3 46 cd 3e d8 ad .&..grs....F.>.. +| 2768: e6 8b 51 ee 41 ed 39 57 65 2e 5c 39 2b eb a7 72 ..Q.A.9We..9+..r +| 2784: b6 4c a8 87 72 a4 16 b4 d9 90 db e9 25 11 53 4e .L..r.......%.SN +| 2800: ef 24 1d b0 21 bc 97 52 38 53 49 c8 31 c1 c4 9f .$..!..R8SI.1... +| 2816: 2b 13 0c 10 5b 4a 0c 6e 07 c0 ec d4 77 f0 f0 38 +...[J.n....w..8 +| 2832: a6 88 d4 28 40 30 76 f4 ab 2e 94 b5 6d 47 79 84 ...(@0v.....mGy. +| 2848: f9 aa 28 32 66 c8 aa cf 17 18 3d 92 0e 66 4e fe ..(2f.....=..fN. +| 2864: 5d 80 51 29 df 97 de a4 c6 57 23 67 84 f4 32 86 ].Q).....W#g..2. +| 2880: 51 03 b6 67 29 54 74 87 da c0 41 e9 3a 3e 07 02 Q..g)Tt...A.:>.. +| 2896: d9 85 dc 55 e7 23 60 80 b0 01 48 cd 59 21 82 fe ...U.#`...H.Y!.. +| 2912: 14 65 1a 5d 9e 5e 2b 69 52 ee 64 01 4d 46 ac 94 .e.].^+iR.d.MF.. +| 2928: 60 04 d9 2c 41 e3 5b 35 e7 cc 75 06 7d ff 48 ae `..,A.[5..u...H. +| 2944: 13 e5 4f 54 f6 78 86 c5 c4 99 58 02 41 87 a9 82 ..OT.x....X.A... +| 2960: 34 95 75 b2 e5 5e 92 23 a1 7b 3c 7b 1d 94 dd 5f 4.u..^.#..<...._ +| 2976: f6 56 07 06 41 12 a0 56 7a 15 01 58 1f 9f 15 1a .V..A..Vz..X.... +| 2992: bd 2e c6 ea b8 29 ae 13 19 a6 40 b0 8d ec 3a cd .....)....@...:. +| 3008: ca 6b b4 d5 96 95 fe 8d 34 23 aa ab df c3 23 fa .k......4#....#. +| 3024: c4 02 eb 10 8c f2 e8 e0 5f d4 e9 4a ae f4 8d 60 ........_..J...` +| 3040: a9 1f 65 40 93 26 bf 1c 49 9d b6 8f e3 f1 c7 0b ..e@.&..I....... +| 3056: a7 bb d1 ae 56 60 5c 80 d8 e0 e4 f0 72 62 45 b7 ....V`......rbE. +| 3072: 31 33 66 a7 17 a4 34 67 f6 55 b1 09 eb 75 8f f0 13f...4g.U...u.. +| 3088: cd c1 86 e3 f5 3a e7 d8 08 da 4c a7 b4 45 8b ce .....:....L..E.. +| 3104: 9c 8d 48 41 b4 3e 6b 2d fb be c3 e7 bf 80 d3 29 ..HA.>k-.......) +| 3120: 9f 8c 5d 7e 48 76 dc 95 cc 30 bf 9f e1 e0 dc c1 ..]~Hv...0...... +| 3136: da 9c 73 18 3c ff 53 49 88 dc ef 90 20 5f 3f 75 ..s.<.SI.... _?u +| 3152: 85 02 0c d4 c3 f4 c1 ed f7 82 05 c4 e9 80 97 65 ...............e +| 3168: 09 2a ac 52 c5 77 6c f4 6b 35 30 c2 fd 38 48 ae .*.R.wl.k50..8H. +| 3184: b6 f6 38 0f 87 1a 66 54 91 3a 5b a7 48 5c 9d 36 ..8...fT.:[.H..6 +| 3200: e5 7f 60 4d 68 19 ff 8d 2e 6a 8a 99 37 72 11 f9 ..`Mh....j..7r.. +| 3216: 5b 29 77 3e 76 cd 57 6b 07 d0 cf d6 b4 cd d0 1e [)w>v.Wk........ +| 3232: dd 6e 1c a1 bd 7f bd 9d 57 0d 14 25 9e 4a 36 8c .n......W..%.J6. +| 3248: 9f 7b b4 f9 db 57 22 f1 0b 47 e8 2e 04 70 e2 d6 .....W...G...p.. +| 3264: 29 0e bb 52 33 80 6b 12 a5 20 97 3b 60 01 a2 8c )..R3.k.. .;`... +| 3280: 74 68 0f b1 b0 44 63 04 fe 69 2f 98 ee d9 e9 39 th...Dc..i/....9 +| 3296: af 0d 7a 79 00 ca d4 3e 96 e1 27 f8 27 e6 a7 d5 ..zy...>..'.'... +| 3312: 0c 7d 0c 13 de 31 bd 75 cf 41 04 1f 03 bb df 1d .....1.u.A...... +| 3328: c7 c0 3b a8 15 d8 15 4b b5 89 ff 4c 16 31 7a 62 ..;....K...L.1zb +| 3344: bb f8 d0 f2 43 fe 03 68 b0 35 7a 33 ae 4e a9 0b ....C..h.5z3.N.. +| 3360: 6a c8 35 d3 2c bf 6c 35 7b 1d 4c 9d 62 c1 96 01 j.5.,.l5..L.b... +| 3376: cf 7f 5f f0 20 ce 16 25 32 bc b2 bb 29 7a 6c f5 .._. ..%2...)zl. +| 3392: 36 17 6e 1b 93 4f 0f f0 de eb 67 a8 be 6a ad 7a 6.n..O....g..j.z +| 3408: e6 42 1f 63 d2 de 72 1b f6 01 08 e8 72 d6 85 81 .B.c..r.....r... +| 3424: 3d ec 80 70 f3 84 5c 54 2a 2d 66 f8 e9 ba d7 7c =..p...T*-f....| +| 3440: af df 2a 43 f4 6c 78 3e 5d e6 92 b4 9f 02 05 bb ..*C.lx>]....... +| 3456: 84 9e ed 44 71 74 21 c9 be 07 3b 98 b1 49 b7 81 ...Dqt!...;..I.. +| 3472: 6a 4a c4 08 28 1c 19 0f ae 36 c2 6c a2 55 af 09 jJ..(....6.l.U.. +| 3488: 47 da e4 a8 32 df 4f 68 83 b1 c8 89 a0 1a e1 72 G...2.Oh.......r +| 3504: b2 dd ab e9 fd 2f c8 79 d1 7e 69 16 e7 72 85 09 ...../.y.~i..r.. +| 3520: 95 77 18 1f 2c d1 51 31 68 88 8f 0a fb a1 4f 3b .w..,.Q1h.....O; +| 3536: 6d fb 24 cc 0a 77 34 cb 49 35 06 07 a6 cd 5e 27 m.$..w4.I5....^' +| 3552: d6 87 cc ae e1 00 cc 69 22 0e 4f 5a 59 cc 8f 30 .......i..OZY..0 +| 3568: c3 65 8f 8d bc 1a 86 8f 02 9b 0a 2c 5b b5 3d fa .e.........,[.=. +| 3584: c9 fd 1c 96 4c 59 e1 ac 43 17 e8 b1 18 57 c8 48 ....LY..C....W.H +| 3600: c7 df e9 d1 d3 11 d9 4b 44 33 d9 da 9d 86 7a cb .......KD3....z. +| 3616: de 17 ce 40 aa b8 2d b6 26 a9 68 e8 72 f3 cb e0 ...@..-.&.h.r... +| 3632: ae ec b1 1e 76 0b 5c 49 93 f5 f1 04 56 fa ac 33 ....v..I....V..3 +| 3648: 65 de d4 36 6b 3a 3c e1 a7 7b b6 66 e1 69 e7 86 e..6k:<....f.i.. +| 3664: c0 42 a2 71 e5 f1 35 3b 38 1e 52 a2 bb 72 39 de .B.q..5;8.R..r9. +| 3680: e2 78 23 3a 71 b3 0b 8d 94 e5 26 80 9d d3 67 76 .x#:q.....&...gv +| 3696: 14 aa 98 b4 f8 42 78 6c a1 8f 7a bd 5a 5f 99 1f .....Bxl..z.Z_.. +| 3712: 14 5a d4 02 bf c3 60 a7 c7 34 8f 81 4b 4a f6 c2 .Z....`..4..KJ.. +| 3728: 21 0e 6e 26 55 79 10 67 13 dc da ac 80 37 ad 95 !.n&Uy.g.....7.. +| 3744: 11 4a 54 b5 f9 da f6 66 26 99 51 98 1e 05 c0 5b .JT....f&.Q....[ +| 3760: 2d 23 b8 a2 5b de c3 f4 4f 2b 3a 22 10 1c 25 cc -#..[...O+:...%. +| 3776: 57 db 91 c4 6c df 33 a1 1f d6 a3 2e 56 de 54 d1 W...l.3.....V.T. +| 3792: 02 ac 2d ff 96 d6 db ec 80 f7 79 ea 81 30 68 d0 ..-.......y..0h. +| 3808: 90 b2 d1 cc 5e ac 50 df 5e 1b 0a 78 01 70 e8 21 ....^.P.^..x.p.! +| 3824: 35 08 64 46 9a 55 91 aa 9e 41 df 4f 5b 2b 3e 1c 5.dF.U...A.O[+>. +| 3840: bc f5 90 37 d7 35 ac 34 8e 70 94 28 46 eb c8 42 ...7.5.4.p.(F..B +| 3856: 10 df 40 b6 c0 76 57 6d 79 d6 f7 b6 cf ca 88 bc ..@..vWmy....... +| 3872: f9 f6 ce ad 69 d8 a6 a7 41 cb cb c6 bf 91 ef ff ....i...A....... +| 3888: 72 6f d2 b0 1e bf db 33 4b cd 1b 81 2d 65 de ff ro.....3K...-e.. +| 3904: 5c 4b 45 f0 57 c3 5e 1f 77 6d 14 b7 ea e4 8c 88 .KE.W.^.wm...... +| 3920: d0 b1 63 86 fc 68 26 e3 b2 7a 5f ca ec 09 00 4b ..c..h&..z_....K +| 3936: dc f0 d4 70 fd c1 10 36 a9 cc fc 0d fd 24 12 13 ...p...6.....$.. +| 3952: d9 bd f0 2f 87 99 00 d1 d4 7c c6 41 f3 59 2a db .../.....|.A.Y*. +| 3968: e1 7c 44 99 65 e7 4b 2f 42 a6 02 ad eb 7b db 14 .|D.e.K/B....... +| 3984: e4 c7 0d bf a7 f1 3d 69 01 cc c0 2e 82 06 fe 13 ......=i........ +| 4000: a3 f2 df 3f fe 65 d6 73 b3 d1 48 0c 00 5f 6c 4e ...?.e.s..H.._lN +| 4016: 1e ff f6 d2 69 08 a0 7a ed 50 16 a5 b1 56 1c f5 ....i..z.P...V.. +| 4032: 14 b0 b8 3a 38 19 a6 e7 ef 28 3d b9 9f 5e 38 39 ...:8....(=..^89 +| 4048: bc 41 c2 b1 fa 0b be 8b e9 9f 2f a0 be 8a eb 7f .A......../..... +| 4064: eb 55 5e fb 3a 36 82 02 fd 50 a6 b2 7d fb e7 5a .U^.:6...P.....Z +| 4080: a4 b2 d9 d9 8b 93 7a 2e ed c3 0c 13 68 22 cf f9 ......z.....h... +| page 18 offset 69632 +| 0: 0d 00 00 00 01 04 3f 00 04 3f 00 00 00 00 00 00 ......?..?...... +| 1072: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 97 ................ +| 1088: 3e 0a 06 00 ae 7c 00 00 18 91 e3 df f3 2f 7b 53 >....|......./.S +| 1104: 6d 25 6e 2a df e7 b0 18 33 2f 73 08 d7 a8 04 75 m%n*....3/s....u +| 1120: 49 89 b0 84 41 57 fb 8f 24 85 4e 24 c6 f1 14 03 I...AW..$.N$.... +| 1136: 9e 8e ef 9d f0 5c 54 69 11 78 d5 0f d6 97 22 1e ......Ti.x...... +| 1152: b5 17 73 66 0d 95 db ce c4 12 cb b0 ae a8 53 70 ..sf..........Sp +| 1168: c7 6b 02 f6 7d 56 49 eb 11 75 da b6 c2 ba a1 f2 .k...VI..u...... +| 1184: 19 8d c3 fc 6f 9d 3e e8 0e 1a 51 37 d7 ff b1 19 ....o.>...Q7.... +| 1200: 50 bd a1 74 26 2b 53 c6 1e 12 75 20 ed 43 51 0e P..t&+S...u .CQ. +| 1216: 28 f7 c4 a3 b5 8c ec de 2c 58 29 8c 74 81 d7 2d (.......,X).t..- +| 1232: a2 8d 0a d3 85 98 6b f2 3e 34 94 08 73 06 e7 06 ......k.>4..s... +| 1248: 04 e3 81 81 2a d8 5c 61 1b e9 7c ff 92 a8 c9 50 ....*..a..|....P +| 1264: 97 a0 8e f6 1f 4b 25 ae 8e fa 84 7a 6f a7 11 42 .....K%....zo..B +| 1280: c2 97 7a 93 42 39 ab 41 9f c1 7c 86 4a 66 06 cd ..z.B9.A..|.Jf.. +| 1296: e3 20 2c 3e ba f5 3c 32 fd a7 4a 73 50 74 a4 50 . ,>..<2..JsPt.P +| 1312: 89 28 57 71 bc 3d 31 77 89 ef cd cd ff b4 fb 2c .(Wq.=1w......., +| 1328: 5f a0 55 da b0 d9 18 d4 f7 a3 e4 f0 bf a8 86 b8 _.U............. +| 1344: 64 97 49 26 1b 4a b7 e7 6f 88 a9 a8 ab 54 1c 48 d.I&.J..o....T.H +| 1360: 0c 77 2c e3 7a 83 7d 05 c8 22 f7 b7 57 92 61 97 .w,.z.......W.a. +| 1376: 69 52 51 f7 8d 4c 2a b5 5e d5 55 f4 fc ae 35 bc iRQ..L*.^.U...5. +| 1392: fa 44 c5 19 ec 62 5e 98 12 bf a8 e1 53 32 76 4c .D...b^.....S2vL +| 1408: fb 4e 80 40 85 39 71 69 bd 55 90 9b 4c 46 b1 06 .N.@.9qi.U..LF.. +| 1424: 84 14 80 f3 2b ba 43 15 7e 12 44 4b 38 2e 01 a8 ....+.C.~.DK8... +| 1440: 2e 58 11 e1 dd 6b 24 9e 6a fb 21 14 f5 ae be 7a .X...k$.j.!....z +| 1456: 9b 26 0d a8 2f 29 8c 5a 63 8a cf 58 36 e1 76 fb .&../).Zc..X6.v. +| 1472: ca 95 7a 0c 74 0d d5 57 04 a4 65 5f 2a 0d 46 ee ..z.t..W..e_*.F. +| 1488: 0c 2e a6 a1 dd 04 b8 1d b7 72 a0 d5 ad cc 8e d3 .........r...... +| 1504: bb 04 bc 39 4d 22 0f 4e 6c b1 4d b4 08 3b 7f a5 ...9M..Nl.M..;.. +| 1520: f1 7d 18 c5 bf 43 86 b3 c1 3b 85 6f 30 84 1a 7b .....C...;.o0... +| 1536: 17 d0 91 d3 6d 99 cc ff ac 64 88 53 d3 ad 1e 5f ....m....d.S..._ +| 1552: ba 4c af 64 80 ae ca c7 27 56 7e 41 02 61 f1 d2 .L.d....'V~A.a.. +| 1568: e4 4b 99 7c e4 18 41 9c f7 b9 e8 5a 3f 6e a3 57 .K.|..A....Z?n.W +| 1584: ca 18 c5 8a a8 39 c6 fe 02 d0 9d 26 37 42 07 3d .....9.....&7B.= +| 1600: 38 4e fe 9a 3b 54 39 20 23 53 8a 84 2f 4a 06 06 8N..;T9 #S../J.. +| 1616: ed 56 dd d8 bf 56 ef ca a5 c0 a4 aa d5 88 41 42 .V...V........AB +| 1632: 8a c0 37 65 f3 c8 4c 87 a5 f3 3d 99 78 2b d7 4e ..7e..L...=.x+.N +| 1648: d7 6e 51 28 3f 5c 93 cb 56 08 91 39 8e 1d fb 26 .nQ(?...V..9...& +| 1664: 5d 80 7c 44 59 c4 d4 b3 5e 0c c1 3f 85 f8 d6 d0 ].|DY...^..?.... +| 1680: 25 0f a8 c4 40 a5 f7 63 ec 2b fc 78 e6 b4 7c 72 %...@..c.+.x..|r +| 1696: 87 f0 6d 2c 00 63 dc 29 4a e5 b5 6b 9e 73 33 b4 ..m,.c.)J..k.s3. +| 1712: 19 03 1a 5c de 8f 98 fa ce 4d e3 1a 62 6b 5b f5 .........M..bk[. +| 1728: 60 d6 4c 13 39 14 06 83 90 09 56 8b 71 3b b9 bc `.L.9.....V.q;.. +| 1744: e3 7e 5e ae f5 3e b7 aa bd 73 d6 f1 47 4a 84 60 .~^..>...s..GJ.` +| 1760: 20 d7 93 ce f0 f2 1a 63 b7 f0 7e 3b 4e 36 1c dd ......c..~;N6.. +| 1776: cc ef 50 7a a9 90 f7 48 05 fb 78 e8 72 71 df 3a ..Pz...H..x.rq.: +| 1792: 41 51 2c 4c 5d 0f cd 51 0e f3 5a a1 e6 81 15 b3 AQ,L]..Q..Z..... +| 1808: bb 48 5e 13 cf 46 4c f1 26 47 4b 51 87 d7 39 a6 .H^..FL.&GKQ..9. +| 1824: 38 c5 72 52 55 97 f6 81 bc 0f a1 95 72 b8 ec 6d 8.rRU.......r..m +| 1840: dd 6d 92 03 b5 0f d6 fd 7e 29 c9 55 50 57 71 2c .m......~).UPWq, +| 1856: 34 35 21 75 6a 4e e6 f0 6a 99 b7 51 b5 3e 0a 6c 45!ujN..j..Q.>.l +| 1872: 1b c0 ba cb 92 90 15 b8 35 b9 6b 78 f2 c6 03 48 ........5.kx...H +| 1888: 66 d6 2e 75 47 b8 eb d0 30 48 c9 4d 67 d1 c1 8a f..uG...0H.Mg... +| 1904: b2 9c a9 c0 3d a1 77 e3 35 e4 85 01 e7 dc 74 bc ....=.w.5.....t. +| 1920: 8d ff f6 f0 ad e1 35 63 75 6d ee 28 53 29 1c 9c ......5cum.(S).. +| 1936: 67 dd ea 7d 1f af 87 c3 2e 8d a4 23 d2 6b db 49 g..........#.k.I +| 1952: 1a 36 10 7b e1 6c 9a 1c a2 5f 17 fb 43 e5 da f5 .6...l..._..C... +| 1968: 2c 60 86 42 1d 28 a1 5f cc 73 b2 5d 69 2b fa 18 ,`.B.(._.s.]i+.. +| 1984: 85 a2 de 30 9a c7 08 42 b9 e3 b1 a1 32 1d 70 31 ...0...B....2.p1 +| 2000: b4 7a cf f4 57 5d 5e 45 53 1d 79 35 7b 4f 9a 2f .z..W]^ES.y5.O./ +| 2016: 80 11 76 23 60 dc 86 e7 f0 74 2d 46 51 40 01 10 ..v#`....t-FQ@.. +| 2032: 13 69 90 a9 fb cb 66 8e 3f e0 a1 4e 99 eb 61 1b .i....f.?..N..a. +| 2048: fe c7 5e b5 f5 0f a3 46 64 19 09 11 c0 83 c7 28 ..^....Fd......( +| 2064: 41 20 81 d3 f5 9c 21 2b ed 06 1a 4a 89 4d 6a e5 A ....!+...J.Mj. +| 2080: 2d 4f 95 d7 3b 95 8c 59 6f 3e 79 51 5f ef b8 2a -O..;..Yo>yQ_..* +| 2096: 43 ef 07 e2 d1 d1 13 38 67 54 88 e2 6f 03 fe 05 C......8gT..o... +| 2112: 10 0e e8 e9 4e 9a 75 92 ec 1f e7 56 61 3b 54 43 ....N.u....Va;TC +| 2128: 08 af e9 d5 56 bc 87 a3 25 6f e2 b8 01 62 1a 30 ....V...%o...b.0 +| 2144: ba 26 8a b1 9a 7e 44 a2 f5 d4 75 2e a0 d0 b8 71 .&...~D...u....q +| 2160: 61 61 92 ff 32 0b a8 94 a5 81 80 d0 3b b5 51 4a aa..2.......;.QJ +| 2176: 01 e2 8e 0a 38 90 19 f7 b4 38 b0 9c 32 e1 6a f0 ....8....8..2.j. +| 2192: f8 7b f0 58 0a d1 19 c0 20 d6 54 fe 28 ac 02 64 ...X.... .T.(..d +| 2208: c4 33 55 01 d2 bd 01 51 87 01 0c 66 bb 6e 1b 94 .3U....Q...f.n.. +| 2224: 9c 24 50 40 5b 2f 64 f9 b5 b6 6b 15 fd f8 e7 05 .$P@[/d...k..... +| 2240: 37 92 95 d3 b4 1e be b3 09 c0 74 f8 ca 03 10 89 7.........t..... +| 2256: 2d 4d f5 56 0f 6e 20 72 a7 5e 1d 9f fd d5 43 af -M.V.n r.^....C. +| 2272: 2e 7b f7 91 99 ed 47 ea c7 a4 76 82 ca 5f 94 20 ......G...v.._. +| 2288: 52 01 b4 21 cb 50 d5 f2 d6 cf 1d 11 0f b2 07 ee R..!.P.......... +| 2304: f2 cf 2b 52 7b 1d e0 be 16 a1 cf 06 52 1b 33 f5 ..+R........R.3. +| 2320: c2 8e ac f1 00 2b 82 cf b4 ae d5 f0 9b 4b 11 14 .....+.......K.. +| 2336: b5 c5 43 f4 08 9d 4a 59 ba a6 52 67 fe e1 bd 2d ..C...JY..Rg...- +| 2352: c2 40 c6 3e d1 8a d1 f5 a0 b4 d0 b1 92 3f 9e 1d .@.>.........?.. +| 2368: 18 5e b4 78 30 45 57 29 30 e1 5a bb ee 7b 1e 99 .^.x0EW)0.Z..... +| 2384: 0d 10 ea 14 7b 5e 36 32 8e e1 a7 e4 76 19 48 a8 .....^62....v.H. +| 2400: c6 19 74 b9 4e 31 81 24 fe 07 e3 86 ec c4 1d 05 ..t.N1.$........ +| 2416: 1f 5a 52 14 0c 08 3f d1 93 fe b1 46 27 87 58 21 .ZR...?....F'.X! +| 2432: 4b d5 ff bb 98 d8 f8 0c 96 66 02 9c 86 f9 0e 0b K........f...... +| 2448: f0 11 7a 99 39 52 38 84 22 04 58 07 f1 95 eb c7 ..z.9R8...X..... +| 2464: 44 2e a5 fe d4 68 a9 98 77 14 4f 8a 44 4c 7d c4 D....h..w.O.DL.. +| 2480: 49 8a d7 89 83 8e e6 1e d0 af b7 41 3d 1a 85 14 I..........A=... +| 2496: ec 3a e5 1b 2c c5 17 77 85 82 19 57 37 94 93 7e .:..,..w...W7..~ +| 2512: 52 16 a3 dd 0a fd 57 1a 57 32 11 4d 71 e3 4b 1b R.....W.W2.Mq.K. +| 2528: c5 02 d7 89 74 85 b0 3d 8d 7b 53 a2 d6 60 99 d4 ....t..=..S..`.. +| 2544: ce f0 1c 3d a3 aa db db c4 80 38 7a cb 12 7e 66 ...=......8z..~f +| 2560: 3f 69 af fa 57 49 35 05 94 33 df fe 91 8a 25 3d ?i..WI5..3....%= +| 2576: 9b 32 71 72 d2 bc bc 23 61 69 9c 68 a7 58 c0 f1 .2qr...#ai.h.X.. +| 2592: 0e 20 a9 d3 d2 a9 11 d7 ee 52 46 70 b7 aa 6b f3 . .......RFp..k. +| 2608: 4a 51 a7 a5 26 92 35 44 f9 cc 7b c7 ec db 5d b6 JQ..&.5D......]. +| 2624: 5c 88 d9 bd 14 df a0 14 35 09 2f c8 76 d4 4c 19 ........5./.v.L. +| 2640: 12 29 b9 dd 9b 21 ed b8 ee 1f f9 38 05 9e 93 aa .)...!.....8.... +| 2656: ab 82 15 69 88 81 f6 4f 1b 72 bb 84 cb 9c 33 ec ...i...O.r....3. +| 2672: 94 4d 44 42 8e 8f 12 91 1f 32 07 09 38 8b 44 be .MDB.....2..8.D. +| 2688: 9e 31 49 9e 76 04 d8 b7 69 ad f1 59 81 5f d7 a0 .1I.v...i..Y._.. +| 2704: 2f 34 94 27 b4 c1 e9 f0 18 a7 43 7e 1e fd 27 5b /4.'......C~..'[ +| 2720: d8 e9 c3 5d be 8f 91 f2 4a cd 33 5f 6c 76 f6 f1 ...]....J.3_lv.. +| 2736: 17 ae 80 87 e7 ec 22 ef 73 8e a7 3a 30 dd 27 3d ........s..:0.'= +| 2752: 6d 95 59 eb f3 7f 97 b7 b9 8d ff 86 ed dd 5d f4 m.Y...........]. +| 2768: 39 3c 6a 13 3d 7a 93 3d 37 ed 8c d6 98 0f 0b 7a 9.....p..$o..... +| 3040: e1 6a a9 0a d8 f6 89 09 51 98 1f 89 1b f1 34 87 .j......Q.....4. +| 3056: a5 b3 22 d0 65 53 bd ae 57 7a 8a 8f a8 a6 10 9e ....eS..Wz...... +| 3072: 72 7c 6e 37 8f 67 db d8 89 54 77 87 6d 63 6e 31 r|n7.g...Tw.mcn1 +| 3088: ef ae 41 51 22 cc 24 08 89 f6 dd 2c f9 cb a4 f8 ..AQ..$....,.... +| 3104: ea f9 42 33 01 fd cf b9 d6 73 aa b4 9d 45 31 eb ..B3.....s...E1. +| 3120: 42 ca df 3a d2 3c c9 41 28 fd e4 a2 2f cf bf ca B..:.<.A(.../... +| 3136: 63 94 a2 74 ee 6b 4c 62 bb 74 5f cc 39 68 b5 e9 c..t.kLb.t_.9h.. +| 3152: 68 47 90 45 85 f0 20 4e 3a fe 4a 4f ab f2 fe c7 hG.E.. N:.JO.... +| 3168: f5 23 56 6e 09 9e c6 3d 36 30 82 62 6f 5f 78 f6 .#Vn...=60.bo_x. +| 3184: f6 07 58 e8 fd 98 09 e5 a5 5b 65 27 43 e6 9e 3d ..X......[e'C..= +| 3200: 98 d5 db 1d 09 35 48 b0 cd 5e 53 a1 d6 b2 4f 85 .....5H..^S...O. +| 3216: e5 4d 80 18 8f 78 6e a0 0e 35 08 7d 1d 5d 3e ab .M...xn..5...]>. +| 3232: c3 5c dc ec c9 e0 22 1a 17 a3 80 40 3d e7 66 df ...........@=.f. +| 3248: b2 38 f4 59 6d 20 e8 83 93 53 8c ac 52 26 ec 60 .8.Ym ...S..R&.` +| 3264: f8 50 85 1c 97 9c ae a3 9f bd af 75 be 73 23 b5 .P.........u.s#. +| 3280: 7f fa 1f 7f 28 16 c7 7c 5f 0a 5b 32 1a c8 45 cf ....(..|_.[2..E. +| 3296: df 2e 8e d1 d9 59 76 d5 6c 8d b5 12 8a c5 77 32 .....Yv.l.....w2 +| 3312: d4 87 02 80 09 b6 43 34 76 09 f9 b4 74 66 ce ee ......C4v...tf.. +| 3328: 26 77 62 11 b0 23 92 5c 72 38 41 e9 70 4f b5 83 &wb..#..r8A.pO.. +| 3344: e4 35 7c b2 2a 38 49 12 48 18 1c 95 5c b1 18 1a .5|.*8I.H....... +| 3360: 51 cd 4b 7b 22 94 0e c7 da c1 30 97 d1 be 42 07 Q.K.......0...B. +| 3376: 94 01 a5 fd 2f 0d 2c 53 33 c0 91 c6 bc af 2c f2 ..../.,S3.....,. +| 3392: f2 07 6e 4a d2 22 3e 3c 18 3c ca 24 bf 42 78 7a ..nJ..><.<.$.Bxz +| 3408: 69 0e a9 12 e3 20 fa 8b ad 75 27 0c c3 82 84 8d i.... ...u'..... +| 3424: 46 af 3e 1e 89 27 4d 7e f7 21 96 b4 6c 17 7f 19 F.>..'M~.!..l... +| 3440: 8a 78 d7 bb 40 67 35 45 2d d4 97 6a 4c e9 4a 58 .x..@g5E-..jL.JX +| 3456: 22 a3 bb 31 d5 4f 26 40 cc fe c8 cd 1d a4 0d 67 ...1.O&@.......g +| 3472: 14 13 05 4d 0e 15 40 ea 7d 62 c6 80 08 b9 f6 b2 ...M..@..b...... +| 3488: 44 58 66 d5 ca b6 f4 20 b0 4a b8 37 64 3d b0 a7 DXf.... .J.7d=.. +| 3504: a7 87 70 26 b2 ea f9 cf 98 03 6e 63 5a fe c4 cd ..p&......ncZ... +| 3520: 80 cb ca f4 a6 02 11 39 4f 6c bf bf a4 8e 99 32 .......9Ol.....2 +| 3536: e3 47 51 3e 85 f6 84 6b 3d 9a fe 2f 96 18 49 2a .GQ>...k=../..I* +| 3552: dc a4 56 77 d1 3f 94 61 2b 58 e7 74 ee c9 16 7b ..Vw.?.a+X.t.... +| 3568: 6f 76 47 da b2 fb 89 75 80 78 05 69 c8 3e f0 97 ovG....u.x.i.>.. +| 3584: 1b 40 f1 48 43 7f cd 0b b1 c6 cf 59 73 4f 3f 33 .@.HC......YsO?3 +| 3600: 2e 32 d2 b6 69 fc 6b f2 20 0f 12 bf f0 98 2e e2 .2..i.k. ....... +| 3616: 4f ed 80 27 00 e2 b1 c2 ca cd 6b de e1 b4 af 9e O..'......k..... +| 3632: df 4f d2 da 6d 9f b4 5d 99 4a c4 59 d4 e1 98 05 .O..m..].J.Y.... +| 3648: 68 00 a5 72 3f 0e 35 29 59 81 8a f9 f2 c0 5a de h..r?.5)Y.....Z. +| 3664: 25 35 d5 60 03 f7 0f 20 3a bb a6 45 fc cc 2d e9 %5.`... :..E..-. +| 3680: fe 37 0b 6c f2 96 19 dc 39 62 6e f7 9c 17 37 8c .7.l....9bn...7. +| 3696: 8a dd 62 a2 a3 e6 b8 a0 cd 61 f5 27 3b 29 e2 f4 ..b......a.';).. +| 3712: 0a 06 21 c3 b9 8e 70 34 59 ca d9 a1 92 db a9 74 ..!...p4Y......t +| 3728: d1 7c a1 ef 64 c2 75 51 02 b3 6a 57 6d 12 f6 eb .|..d.uQ..jWm... +| 3744: 3b 41 ce a5 ca 69 62 3e c3 55 eb 1c 80 01 79 05 ;A...ib>.U....y. +| 3760: a4 51 7c 45 00 76 41 5e 57 1a 88 aa 15 c8 42 82 .Q|E.vA^W.....B. +| 3776: d5 83 74 97 93 b4 28 df 4d 82 7d 6f ef 6a d4 e4 ..t...(.M..o.j.. +| 3792: 03 e7 20 4a e8 84 54 0c 6e 5f 8e b0 d1 0d 67 aa .. J..T.n_....g. +| 3808: f9 20 21 ea c0 4e 5c e3 f8 65 d7 67 0a 7a e0 0b . !..N...e.g.z.. +| 3824: d8 1e 53 95 bf 24 39 12 d4 30 8c 2c b4 13 2e bd ..S..$9..0.,.... +| 3840: f8 81 d5 15 35 d7 3f f2 23 a4 2c 1b cd 29 e8 88 ....5.?.#.,..).. +| 3856: a8 f1 cc 3f 9b 72 d0 8e c0 cb 80 ca c9 68 1a ca ...?.r.......h.. +| 3872: 38 09 59 e5 a6 33 95 57 55 c9 dd c4 8a 8d 36 e5 8.Y..3.WU.....6. +| 3888: 0c 40 95 77 63 4b 82 53 98 d0 bd cd 57 b6 f7 2b .@.wcK.S....W..+ +| 3904: fe 1c 25 d4 95 3d 4c 63 b7 fb 94 37 3f 6d 96 28 ..%..=Lc...7?m.( +| 3920: 1d 13 76 d2 ab 9a 7e 8f f5 a2 e6 15 0b 1c 10 14 ..v...~......... +| 3936: c3 b5 c0 1c 52 af 2a 32 35 05 4a 0c f1 cb ed 5e ....R.*25.J....^ +| 3952: 42 6c 31 8b dc 78 4b 68 5a b7 0f e3 6c d6 da e6 Bl1..xKhZ...l... +| 3968: 59 f3 34 43 47 1c 2b f6 f4 eb 6e 12 4d 60 f4 d9 Y.4CG.+...n.M`.. +| 3984: 39 2f 84 a5 8f f0 03 e7 ce b6 10 d0 a9 f9 69 76 9/............iv +| 4000: 94 29 2e 60 db fc e7 5f e6 c5 5b b9 c5 a7 1f 95 .).`..._..[..... +| 4016: 0d 66 aa 55 7b f5 ec 67 c6 f7 26 52 c4 02 60 31 .f.U...g..&R..`1 +| 4032: 7d 5e de ba 06 2f 09 1d a1 db 61 83 c8 13 be 98 .^.../....a..... +| 4048: 14 9d c9 15 1e 33 04 de a5 53 ed 42 ab 45 b1 f3 .....3...S.B.E.. +| 4064: 09 a7 47 54 14 bc a0 88 80 58 94 0a 78 f2 31 3d ..GT.....X..x.1= +| 4080: bc 0d 85 bf da a0 7e d2 be e4 1c cc b4 45 a8 bd ......~......E.. +| page 19 offset 73728 +| 0: 0d 0f e6 00 03 0c 8a 00 0c 8a 0f ec 0c 94 00 00 ................ +| 3200: 00 00 00 00 00 00 00 00 00 00 08 01 03 00 16 2e ................ +| 3216: b1 7d 24 24 86 4a 84 80 80 80 80 01 04 00 8d 18 ..$$.J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 00 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 3d ......0000000..= +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 07 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 d3 02 01 02 nable........... +| 3456: 02 01 02 02 02 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 54 01 02 03 01 02 03 ....gcc..T...... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 13 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 01 f3 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 08 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 ad 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 01 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0e fc 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 00 00 00 06 14 24 12 0a 03 00 D..@.......$.... +| 4080: 2a 00 00 00 00 01 02 08 00 02 01 01 01 02 01 07 *............... +| page 20 offset 77824 +| 0: 0d 00 00 00 01 00 22 00 00 22 00 00 00 00 00 00 ................ +| 32: 00 00 9f 56 88 80 80 80 80 01 04 00 bf 30 00 00 ...V.........0.. +| 48: 0e 9c 02 30 30 01 0c 74 81 44 06 81 0b 01 0a 81 ...00..t.D...... +| 64: 03 58 58 04 01 10 82 66 10 0f 0b 81 14 24 01 0c .XX....f.....$.. +| 80: 81 14 47 4a 20 6e 01 0e 71 81 41 05 82 0f 03 01 ..GJ n..q.A..... +| 96: 0c 53 54 4c 2c 70 7d 01 12 81 43 06 1e 37 42 37 .STL,p....C..7B7 +| 112: 77 3e 01 14 81 22 07 81 30 81 5d 03 05 0d 01 08 w>......0.]..... +| 128: 82 01 81 21 01 0e 81 45 54 51 81 0a 61 02 01 30 ...!...ETQ..a..0 +| 144: 01 04 82 6e 02 01 36 08 04 83 7e 02 01 38 02 04 ...n..6...~..8.. +| 160: 83 4f 02 01 62 06 04 81 11 03 01 36 08 04 81 5b .O..b......6...[ +| 176: 03 01 64 02 04 84 5d 02 01 63 05 04 81 02 02 01 ..d...]..c...... +| 192: 64 07 04 81 67 02 01 65 0a 04 83 0c 03 02 70 77 d...g..e......pw +| 208: 04 04 83 70 02 02 66 79 08 04 82 25 02 01 68 0a ...p..fy...%..h. +| 224: 04 81 34 02 01 6b 03 02 4d 03 04 84 32 02 01 70 ..4..k..M...2..p +| 240: 01 04 81 15 02 01 73 07 04 83 4d 02 02 37 02 01 ......s...M..7.. +| 256: 74 02 02 40 03 01 68 07 04 83 45 02 02 76 69 09 t..@..h...E..vi. +| 272: 04 82 66 03 01 6c 09 04 82 34 02 01 79 01 04 84 ..f..l...4..y... +| 288: 13 02 02 c2 ba 0a 04 81 68 03 01 bc 0a 04 83 0d ........h....... +| 304: 02 04 ca 80 69 6c 09 04 81 5a 02 02 d3 84 05 04 ....il...Z...... +| 320: 83 0a 01 01 31 01 16 18 1a 30 81 4c 31 12 54 81 ....1....0.L1.T. +| 336: 00 1d 01 0e 02 25 82 4b 1d 3f 76 01 0c 43 79 4b .....%.K.?v..CyK +| 352: 5b 0b 76 01 0e 35 1a 81 08 82 77 10 01 0c 81 24 [.v..5....w....$ +| 368: 18 07 74 6f 01 08 29 81 08 69 01 10 81 0e 0d 4a ..to..)..i.....J +| 384: 81 01 07 58 01 08 6b 51 76 7a 01 0c 20 81 18 6c ...X..kQvz.. ..l +| 400: 2c 18 01 08 83 7c 7b 0e 02 01 32 08 02 48 03 04 ,....|....2..H.. +| 416: f3 a1 97 a5 03 04 83 71 02 01 33 02 04 81 2f 03 .......q..3.../. +| 432: 01 66 09 04 83 15 02 03 61 ca b1 08 04 81 48 02 .f......a.....H. +| 448: 01 63 01 02 1c 02 03 65 c2 aa 06 02 5a 02 01 68 .c.....e....Z..h +| 464: 03 04 81 6e 02 01 69 0a 04 82 5a 03 01 69 08 04 ...n..i...Z..i.. +| 480: 81 65 02 01 6a 02 02 64 06 04 84 12 02 01 6c 01 .e..j..d......l. +| 496: 02 1f 02 01 6e 03 04 83 24 02 03 6f c2 b3 07 04 ....n...$..o.... +| 512: 84 52 02 03 72 c9 9b 04 04 82 6e 02 01 73 07 04 .R..r.....n..s.. +| 528: 82 10 02 01 77 0a 02 31 02 01 79 07 04 81 7f 02 ....w..1..y..... +| 544: 02 7a 62 09 04 83 49 02 02 c2 b9 02 04 83 01 03 .zb...I......... +| 560: 02 bd 75 09 04 83 45 02 02 c6 aa 02 04 83 08 02 ..u...E......... +| 576: 03 d7 92 6a 08 02 04 01 01 32 01 0c 81 7d 72 81 ...j.....2....r. +| 592: 4e 36 01 0e 1a 81 10 81 10 11 2e 01 08 78 49 82 N6...........xI. +| 608: 3b 01 0e 73 81 64 05 1b 81 69 01 1a 18 06 7e 2c ;..s.d...i....~, +| 624: 21 12 20 5a 28 69 06 1e 06 01 0c 81 48 05 31 81 !. Z(i......H.1. +| 640: 24 01 14 25 81 42 3e 81 16 81 37 13 13 01 0c 21 $..%.B>...7....! +| 656: 0d 0f 3e 82 03 01 08 6b 60 81 27 01 10 2d 81 41 ..>....k`.'..-.A +| 672: 06 69 81 08 30 02 01 32 03 04 81 01 02 01 33 08 .i..0..2......3. +| 688: 02 3c 02 01 35 0a 04 84 64 02 01 38 05 04 83 41 .<..5...d..8...A +| 704: 02 02 61 69 09 04 82 49 02 03 66 c8 ab 09 04 82 ..ai...I..f..... +| 720: 68 02 01 68 04 04 83 27 02 01 6c 04 04 84 18 03 h..h...'..l..... +| 736: 02 72 6e 06 04 82 30 02 01 6e 04 04 84 55 03 01 .rn...0..n...U.. +| 752: 6d 05 04 83 30 03 03 76 c2 bd 09 02 57 02 01 71 m...0..v....W..q +| 768: 02 04 84 22 03 04 84 11 03 05 72 f0 92 bc bc 0a ..........r..... +| 784: 04 82 40 02 01 72 04 04 83 5b 02 01 73 05 02 6f ..@..r...[..s..o +| 800: 03 04 83 6f 02 01 75 04 02 55 02 01 78 03 04 82 ...o..u..U..x... +| 816: 56 02 01 7a 06 04 85 01 03 02 25 02 02 c2 b2 07 V..z......%..... +| 832: 04 83 7a 03 03 b3 ce bc 03 04 84 2b 03 03 bc c2 ..z........+.... +| 848: b2 09 04 83 53 02 04 ce a2 38 6e 07 04 82 23 03 ....S....8n...#. +| 864: 01 bc 08 04 83 7f 02 03 d2 b7 69 0a 04 84 19 02 ..........i..... +| 880: 02 d6 84 09 04 81 0e 01 01 33 01 06 2a 83 69 01 .........3..*.i. +| 896: 18 0f 0e 25 42 81 71 81 21 17 07 03 0d 01 1a 15 ...%B.q.!....... +| 912: 13 42 03 81 40 45 21 60 4d 1a 04 0e 01 0a 83 18 .B..@E!`M....... +| 928: 81 11 0a 01 08 64 83 24 24 01 16 62 1d 0c 06 62 .....d.$$..b...b +| 944: 50 2c 73 17 15 4d 01 0e 1b 82 15 4d 31 3a 4f 01 P,s..M.....M1:O. +| 960: 0e 22 63 51 81 7f 81 22 01 0e 21 81 0a 3e 42 82 ..cQ......!..>B. +| 976: 08 01 12 05 82 0a 33 19 10 81 37 63 02 01 32 05 ......3...7c..2. +| 992: 04 82 3c 02 01 36 02 04 84 0f 02 01 38 06 04 84 ..<..6......8... +| 1008: 1f 02 01 39 06 04 82 06 02 01 61 08 04 81 00 03 ...9......a..... +| 1024: 03 33 6f 62 01 02 69 02 01 65 02 04 84 05 07 04 .3ob..i..e...... +| 1040: 84 0e 02 01 67 09 04 82 44 02 01 6a 07 02 55 02 ....g...D..j..U. +| 1056: 01 6b 02 04 84 04 07 04 84 44 02 01 6c 06 04 84 .k.......D..l... +| 1072: 0d 02 01 6e 06 04 84 10 03 01 77 03 02 46 02 01 ...n......w..F.. +| 1088: 74 08 04 83 33 03 01 77 03 04 81 5b 02 01 75 0a t...3..w...[..u. +| 1104: 04 81 75 02 01 78 01 04 83 48 03 02 c2 b2 02 04 ..u..x...H...... +| 1120: 81 77 02 04 c2 b3 c2 b3 09 02 1b 02 06 f2 98 9e .w.............. +| 1136: a2 61 6c 08 04 83 31 01 01 34 01 0e 81 1c 82 11 .al...1..4...... +| 1152: 4e 81 06 01 10 4a 12 64 39 18 81 24 72 01 04 81 N....J.d9..$r... +| 1168: 0b 01 10 81 08 81 50 48 2a 79 15 01 14 13 72 07 ......PH*y....r. +| 1184: 2f 42 81 21 50 81 02 01 10 6d 46 3e 81 0c 81 17 /B.!P....mF>.... +| 1200: 23 01 08 0a 5f 83 38 01 0a 57 81 3e 82 0f 01 10 #..._.8..W.>.... +| 1216: 81 07 81 7e 0c 81 14 1d 01 08 1f 82 42 4e 02 01 ...~........BN.. +| 1232: 30 01 04 81 36 04 04 81 39 03 02 c2 bd 02 04 81 0...6...9....... +| 1248: 1e 02 01 31 05 04 84 31 02 01 35 0a 04 81 2b 02 ...1...1..5...+. +| 1264: 06 37 c2 b9 eb a0 86 04 04 83 0d 02 02 61 6f 04 .7...........ao. +| 1280: 04 83 39 02 01 63 07 04 83 3c 03 01 67 0a 04 84 ..9..c...<..g... +| 1296: 6a 02 01 65 07 06 68 82 7d 02 03 66 c2 be 07 04 j..e..h....f.... +| 1312: 81 3d 02 01 67 08 02 58 01 04 83 16 02 01 6c 03 .=..g..X......l. +| 1328: 04 81 73 02 01 6d 08 04 81 42 02 01 6e 04 04 83 ..s..m...B..n... +| 1344: 75 02 03 6f 65 32 06 04 84 4f 02 01 70 05 04 81 u..oe2...O..p... +| 1360: 56 03 01 77 09 04 81 47 02 02 72 32 05 02 0b 02 V..w...G..r2.... +| 1376: 01 73 01 02 47 04 04 84 09 02 01 74 07 04 85 05 .s..G......t.... +| 1392: 02 01 75 01 02 5b 02 04 76 e5 b9 83 07 04 82 03 ..u..[..v....... +| 1408: 02 01 78 01 04 81 11 02 02 c6 bf 02 04 81 3b 02 ..x...........;. +| 1424: 02 ce bc 06 04 84 3a 02 03 e1 83 93 03 04 81 2b ......:........+ +| 1440: 01 01 35 01 0e 81 7b 61 23 71 55 0c 01 08 82 30 ..5....a#qU....0 +| 1456: 2a 7b 01 0c 81 45 81 20 72 38 01 0c 29 82 25 55 *....E. r8..).%U +| 1472: 4c 21 01 0c 81 53 38 25 3a 63 01 08 83 55 81 0f L!...S8%:c...U.. +| 1488: 01 0e 81 32 5b 81 1b 31 6f 01 16 1e 6d 17 0e 17 ...2[..1o...m... +| 1504: 1f 6b 0b 3e 3d 3a 01 12 32 81 4a 81 04 55 47 20 .k.>=:..2.J..UG +| 1520: 09 01 10 82 4b 81 0a 17 3e 05 2a 02 01 31 08 04 ....K...>.*..1.. +| 1536: 84 04 02 02 36 68 06 02 18 02 01 61 02 04 84 3d ....6h.....a...= +| 1552: 08 04 81 38 03 01 34 01 02 71 02 03 63 75 6d 0a ...8..4..q..cum. +| 1568: 04 81 3a 02 01 64 09 02 52 01 04 82 48 02 01 68 ..:..d..R...H..h +| 1584: 08 02 30 02 04 83 4e 02 01 6e 01 02 42 02 05 6f ..0...N..n..B..o +| 1600: 63 6c dc ab 08 02 11 03 04 67 69 c2 be 03 04 82 cl.......gi..... +| 1616: 29 02 01 71 07 04 82 12 02 01 79 06 04 84 1b 02 )..q......y..... +| 1632: 02 7a 33 09 04 83 4c 02 02 c2 b2 05 04 84 17 03 .z3...L......... +| 1648: 01 b9 09 02 35 04 02 6b 78 0a 04 81 31 03 01 bc ....5..kx...1... +| 1664: 0a 02 41 02 08 cb 8a f0 9c bb a1 67 68 02 04 81 ..A........gh... +| 1680: 75 01 01 36 01 10 55 77 30 36 21 31 69 60 01 0e u..6..Uw06!1i`.. +| 1696: 3b 0b 3e 4e 1c 2c 1b 01 16 44 32 54 07 2f 0e 81 ;.>N.,...D2T./.. +| 1712: 11 81 03 14 01 04 81 63 01 0e 81 16 82 5c 15 20 .......c....... +| 1728: 19 01 12 0d 1f 43 6e 39 06 18 81 54 01 0c 81 40 .....Cn9...T...@ +| 1744: 2c 81 0f 4d 01 10 16 82 5e 0e 08 0d 78 51 01 12 ,..M....^...xQ.. +| 1760: 1d 81 73 81 22 29 17 66 1a 01 0a 81 3f 81 3f 19 ..s..).f....?.?. +| 1776: 02 01 30 0a 04 83 47 03 01 6d 05 04 82 3e 02 01 ..0...G..m...>.. +| 1792: 32 07 04 81 5e 03 04 82 19 02 04 34 71 6c 38 01 2...^......4ql8. +| 1808: 04 83 6b 02 02 37 67 08 04 84 53 02 01 64 04 02 ..k..7g...S..d.. +| 1824: 7b 03 01 71 02 04 81 71 02 01 65 01 04 83 66 05 ...q...q..e...f. +| 1840: 04 84 5c 02 01 69 09 04 81 3a 02 01 6a 03 04 83 .....i...:..j... +| 1856: 35 02 01 6b 06 04 83 6d 01 04 83 5f 02 04 84 0f 5..k...m..._.... +| 1872: 02 01 73 03 04 83 2d 02 01 74 06 04 84 0e 02 04 ..s...-..t...... +| 1888: 82 02 02 01 75 05 04 83 73 03 01 37 03 04 84 32 ....u...s..7...2 +| 1904: 02 01 76 03 04 83 0f 02 01 77 08 04 82 33 03 01 ..v......w...3.. +| 1920: 6f 01 02 0b 02 01 78 02 02 36 02 02 c2 b3 03 04 o.....x..6...... +| 1936: 81 07 02 04 84 29 02 02 ca a9 05 04 83 19 02 03 .....).......... +| 1952: ce bc 79 08 04 84 4b 02 02 d0 b8 05 04 81 06 02 ..y...K......... +| 1968: 06 f1 b5 87 92 35 6b 05 04 82 34 01 01 37 01 14 .....5k...4..7.. +| 1984: 81 3d 4b 17 08 16 81 03 81 20 01 0e 42 81 27 32 .=K...... ..B.'2 +| 2000: 20 0c 7a 01 0a 4c 83 12 40 19 01 16 06 21 6f 81 .z..L..@....!o. +| 2016: 02 70 36 0d 49 2e 2d 01 14 81 12 19 19 60 06 4c .p6.I.-......`.L +| 2032: 44 17 4f 01 12 03 81 12 04 81 23 81 4b 0a 01 12 D.O.......#.K... +| 2048: 0b 48 50 22 10 7f 81 38 56 01 0e 27 38 81 21 82 .HP....8V..'8.!. +| 2064: 08 2a 01 08 81 6b 41 24 01 0c 81 7c 73 81 3e 05 .*...kA$...|s.>. +| 2080: 02 02 34 67 05 04 83 5d 02 01 37 05 04 81 2f 02 ..4g...]..7.../. +| 2096: 02 73 02 01 38 01 02 21 02 01 39 09 02 4a 02 01 .s..8..!..9..J.. +| 2112: 62 0a 02 70 03 02 71 7a 01 04 83 26 02 01 64 0a b..p..qz...&..d. +| 2128: 04 84 04 02 01 65 0a 02 78 02 01 68 02 04 82 26 .....e..x..h...& +| 2144: 02 02 69 35 06 04 82 02 02 01 6b 02 04 82 0a 03 ..i5......k..... +| 2160: 02 c2 bc 05 04 82 08 02 01 6d 08 02 54 02 01 6e .........m..T..n +| 2176: 06 04 82 32 02 03 6f 74 61 02 04 81 2a 02 05 71 ...2..ota...*..q +| 2192: f2 96 8c 90 01 04 83 5a 02 01 72 09 04 83 31 02 .......Z..r...1. +| 2208: 01 74 03 02 71 04 04 83 62 02 02 79 65 04 04 83 .t..q...b..ye... +| 2224: 26 02 02 c2 aa 05 04 82 19 03 01 b3 02 02 10 02 &............... +| 2240: 04 83 48 03 01 bd 01 04 84 10 02 02 cf ad 06 04 ..H............. +| 2256: 82 04 02 04 f0 97 97 98 08 04 83 1f 02 04 f3 a5 ................ +| 2272: b3 87 08 04 81 64 01 01 38 01 0a 16 82 49 81 30 .....d..8....I.0 +| 2288: 01 0e 81 19 27 34 53 82 19 01 12 20 1f 5c 5f 81 ....'4S.... .._. +| 2304: 50 81 17 04 01 10 0e 81 2e 2d 51 67 81 16 01 0c P........-Qg.... +| 2320: 81 62 09 81 79 10 01 08 63 04 81 1c 01 0a 81 2e .b..y...c....... +| 2336: 5d 81 0f 01 0a 0b 81 40 81 63 01 10 27 81 17 81 ]......@.c..'... +| 2352: 2b 48 6c 4c 01 10 81 23 4d 64 0a 7f 81 01 02 01 +HlL...#Md...... +| 2368: 31 03 02 6b 02 01 36 06 04 83 79 02 01 37 08 04 1..k..6...y..7.. +| 2384: 84 47 02 04 39 c2 bc 61 09 04 84 61 02 05 64 c2 .G..9..a...a..d. +| 2400: b9 da 9c 01 04 81 73 02 01 66 01 04 81 58 08 04 ......s..f...X.. +| 2416: 82 39 02 02 67 74 0a 04 81 5f 02 01 68 09 04 83 .9..gt..._..h... +| 2432: 2a 02 01 69 0a 04 83 6a 03 01 78 02 04 83 11 02 *..i...j..x..... +| 2448: 01 6a 05 04 83 26 02 06 0d 82 58 03 01 70 07 04 .j...&....X..p.. +| 2464: 83 2b 02 01 6b 07 08 83 58 81 12 02 01 6e 0a 02 .+..k...X....n.. +| 2480: 71 02 01 77 0a 04 83 04 02 03 c2 ba 31 06 04 84 q..w........1... +| 2496: 4e 03 01 bd 07 04 83 18 02 02 d3 b5 05 02 68 01 N.............h. +| 2512: 01 39 01 0a 53 81 03 81 24 01 0e 13 82 3d 81 1e .9..S...$....=.. +| 2528: 37 2d 01 0c 6a 15 09 13 82 58 01 08 83 0f 81 3f 7-..j....X.....? +| 2544: 01 10 4f 81 60 81 29 64 15 0c 01 0e 4b 0d 62 0d ..O.`.)d....K.b. +| 2560: 81 00 5e 01 0a 37 81 1b 59 56 01 0e 81 0c 6b 6d ..^..7..YV....km +| 2576: 81 67 1d 01 10 38 2f 63 38 0c 37 20 6b 01 10 7e .g...8/c8.7 k..~ +| 2592: 10 18 81 4a 81 64 23 02 01 32 02 04 83 2c 02 01 ...J.d#..2...,.. +| 2608: 39 04 04 81 10 01 04 82 35 02 04 62 6e c7 9c 0a 9.......5..bn... +| 2624: 04 84 2b 02 02 65 63 06 04 83 0a 03 01 6a 02 04 ..+..ec......j.. +| 2640: 84 4d 02 01 66 01 04 83 4d 02 04 81 0a 02 03 68 .M..f...M......h +| 2656: ce bc 0a 04 83 41 02 01 6c 06 02 54 02 01 6d 08 .....A..l..T..m. +| 2672: 04 81 14 02 02 6f 6c 0a 04 84 0a 02 01 70 02 04 .....ol......p.. +| 2688: 82 2b 05 04 81 49 02 05 71 69 c2 bd 75 0a 02 46 .+...I..qi..u..F +| 2704: 02 02 72 38 0a 04 82 26 02 01 74 04 04 81 0b 04 ..r8...&..t..... +| 2720: 04 84 6e 02 01 77 03 04 81 6a 01 04 82 6b 03 01 ..n..w...j...k.. +| 2736: 65 09 04 82 58 02 01 78 06 02 6e 02 02 c2 b3 08 e...X..x..n..... +| 2752: 06 76 81 10 03 01 b9 04 02 47 03 01 bc 0a 04 83 .v.......G...... +| 2768: 03 02 02 ce bc 02 06 83 3f 19 02 03 d9 be 7a 03 ........?.....z. +| 2784: 04 82 5d 01 01 61 01 1e 17 50 16 12 12 2d 32 81 ..]..a...P...-2. +| 2800: 1e 56 81 06 04 06 1b 01 16 43 5c 52 64 19 15 27 .V.......C.Rd..' +| 2816: 1d 29 0e 52 01 1c 1d 03 81 1c 13 6b 3d 64 14 32 .).R.......k=d.2 +| 2832: 16 29 09 0a 01 22 32 13 07 81 08 03 22 05 17 20 .)....2........ +| 2848: 07 0e 7a 15 71 22 22 01 1c 30 05 06 0b 22 40 11 ..z.q....0....@. +| 2864: 49 1c 0c 15 81 1f 27 01 0c 4a 14 1c 7a 36 66 01 I.....'..J..z6f. +| 2880: 1c 10 19 81 0a 81 1c 81 02 10 54 33 09 0c 07 01 ..........T3.... +| 2896: 1e 38 2e 33 3a 22 1e 2e 35 3a 0f 07 77 15 0b 2f .8.3:...5:..w../ +| 2912: 01 2a 08 5a 35 13 0b 07 05 04 17 42 32 14 04 09 .*.Z5......B2... +| 2928: 1d 0b 09 06 42 72 1c 01 1e 22 0a 15 2b 03 04 6c ....Br......+..l +| 2944: 05 81 3d 14 19 58 21 4a 02 01 32 08 04 83 64 03 ..=..X!J..2...d. +| 2960: 01 67 07 04 82 5c 02 02 34 66 03 04 83 6d 03 01 .g......4f...m.. +| 2976: 73 08 04 81 18 02 05 37 6a 77 c2 b9 0a 04 83 1d s......7jw...... +| 2992: 02 02 38 7a 0a 04 82 3a 02 01 39 05 02 5c 03 02 ..8z...:..9..... +| 3008: c2 b3 06 02 70 02 01 61 01 04 83 03 02 01 66 02 ....p..a......f. +| 3024: 04 81 5b 02 04 82 37 02 01 68 04 04 82 78 01 02 ..[...7..h...x.. +| 3040: 67 02 01 69 04 04 84 0e 06 04 82 41 02 01 6a 01 g..i.......A..j. +| 3056: 04 82 4c 02 02 4e 03 04 81 73 03 01 6c 02 04 81 ..L..N...s..l... +| 3072: 5f 02 01 6b 08 04 81 26 02 05 6d c2 b3 6f 76 06 _..k...&..m..ov. +| 3088: 04 82 7c 02 01 6e 08 02 1c 02 01 6f 02 04 83 24 ..|..n.....o...$ +| 3104: 07 04 83 77 02 01 70 04 04 82 7e 02 01 71 0a 04 ...w..p...~..q.. +| 3120: 81 19 02 06 72 f1 b6 bc b3 68 01 04 83 0e 02 01 ....r....h...... +| 3136: 73 03 04 82 17 01 08 81 70 81 65 03 04 39 c2 ba s.......p.e..9.. +| 3152: 69 06 04 81 65 03 01 76 02 04 81 26 02 01 74 01 i...e..v...&..t. +| 3168: 02 4c 02 01 75 01 04 84 67 02 02 77 69 0a 02 09 .L..u...g..wi... +| 3184: 02 04 78 d2 b9 73 04 04 83 42 02 03 7a c2 b3 08 ..x..s...B..z... +| 3200: 02 43 02 02 c2 aa 01 04 82 61 06 04 84 10 02 02 .C.......a...... +| 3216: c6 88 07 02 1f 02 04 c8 b8 dd bf 05 04 81 72 02 ..............r. +| 3232: 02 c9 ad 07 04 84 2f 02 02 ce bc 07 02 05 02 02 ....../......... +| 3248: cf 85 0a 04 84 36 02 02 d9 a3 02 04 81 72 02 04 .....6.......r.. +| 3264: f1 a9 82 9b 0a 04 82 72 01 01 62 01 1c 81 17 06 .......r..b..... +| 3280: 4a 46 28 2a 0a 4a 10 36 05 52 20 01 1c 1f 15 5f JF(*.J.6.R ...._ +| 3296: 17 12 2b 46 03 81 2b 0f 1b 1f 4f 01 26 6d 68 10 ..+F..+...O.&mh. +| 3312: 04 08 06 20 04 10 05 20 05 07 6d 2e 09 14 05 24 ... ... ..m....$ +| 3328: 01 18 21 2b 11 12 81 0f 3e 22 11 28 51 28 01 18 ..!+....>..(Q(.. +| 3344: 12 11 4e 36 81 5a 23 3b 1a 16 26 2a 01 20 34 79 ..N6.Z#;..&*. 4y +| 3360: 1e 32 14 05 27 17 0c 4a 49 46 09 03 0d 2f 01 0e .2..'..JIF.../.. +| 3376: 15 35 6f 3c 24 5e 2e 01 14 15 40 49 25 2c 0a 1d .5o<$^....@I%,.. +| 3392: 67 70 2e 01 1c 06 03 0b 0e 76 81 0e 15 81 25 0b gp.......v....%. +| 3408: 39 2d 1a 01 14 27 1e 81 01 27 81 54 76 14 3e 02 9-...'...'.Tv.>. +| 3424: 02 31 62 08 04 83 1d 02 01 33 0a 04 83 35 02 01 .1b......3...5.. +| 3440: 37 02 04 82 6b 02 01 39 0a 02 29 02 01 61 03 04 7...k..9..)..a.. +| 3456: 82 72 02 01 64 01 04 81 05 02 04 84 1f 02 01 65 .r..d..........e +| 3472: 01 04 81 69 02 01 66 04 04 82 50 05 04 82 10 02 ...i..f...P..... +| 3488: 01 69 01 02 0d 02 01 6b 0a 04 81 0a 02 01 6c 02 .i.....k......l. +| 3504: 04 81 79 04 04 84 6e 03 01 31 0a 04 84 66 02 01 ..y...n..1...f.. +| 3520: 6f 0a 04 83 48 02 01 73 01 02 3d 03 02 10 02 01 o...H..s..=..... +| 3536: 74 02 04 82 0e 02 01 75 06 04 84 74 03 01 32 01 t......u...t..2. +| 3552: 04 82 0d 02 01 76 06 04 83 3a 02 01 78 08 04 82 .....v...:..x... +| 3568: 73 03 01 6c 09 04 84 1c 03 01 78 02 04 82 08 03 s..l......x..... +| 3584: 02 7a 69 0a 04 83 73 02 01 79 03 04 82 42 02 01 .zi...s..y...B.. +| 3600: 7a 08 04 83 03 02 06 c2 b9 e3 b1 a1 32 0a 04 81 z...........2... +| 3616: 46 03 01 bc 04 04 83 36 03 01 be 04 04 84 3f 02 F......6......?. +| 3632: 02 c6 80 0a 04 84 00 02 03 d4 a3 76 07 04 84 33 ...........v...3 +| 3648: 02 04 da ba 71 6f 08 04 84 11 02 02 dd a8 08 02 ....qo.......... +| 3664: 51 02 06 eb 9e ac 7a 73 79 06 02 32 01 01 63 01 Q.....zsy..2..c. +| 3680: 18 04 39 12 3a 05 34 56 46 59 81 0b 43 01 22 2a ..9.:.4VFY..C..* +| 3696: 1d 1f 53 2b 60 22 12 0d 13 1c 2e 35 05 03 0f 54 ..S+`......5...T +| 3712: 01 14 81 1e 81 1f 17 03 24 2a 81 0b 01 24 34 20 ........$*...$4 +| 3728: 06 21 0a 30 05 08 03 62 0c 26 10 6c 3b 0d 44 3a .!.0...b.&.l;.D: +| 3744: 01 1a 15 0b 77 04 45 3f 34 37 3f 08 13 36 53 01 ....w.E?47?..6S. +| 3760: 1c 02 38 33 49 1e 2e 81 19 43 33 3e 24 15 08 01 ..83I....C3>$... +| 3776: 1a 09 81 55 40 43 09 21 27 4c 04 5b 07 07 07 07 ...U@C.!'L.[.... +| 3792: 07 07 07 07 07 08 08 07 0a 07 0a 06 07 08 07 07 ................ +| 3808: 08 07 0a 08 56 06 0a 07 07 09 06 08 07 07 07 0a ....V........... +| 3824: 06 07 09 09 07 06 07 08 08 08 08 08 5e 07 06 07 ............^... +| 3840: 07 08 09 07 07 08 07 07 08 0b 0b 07 0a 06 07 0a ................ +| 3856: 08 09 09 0a 07 09 08 65 07 07 07 07 07 08 0b 07 .......e........ +| 3872: 06 0b 07 07 06 07 07 07 07 08 09 0c 57 0b 08 07 ............W... +| 3888: 07 0c 08 07 07 08 09 0a 07 07 07 09 07 07 07 0a ................ +| 3904: 07 06 0a 07 08 08 09 5b 07 07 0b 06 09 0a 0a 06 .......[........ +| 3920: 0a 0a 07 07 08 08 06 08 06 0e 5f 07 07 0b 0a 08 .........._..... +| 3936: 06 07 0b 07 07 0f 07 0b 07 07 07 07 06 06 0c 08 ................ +| 3952: 09 08 0c 65 08 0a 06 06 06 08 07 06 07 08 07 08 ...e............ +| 3968: 06 07 09 0b 07 0a 08 08 0a 07 08 0a 0a 58 06 07 .............X.. +| 3984: 07 0a 0b 0b 08 07 07 07 0c 07 09 06 07 09 07 07 ................ +| 4000: 58 07 0b 0a 08 07 0b 09 06 07 08 0b 0a 08 0b 0b X............... +| 4016: 07 06 09 06 07 09 09 81 25 07 07 08 07 0b 08 06 ........%....... +| 4032: 07 07 0b 0a 0b 0e 07 07 0b 06 0b 07 07 0c 0d 0a ................ +| 4048: 07 06 07 07 0a 08 0c 07 0a 08 07 08 08 0a 81 17 ................ +| 4064: 08 07 07 06 07 0b 07 0b 06 07 0b 07 07 09 07 07 ................ +| 4080: 07 07 07 07 07 08 07 07 0c 07 07 08 09 0a 07 0b ................ +| page 21 offset 81920 +| 0: 0d 00 00 00 01 00 21 00 00 21 00 00 00 00 00 00 ......!..!...... +| 32: 00 9f 57 88 80 80 80 80 02 04 00 bf 32 00 08 0e ..W.........2... +| 48: 8c 3a 14 04 29 08 18 17 4a 44 3a 19 4e 5f 56 16 .:..)...JD:.N_V. +| 64: 08 30 25 01 1c 0e 13 32 1b 0b 41 03 2f 81 34 3c .0%....2..A./.4< +| 80: 14 09 21 01 14 81 01 06 10 30 1f 28 0f 51 5f 04 ..!......0.(.Q_. +| 96: 30 63 33 74 05 02 69 02 02 34 76 0a 04 83 63 02 0c3t..i..4v...c. +| 112: 04 36 e7 8e a1 03 04 84 03 02 04 61 77 c2 b9 05 .6.........aw... +| 128: 04 82 56 02 01 63 03 04 82 02 04 04 82 0a 02 02 ..V..c.......... +| 144: 64 30 09 02 40 02 01 67 02 06 61 83 38 02 04 82 d0..@..g..a.8... +| 160: 70 03 04 84 62 03 01 73 02 04 84 5f 02 01 68 01 p...b..s..._..h. +| 176: 04 81 76 03 01 32 07 04 84 3f 02 01 69 09 02 7e ..v..2...?..i..~ +| 192: 02 01 6a 01 04 81 57 02 01 6b 01 04 82 33 02 01 ..j...W..k...3.. +| 208: 6d 05 02 7b 02 01 6e 06 04 82 51 02 01 6f 06 02 m.....n...Q..o.. +| 224: 0e 03 01 65 01 04 81 3c 02 02 70 63 06 02 1e 02 ...e...<..pc.... +| 240: 01 71 02 04 84 3b 08 02 1a 02 01 72 09 06 36 81 .q...;.....r..6. +| 256: 38 02 01 75 02 04 83 3b 03 02 54 02 01 77 04 04 8..u...;..T..w.. +| 272: 83 5c 06 04 83 21 02 01 78 07 04 81 21 02 01 79 .....!..x...!..y +| 288: 06 02 3a 02 02 7a 31 02 04 84 2a 02 02 c2 b2 02 ..:..z1...*..... +| 304: 04 81 01 03 01 b3 07 04 82 5e 03 02 bd 7a 06 04 .........^...z.. +| 320: 82 08 02 02 c6 9e 0a 04 83 4d 02 02 ce bc 05 04 .........M...... +| 336: 84 70 02 02 d3 86 06 04 82 33 02 02 d4 a7 01 02 .p.......3...... +| 352: 7c 02 02 d8 b8 06 04 82 53 02 02 df a4 04 02 7c |.......S......| +| 368: 02 03 e0 b7 ae 04 04 84 5e 02 03 e6 91 9a 03 04 ........^....... +| 384: 81 18 02 04 f0 90 ac 8e 04 04 82 39 01 01 64 01 ...........9..d. +| 400: 16 23 2f 1a 6d 81 47 65 09 1b 42 25 01 10 0d 82 .#/.m.Ge..B%.... +| 416: 18 81 3c 50 06 34 01 10 60 31 5a 14 18 16 5f 18 ....J.Z.!. +| 464: 18 0a 0a 67 3d 81 37 40 68 0e 30 17 18 01 12 08 ...g=.7@h.0..... +| 480: 6a 54 81 0c 49 14 4f 21 01 16 26 82 0d 3c 0c 60 jT..I.O!..&..<.` +| 496: 19 27 51 05 07 01 16 24 20 07 1a 81 1c 1f 24 43 .'Q....$ .....$C +| 512: 81 5b 01 16 42 21 05 81 07 0d 36 5b 21 81 15 02 .[..B!....6[!... +| 528: 01 30 01 04 84 4d 02 01 32 04 04 84 38 02 01 34 .0...M..2...8..4 +| 544: 06 04 83 1e 02 04 84 3c 01 04 81 2c 02 01 62 08 .......<...,..b. +| 560: 04 82 51 03 01 61 05 04 84 3e 02 01 63 06 04 84 ..Q..a...>..c... +| 576: 24 03 04 83 40 02 01 64 01 08 81 18 83 2e 02 01 $...@..d........ +| 592: 65 03 04 82 46 02 01 66 09 04 84 33 02 01 67 03 e...F..f...3..g. +| 608: 04 81 55 02 07 68 c2 b9 62 71 61 30 02 02 48 02 ..U..h..bqa0..H. +| 624: 01 69 04 04 82 4b 03 01 78 03 04 81 3c 02 02 6a .i...K..x...<..j +| 640: 34 05 02 47 02 01 6b 01 04 82 29 03 01 38 0a 02 4..G..k...)..8.. +| 656: 49 02 01 6c 02 04 84 1a 01 04 84 58 07 04 82 2c I..l.......X..., +| 672: 02 01 6d 06 04 83 0b 02 04 84 0c 03 02 c2 bc 08 ..m............. +| 688: 04 83 17 02 01 6e 02 04 83 70 02 01 6f 04 04 81 .....n...p..o... +| 704: 56 03 01 61 03 02 5b 03 04 65 79 64 37 06 04 81 V..a..[..eyd7... +| 720: 38 03 03 ee a9 bf 08 02 45 02 01 71 04 04 81 75 8.......E..q...u +| 736: 03 01 74 09 04 83 66 02 01 72 06 02 4f 02 01 73 ..t...f..r..O..s +| 752: 01 04 81 26 07 02 2d 02 01 74 02 02 14 02 02 3c ...&..-..t.....< +| 768: 04 04 84 5d 02 02 77 64 02 02 31 02 01 78 02 04 ...]..wd..1..x.. +| 784: 84 59 03 04 84 65 02 01 79 02 02 58 04 04 81 49 .Y...e..y..X...I +| 800: 01 04 83 17 03 02 7f 02 02 c2 b3 04 02 05 03 01 ................ +| 816: b9 01 04 84 09 03 01 be 0a 04 82 59 02 02 ce bc ...........Y.... +| 832: 06 04 81 02 02 02 d7 9d 06 04 83 31 02 04 df 84 ...........1.... +| 848: da 8c 08 04 82 67 02 04 e1 b5 b6 6b 0a 04 81 7b .....g.....k.... +| 864: 02 03 e3 a1 92 03 04 83 66 02 04 f0 9b 93 a9 07 ........f....... +| 880: 04 85 0b 04 02 a8 81 06 04 82 3e 01 01 65 01 24 ..........>..e.$ +| 896: 43 34 18 15 08 09 51 19 5b 17 28 26 11 06 23 38 C4....Q.[.(&..#8 +| 912: 2b 2b 01 1e 1c 32 20 17 0f 2c 03 0c 1b 0c 6f 25 ++...2 ..,....o% +| 928: 73 16 40 01 1e 07 12 81 5a 44 19 06 03 24 5c 0e s.@.....ZD...$.. +| 944: 16 18 39 0d 01 26 0d 09 06 41 42 09 24 1d 25 0c ..9..&...AB.$.%. +| 960: 06 1d 63 14 26 18 7d 0a 1d 01 14 25 56 29 0a 81 ..c.&......%V).. +| 976: 1a 1b 37 5d 3d 01 16 1c 81 26 25 7f 41 52 46 08 ..7]=....&%.ARF. +| 992: 1b 16 01 12 81 0f 6b 7b 0f 2d 03 7f 07 01 18 19 ......k..-...... +| 1008: 28 81 01 50 36 41 16 81 22 17 03 01 1c 16 1a 81 (..P6A.......... +| 1024: 1f 10 17 41 4c 1c 0b 66 49 10 06 01 18 54 82 38 ...AL..fI....T.8 +| 1040: 13 2a 0b 14 4c 15 0f 36 0b 02 01 30 07 04 82 28 .*..L..6...0...( +| 1056: 02 01 31 09 04 81 2b 01 04 83 38 02 01 33 03 04 ..1...+...8..3.. +| 1072: 81 76 02 01 61 08 04 84 6b 03 05 7a ea 86 b3 6a .v..a...k..z...j +| 1088: 06 04 82 52 02 01 62 0a 04 83 19 02 01 63 02 04 ...R..b......c.. +| 1104: 82 4d 03 02 c2 be 04 04 82 29 02 01 64 05 04 81 .M.......)..d... +| 1120: 7a 03 04 81 72 02 02 66 61 04 02 41 03 01 77 06 z...r..fa..A..w. +| 1136: 02 3e 02 04 82 10 02 01 67 05 04 84 48 02 01 69 .>......g...H..i +| 1152: 01 04 83 12 02 02 6c 62 07 04 81 1d 02 01 6d 01 ......lb......m. +| 1168: 04 83 6e 02 01 6f 05 02 09 02 04 81 16 02 01 70 ..n..o.........p +| 1184: 05 06 07 82 72 03 01 38 07 04 84 46 02 01 73 02 ....r..8...F..s. +| 1200: 04 84 2d 05 04 81 58 03 04 81 4a 03 02 c2 bd 0a ..-...X...J..... +| 1216: 04 83 2d 02 01 74 02 04 84 55 05 04 85 02 02 01 ..-..t...U...... +| 1232: 75 02 04 82 62 02 01 76 01 04 84 31 02 01 77 09 u...b..v...1..w. +| 1248: 04 82 15 02 01 78 01 02 45 05 04 82 14 03 04 83 .....x..E....... +| 1264: 01 02 01 79 06 04 81 0c 02 02 7a 78 09 04 81 6d ...y......zx...m +| 1280: 02 04 c2 bc 61 72 04 04 83 20 04 01 6f 01 02 70 ....ar... ..o..p +| 1296: 02 02 c9 9e 01 04 82 02 02 02 ca 83 05 04 82 3f ...............? +| 1312: 03 01 9b 05 02 52 03 01 aa 07 04 83 65 02 02 cd .....R......e... +| 1328: bb 01 04 83 07 02 02 ce b2 06 02 5b 02 03 cf a7 ...........[.... +| 1344: 78 03 04 81 3b 02 03 ef a9 89 08 04 82 20 02 04 x...;........ .. +| 1360: f5 87 b0 93 02 04 84 18 01 01 66 01 1c 2b 3d 5b ..........f..+=[ +| 1376: 23 45 2d 41 17 0e 81 1d 0f 15 1f 01 16 05 81 20 #E-A........... +| 1392: 0d 27 81 0f 3e 28 2b 13 01 14 62 2c 44 26 17 04 .'..>(+...b,D&.. +| 1408: 23 50 6d 5b 01 1c 30 5b 81 27 07 29 38 15 4d 11 #Pm[..0[.'.)8.M. +| 1424: 2e 20 17 20 01 1a 19 65 4f 14 2e 55 0e 12 39 13 . . ...eO..U..9. +| 1440: 1d 70 12 01 20 1a 0d 2f 05 81 19 2b 42 05 37 4c .p.. ../...+B.7L +| 1456: 21 22 04 25 2b 01 14 45 3e 05 10 81 47 6c 81 11 !..%+..E>...Gl.. +| 1472: 11 01 1e 78 04 12 18 0a 05 10 14 6b 05 0a 04 82 ...x.......k.... +| 1488: 05 15 01 1e 05 23 4b 18 24 31 20 27 12 2e 81 0e .....#K.$1 '.... +| 1504: 31 18 18 01 10 55 7e 29 2a 06 19 65 38 02 01 30 1....U~)*..e8..0 +| 1520: 09 02 14 02 01 31 02 04 84 00 02 01 36 06 04 84 .....1......6... +| 1536: 7a 02 01 61 07 04 83 71 02 01 62 05 04 83 10 03 z..a...q..b..... +| 1552: 02 34 02 02 63 6f 07 04 83 27 02 01 64 07 04 81 .4..co...'..d... +| 1568: 0c 03 04 81 55 03 01 6f 03 02 7e 02 01 69 02 04 ....U..o..~..i.. +| 1584: 81 08 03 02 05 03 04 82 70 02 01 6a 05 02 41 03 ........p..j..A. +| 1600: 02 6f 30 04 02 5e 02 01 6b 02 04 82 13 02 01 6c .o0..^..k......l +| 1616: 02 04 83 2d 08 04 81 20 02 01 6e 09 04 82 69 02 ...-... ..n...i. +| 1632: 02 6f 38 02 04 83 73 02 01 71 0a 04 81 4f 02 01 .o8...s..q...O.. +| 1648: 74 09 04 83 2c 02 01 75 03 04 83 79 03 03 68 c2 t...,..u...y..h. +| 1664: b3 03 04 81 09 02 02 77 64 06 04 81 79 02 01 78 .......wd...y..x +| 1680: 03 04 83 41 03 02 19 02 01 79 02 04 82 1a 07 02 ...A.....y...... +| 1696: 72 03 01 6a 06 04 83 1a 02 01 7a 05 04 84 6d 01 r..j......z...m. +| 1712: 04 81 24 02 03 c2 aa 75 0a 04 84 72 03 01 b3 04 ..$....u...r.... +| 1728: 04 82 02 03 04 82 2c 03 01 ba 07 04 26 51 03 02 ......,.....&Q.. +| 1744: bd 65 07 04 81 1a 03 01 be 09 02 44 02 02 ca 97 .e.........D.... +| 1760: 02 04 84 30 02 03 d5 a7 6f 04 02 36 03 02 bd 72 ...0....o..6...r +| 1776: 07 04 81 03 02 05 f3 b2 a7 91 68 09 04 82 22 01 ..........h..... +| 1792: 01 67 01 16 81 41 81 1f 4c 0f 0d 0d 6f 3d 04 01 .g...A..L...o=.. +| 1808: 18 12 08 5e 05 16 4f 42 61 22 45 3c 27 01 20 08 ...^..OBa.E<'. . +| 1824: 13 27 14 17 1f 08 3e 0d 41 81 1c 3e 06 1c 1e 01 .'....>.A..>.... +| 1840: 16 0c 11 6b 26 54 05 14 03 81 6b 12 01 1a 36 41 ...k&T....k...6A +| 1856: 11 0b 0c 36 7c 08 0e 0c 52 4f 50 01 0c 81 0e 22 ...6|...ROP..... +| 1872: 3d 72 1f 01 22 07 0e 22 3c 4d 1c 09 03 2d 22 1e =r.........?.....u.. +| 2656: be 6b 05 04 83 0c 02 04 c3 b8 66 70 01 04 82 2f .k........fp.../ +| 2672: 02 02 ce bc 08 04 81 04 02 03 cf a7 6b 06 04 82 ............k... +| 2688: 4d 02 03 d9 84 72 03 02 13 02 02 da 90 06 04 84 M....r.......... +| 2704: 73 02 04 f0 ad 93 a0 06 02 09 01 01 69 01 20 37 s...........i. 7 +| 2720: 4d 55 11 49 26 3c 17 0d 4a 26 03 05 0c 20 43 01 MU.I&<..J&... C. +| 2736: 10 59 08 37 5a 56 72 81 05 01 1a 16 33 32 0a 03 .Y.7ZVr.....32.. +| 2752: 18 81 17 5f 59 27 37 08 01 24 0a 70 1f 07 40 3b ..._Y'7..$.p..@; +| 2768: 15 20 09 15 30 0f 17 06 36 81 08 17 01 1c 32 0a . ..0...6.....2. +| 2784: 05 1a 5f 1c 32 78 03 15 17 81 2b 1e 01 1a 04 03 .._.2x....+..... +| 2800: 2b 07 0c 38 43 6c 0f 81 77 0e 0e 01 1a 1c 3d 81 +..8Cl..w.....=. +| 2816: 0d 0d 17 1d 4c 63 44 2e 11 2b 01 18 2b 2d 49 72 ....LcD..+..+-Ir +| 2832: 1d 22 17 31 19 27 32 4c 01 1c 0b 67 27 20 81 1c ...1.'2L...g' .. +| 2848: 47 31 29 0d 0b 12 4c 07 01 18 35 81 0b 08 0e 5f G1)...L...5...._ +| 2864: 11 18 37 81 09 09 02 01 30 09 02 19 02 02 31 63 ..7.....0.....1c +| 2880: 09 02 7f 02 01 32 04 02 2c 05 04 83 6e 02 01 35 .....2..,...n..5 +| 2896: 09 04 83 7a 03 01 79 05 02 0f 02 02 36 73 05 04 ...z..y.....6s.. +| 2912: 83 3d 02 03 38 6a 70 07 04 84 39 02 01 61 07 04 .=..8jp...9..a.. +| 2928: 83 44 02 01 62 0a 04 84 37 02 01 64 01 04 85 00 .D..b...7..d.... +| 2944: 07 04 82 36 03 02 cf a7 01 04 82 5f 02 01 66 06 ...6......._..f. +| 2960: 04 82 35 03 03 71 70 6e 04 04 82 14 02 02 67 7a ..5..qpn......gz +| 2976: 02 04 82 00 02 01 68 01 02 14 02 01 69 05 02 48 ......h.....i..H +| 2992: 02 01 6a 01 04 83 7a 07 02 12 01 02 33 02 01 6d ..j...z.....3..m +| 3008: 04 04 83 76 03 01 68 08 04 82 63 02 01 6f 03 06 ...v..h...c..o.. +| 3024: 2c 81 52 06 04 81 4f 02 01 70 06 04 83 00 03 02 ,.R...O..p...... +| 3040: c2 be 06 04 83 37 02 01 72 09 04 82 78 03 03 71 .....7..r...x..q +| 3056: 61 6c 0a 02 3e 03 02 c2 be 05 04 84 67 02 01 73 al..>.......g..s +| 3072: 09 04 81 0d 02 01 74 04 02 58 02 01 75 08 04 81 ......t..X..u... +| 3088: 17 02 01 76 04 04 83 4b 06 04 84 70 02 01 7a 02 ...v...K...p..z. +| 3104: 04 83 7e 02 02 c2 aa 03 02 61 03 01 b2 08 04 84 ..~......a...... +| 3120: 13 03 04 b9 c2 b2 6a 01 02 19 03 02 bc 63 05 04 ......j......c.. +| 3136: 84 12 02 02 c7 86 09 04 84 11 02 02 ce bc 02 02 ................ +| 3152: 17 02 03 cf ab 6e 07 02 29 02 06 d5 b1 30 d1 bb .....n..)....0.. +| 3168: 6e 01 02 4e 02 03 eb 8b a7 08 04 81 6b 03 02 b5 n..N........k... +| 3184: bc 07 04 83 08 01 01 6a 01 14 81 07 79 0f 79 32 .......j....y.y2 +| 3200: 33 13 81 16 01 16 2c 0a 81 1b 24 2f 05 81 19 19 3.....,...$/.... +| 3216: 2d 01 16 81 4c 03 14 4f 81 08 2d 05 2f 21 01 18 -...L..O..-./!.. +| 3232: 27 1f 0e 81 16 0c 82 4b 04 12 06 0e 01 1e 26 05 '......K......&. +| 3248: 22 14 55 05 09 81 16 21 05 19 37 74 32 01 1e 37 ..U....!..7t2..7 +| 3264: 07 1e 03 10 81 2c 38 07 1a 2a 2b 81 0d 08 01 1e .....,8..*+..... +| 3280: 2b 19 31 3c 4c 03 1a 14 04 1c 06 76 04 48 62 01 +.1-......p$D..b +| 3760: 12 3b 50 15 34 03 16 01 0c 32 08 08 0a 0a 0b 07 .;P.4....2...... +| 3776: 10 07 07 07 06 07 07 06 07 06 07 07 0a 08 0a 0b ................ +| 3792: 07 06 08 08 07 08 08 08 08 07 08 07 09 09 0a 81 ................ +| 3808: 03 07 07 0f 07 07 0b 09 07 07 07 0c 07 07 07 07 ................ +| 3824: 06 0f 0b 08 07 07 06 0a 08 07 07 06 0a 0d 07 0b ................ +| 3840: 11 07 07 07 08 08 0a 0a 09 0a 08 81 1e 07 0b 07 ................ +| 3856: 07 0b 07 07 08 0b 07 0a 07 07 08 07 0a 08 07 0f ................ +| 3872: 08 0b 07 07 07 0e 07 08 0a 06 08 08 06 07 08 07 ................ +| 3888: 09 09 0a 81 15 06 07 07 07 0a 08 0b 06 0e 06 07 ................ +| 3904: 07 0b 07 08 07 07 07 09 08 0a 0a 07 0b 09 0b 07 ................ +| 3920: 08 06 08 08 08 0b 81 0d 08 08 07 07 07 0b 0b 0b ................ +| 3936: 08 07 07 07 0f 0b 06 07 07 07 08 0c 07 07 08 08 ................ +| 3952: 0e 09 0b 07 06 08 06 07 07 08 09 07 0a 0a 81 1f ................ +| 3968: 07 09 08 06 0b 0a 0d 07 0a 07 08 0a 09 07 08 07 ................ +| 3984: 07 09 06 0d 09 07 0a 08 08 0b 0b 06 08 0a 08 09 ................ +| 4000: 08 08 09 81 1c 06 07 0a 07 06 08 09 07 07 0b 08 ................ +| 4016: 07 09 08 06 06 0d 07 07 0c 07 08 07 08 08 07 06 ................ +| 4032: 07 0b 07 07 07 09 08 08 07 08 0b 09 08 81 16 09 ................ +| 4048: 07 07 07 07 08 07 09 07 07 07 07 07 07 07 0a 07 ................ +| 4064: 07 07 07 09 07 09 08 07 07 07 07 07 07 07 09 09 ................ +| 4080: 0a 07 06 07 0b 09 07 09 07 08 09 09 0c 0b 07 07 ................ +| page 22 offset 86016 +| 0: 0d 00 00 00 01 00 24 00 00 24 00 00 00 00 00 00 ......$..$...... +| 32: 00 00 00 00 9f 54 88 80 80 80 80 03 04 00 bf 2c .....T........., +| 48: 00 0a 0e 8b 61 42 11 82 12 4d 05 12 81 47 48 81 ....aB...M...GH. +| 64: 1c 3a 41 17 25 01 2a 42 21 1d 1f 07 44 65 2c 0f .:A.%.*B!...De,. +| 80: 18 20 18 08 03 19 04 10 19 0b 17 26 01 1a 14 1f . .........&.... +| 96: 11 1f 2b 10 82 08 26 2b 76 0c 27 01 22 4c 23 06 ..+...&+v.'..L#. +| 112: 81 10 2e 2e 03 37 4b 13 14 14 1b 15 24 0d 01 18 .....7K.....$... +| 128: 3a 0e 4b 15 7b 2a 4a 13 24 81 10 0a 01 18 11 16 :.K..*J.$....... +| 144: 28 1f 57 54 15 16 6b 50 2f 04 04 30 6b 30 34 05 (.WT..kP/..0k04. +| 160: 04 83 1c 02 01 31 04 02 39 02 02 22 01 04 82 48 .....1..9......H +| 176: 02 01 32 09 04 81 25 02 01 34 01 04 81 1a 02 01 ..2...%..4...... +| 192: 35 01 02 09 03 01 30 09 04 83 29 02 01 37 05 04 5.....0...)..7.. +| 208: 82 70 02 01 38 06 04 83 2f 02 01 39 03 04 83 29 .p..8.../..9...) +| 224: 02 01 61 09 02 67 02 01 62 02 04 82 57 02 01 63 ..a..g..b...W..c +| 240: 01 04 82 24 02 02 64 33 09 04 84 06 02 01 65 09 ...$..d3......e. +| 256: 04 84 46 02 01 66 03 04 84 15 01 04 84 69 04 04 ..F..f.......i.. +| 272: 84 36 02 01 67 06 04 84 7e 02 08 83 35 81 2c 02 .6..g...~...5.,. +| 288: 01 68 03 04 82 3a 05 08 81 2e 81 20 02 01 69 02 .h...:..... ..i. +| 304: 04 81 69 02 01 6a 06 02 46 01 08 82 1e 82 5b 02 ..i..j..F.....[. +| 320: 04 84 20 02 01 6b 09 04 81 2e 02 01 6c 08 04 84 .. ..k......l... +| 336: 56 03 01 62 0a 04 83 3f 02 02 6d 6a 01 04 82 15 V..b...?..mj.... +| 352: 02 04 6f 69 72 33 01 04 82 25 02 01 72 01 02 44 ..oir3...%..r..D +| 368: 03 01 35 06 04 81 3b 02 02 74 74 04 04 81 79 02 ..5...;..tt...y. +| 384: 01 78 01 04 81 5b 02 02 7a 02 01 79 04 02 19 02 .x...[..z..y.... +| 400: 03 7a 71 6b 03 04 83 37 02 02 c2 b3 01 02 31 04 .zqk...7......1. +| 416: 04 82 04 03 07 ba 6e f5 9a 8b 98 76 02 02 5b 02 ......n....v..[. +| 432: 02 cb 8f 06 04 84 28 02 02 ce bc 01 04 81 66 06 ......(.......f. +| 448: 04 81 1b 02 04 83 47 04 06 ce bc ce bf df bc 09 ......G......... +| 464: 04 82 4f 02 02 dd 9c 03 04 81 62 02 04 f2 86 b3 ..O.......b..... +| 480: bb 01 04 82 08 01 01 6c 01 12 81 34 81 00 0d 2a .......l...4...* +| 496: 81 48 13 01 1a 0e 14 4f 81 17 14 12 10 75 68 36 .H.....O.....uh6 +| 512: 0b 1d 01 16 12 22 23 51 1d 7c 25 08 20 4f 7a 01 ......#Q.|%. Oz. +| 528: 12 08 0b 81 7c 71 81 12 0b 0b 01 0e 4e 50 55 82 ....|q......NPU. +| 544: 1e 2e 23 01 1c 0f 10 24 07 33 06 81 0c 73 42 1d ..#....$.3...sB. +| 560: 81 04 16 01 16 5e 62 6a 06 32 81 03 2f 0f 08 4e .....^bj.2../..N +| 576: 01 06 81 2c 22 01 12 53 81 1e 6e 40 31 0b 1c 41 ...,...S..n@1..A +| 592: 01 18 79 14 11 17 13 81 0f 81 15 19 35 40 02 01 ..y.........5@.. +| 608: 30 01 04 81 0d 03 04 83 6b 02 01 33 07 04 84 66 0.......k..3...f +| 624: 03 04 83 08 02 01 35 08 04 83 43 01 04 83 50 02 ......5...C...P. +| 640: 01 36 09 02 3b 02 01 37 07 04 84 38 02 01 61 0a .6..;..7...8..a. +| 656: 04 83 17 03 04 e1 8d 93 69 01 04 81 0e 02 01 62 ........i......b +| 672: 03 04 84 16 04 04 83 0e 02 01 63 05 04 81 79 05 ..........c...y. +| 688: 04 84 5d 02 01 64 07 04 81 4a 03 02 6d 6e 08 02 ..]..d...J..mn.. +| 704: 09 02 01 65 06 04 84 3e 03 02 76 7a 02 02 76 02 ...e...>..vz..v. +| 720: 01 66 0a 02 47 03 01 79 09 04 81 21 02 02 67 6f .f..G..y...!..go +| 736: 05 02 38 02 02 68 77 03 04 82 27 02 01 69 02 02 ..8..hw...'..i.. +| 752: 6f 04 02 6f 02 01 6b 07 04 82 61 02 01 6c 06 08 o..o..k...a..l.. +| 768: 82 2e 81 5a 02 01 6d 01 04 84 25 03 04 83 31 02 ...Z..m...%...1. +| 784: 06 2f 82 2e 03 01 68 08 04 83 73 02 01 6e 09 04 ./....h...s..n.. +| 800: 84 5a 02 01 6f 05 04 82 39 02 01 70 04 04 82 19 .Z..o...9..p.... +| 816: 05 02 0f 02 01 71 07 04 83 39 03 02 63 71 06 04 .....q...9..cq.. +| 832: 81 5e 03 01 67 07 04 83 73 03 01 70 08 04 84 6a .^..g...s..p...j +| 848: 02 01 73 03 04 83 40 02 01 74 07 04 84 7c 03 06 ..s...@..t...|.. +| 864: 35 69 7a 70 c2 bd 01 04 83 1a 02 01 75 01 08 82 5izp........u... +| 880: 2e 81 13 05 04 82 69 03 01 70 03 04 84 50 02 01 ......i..p...P.. +| 896: 76 01 04 82 52 09 04 82 64 02 02 77 73 03 04 81 v...R...d..ws... +| 912: 27 02 01 78 08 02 1f 01 04 83 64 02 01 79 04 08 '..x......d..y.. +| 928: 81 49 81 44 05 04 84 01 02 03 c2 aa 76 02 04 82 .I.D........v... +| 944: 67 03 01 b9 02 02 4f 06 02 31 03 01 bd 05 04 82 g.....O..1...... +| 960: 3d 02 02 ce bc 01 02 2c 01 01 6d 01 12 0c 47 83 =......,..m...G. +| 976: 0a 07 52 16 41 05 01 18 50 05 74 25 21 81 16 16 ..R.A...P.t%!... +| 992: 05 24 16 6c 01 22 81 05 18 06 12 2a 59 09 1f 09 .$.l.......*Y... +| 1008: 3d 27 39 17 3a 1f 04 01 14 72 81 1a 24 81 38 20 ='9.:....r..$.8 +| 1024: 27 19 2b 01 18 81 2b 24 63 0b 11 5d 2d 0c 41 1a '.+...+$c..]-.A. +| 1040: 30 01 1e 15 04 3c 3a 15 0e 28 18 06 56 81 37 20 0....<:..(..V.7 +| 1056: 0d 6b 01 1e 63 3f 42 73 0a 26 0c 6f 0f 15 13 0b .k..c?Bs.&.o.... +| 1072: 04 25 33 01 16 24 52 16 3a 21 1b 63 26 31 20 34 .%3..$R.:!.c&1 4 +| 1088: 01 0e 1a 60 13 19 3d 82 1f 01 1e 5b 07 26 07 20 ...`..=....[.&. +| 1104: 03 5a 69 6c 27 0b 22 24 20 0f 02 01 31 01 04 82 .Zil'..$ ...1... +| 1120: 6f 02 01 32 03 04 82 41 01 04 81 43 02 01 33 09 o..2...A...C..3. +| 1136: 04 81 23 03 05 7a 68 c2 bd 73 06 04 81 3a 02 01 ..#..zh..s...:.. +| 1152: 35 07 04 83 6c 02 01 38 01 04 83 1c 02 02 62 6f 5...l..8......bo +| 1168: 06 04 83 16 02 01 63 07 02 5b 03 06 6e 31 cf af ......c..[..n1.. +| 1184: 61 71 0a 04 83 33 03 02 c2 be 03 04 83 2b 02 01 aq...3.......+.. +| 1200: 64 09 04 81 19 01 04 83 09 03 01 62 0a 04 82 56 d..........b...V +| 1216: 03 01 6a 03 04 84 4e 02 01 65 05 04 82 3a 02 04 ..j...N..e...:.. +| 1232: 82 53 02 01 66 09 04 82 7a 02 01 67 02 04 82 69 .S..f...z..g...i +| 1248: 08 04 81 35 02 01 68 09 04 83 2f 02 01 69 01 04 ...5..h.../..i.. +| 1264: 81 62 02 01 6a 0a 04 81 59 02 01 6b 09 04 81 5f .b..j...Y..k..._ +| 1280: 02 01 6c 08 04 84 0d 02 01 6d 05 04 84 32 02 01 ..l......m...2.. +| 1296: 6e 01 04 83 05 02 01 6f 07 04 82 6f 02 01 70 04 n......o...o..p. +| 1312: 04 83 60 03 04 82 0d 03 03 c2 b9 70 02 04 81 43 ..`........p...C +| 1328: 02 01 71 07 04 81 13 03 04 82 35 02 01 72 03 04 ..q.......5..r.. +| 1344: 84 51 02 02 73 65 04 04 84 14 02 01 74 01 04 83 .Q..se......t... +| 1360: 5f 02 01 75 03 02 0b 03 02 c2 be 08 04 82 3a 02 _..u..........:. +| 1376: 01 76 06 04 84 43 02 01 77 01 04 82 18 01 04 81 .v...C..w....... +| 1392: 32 02 01 78 08 04 81 47 02 01 7a 01 04 82 44 02 2..x...G..z...D. +| 1408: 02 c2 b3 05 04 83 12 03 04 83 14 04 01 37 04 04 .............7.. +| 1424: 83 6c 03 03 ba dd af 05 04 82 41 02 02 c3 9f 07 .l........A..... +| 1440: 04 82 34 02 03 c6 80 78 0a 04 83 14 02 02 ca 95 ..4....x........ +| 1456: 07 04 83 0c 02 03 ce b0 67 03 02 6e 03 01 bc 05 ........g..n.... +| 1472: 02 17 04 02 6c 02 03 e7 a4 94 05 04 82 51 01 01 ....l........Q.. +| 1488: 6e 01 18 56 26 2f 54 29 5a 25 11 28 24 81 0f 01 n..V&/T)Z%.($... +| 1504: 1c 06 39 06 13 59 09 07 81 39 16 0c 39 27 4a 01 ..9..Y...9..9'J. +| 1520: 14 2a 03 53 23 81 0d 7d 5b 06 03 01 18 1e 03 81 .*.S#...[....... +| 1536: 02 34 37 29 15 34 81 30 36 01 1a 81 25 19 07 21 .47).4.06...%..! +| 1552: 07 33 12 82 02 45 04 05 01 12 82 0f 42 55 1d 08 .3...E......BU.. +| 1568: 1a 44 3d 01 16 18 17 36 54 6e 1b 49 1f 30 09 1c .D=....6Tn.I.0.. +| 1584: 01 16 68 81 00 08 48 3a 13 50 3c 28 27 01 1c 28 ..h...H:.P<('..( +| 1600: 03 06 12 5b 06 30 38 04 65 54 1a 0b 4d 01 20 03 ...[.08.eT..M. . +| 1616: 09 3d 27 04 11 59 11 1a 0b 73 53 81 03 04 26 02 .='..Y...sS...&. +| 1632: 01 32 02 04 83 3e 02 01 36 01 02 73 09 04 81 14 .2...>..6..s.... +| 1648: 02 01 37 0a 04 83 30 02 01 62 08 04 84 4d 02 02 ..7...0..b...M.. +| 1664: 63 7a 0a 04 84 08 02 01 64 07 04 83 61 03 02 7a cz......d...a..z +| 1680: 67 09 02 58 02 02 65 69 09 02 4e 02 05 67 eb 9b g..X..ei..N..g.. +| 1696: a1 72 09 04 81 06 02 01 68 01 04 82 13 02 01 69 .r......h......i +| 1712: 08 04 84 39 01 04 82 45 02 01 6a 0a 04 83 72 02 ...9...E..j...r. +| 1728: 02 6b 70 07 04 81 25 02 01 6c 0a 02 5a 02 01 6d .kp...%..l..Z..m +| 1744: 03 08 82 58 81 11 01 04 84 2c 02 01 6e 07 02 74 ...X.....,..n..t +| 1760: 02 01 6f 05 04 82 7d 02 01 71 0a 02 7c 02 02 72 ..o......q..|..r +| 1776: 75 05 04 83 09 02 01 74 03 04 83 1d 02 01 75 07 u......t......u. +| 1792: 04 84 21 02 01 76 07 04 84 49 02 01 77 03 02 09 ..!..v...I..w... +| 1808: 02 04 81 01 02 01 78 06 04 81 0f 02 01 7a 04 04 ......x......z.. +| 1824: 81 3e 02 04 c2 aa 64 6f 05 04 83 11 03 02 bc 64 .>....do.......d +| 1840: 09 04 81 7b 02 05 c6 80 39 63 64 01 04 83 62 03 ........9cd...b. +| 1856: 01 92 09 04 82 42 02 02 ce bc 09 02 4d 02 02 cf .....B......M... +| 1872: 97 06 04 83 42 02 02 d1 8b 02 02 2d 02 05 dd b1 ....B......-.... +| 1888: ca b2 70 01 02 1b 02 03 e6 b5 96 09 04 81 50 02 ..p...........P. +| 1904: 04 e8 b8 aa 6e 08 04 82 1a 02 05 f0 90 b2 b2 6c ....n..........l +| 1920: 01 02 59 02 04 f1 8f 99 80 0a 04 83 25 01 01 6f ..Y.........%..o +| 1936: 01 26 05 6b 0f 30 12 1f 16 5d 19 08 20 08 33 33 .&.k.0...].. .33 +| 1952: 1e 12 31 31 06 01 16 04 63 33 67 24 2b 81 0a 5d ..11....c3g$+..] +| 1968: 2c 0b 01 14 18 0f 36 82 49 0c 1c 08 27 4b 01 1a ,.....6.I...'K.. +| 1984: 4b 24 3a 13 64 07 14 0b 43 08 68 2e 3c 01 26 1a K$:.d...C.h.<.&. +| 2000: 0b 06 58 10 3d 1b 0a 2f 3e 27 36 1b 1b 21 05 25 ..X.=../>'6..!.% +| 2016: 0d 41 01 12 81 16 4b 2e 7c 7d 32 1f 1e 01 1a 2e .A....K.|.2..... +| 2032: 20 62 72 11 22 05 20 0a 2c 24 60 5e 01 2a 0e 5e br... .,$`^.*.^ +| 2048: 13 22 2d 1c 0c 1f 0c 10 0c 47 05 4e 03 15 06 33 ..-......G.N...3 +| 2064: 1d 06 29 01 10 81 08 81 0b 81 48 53 11 01 20 14 ..).......HS.. . +| 2080: 25 81 17 10 08 08 47 2a 56 2b 2f 20 05 26 22 02 %.....G*V+/ .&.. +| 2096: 01 30 0a 02 5e 03 02 39 76 08 04 82 5c 02 01 31 .0..^..9v......1 +| 2112: 04 02 54 02 01 33 01 04 81 2d 02 02 36 37 09 02 ..T..3...-..67.. +| 2128: 45 03 01 61 04 04 84 16 02 01 37 0a 04 84 5e 02 E..a......7...^. +| 2144: 03 39 c2 b3 01 04 83 2c 02 01 61 08 04 82 4b 02 .9.....,..a...K. +| 2160: 01 62 08 04 83 08 03 02 65 30 07 04 84 0d 02 01 .b......e0...... +| 2176: 64 02 04 82 5e 02 04 81 07 02 05 65 70 6b 7a 35 d...^......epkz5 +| 2192: 02 02 4e 03 03 db 8a 69 01 04 83 0b 02 01 66 05 ..N....i......f. +| 2208: 04 84 34 03 01 68 04 04 84 51 02 01 67 05 02 53 ..4..h...Q..g..S +| 2224: 02 04 82 78 02 01 68 09 04 83 6f 02 01 69 03 04 ...x..h...o..i.. +| 2240: 81 42 02 01 6a 01 04 84 55 03 04 84 10 02 04 81 .B..j...U....... +| 2256: 56 03 02 72 74 06 04 83 20 02 02 6c 7a 01 04 83 V..rt... ..lz... +| 2272: 38 02 01 6e 01 02 15 02 01 6f 01 04 81 7c 01 02 8..n.....o...|.. +| 2288: 27 02 01 73 01 02 36 04 02 2e 03 03 65 ce bc 08 '..s..6.....e... +| 2304: 04 82 43 02 01 74 09 04 82 7f 02 01 75 07 04 82 ..C..t......u... +| 2320: 13 03 01 66 07 04 83 21 03 02 6a 69 07 04 83 7e ...f...!..ji...~ +| 2336: 03 01 6b 01 04 84 26 02 01 76 0a 04 82 7f 03 05 ..k...&..v...... +| 2352: 67 da b2 65 75 0a 04 84 13 03 06 c2 be c9 87 6b g..eu..........k +| 2368: 61 07 04 81 5b 02 02 77 64 01 02 3c 02 01 7a 06 a...[..wd..<..z. +| 2384: 04 81 21 01 04 81 29 02 04 84 4b 03 02 79 30 09 ..!...)...K..y0. +| 2400: 04 83 7d 03 02 ce bc 02 04 81 6a 02 02 c2 aa 05 ..........j..... +| 2416: 02 50 03 01 b2 02 04 81 12 05 02 3e 04 01 63 07 .P.........>..c. +| 2432: 04 84 04 03 03 b3 62 61 01 04 84 3e 03 02 ba 6f ......ba...>...o +| 2448: 02 04 84 51 02 02 c4 91 04 04 84 36 02 05 ce b9 ...Q.......6.... +| 2464: 65 c9 97 07 04 83 76 02 04 ef 82 ac 6d 09 02 5d e.....v.....m..] +| 2480: 02 05 f0 92 bd b3 6f 03 04 81 25 03 04 99 ba 8f ......o...%..... +| 2496: 64 02 04 81 1f 02 05 f3 8a bd b1 6d 09 04 81 28 d..........m...( +| 2512: 01 01 70 01 18 46 7c 1f 14 0a 1b 2e 1c 2d 14 6b ..p..F|......-.k +| 2528: 07 01 18 19 81 05 2e 28 40 11 2b 22 0b 07 18 01 .......(@.+..... +| 2544: 14 40 13 81 36 12 2d 7f 1f 1b 1a 01 12 82 76 1e .@..6.-.......v. +| 2560: 1d 12 81 03 07 12 01 10 5a 81 38 19 36 52 26 18 ........Z.8.6R&. +| 2576: 01 12 64 5d 40 25 05 57 33 31 11 01 1e 36 36 25 ..d]@%.W31...66% +| 2592: 1b 28 07 22 42 13 0f 07 1c 5a 73 51 01 24 07 2d .(..B....ZsQ.$.- +| 2608: 06 03 81 00 22 03 1d 19 18 26 2c 16 23 4f 76 08 .........&,.#Ov. +| 2624: 01 1c 11 1b 54 78 23 1a 81 13 75 04 0c 14 12 0c ....Tx#...u..... +| 2640: 01 12 24 0d 81 4d 11 7f 22 35 2e 02 01 30 02 04 ..$..M...5...0.. +| 2656: 83 5e 02 01 31 0a 04 81 47 03 05 e7 b2 a5 c3 9f .^..1...G....... +| 2672: 04 04 83 16 02 01 35 08 02 35 02 02 36 79 09 04 ......5..5..6y.. +| 2688: 82 24 02 01 37 01 04 81 30 03 02 6d 65 04 04 82 .$..7...0..me... +| 2704: 0e 02 02 38 63 06 02 41 03 02 c9 aa 06 04 84 5b ...8c..A.......[ +| 2720: 02 01 61 03 04 83 0b 03 04 84 4c 03 04 83 5f 03 ..a.......L..._. +| 2736: 01 6c 07 04 83 7f 02 01 63 04 04 84 7b 02 01 64 .l......c......d +| 2752: 04 04 83 5d 02 01 65 06 04 83 4c 03 01 74 08 04 ...]..e...L..t.. +| 2768: 82 0b 02 01 68 03 02 06 03 04 71 e7 8e bb 08 04 ....h.....q..... +| 2784: 81 43 03 01 74 07 04 82 30 02 03 69 ce bc 07 04 .C..t...0..i.... +| 2800: 81 3c 02 03 6a 62 6c 09 02 4b 02 01 6c 09 04 81 .<..jbl..K..l... +| 2816: 60 02 01 6d 05 04 82 6f 04 04 81 62 03 04 6c 74 `..m...o...b..lt +| 2832: c2 b9 07 04 85 08 03 02 c2 be 08 04 83 2b 02 02 .............+.. +| 2848: 6e 73 08 04 81 1f 03 01 75 04 04 83 72 02 01 6f ns......u...r..o +| 2864: 06 08 81 14 83 2f 03 02 ce bc 0a 04 83 67 03 03 ...../.......g.. +| 2880: cf 80 66 07 02 28 02 01 70 01 04 82 2a 03 01 7a ..f..(..p...*..z +| 2896: 08 04 81 0d 02 01 71 07 04 81 37 02 01 73 05 02 ......q...7..s.. +| 2912: 59 03 01 6d 07 02 7a 02 01 74 05 04 82 62 02 02 Y..m..z..t...b.. +| 2928: 75 6a 03 04 81 37 02 02 77 68 08 04 84 14 02 01 uj...7..wh...... +| 2944: 78 01 04 83 6f 02 01 79 01 04 82 66 06 04 81 36 x...o..y...f...6 +| 2960: 02 01 7a 0a 04 81 15 03 05 6f f0 99 8a a6 08 04 ..z......o...... +| 2976: 82 06 02 02 c2 b2 04 04 83 38 03 03 b3 dd 9d 05 .........8...... +| 2992: 04 83 6f 03 01 b9 06 04 83 66 03 01 ba 01 04 81 ..o......f...... +| 3008: 0c 07 04 83 10 04 01 69 06 04 81 39 03 01 bd 04 .......i...9.... +| 3024: 04 82 00 06 02 16 02 02 da 99 02 04 84 49 02 07 .............I.. +| 3040: dd 9b 6e e6 89 a0 75 01 04 81 6e 02 09 f0 92 ab ..n...u...n..... +| 3056: ab ec 8c 8d ca 81 01 04 81 46 01 01 71 01 1a 1a .........F..q... +| 3072: 08 41 65 0b 22 18 69 2f 81 0e 13 05 01 18 08 2d .Ae...i/.......- +| 3088: 07 04 81 70 23 18 20 1a 0c 74 01 16 1f 11 05 6b ...p#. ..t.....k +| 3104: 2f 18 47 13 16 30 4c 01 22 12 05 2d 39 05 07 61 /.G..0L....-9..a +| 3120: 0b 06 0a 56 3f 56 2e 10 0d 50 01 16 1f 1f 3c 2b ...V?V...P....<+ +| 3136: 32 0c 43 4b 81 18 5a 01 18 16 0d 0e 32 40 42 1c 2.CK..Z.....2@B. +| 3152: 4a 24 12 17 13 01 20 03 6e 0e 1c 0f 04 0b 3b 05 J$.... .n.....;. +| 3168: 52 4e 0e 23 26 50 20 01 12 59 09 72 4f 47 44 15 RN.#&P ..Y.rOGD. +| 3184: 41 28 01 16 5e 36 24 0e 19 81 15 07 81 26 16 01 A(..^6$......&.. +| 3200: 18 81 0e 0f 5e 81 01 04 04 1a 1a 44 50 02 02 31 ....^......DP..1 +| 3216: 68 09 04 83 76 02 01 32 07 04 81 70 02 02 33 7a h...v..2...p..3z +| 3232: 05 04 84 4a 02 01 34 08 04 81 02 03 01 69 08 04 ...J..4......i.. +| 3248: 81 0b 03 01 76 02 04 81 62 02 01 37 04 04 81 78 ....v...b..7...x +| 3264: 05 04 82 51 01 02 15 02 03 38 75 77 06 04 82 1b ...Q.....8uw.... +| 3280: 02 01 39 04 02 20 03 04 84 03 03 01 74 03 04 81 ..9.. ......t... +| 3296: 60 02 01 61 04 02 24 01 04 83 63 02 01 62 09 02 `..a..$...c..b.. +| 3312: 15 02 01 63 05 06 51 84 0e 03 03 c2 be 62 04 04 ...c..Q......b.. +| 3328: 82 3d 02 02 64 70 03 04 83 36 02 01 66 06 04 82 .=..dp...6..f... +| 3344: 66 02 02 6a 38 02 04 84 0a 02 02 6d 61 05 04 81 f..j8......ma... +| 3360: 30 03 01 6e 08 04 83 71 02 01 6e 01 02 60 02 01 0..n...q..n..`.. +| 3376: 70 02 04 82 43 02 01 71 05 06 82 48 49 02 01 73 p...C..q...HI..s +| 3392: 06 04 81 1f 02 01 74 09 04 81 70 02 01 75 05 04 ......t...p..u.. +| 3408: 83 44 02 01 76 05 04 84 08 04 04 82 1d 02 01 77 .D..v..........w +| 3424: 09 02 69 02 02 78 69 08 04 82 79 02 01 7a 05 04 ..i..xi...y..z.. +| 3440: 84 22 03 09 66 cf 9f 71 67 36 e2 b1 a5 02 04 81 ....f..qg6...... +| 3456: 21 02 03 c2 aa 6a 01 04 81 12 03 01 b3 03 02 35 !....j.........5 +| 3472: 02 04 84 1f 04 04 84 19 03 04 b9 d4 87 68 02 04 .............h.. +| 3488: 81 42 03 01 ba 06 04 84 44 04 02 64 67 03 04 84 .B......D..dg... +| 3504: 4b 03 01 bc 04 04 81 42 03 01 be 01 02 48 02 02 K......B.....H.. +| 3520: ce bc 0a 04 81 2e 02 02 cf a7 02 04 83 6e 02 04 .............n.. +| 3536: e7 a4 ba 67 07 04 83 59 01 01 72 01 20 41 1d 3a ...g...Y..r. A.: +| 3552: 0b 05 04 12 1f 3c 03 2f 81 05 81 2e 13 01 14 0c .....<./........ +| 3568: 81 3f 17 81 07 63 08 81 07 01 12 81 06 71 17 0b .?...c.......q.. +| 3584: 46 81 5a 13 01 18 02 81 06 0f 49 42 2d 39 2e 2c F.Z.......IB-9., +| 3600: 45 27 01 16 06 19 27 1d 3f 53 81 3a 45 0d 20 01 E'....'.?S.:E. . +| 3616: 14 1b 07 2a 76 37 36 09 82 4d 04 01 14 0c 30 2d ...*v76..M....0- +| 3632: 2d 51 38 31 49 49 74 01 16 29 17 0a 2f 72 1d 4a -Q81IIt..)../r.J +| 3648: 81 27 11 18 01 14 63 81 1c 5f 04 4d 38 19 18 0e .'....c.._.M8... +| 3664: 01 1c 56 2f 25 5f 07 05 03 28 24 5d 2b 4d 42 13 ..V/%_...($]+MB. +| 3680: 02 02 30 71 01 04 84 03 02 01 33 08 04 83 16 01 ..0q......3..... +| 3696: 04 83 3d 02 01 35 03 02 58 02 01 37 04 04 83 0c ..=..5..X..7.... +| 3712: 03 01 64 04 04 84 46 02 02 38 61 0a 04 83 66 03 ..d...F..8a...f. +| 3728: 02 73 69 09 04 82 5f 02 01 39 09 04 84 17 02 02 .si..._..9...... +| 3744: 61 6a 03 04 83 12 02 01 62 07 04 82 72 03 01 65 aj......b...r..e +| 3760: 09 04 83 14 04 01 68 07 04 82 24 6a 09 0d 07 07 ......h...$j.... +| 3776: 06 07 07 07 07 06 07 07 08 07 0f 0d 0d 07 10 07 ................ +| 3792: 07 07 08 0a 06 07 08 0a 06 09 0b 0c 08 10 0c 08 ................ +| 3808: 0a 79 0b 0b 0b 06 07 07 0a 0b 0b 07 07 07 07 06 .y.............. +| 3824: 07 07 08 09 07 09 10 07 07 07 0a 07 08 07 07 07 ................ +| 3840: 07 0c 0d 07 0b 08 0a 0d 09 09 07 07 81 12 07 0b ................ +| 3856: 07 0b 07 07 08 06 0c 08 0b 07 07 0b 07 0b 07 07 ................ +| 3872: 07 07 07 07 07 07 0b 09 0b 07 08 07 06 08 07 0b ................ +| 3888: 07 07 0c 07 09 08 09 08 08 09 09 81 11 07 0a 07 ................ +| 3904: 07 08 07 07 07 0b 07 0b 07 08 06 0d 06 07 06 08 ................ +| 3920: 07 07 07 0a 07 07 0a 08 0b 07 07 08 07 0a 09 0a ................ +| 3936: 0a 0a 81 22 06 08 06 07 07 07 07 09 07 07 08 0b ................ +| 3952: 0a 09 07 07 0a 07 07 0f 08 08 06 0a 09 09 07 07 ................ +| 3968: 07 08 07 07 0b 0c 07 0f 08 08 07 0a 07 09 08 08 ................ +| 3984: 0b 09 0b 0a 0b 81 0b 07 07 0b 06 08 07 08 07 08 ................ +| 4000: 0f 07 07 07 07 07 06 0a 07 09 08 07 0b 0a 08 08 ................ +| 4016: 07 09 08 08 07 07 07 06 06 07 08 08 07 0b 07 0b ................ +| 4032: 08 09 07 0b 07 0a 08 0d 0f 81 13 08 07 08 07 07 ................ +| 4048: 07 0e 09 0a 07 0a 06 08 09 08 07 08 08 07 06 07 ................ +| 4064: 08 07 07 07 0b 06 08 07 0f 09 0e 0a 07 08 07 06 ................ +| 4080: 08 08 0a 81 08 08 0b 06 07 07 08 08 07 08 07 07 ................ +| page 23 offset 90112 +| 0: 0d 00 00 00 01 00 22 00 00 22 00 00 00 00 00 00 ................ +| 32: 00 00 9f 56 88 80 80 80 80 04 04 00 bf 30 00 00 ...V.........0.. +| 48: 0e 8c 03 30 72 64 04 04 81 24 02 01 66 06 04 81 ...0rd...$..f... +| 64: 05 02 04 81 0f 03 01 70 0a 04 82 45 02 01 67 01 .......p...E..g. +| 80: 04 84 1a 02 04 81 03 07 04 82 13 02 01 68 05 04 .............h.. +| 96: 83 1d 02 01 6a 02 04 83 03 02 06 82 26 39 03 01 ....j.......&9.. +| 112: 67 01 04 81 0a 02 02 6c 70 09 02 48 03 03 c9 be g......lp..H.... +| 128: 6e 05 04 83 58 02 01 6e 07 04 83 10 03 02 30 6d n...X..n......0m +| 144: 07 04 84 20 02 03 6f d2 b1 09 04 84 43 02 01 70 ... ..o.....C..p +| 160: 07 02 24 01 02 7d 02 01 71 07 04 83 2f 03 04 81 ..$.....q.../... +| 176: 18 03 01 62 08 04 82 39 02 02 72 75 0a 04 81 24 ...b...9..ru...$ +| 192: 02 01 75 02 04 81 4e 05 04 83 3d 02 01 76 03 02 ..u...N...=..v.. +| 208: 64 03 02 6b 6c 08 04 82 2f 02 01 78 05 04 84 66 d..kl.../..x...f +| 224: 02 02 79 36 02 02 39 02 02 c2 b2 04 04 84 7c 04 ..y6..9.......|. +| 240: 02 dd ab 09 04 83 71 03 02 b3 31 09 04 82 33 03 ......q...1...3. +| 256: 01 bc 04 02 6e 03 01 bd 07 04 85 09 04 01 75 07 ....n.........u. +| 272: 02 0e 02 02 ce bc 07 02 6b 02 02 d1 9e 0a 04 84 ........k....... +| 288: 51 02 03 d5 b1 36 09 02 1e 02 04 e4 8d ac 38 01 Q....6........8. +| 304: 02 61 02 04 f0 96 85 81 09 04 83 5e 03 03 9a 87 .a.........^.... +| 320: 8d 05 04 83 51 01 01 73 01 30 0f 1c 12 09 29 0d ....Q..s.0....). +| 336: 07 2e 81 01 11 2a 22 1d 04 2e 0f 08 04 23 29 0b .....*.......#). +| 352: 3b 15 01 16 2f 2f 81 20 81 07 13 56 26 3b 25 01 ;...//. ...V&;%. +| 368: 28 49 03 0a 61 04 08 2e 03 22 10 38 34 0c 09 29 (I..a......84..) +| 384: 0c 0c 34 28 0b 01 1e 0f 3c 5b 13 0a 34 11 5a 73 ..4(....<[..4.Zs +| 400: 1e 22 03 1a 54 03 01 22 08 06 3c 41 06 2c 2b 16 ....T......^btG7 +| 1328: 20 3d 07 20 0e 01 1e 10 69 42 48 08 04 08 56 08 =. ....iBH...V. +| 1344: 66 21 10 07 1a 34 01 18 1f 28 31 41 81 06 36 19 f!...4...(1A..6. +| 1360: 53 0d 0f 0c 01 1c 16 0a 0e 04 52 17 22 48 7b 43 S.........R..H.C +| 1376: 3e 04 47 44 01 1e 25 4b 0b 30 2c 0a 0c 6b 71 15 >.GD..%K.0,..kq. +| 1392: 03 4c 19 0f 16 01 22 2b 13 28 1d 24 4c 0f 46 0f .L.....+.(.$L.F. +| 1408: 2f 0b 1c 03 0e 4a 4a 33 01 10 19 29 81 24 0a 82 /....JJ3...).$.. +| 1424: 0c 46 02 01 30 05 04 82 5d 02 01 32 07 04 83 1f .F..0...]..2.... +| 1440: 02 02 33 6c 06 04 84 6f 02 01 35 08 02 33 02 06 ..3l...o..5..3.. +| 1456: 37 6e 66 ef 98 bd 06 04 81 35 02 02 61 76 03 06 7nf......5..av.. +| 1472: 82 12 73 02 01 63 02 02 57 02 04 83 09 03 01 69 ..s..c..W......i +| 1488: 05 02 66 02 01 64 06 04 81 01 02 01 65 06 02 3b ..f..d......e..; +| 1504: 03 01 71 01 04 83 6c 02 01 66 01 04 84 7c 01 04 ..q...l..f...|.. +| 1520: 83 5c 03 03 69 c2 b2 08 04 84 74 03 03 73 c2 bd ....i.....t..s.. +| 1536: 03 04 84 0a 02 01 67 06 08 82 1d 82 56 04 04 81 ......g.....V... +| 1552: 33 02 01 68 09 02 51 02 01 69 0a 02 08 02 01 6a 3..h..Q..i.....j +| 1568: 06 02 6c 03 01 6e 0a 04 81 2c 02 01 6c 08 04 83 ..l..n...,..l... +| 1584: 15 02 01 6d 04 04 84 24 03 01 33 01 04 81 3a 03 ...m...$..3...:. +| 1600: 03 6e c2 b9 07 04 81 6b 02 01 6f 06 04 83 28 02 .n.....k..o...(. +| 1616: 03 70 77 71 0a 04 81 2a 02 01 71 0a 04 84 34 02 .pwq...*..q...4. +| 1632: 01 73 05 04 81 7f 03 02 6a 73 01 04 83 08 02 01 .s......js...... +| 1648: 74 02 04 81 60 01 04 83 5e 06 04 82 37 02 01 76 t...`...^...7..v +| 1664: 08 04 81 2d 02 01 77 01 04 82 12 01 02 46 02 03 ...-..w......F.. +| 1680: 78 73 66 04 02 37 03 02 7a 62 02 04 83 1a 02 01 xsf..7..zb...... +| 1696: 79 04 02 1a 03 04 84 2a 02 04 84 22 02 01 7a 05 y......*......z. +| 1712: 04 83 4d 02 02 c2 b2 09 04 83 04 03 01 b9 04 04 ..M............. +| 1728: 82 3e 03 02 bd 6c 07 04 84 06 03 02 be 73 0a 04 .>...l.......s.. +| 1744: 83 5b 02 06 c9 82 71 c6 9a 61 05 02 7a 03 04 89 .[....q..a..z... +| 1760: e3 a1 be 04 04 83 05 02 04 ca aa 33 67 04 04 84 ...........3g... +| 1776: 61 02 03 d6 83 6b 08 04 81 77 02 02 da b0 0a 02 a....k...w...... +| 1792: 32 03 01 b6 0a 02 13 02 03 e3 9a bb 04 04 84 42 2..............B +| 1808: 02 04 f0 b6 8e b7 06 04 83 5a 01 01 76 01 1a 7f .........Z..v... +| 1824: 38 16 7c 3b 26 23 08 05 47 1c 19 20 01 1c 28 45 8.|;&#..G.. ..(E +| 1840: 20 0a 81 0a 45 15 44 05 03 1d 26 44 01 10 0c 3e ...E.D...&D...> +| 1856: 26 3c 6d 82 43 04 01 1c 04 15 3a 56 62 39 16 03 &..xqg2.. +| 2192: 84 72 03 02 76 76 01 04 82 1f 02 03 79 75 6f 04 .r..vv......yuo. +| 2208: 04 84 77 03 04 f6 84 93 a8 05 04 83 6c 02 01 7a ..w.........l..z +| 2224: 02 02 7c 02 04 83 21 05 04 83 07 02 02 c2 aa 01 ..|...!......... +| 2240: 04 81 2f 03 01 bc 0a 04 81 65 02 02 c3 be 01 04 ../......e...... +| 2256: 82 04 02 02 df 94 07 04 82 5f 02 04 ea 80 84 6e ........._.....n +| 2272: 04 04 84 73 03 03 84 b1 75 06 04 84 56 02 03 ef ...s....u...V... +| 2288: 89 9e 07 04 83 7b 02 04 f0 92 ab 9a 0a 04 84 60 ...............` +| 2304: 01 01 77 01 18 49 1d 46 47 07 65 21 05 81 13 1a ..w..I.FG.e!.... +| 2320: 06 01 16 81 7e 5a 26 1b 12 05 0f 44 03 57 01 16 ....~Z&....D.W.. +| 2336: 14 4d 31 81 0d 2c 2a 07 03 81 2c 01 14 23 0a 2a .M1..,*...,..#.* +| 2352: 66 18 65 76 21 5a 5f 01 14 14 81 4c 11 10 24 81 f.ev!Z_....L..$. +| 2368: 49 48 28 01 16 7b 0a 4c 08 10 22 81 2d 03 69 38 IH(....L....-.i8 +| 2384: 01 0e 3c 6e 1f 81 6a 2a 16 01 1a 3b 81 25 0d 28 ....d......i +| 2592: 06 04 83 60 03 01 35 0a 04 82 3d 02 01 6b 09 04 ...`..5...=..k.. +| 2608: 83 34 03 01 70 05 04 81 6e 02 01 6c 09 04 83 28 .4..p...n..l...( +| 2624: 02 01 6d 04 04 82 6f 04 04 83 09 01 04 84 48 02 ..m...o.......H. +| 2640: 04 6e 79 68 31 03 04 83 4d 02 01 6f 03 02 2d 03 .nyh1...M..o..-. +| 2656: 01 6e 09 02 61 02 01 70 04 04 82 67 02 03 71 63 .n..a..p...g..qc +| 2672: 79 09 04 81 0b 03 02 c2 bc 0a 02 30 02 01 72 08 y..........0..r. +| 2688: 02 5e 02 01 73 08 04 81 07 02 01 75 05 02 6c 05 .^..s......u..l. +| 2704: 04 84 56 02 01 76 08 04 81 01 02 01 78 05 04 84 ..V..v......x... +| 2720: 61 02 04 84 6c 02 01 7a 0a 04 83 2e 02 02 c2 ba a...l..z........ +| 2736: 04 04 81 39 03 01 bc 01 04 83 23 02 02 c9 a8 03 ...9......#..... +| 2752: 04 83 38 02 04 ce bc 62 68 02 04 82 25 02 02 d4 ..8....bh...%... +| 2768: 85 08 04 82 2d 02 02 db 91 09 04 84 29 01 01 78 ....-.......)..x +| 2784: 01 0a 27 82 4d 5c 23 01 1c 11 62 38 15 17 67 05 ..'.M.#...b8..g. +| 2800: 3e 05 33 1b 29 1c 26 01 16 0d 81 33 0c 0c 09 61 >.3.).&....3...a +| 2816: 4e 1a 54 47 01 18 22 43 09 60 11 2a 08 1b 81 30 N.TG...C.`.*...0 +| 2832: 27 44 01 24 2d 06 1c 19 2a 5e 22 24 04 2a 5c 0b 'D.$-...*^.$.*.. +| 2848: 42 0d 0f 37 06 2d 01 1a 4c 28 2d 47 14 46 18 45 B..7.-..L(-G.F.E +| 2864: 0e 2c 04 81 1a 01 14 1d 33 75 70 62 32 75 26 2d .,......3upb2u&- +| 2880: 08 01 18 41 0b 0a 0c 1f 17 27 61 81 37 3b 21 01 ...A.....'a.7;!. +| 2896: 1e 39 12 0e 1b 17 17 08 4c 04 81 18 0a 81 12 1a .9......L....... +| 2912: 01 22 0d 11 30 32 1f 5d 31 08 1e 2e 23 3b 03 49 ....02.]1...#;.I +| 2928: 05 6d 03 02 01 30 01 04 81 47 04 02 22 03 02 65 .m...0...G.....e +| 2944: 77 0a 04 82 16 02 01 32 04 08 81 55 81 2e 01 06 w......2...U.... +| 2960: 57 3a 73 02 01 36 0a 02 4f 03 01 7a 04 04 81 27 W:s..6..O..z...' +| 2976: 02 01 61 05 04 82 21 02 01 62 01 02 7e 02 01 63 ..a...!..b..~..c +| 2992: 05 04 81 0c 03 02 c2 b3 08 04 81 24 02 01 65 05 ...........$..e. +| 3008: 04 84 5e 01 04 81 04 01 04 82 3e 01 04 83 46 02 ..^.......>...F. +| 3024: 01 67 02 04 84 09 02 01 68 01 04 83 04 02 01 69 .g......h......i +| 3040: 04 04 82 31 04 04 82 5a 02 01 6a 01 04 81 21 02 ...1...Z..j...!. +| 3056: 01 6b 08 04 83 5e 03 02 68 7a 0a 04 84 67 03 04 .k...^..hz...g.. +| 3072: 77 78 64 6f 04 04 81 4c 02 01 6d 04 04 83 03 03 wxdo...L..m..... +| 3088: 01 37 01 04 83 75 03 01 67 07 04 82 04 02 01 6e .7...u..g......n +| 3104: 06 02 2c 04 04 83 52 02 01 6f 05 04 81 14 02 03 ..,...R..o...... +| 3120: 70 6d 73 08 04 83 68 02 01 71 06 04 84 7c 02 01 pms...h..q...|.. +| 3136: 72 02 04 84 20 03 04 83 33 01 04 82 13 02 04 82 r... ...3....... +| 3152: 17 03 01 69 04 04 81 1c 03 01 76 04 04 84 71 02 ...i......v...q. +| 3168: 01 73 04 04 82 53 02 01 75 01 04 81 3b 05 04 82 .s...S..u...;... +| 3184: 2a 02 01 77 03 04 82 4f 02 01 79 02 02 60 02 01 *..w...O..y..`.. +| 3200: 7a 08 04 81 34 02 04 c2 aa 6b 61 04 04 82 7b 02 z...4....ka..... +| 3216: 02 c6 b4 0a 04 81 02 02 02 ca 83 03 02 5a 02 02 .............Z.. +| 3232: d4 ae 04 04 81 45 02 02 d7 bb 0a 04 83 78 02 02 .....E.......x.. +| 3248: dc 96 06 04 82 1f 02 02 df 95 05 04 82 11 02 03 ................ +| 3264: ea b7 a6 07 04 83 6e 02 04 f0 93 a0 b9 06 04 82 ......n......... +| 3280: 6c 02 05 f5 af 82 82 73 07 02 41 01 01 79 01 1e l......s..A..y.. +| 3296: 07 21 30 08 81 04 17 25 0a 0c 81 17 27 81 25 01 .!0....%....'.%. +| 3312: 1a 03 5d 21 81 16 07 3f 27 16 3f 14 08 18 01 12 ..]!...?'.?..... +| 3328: 68 4f 0e 25 34 3a 3d 81 16 01 12 09 81 0a 3c 72 hO.%4:=...............n..t... +| 3648: 17 02 03 c5 a7 33 0a 04 84 55 02 02 c7 9d 04 04 .....3...U...... +| 3664: 81 60 02 08 ca 81 30 68 f0 90 90 b2 09 04 84 2e .`....0h........ +| 3680: 03 01 b5 02 04 82 33 02 03 d1 9e 6c 02 04 81 1a ......3....l.... +| 3696: 02 02 d4 93 04 04 81 05 02 02 d8 ab 03 04 82 49 ...............I +| 3712: 03 02 b8 73 01 04 84 1d 02 03 e2 af 9d 07 02 4d ...s...........M +| 3728: 02 03 ec 8f ad 03 04 84 47 01 01 7a 01 22 1d 42 ........G..z...B +| 3744: 10 5f 1b 08 63 1e 07 0a 30 07 0b 26 31 61 07 01 ._..c...0..&1a.. +| 3760: 1c 0a 37 29 0a 68 40 38 0a 4a 04 08 0b 07 0f 07 ..7).h@8.J...... +| 3776: 0c 07 07 09 07 08 09 09 0b 07 08 0b 06 08 07 07 ................ +| 3792: 08 08 08 06 07 06 07 08 08 09 0a 09 81 31 06 07 .............1.. +| 3808: 0d 06 06 07 0a 07 08 0e 08 07 07 0b 06 07 09 07 ................ +| 3824: 0b 07 07 07 0c 07 07 07 07 08 09 08 07 0a 81 14 ................ +| 3840: 07 07 06 0b 08 06 07 09 07 07 0f 0e 06 0b 0b 0a ................ +| 3856: 0c 07 06 06 09 06 08 0b 09 07 08 0f 0b 07 0c 08 ................ +| 3872: 06 07 07 09 08 09 07 09 0a 81 20 07 07 08 06 0c .......... ..... +| 3888: 09 0a 06 07 06 07 0b 09 09 0d 06 06 06 07 07 07 ................ +| 3904: 07 09 07 09 07 07 08 0f 07 0a 08 08 0e 07 08 07 ................ +| 3920: 08 08 0b 0a 0a 09 07 06 09 0a 81 10 07 06 09 0a ................ +| 3936: 07 08 07 07 08 07 0d 0a 07 07 0a 07 08 07 13 0a ................ +| 3952: 08 08 08 08 07 09 0a 08 09 0a 0e 08 07 08 08 0a ................ +| 3968: 09 09 0a 81 08 07 08 08 0b 08 07 07 08 0b 07 0b ................ +| 3984: 0b 0a 07 07 07 07 07 07 07 07 07 07 0f 0a 06 06 ................ +| 4000: 07 09 07 06 07 0a 07 0b 07 08 07 08 0a 08 08 81 ................ +| 4016: 16 0a 08 0e 06 07 07 06 07 08 13 07 07 0b 07 07 ................ +| 4032: 08 0a 07 07 07 0a 07 09 07 13 07 07 07 0b 07 06 ................ +| 4048: 07 0a 08 07 08 08 08 08 09 0a 0a 81 06 07 0e 07 ................ +| 4064: 0b 06 0b 0b 0b 07 08 07 0f 07 06 08 07 09 0b 0a ................ +| 4080: 06 0e 07 08 09 07 09 08 0e 07 09 08 08 08 08 09 ................ +| page 24 offset 94208 +| 0: 0d 00 00 00 01 00 28 00 00 28 00 00 00 00 00 00 ......(..(...... +| 32: 00 00 00 00 00 00 00 00 9f 50 88 80 80 80 80 05 .........P...... +| 48: 04 00 bf 24 00 09 0e 5b 14 06 1c 0d 12 03 18 37 ...$...[.......7 +| 64: 1f 05 55 32 10 15 5a 81 07 4b 04 01 16 07 0e 5f ..U2..Z..K....._ +| 80: 0b 46 0b 22 81 41 24 23 01 28 2a 07 11 09 15 0b .F...A$#.(*..... +| 96: 0f 0c 09 0a 1e 05 1f 5e 07 19 1c 1e 33 3e 01 18 .......^....3>.. +| 112: 26 0c 1b 43 08 38 11 05 81 1b 5f 20 01 1e 06 1c &..C.8...._ .... +| 128: 36 27 33 20 53 4f 06 03 1a 2f 2c 40 6d 01 18 72 6'3 SO.../,@m..r +| 144: 81 0d 20 3a 23 05 34 19 23 2b 36 01 12 2c 4d 0c .. :#.4.#+6..,M. +| 160: 59 82 04 46 40 0f 01 1a 28 15 32 33 2e 52 0f 49 Y..F@...(.23.R.I +| 176: 04 09 81 33 26 03 30 7a 34 01 04 82 05 02 01 36 ...3&.0z4......6 +| 192: 06 04 81 5a 02 04 37 ce bc 74 09 04 81 17 02 01 ...Z..7..t...... +| 208: 38 01 04 83 79 03 03 dd 9a 73 02 04 84 12 02 01 8...y....s...... +| 224: 61 04 04 82 1c 02 01 62 01 04 83 4e 07 04 82 5b a......b...N...[ +| 240: 02 01 63 0a 02 4e 02 01 64 04 04 84 7d 02 01 65 ..c..N..d......e +| 256: 01 04 84 2a 02 01 67 01 04 81 75 02 04 68 d0 bf ...*..g...u..h.. +| 272: 70 09 04 81 4a 02 01 69 01 04 84 34 05 04 82 48 p...J..i...4...H +| 288: 02 01 6a 02 02 1d 04 04 81 54 02 01 6b 03 04 81 ..j......T..k... +| 304: 0d 02 01 6c 09 04 83 54 02 01 6e 06 02 0b 01 04 ...l...T..n..... +| 320: 84 1c 02 01 6f 0a 02 26 03 01 6a 08 04 82 26 03 ....o..&..j...&. +| 336: 05 73 75 c2 bd 61 03 02 38 02 02 70 37 06 04 81 .su..a..8..p7... +| 352: 7e 02 01 71 01 04 83 67 03 01 65 06 04 84 08 02 ~..q...g..e..... +| 368: 01 72 0a 04 82 1e 02 01 74 01 04 82 6c 02 02 5d .r......t...l..] +| 384: 04 04 84 17 03 04 82 77 02 01 75 04 08 82 22 82 .......w..u..... +| 400: 0b 02 02 76 6c 03 04 83 25 02 01 77 03 04 81 24 ...vl...%..w...$ +| 416: 03 04 82 42 02 01 78 06 04 82 4a 03 02 c2 b2 02 ...B..x...J..... +| 432: 04 83 21 02 01 79 09 04 83 43 02 01 7a 04 02 51 ..!..y...C..z..Q +| 448: 02 02 c2 aa 02 02 51 03 01 b3 09 04 81 2a 03 01 ......Q......*.. +| 464: b9 06 04 82 73 03 02 bd 73 04 02 4e 04 05 76 c2 ....s...s..N..v. +| 480: be 67 65 08 02 1d 04 03 78 de bc 02 04 83 6d 04 .ge.....x.....m. +| 496: 01 7a 09 04 84 1d 02 04 ca a2 36 32 07 04 83 2e .z........62.... +| 512: 02 02 d8 aa 04 04 84 31 02 02 df 8a 02 02 73 01 .......1......s. +| 528: 02 c2 aa 01 08 81 7f 82 01 01 06 56 83 70 02 06 ...........V.p.. +| 544: 74 82 6d 01 06 81 5d 45 01 04 82 7b 01 0c 81 46 t.m...]E.......F +| 560: 13 6a 82 12 01 0c 82 24 63 15 35 74 01 12 81 14 .j.....$c.5t.... +| 576: 81 24 1f 3d 7c 15 1c 01 0a 82 39 1a 81 6f 03 02 .$.=|.....9..o.. +| 592: 30 65 07 04 83 3a 03 01 63 03 04 81 7d 03 01 65 0e...:..c......e +| 608: 09 02 03 03 01 66 01 02 6c 04 02 66 7a 03 04 82 .....f..l..fz... +| 624: 1c 03 01 6b 0a 04 82 46 03 01 7a 08 04 84 2d 03 ...k...F..z...-. +| 640: 03 c2 bd 73 0a 04 81 10 05 02 c2 bc 03 02 03 04 ...s............ +| 656: 01 be 02 04 84 35 03 04 f0 9f ad ae 08 04 83 41 .....5.........A +| 672: 02 01 b2 01 0a 3e 77 21 82 5f 01 08 81 74 82 68 .....>w!._...t.h +| 688: 01 0e 39 81 0c 81 4b 1c 4e 01 0e 83 46 06 3b 1d ..9...K.N...F.;. +| 704: 0e 38 01 0e 81 50 15 81 53 35 57 01 08 82 7d 26 .8...P..S5W....& +| 720: 1c 01 04 2f 0e 01 08 66 5d 83 24 01 0e 43 81 10 .../...f].$..C.. +| 736: 24 82 77 04 01 0a 82 0a 81 61 1f 03 01 35 08 04 $.w......a...5.. +| 752: 81 11 03 01 61 09 04 82 0b 03 01 64 07 04 81 7e ....a......d...~ +| 768: 03 01 66 05 04 82 29 03 01 67 03 04 84 17 03 04 ..f...)..g...... +| 784: 69 34 d8 a1 05 04 82 42 03 01 73 04 04 84 3d 03 i4.....B..s...=. +| 800: 01 76 06 04 81 70 04 03 76 c2 ba 07 04 81 34 03 .v...p..v.....4. +| 816: 01 78 06 04 83 48 03 03 c2 b3 36 07 02 7f 03 02 .x...H....6..... +| 832: d4 85 04 04 81 2d 03 02 d9 bc 05 04 83 20 03 04 .....-....... .. +| 848: ee b5 9c 70 03 04 83 57 03 05 f6 80 80 bb 70 05 ...p...W......p. +| 864: 04 81 6a 02 01 b3 01 06 82 41 12 01 08 67 0d 82 ..j......A...g.. +| 880: 69 01 0e 81 51 22 5c 7d 81 00 01 10 68 05 2f 81 i...Q.......h./. +| 896: 4e 81 78 03 01 0a 74 81 58 81 72 01 0c 4e 72 07 N.x...t.X.r..Nr. +| 912: 79 81 7d 01 10 44 3b 58 66 82 08 0c 2f 01 06 0c y....D;Xf.../... +| 928: 65 36 01 04 81 02 01 08 5d 43 82 10 03 01 61 06 e6......]C....a. +| 944: 04 83 3c 03 01 63 07 04 82 32 03 01 68 03 04 82 ..<..c...2..h... +| 960: 76 03 03 6a 77 6d 0a 04 84 35 03 01 70 01 04 84 v..jwm...5..p... +| 976: 70 03 01 74 01 06 84 05 0a 06 02 47 04 04 6a 75 p..t.......G..ju +| 992: ce bc 07 04 82 4d 03 02 75 65 07 04 81 05 03 01 .....M..ue...... +| 1008: 77 01 04 81 59 04 03 6f db 85 08 02 3d 03 04 c2 w...Y..o....=... +| 1024: be c2 b9 01 04 81 04 03 07 c4 a7 67 d5 bb 7a 68 ...........g..zh +| 1040: 06 04 84 05 03 02 c7 a5 02 04 81 10 02 01 b9 01 ................ +| 1056: 0c 1e 81 43 0b 81 56 01 08 82 70 81 27 01 06 34 ...C..V...p.'..4 +| 1072: 83 38 01 0c 81 0f 1d 04 81 74 01 06 82 58 65 01 .8.......t...Xe. +| 1088: 10 27 55 7a 4b 81 00 2c 29 01 0e 04 1f 24 09 83 .'UzK..,)....$.. +| 1104: 44 04 01 08 65 2f 41 2b 01 0c 34 0c 81 17 83 0f D...e/A+..4..... +| 1120: 01 04 82 69 03 02 61 73 02 02 4d 03 01 65 01 04 ...i..as..M..e.. +| 1136: 81 2e 03 02 69 61 04 04 83 06 04 03 69 69 67 08 ....ia......iig. +| 1152: 02 1b 03 03 6b c2 ba 06 04 82 49 03 01 6d 04 04 ....k.....I..m.. +| 1168: 84 67 03 01 6f 02 04 82 4b 03 01 70 06 04 84 34 .g..o...K..p...4 +| 1184: 03 04 72 61 76 76 04 02 67 03 03 73 c2 be 07 04 ..ravv..g..s.... +| 1200: 82 17 03 01 76 03 04 81 56 03 01 78 02 04 84 24 ....v...V..x...$ +| 1216: 03 03 79 69 38 05 04 84 68 03 05 c2 aa 63 c2 b2 ..yi8...h....c.. +| 1232: 08 02 69 04 01 b9 06 04 82 75 04 01 ba 01 04 81 ..i......u...... +| 1248: 51 04 01 bc 0a 04 81 0f 03 02 c5 a7 0a 04 84 71 Q..............q +| 1264: 03 05 c9 99 ee 84 af 03 04 84 10 04 06 9f db a5 ................ +| 1280: 30 71 68 07 04 82 46 03 05 ca 8b 64 78 66 0a 04 0qh...F....dxf.. +| 1296: 84 01 03 02 dd 9b 0a 04 82 4e 03 02 de bd 08 04 .........N...... +| 1312: 82 03 03 07 eb b1 bd f2 84 82 8d 04 04 84 23 02 ..............#. +| 1328: 01 ba 01 0c 2d 81 55 09 82 17 01 04 07 64 01 0e ....-.U......d.. +| 1344: 33 81 73 0e 36 81 63 01 0e 2f 60 82 09 5b 5f 2c 3.s.6.c../`..[_, +| 1360: 01 08 82 6d 81 69 01 0c 81 2d 3c 35 81 0b 01 10 ...m.i...-<5.... +| 1376: 4f 81 08 03 82 4f 29 27 01 0c 0f 03 55 82 6d 10 O....O)'....U.m. +| 1392: 01 08 0d 82 16 1e 01 06 2c 82 56 03 01 33 05 04 ........,.V..3.. +| 1408: 82 25 03 01 36 07 04 82 73 03 01 63 0a 02 48 03 .%..6...s..c..H. +| 1424: 02 66 6f 07 04 84 60 03 01 67 03 04 84 27 03 01 .fo...`..g...'.. +| 1440: 6c 0a 02 63 03 01 6d 07 04 82 75 03 01 6f 08 04 l..c..m...u..o.. +| 1456: 82 4d 04 04 61 74 d4 a5 02 04 83 3c 03 02 d9 a8 .M..at.....<.... +| 1472: 04 04 83 4e 03 03 db 93 61 02 04 81 52 03 03 e1 ...N....a...R... +| 1488: bf a7 08 04 83 42 02 01 bc 01 06 81 6c 2f 01 06 .....B......l/.. +| 1504: 81 31 32 01 06 30 81 02 01 0c 81 02 81 73 04 67 .12..0.......s.g +| 1520: 01 06 81 6c 20 01 0e 57 6d 04 81 1f 81 24 01 04 ...l ..Wm....$.. +| 1536: 83 54 01 0c 70 49 81 59 81 11 01 0e 56 81 44 0d .T..pI.Y....V.D. +| 1552: 3d 81 23 01 0a 83 15 81 6b 06 03 02 39 6d 0a 02 =.#.....k...9m.. +| 1568: 59 03 03 62 c2 aa 03 04 82 2f 03 01 63 02 04 84 Y..b...../..c... +| 1584: 45 04 05 79 76 7a c2 ba 04 04 81 19 03 01 66 07 E..yvz........f. +| 1600: 04 81 74 03 01 69 08 04 83 59 03 02 6f 37 09 04 ..t..i...Y..o7.. +| 1616: 84 38 03 01 72 04 04 81 3f 03 01 73 07 02 69 03 .8..r...?..s..i. +| 1632: 03 79 76 62 04 02 76 03 02 c2 b2 06 04 84 48 04 .yvb..v.......H. +| 1648: 03 b3 34 79 02 04 81 3d 04 01 bc 08 04 82 19 03 ..4y...=........ +| 1664: 04 ce bc ce bc 01 04 84 4a 03 04 e7 a8 a1 79 01 ........J.....y. +| 1680: 02 03 02 01 bd 01 06 58 82 1d 01 0a 82 7b 81 02 .......X........ +| 1696: 57 01 04 0a 6b 01 08 81 5f 70 13 01 12 0e 38 65 W...k..._p....8e +| 1712: 11 34 52 4c 5c 75 01 06 82 56 3e 01 02 23 01 0a .4RL.u...V>..#.. +| 1728: 81 29 08 83 2f 01 0c 81 43 81 48 2f 03 01 08 83 .)../...C.H/.... +| 1744: 5a 76 3a 03 01 30 04 04 81 71 03 01 32 01 04 82 Zv:..0...q..2... +| 1760: 1d 03 01 36 02 04 84 34 03 01 61 09 04 82 41 03 ...6...4..a...A. +| 1776: 01 68 07 02 38 03 01 6a 08 04 84 15 03 01 6c 08 .h..8..j......l. +| 1792: 04 82 66 03 01 6d 08 04 82 69 03 01 6f 01 04 82 ..f..m...i..o... +| 1808: 40 02 04 84 2d 03 02 78 03 01 73 04 08 81 29 82 @...-..x..s...). +| 1824: 08 03 03 75 75 6b 06 04 84 21 03 01 78 05 04 83 ...uuk...!..x... +| 1840: 61 04 04 c2 be c2 bc 06 04 82 5c 03 01 79 01 04 a............y.. +| 1856: 83 46 03 02 c2 b3 06 02 40 02 01 be 01 08 81 0f .F......@....... +| 1872: 81 71 01 0e 81 25 60 81 39 1d 70 01 06 1b 83 53 .q...%`.9.p....S +| 1888: 01 06 83 37 52 01 10 6b 65 18 2a 81 4d 47 33 01 ...7R..ke.*.MG3. +| 1904: 06 82 79 07 01 02 46 01 08 03 13 81 64 01 0c 5a ..y...F.....d..Z +| 1920: 65 76 82 33 04 01 08 82 61 82 19 03 02 35 64 01 ev.3....a....5d. +| 1936: 04 84 17 03 01 37 06 04 82 10 03 01 62 08 02 5b .....7......b..[ +| 1952: 03 01 66 07 04 83 04 03 01 67 06 02 36 03 01 6a ..f......g..6..j +| 1968: 01 04 83 3e 08 04 83 59 03 01 71 01 04 82 47 03 ...>...Y..q...G. +| 1984: 01 72 08 06 81 73 32 03 01 75 09 04 82 35 03 02 .r...s2..u...5.. +| 2000: 77 76 07 04 83 68 03 01 78 05 02 72 03 02 c2 b3 wv...h..x..r.... +| 2016: 0a 04 81 7e 04 01 bd 05 04 84 35 03 02 cb 8a 03 ...~......5..... +| 2032: 04 84 46 03 02 ce bc 02 04 82 72 01 05 c3 9f ca ..F.......r..... +| 2048: 83 74 03 04 81 58 02 01 a6 08 04 84 26 02 04 82 .t...X......&... +| 2064: 1d 02 01 be 06 04 81 4f 03 04 83 1d 01 02 c4 a7 .......O........ +| 2080: 01 02 0e 05 04 81 7c 04 04 84 09 02 01 b3 01 04 ......|......... +| 2096: 81 43 09 04 82 70 03 02 67 72 07 04 83 06 02 01 .C...p..gr...... +| 2112: b8 06 04 83 09 03 04 78 6d 77 75 07 04 82 69 01 .......xmwu...i. +| 2128: 02 c5 80 06 04 83 40 02 01 82 06 04 82 6a 03 04 ......@......j.. +| 2144: 82 1a 02 01 89 07 08 81 39 81 6c 02 01 8b 06 04 ........9.l..... +| 2160: 81 1a 03 01 6a 02 02 26 02 01 93 04 04 83 49 03 ....j..&......I. +| 2176: 02 6e 70 03 02 0f 01 02 c6 83 01 04 82 5b 06 04 .np..........[.. +| 2192: 83 6a 02 01 85 02 04 82 0d 03 01 31 09 02 41 03 .j.........1..A. +| 2208: 01 6b 0a 04 84 0d 02 01 88 04 06 82 16 7f 02 01 .k.............. +| 2224: 95 06 04 81 50 02 04 84 18 01 04 84 3f 03 01 78 ....P.......?..x +| 2240: 06 04 84 11 02 01 99 05 04 83 38 01 04 81 4a 02 ..........8...J. +| 2256: 01 9a 04 04 81 16 02 02 9b 62 03 02 10 02 01 a3 .........b...... +| 2272: 04 04 82 0d 01 04 84 64 02 01 a5 05 06 81 3d 61 .......d......=a +| 2288: 03 02 79 69 05 04 82 23 02 01 a8 09 04 83 44 02 ..yi...#......D. +| 2304: 01 ab 03 04 81 13 02 02 ad 75 06 04 82 07 02 01 .........u...... +| 2320: b4 05 04 82 74 03 04 75 61 65 6c 05 04 82 68 02 ....t..uael...h. +| 2336: 01 b6 03 04 81 6b 02 01 ba 05 04 83 72 02 01 bd .....k......r... +| 2352: 06 04 84 15 02 02 be 74 02 04 84 57 03 01 79 07 .......t...W..y. +| 2368: 04 82 5d 02 02 bf 67 09 04 81 42 03 01 72 03 04 ..]...g...B..r.. +| 2384: 82 7a 03 01 77 0a 02 3c 01 02 c7 80 02 04 84 1d .z..w..<........ +| 2400: 03 01 31 04 04 83 1d 02 03 81 c9 ab 06 04 83 4d ..1............M +| 2416: 02 01 82 09 04 83 25 02 03 86 d3 97 05 02 34 02 ......%.......4. +| 2432: 02 89 78 02 04 83 19 02 01 8c 03 02 56 02 04 81 ..x.........V... +| 2448: 3b 02 04 82 16 02 01 9c 03 04 82 55 04 04 81 08 ;..........U.... +| 2464: 02 01 9d 07 04 81 1c 03 01 68 04 04 83 11 02 01 .........h...... +| 2480: 9f 05 04 81 0e 02 01 a1 01 04 84 79 08 04 82 27 ...........y...' +| 2496: 02 01 a3 04 04 81 5c 06 02 33 02 02 a5 76 0a 04 .........3...v.. +| 2512: 82 07 02 08 ad 7a c8 bf e0 a4 8a 64 04 04 81 03 .....z.....d.... +| 2528: 02 01 af 05 04 84 25 02 01 b3 03 02 5c 02 01 bb ......%......... +| 2544: 06 02 06 03 01 68 03 04 81 2c 03 01 71 04 04 81 .....h...,..q... +| 2560: 69 02 01 bd 03 06 81 7a 6f 02 04 84 2f 01 03 c8 i......zo.../... +| 2576: 9d 36 02 04 83 43 02 02 a1 70 01 04 82 4a 02 01 .6...C...p...J.. +| 2592: a5 02 04 83 45 02 01 ad 01 04 81 10 02 01 b1 02 ....E........... +| 2608: 04 84 4c 07 04 84 03 02 01 bc 08 04 81 3b 03 01 ..L..........;.. +| 2624: 31 03 04 83 45 02 01 bf 07 04 81 17 01 02 c9 80 1...E........... +| 2640: 01 02 3a 01 04 83 62 02 01 82 02 04 81 55 04 04 ..:...b......U.. +| 2656: 84 6d 02 02 50 02 01 8f 02 04 84 16 06 04 83 57 .m..P..........W +| 2672: 02 01 91 04 04 83 4f 02 01 93 0a 04 81 1d 03 01 ......O......... +| 2688: 69 05 04 83 37 03 01 79 08 04 82 21 02 01 96 04 i...7..y...!.... +| 2704: 04 82 13 04 02 6c 02 04 83 29 02 01 97 04 02 2d .....l...).....- +| 2720: 02 03 99 6d 74 05 04 81 73 02 01 9f 09 04 84 63 ...mt...s......c +| 2736: 02 01 a0 05 02 6a 02 02 76 02 01 a2 02 02 2e 02 .....j..v....... +| 2752: 01 a3 01 04 82 16 02 01 a5 04 04 81 31 06 04 83 ............1... +| 2768: 0f 02 01 a8 03 04 83 47 02 01 a9 06 06 82 0d 49 .......G.......I +| 2784: 02 01 af 0a 04 83 1c 02 02 b2 34 04 04 81 3c 02 ..........4...<. +| 2800: 04 b5 74 c2 b2 03 04 81 28 02 01 ba 09 04 83 62 ..t.....(......b +| 2816: 02 01 bb 07 04 84 73 02 01 bc 06 04 81 2a 02 01 ......s......*.. +| 2832: bd 04 04 81 68 02 01 be 09 04 83 67 03 01 73 08 ....h......g..s. +| 2848: 04 83 5f 01 02 ca 80 07 04 84 0b 03 01 77 02 04 .._..........w.. +| 2864: 82 59 02 01 81 06 04 82 1c 02 01 83 08 04 83 1b .Y.............. +| 2880: 01 04 81 71 02 01 86 07 04 81 79 02 01 88 08 02 ...q......y..... +| 2896: 62 01 04 81 0a 02 01 89 04 04 81 62 02 01 8a 03 b..........b.... +| 2912: 04 83 11 01 04 83 50 02 01 8c 06 04 84 2e 02 01 ......P......... +| 2928: 8e 03 04 82 3e 04 04 82 33 03 01 68 07 04 82 3b ....>...3..h...; +| 2944: 02 01 8f 04 04 82 63 03 01 73 08 02 7e 02 01 90 ......c..s..~... +| 2960: 08 02 20 02 01 91 01 04 83 52 02 01 94 06 04 83 .. ......R...... +| 2976: 36 02 02 95 7a 0a 02 51 02 04 99 75 73 73 03 02 6...z..Q...uss.. +| 2992: 36 02 01 9c 06 04 81 27 02 01 a5 07 04 81 7b 03 6......'........ +| 3008: 07 e0 a4 aa d5 b8 61 62 0a 02 77 02 02 a6 6f 07 ......ab..w...o. +| 3024: 04 81 07 03 03 ec b2 be 02 04 82 39 02 02 a7 33 ...........9...3 +| 3040: 08 02 61 02 01 a9 03 04 84 07 02 01 b6 0a 04 84 ..a............. +| 3056: 02 02 01 b7 05 04 81 4a 03 03 6a 69 6a 05 04 81 .......J..jij... +| 3072: 48 02 01 b8 09 04 83 0a 03 04 c4 91 d8 bd 03 02 H............... +| 3088: 1c 02 01 b9 09 02 4c 02 04 ba c9 b1 37 07 04 81 ......L.....7... +| 3104: 4e 02 02 bb 70 01 04 84 4c 02 01 bc 01 04 83 0c N...p...L....... +| 3120: 02 02 be 72 05 02 04 01 02 cb 80 0a 04 84 52 02 ...r..........R. +| 3136: 01 88 07 04 83 1d 02 01 8a 08 02 7c 02 01 a1 09 ...........|.... +| 3152: 04 81 1b 03 01 30 06 04 81 52 03 01 6a 01 04 82 .....0...R..j... +| 3168: 34 02 01 a2 01 04 83 30 02 03 a3 71 64 04 02 7e 4......0...qd..~ +| 3184: 02 01 a4 0a 04 83 34 02 01 ae 06 04 84 7f 03 01 ......4......... +| 3200: 6e 06 04 83 68 01 02 cd b1 02 04 82 55 03 01 6a n...h.......U..j +| 3216: 01 04 83 5d 02 01 b7 08 04 81 58 03 01 75 07 04 ...]......X..u.. +| 3232: 84 09 02 01 b8 01 04 82 0e 06 02 22 03 04 82 4f ...............O +| 3248: 03 02 dd af 05 04 84 3f 02 01 bc 01 04 84 3b 02 .......?......;. +| 3264: 01 bd 09 04 81 61 01 04 83 3c 02 03 bf 64 67 02 .....a...<...dg. +| 3280: 04 82 0c 01 05 ce 81 66 da 94 01 02 10 03 01 6b .......f.......k +| 3296: 0a 04 82 7e 02 01 8d 08 04 83 02 02 07 ad 69 f0 ...~..........i. +| 3312: 98 a6 a7 61 09 04 84 41 03 01 72 03 04 82 62 02 ...a...A..r...b. +| 3328: 01 ae 06 04 83 5d 02 04 81 44 03 01 37 06 04 81 .....]...D..7... +| 3344: 08 02 05 af 64 cf 8d 32 01 02 11 02 01 b1 05 04 ....d..2........ +| 3360: 84 52 03 01 61 08 04 81 68 03 01 68 08 04 83 5a .R..a...h..h...Z +| 3376: 02 01 b2 05 04 84 43 01 02 7e 03 04 83 23 02 04 ......C..~...#.. +| 3392: b3 71 30 73 03 04 83 76 02 01 b5 02 04 82 04 02 .q0s...v........ +| 3408: 01 b6 0a 04 84 6e 02 01 b7 03 04 81 36 02 01 b8 .....n......6... +| 3424: 01 04 84 60 02 02 3a 03 04 84 76 01 02 50 03 02 ...`..:...v..P.. +| 3440: 32 6a 09 04 81 0c 03 01 68 03 02 11 02 01 b9 04 2j......h....... +| 3456: 04 84 0f 02 04 bb d2 b5 62 08 04 81 10 02 01 bc ........b....... +| 3472: 01 0e 4d 81 25 3f 1b 81 54 01 0c 55 81 38 06 82 ..M.%?..T..U.8.. +| 3488: 1b 01 08 21 82 66 37 01 08 65 09 83 3c 02 0a 82 ...!.f7..e..<... +| 3504: 21 08 81 52 01 08 83 4e 81 29 01 0e 47 15 81 21 !..R...N.)..G..! +| 3520: 81 0f 65 01 10 12 20 2d 38 81 00 81 73 01 10 0e ..e... -8...s... +| 3536: 33 6c 2d 3e 81 4e 07 03 01 67 03 04 83 52 03 01 3l->.N...g...R.. +| 3552: 6b 08 04 83 72 03 02 6c 32 02 04 81 51 03 03 6d k...r..l2...Q..m +| 3568: 67 79 09 04 82 67 03 01 70 04 04 81 44 03 02 71 gy...g..p...D..q +| 3584: 6a 0a 04 81 6d 03 01 79 01 04 81 5e 03 02 c2 bd j...m..y...^.... +| 3600: 07 04 81 3a 04 01 be 08 02 18 03 02 ce bc 07 02 ...:............ +| 3616: 34 02 01 bd 03 02 28 03 01 67 04 04 82 09 03 02 4.....(..g...... +| 3632: 6e 6c 01 02 22 03 01 78 02 04 83 5f 02 01 bf 02 nl.....x..._.... +| 3648: 02 16 03 01 7a 06 04 82 05 01 02 cf 80 03 02 69 ....z..........i +| 3664: 03 01 6e 05 04 83 40 02 02 81 6c 06 04 84 5d 03 ..n...@...l...]. +| 3680: 02 ca ab 04 04 81 5d 02 01 84 03 04 81 2e 03 04 ......]......... +| 3696: 82 6e 02 01 85 06 04 82 40 02 01 88 04 02 62 02 .n......@.....b. +| 3712: 03 8a 75 6a 03 04 83 50 02 01 8b 05 04 82 60 81 ..uj...P......`. +| 3728: 01 08 07 0a 07 09 07 0b 06 07 07 07 0a 0b 0a 07 ................ +| 3744: 07 0a 06 07 0a 08 07 07 07 12 09 08 0b 07 08 07 ................ +| 3760: 06 07 07 07 07 0a 09 07 0a 08 07 3f 08 07 06 06 ...........?.... +| 3776: 08 07 07 09 07 07 0a 4b 07 07 07 07 07 0a 07 07 .......K........ +| 3792: 09 07 08 08 08 0a 0b 49 07 07 07 09 07 0b 0a 08 .......I........ +| 3808: 07 08 0a 0d 08 48 07 07 08 08 09 07 07 07 09 09 .....H.......... +| 3824: 07 07 09 0a 07 07 07 08 0b 0c 0b 08 08 0d 4c 07 ..............L. +| 3840: 07 06 08 07 06 07 07 0a 08 09 09 44 07 09 07 0b ...........D.... +| 3856: 07 07 08 07 06 08 08 09 07 0a 09 41 07 07 07 07 ...........A.... +| 3872: 06 07 07 07 0e 09 09 07 0a 07 07 42 08 07 06 07 ...........B.... +| 3888: 06 0b 07 08 07 08 06 08 07 08 08 0b 0b 0b 0f 0b ................ +| 3904: 08 07 0a 08 0b 09 07 06 07 07 0c 07 06 07 08 0f ................ +| 3920: 07 0b 07 07 0b 08 08 07 07 08 07 0a 07 07 07 08 ................ +| 3936: 07 08 07 06 08 07 09 07 08 08 0e 0b 07 07 07 0b ................ +| 3952: 0a 08 0e 07 06 06 07 07 0c 09 08 07 07 0b 07 07 ................ +| 3968: 07 0b 0e 0b 07 07 07 07 0e 06 09 07 09 06 07 0b ................ +| 3984: 07 08 07 08 0a 07 07 07 07 07 07 08 07 07 0b 07 ................ +| 4000: 0a 07 0b 07 0b 07 07 06 06 07 07 07 09 07 07 0c ................ +| 4016: 08 09 07 07 07 07 09 07 09 06 0a 08 07 07 08 07 ................ +| 4032: 06 07 07 07 07 08 07 07 07 08 07 07 07 0e 08 07 ................ +| 4048: 0b 09 0a 07 07 0d 07 0b 07 0a 07 07 07 0e 0a 07 ................ +| 4064: 07 07 11 08 06 07 0a 4a 07 07 08 09 07 08 07 08 .......J........ +| 4080: 06 07 06 07 07 07 06 07 07 07 08 08 0b 07 06 09 ................ +| page 25 offset 98304 +| 0: 0d 00 00 00 01 00 29 00 00 29 00 00 00 00 00 00 ......)..)...... +| 32: 00 00 00 00 00 00 00 00 00 9f 4f 88 80 80 80 80 ..........O..... +| 48: 06 04 00 bf 22 00 00 0e 17 04 30 cf 8c 70 03 02 ..........0..p.. +| 64: 0e 02 01 8d 05 04 82 5c 03 01 63 07 04 84 19 02 ..........c..... +| 80: 01 8e 09 02 66 02 01 93 06 04 83 30 02 01 99 06 ....f......0.... +| 96: 04 84 75 04 04 84 07 03 02 c6 a3 05 04 84 01 02 ..u............. +| 112: 01 9b 08 02 42 02 01 9d 07 02 30 03 02 0b 02 01 ....B.....0..... +| 128: 9f 06 04 81 06 03 02 da a9 07 02 66 02 01 a5 09 ...........f.... +| 144: 02 0c 02 01 ab 02 04 81 39 02 01 ad 01 02 6d 03 ........9.....m. +| 160: 04 82 62 02 01 b2 0a 04 83 36 02 03 b3 d6 88 07 ..b......6...... +| 176: 04 84 78 02 01 b8 05 04 84 02 05 04 81 5d 01 02 ..x..........].. +| 192: d0 b1 0a 02 5f 03 01 7a 03 02 76 02 02 b5 35 08 ...._..z..v...5. +| 208: 04 83 63 03 01 75 07 04 84 35 02 04 b8 71 61 61 ..c..u...5...qaa +| 224: 0a 04 81 6b 02 01 bb 08 04 84 4a 02 01 bd 0a 06 ...k......J..... +| 240: 6f 83 6d 01 02 d1 80 06 04 83 65 02 01 84 07 04 o.m.......e..... +| 256: 82 1b 02 01 85 09 04 82 3f 02 02 86 6f 08 04 82 ........?...o... +| 272: 5f 02 01 88 03 04 81 29 02 01 89 02 04 83 14 08 _......)........ +| 288: 04 84 6f 03 01 63 02 04 84 1c 02 01 8a 0a 06 82 ..o..c.......... +| 304: 14 2d 03 02 6c 69 04 04 83 57 02 01 8e 07 04 84 .-..li...W...... +| 320: 48 02 06 90 e1 91 8f 6a 7a 05 04 83 46 02 03 91 H......jz...F... +| 336: 64 37 03 04 82 01 02 01 93 0a 04 82 1f 02 01 98 d7.............. +| 352: 05 04 84 30 02 03 9b 63 66 05 04 81 55 02 05 9f ...0...cf...U... +| 368: 78 6a d1 87 09 04 81 4d 02 02 a3 78 06 02 07 02 xj.....M...x.... +| 384: 04 a5 e1 a9 8f 06 02 10 02 02 af 76 09 04 83 13 ...........v.... +| 400: 03 02 7a 33 07 04 84 1f 02 01 b1 05 04 84 0e 02 ..z3............ +| 416: 02 b7 71 0a 04 82 7b 02 02 bf 62 0a 04 83 6f 01 ..q.......b...o. +| 432: 04 d2 8b 67 73 06 04 83 2b 02 04 97 e3 86 b2 07 ...gs...+....... +| 448: 04 84 12 02 01 9f 02 04 81 05 03 02 24 02 01 a7 ............$... +| 464: 04 04 84 1f 02 01 a9 04 04 81 47 06 06 23 82 23 ..........G..#.# +| 480: 02 02 b1 78 06 02 43 02 01 b9 02 04 82 44 02 01 ...x..C......D.. +| 496: bd 02 04 83 3d 08 04 81 76 02 01 bf 0a 04 85 04 ....=...v....... +| 512: 01 02 d3 8e 06 04 85 02 02 02 8f 68 03 02 75 02 ...........h..u. +| 528: 01 91 08 02 28 02 01 93 05 02 16 02 01 9d 06 04 ....(........... +| 544: 82 5e 02 02 9f 33 06 04 83 78 02 03 a7 6c 78 06 .^...3...x...lx. +| 560: 04 83 33 03 09 da a9 ec 82 b2 e1 87 90 71 06 04 ..3..........q.. +| 576: 84 7b 02 01 ab 06 04 83 27 02 01 ad 03 02 72 02 ........'.....r. +| 592: 01 af 03 04 82 0c 02 01 b5 0a 04 81 7d 02 01 bb ................ +| 608: 0a 02 58 03 01 6a 03 04 81 70 01 02 d4 81 04 04 ..X..j...p...... +| 624: 84 0d 02 01 85 06 04 81 3d 02 02 8b 35 07 04 83 ........=...5... +| 640: 50 02 02 8d 77 03 02 79 02 01 95 0a 04 84 5c 02 P...w..y........ +| 656: 03 97 6a 6c 0a 04 83 7a 02 01 99 09 04 82 26 02 ..jl...z......&. +| 672: 02 9b 71 05 04 82 01 02 02 9d 67 04 04 84 5f 02 ..q.......g..._. +| 688: 01 a7 04 04 82 4a 01 02 d5 a3 0a 04 81 00 03 01 .....J.......... +| 704: 39 06 04 81 51 02 01 a5 06 02 5e 03 01 67 02 04 9...Q.....^..g.. +| 720: 81 0d 02 01 a9 08 04 84 1e 01 02 75 02 01 aa 04 ...........u.... +| 736: 04 81 0a 02 01 ab 03 04 83 21 02 01 ac 06 02 08 .........!...... +| 752: 02 01 ad 0a 02 57 02 02 b3 74 0a 04 84 40 02 03 .....W...t...@.. +| 768: b4 61 6e 08 04 83 1a 03 02 6d 76 02 04 81 0a 02 .an......mv..... +| 784: 01 ba 06 04 83 51 02 01 bb 03 04 82 61 03 01 67 .....Q......a..g +| 800: 08 04 83 28 02 01 bc 01 02 62 01 02 d6 80 01 04 ...(.....b...... +| 816: 84 53 02 02 82 7a 08 02 7f 02 01 83 06 02 50 02 .S...z........P. +| 832: 04 81 32 02 01 86 09 04 82 52 02 01 87 09 04 83 ..2......R...... +| 848: 7b 02 01 8c 03 04 83 19 01 03 d7 89 74 0a 04 82 ............t... +| 864: 37 02 01 92 07 04 81 23 02 01 93 0a 04 81 12 02 7......#........ +| 880: 01 9a 07 02 58 02 01 9e 03 04 81 19 02 01 a0 0a ....X........... +| 896: 04 82 5e 03 01 6c 01 02 0a 03 04 78 c2 b9 63 05 ..^..l.....x..c. +| 912: 02 02 02 01 a8 0a 02 07 02 01 ab 01 04 81 01 02 ................ +| 928: 03 af c3 b0 05 04 82 4e 02 04 b7 69 c2 be 04 04 .......N...i.... +| 944: 84 44 02 05 bd e4 98 a5 6c 03 04 83 51 01 03 d8 .D......l...Q... +| 960: 9c 38 08 04 84 73 02 02 9d 73 0a 04 83 0b 02 01 .8...s...s...... +| 976: aa 02 04 84 56 02 02 ab 7a 03 04 82 7f 02 01 ac ....V...z....... +| 992: 09 04 82 40 02 02 ad 61 03 04 81 14 03 03 c6 8c ...@...a........ +| 1008: 71 09 04 82 56 02 02 b7 69 0a 04 82 5c 02 03 b8 q...V...i....... +| 1024: 66 66 05 04 81 11 03 01 7a 05 04 82 7a 02 01 be ff......z...z... +| 1040: 05 04 81 2a 02 02 bf 76 0a 02 76 01 02 d9 81 01 ...*...v..v..... +| 1056: 04 83 76 02 01 83 02 02 3c 03 01 36 05 04 81 54 ..v.....<..6...T +| 1072: 02 01 85 09 04 82 73 02 01 86 06 04 84 53 02 03 ......s......S.. +| 1088: a7 6a 61 01 04 82 0f 02 01 a8 04 02 38 02 01 b1 .ja.........8... +| 1104: 01 04 83 21 03 03 dc 94 6a 02 02 2b 02 01 b2 07 ...!....j..+.... +| 1120: 02 1a 02 01 b6 04 04 82 17 02 01 ba 04 04 83 7b ................ +| 1136: 03 01 71 03 04 82 77 02 01 bd 09 04 84 4f 01 04 ..q...w......O.. +| 1152: 82 49 01 07 da 80 e8 b6 8a 67 68 05 04 83 04 02 .I.......gh..... +| 1168: 03 85 d1 9c 06 04 81 59 02 01 8f 05 04 81 2c 02 .......Y......,. +| 1184: 01 93 01 04 83 3d 02 04 9a ed 90 8b 01 04 83 44 .....=.........D +| 1200: 02 02 9c 73 09 04 83 21 02 01 a0 0a 04 85 03 02 ...s...!........ +| 1216: 01 a6 05 02 7e 03 04 81 19 02 02 a9 77 05 04 82 ....~.......w... +| 1232: 69 02 01 ac 03 02 4f 04 04 82 76 03 03 67 c2 bc i.....O...v..g.. +| 1248: 07 04 83 38 02 01 b7 03 04 82 5b 02 01 bf 09 02 ...8......[..... +| 1264: 6a 01 02 db 83 08 04 84 55 02 02 93 70 02 04 81 j.......U...p... +| 1280: 47 02 01 a5 05 04 82 12 02 01 a6 01 02 52 01 02 G............R.. +| 1296: dc 8e 01 02 6a 02 02 96 6b 06 04 84 29 02 01 9a ....j...k...)... +| 1312: 05 04 84 05 03 04 82 3e 02 01 9f 09 04 81 64 02 .......>......d. +| 1328: 03 a4 76 77 0a 04 84 0f 02 01 a6 06 04 83 52 02 ..vw..........R. +| 1344: 01 aa 01 04 83 70 02 01 ae 01 04 83 33 01 02 dd .....p......3... +| 1360: 8c 03 04 82 1f 02 02 90 78 09 04 82 4b 02 01 96 ........x...K... +| 1376: 05 04 81 10 03 01 6c 06 04 83 24 02 02 9c 6a 04 ......l...$...j. +| 1392: 04 82 1b 02 05 a0 79 c9 a8 75 01 02 34 02 01 a8 ......y..u..4... +| 1408: 05 02 2c 02 01 b2 06 04 84 3f 02 02 ba 67 03 02 ..,......?...g.. +| 1424: 2f 02 05 bf e0 a6 ad 77 08 02 2f 01 02 de 83 05 /......w../..... +| 1440: 02 65 02 01 84 06 04 81 00 02 01 85 01 04 82 60 .e.............` +| 1456: 01 04 81 57 02 01 8f 03 04 83 3b 02 01 90 03 02 ...W......;..... +| 1472: 7b 02 01 91 07 04 83 5d 02 03 9c 78 61 07 04 81 .......]...xa... +| 1488: 00 02 01 9e 08 04 82 00 02 02 9f 6f 04 04 83 51 ...........o...Q +| 1504: 02 01 a3 04 04 81 2f 02 02 a5 73 0a 04 84 7a 02 ....../...s...z. +| 1520: 02 b1 66 08 04 81 5a 02 02 b7 78 02 02 78 02 01 ..f...Z...x..x.. +| 1536: b9 05 04 83 08 02 01 ba 0a 04 84 76 02 01 bb 07 ...........v.... +| 1552: 02 5c 02 01 bc 08 02 13 02 01 bf 08 02 2a 01 03 .............*.. +| 1568: df 82 66 08 04 82 2c 02 01 87 01 04 83 0a 02 01 ..f...,......... +| 1584: 8d 05 04 84 0c 02 05 91 72 6b 69 61 03 02 45 02 ........rkia..E. +| 1600: 03 97 de a4 09 04 82 6b 02 02 98 62 06 04 81 18 .......k...b.... +| 1616: 02 01 a0 0a 04 82 4a 02 01 bf 05 02 39 01 06 e0 ......J.....9... +| 1632: a7 9d 31 76 63 05 04 83 00 02 02 af aa 02 04 81 ..1vc........... +| 1648: 2b 02 03 b7 ae 65 01 04 84 36 02 02 bb 98 0a 04 +....e...6...... +| 1664: 82 23 01 04 e1 80 88 63 06 04 83 5c 03 01 a2 04 .#.....c........ +| 1680: 02 5c 02 02 85 9b 05 04 81 3e 02 04 8b b3 db 8f .........>...... +| 1696: 07 02 56 02 04 91 b8 64 66 08 04 82 49 02 02 95 ..V....df...I... +| 1712: 9d 08 04 81 4e 03 01 a2 03 04 81 77 02 02 9a a0 ....N......w.... +| 1728: 01 04 82 39 02 03 9d 90 69 03 02 41 02 03 a6 9a ...9....i..A.... +| 1744: 68 04 02 4c 02 05 a7 9c c7 ad 6e 06 04 83 7b 02 h..L......n..... +| 1760: 03 a8 b2 38 04 02 5f 02 02 af 88 07 02 19 02 03 ...8.._......... +| 1776: bd 96 71 03 04 81 11 01 03 e2 83 80 05 02 5f 02 ..q..........._. +| 1792: 02 b1 a6 01 04 83 34 01 04 84 06 02 02 b3 9d 01 ......4......... +| 1808: 04 83 0f 02 03 ba 9a 64 01 04 84 4f 01 03 e3 81 .......d...O.... +| 1824: 81 0a 02 21 02 02 84 8c 03 04 81 10 02 02 90 a1 ...!............ +| 1840: 02 04 81 1b 02 02 9c a3 02 04 82 2e 02 02 a5 89 ................ +| 1856: 04 04 82 23 02 02 a6 98 07 04 83 00 02 09 ab a8 ...#............ +| 1872: 77 31 63 6a c2 aa 35 04 04 81 38 03 01 aa 01 04 w1cj..5...8..... +| 1888: 82 1e 02 02 ac a4 02 02 37 02 05 b9 8e 70 34 79 ........7....p4y +| 1904: 0a 04 84 30 02 02 bd bf 01 04 83 19 01 05 e4 80 ...0............ +| 1920: 83 79 69 04 04 83 15 03 01 bb 01 02 63 02 03 8a .yi.........c... +| 1936: 8d 36 0a 04 84 57 02 05 8c 88 d0 b1 63 09 04 84 .6...W......c... +| 1952: 49 02 02 90 8e 02 04 84 60 02 02 a1 98 03 04 84 I.......`....... +| 1968: 0f 02 02 b1 a2 08 04 83 30 01 04 e5 85 87 68 02 ........0.....h. +| 1984: 04 84 19 02 03 8a a8 39 0a 02 6e 02 03 8d be 38 .......9..n....8 +| 2000: 09 04 81 78 02 04 8f 85 68 30 05 04 84 5a 03 01 ...x....h0...Z.. +| 2016: a7 09 04 82 09 02 03 92 8e 7a 04 02 28 02 02 95 .........z..(... +| 2032: 89 01 02 38 03 01 be 07 04 84 4d 02 08 9e 9e f4 ...8......M..... +| 2048: 95 95 98 68 71 07 04 84 13 02 02 9f 9f 04 04 84 ...hq........... +| 2064: 35 02 02 a0 a2 02 02 24 03 05 b4 f0 90 b1 92 0a 5......$........ +| 2080: 04 82 15 02 03 a3 a1 64 07 04 84 22 02 02 a6 bb .......d........ +| 2096: 04 04 84 52 02 05 a8 af 75 c2 bd 01 04 81 25 02 ...R....u.....%. +| 2112: 02 aa b3 06 04 81 66 02 03 ae be 7a 0a 02 4d 02 ......f....z..M. +| 2128: 08 af 96 66 6b f3 93 af 8f 04 04 83 02 02 03 bd ...fk........... +| 2144: aa 6a 07 02 62 01 04 e6 80 83 69 08 04 84 19 02 .j..b.....i..... +| 2160: 02 81 bc 0a 04 81 25 02 04 86 aa 32 75 06 04 82 ......%....2u... +| 2176: 65 02 05 96 b7 69 35 30 07 02 78 02 03 9d 98 73 e....i50..x....s +| 2192: 06 02 7f 02 02 a3 97 02 04 81 7f 02 05 a5 80 e7 ................ +| 2208: a0 b6 01 04 81 37 03 01 87 08 04 82 61 02 02 b3 .....7......a... +| 2224: b2 03 04 83 1e 02 02 b8 a0 0a 04 84 2e 02 02 bc ................ +| 2240: af 0a 04 83 71 02 02 bf 91 09 04 84 42 01 03 e7 ....q.......B... +| 2256: 81 b4 07 04 84 5e 02 02 86 8e 01 04 84 2c 02 02 .....^.......,.. +| 2272: 8b a9 09 04 82 02 02 03 8d 9d 30 05 04 84 1a 02 ..........0..... +| 2288: 02 8e b6 08 04 83 60 02 02 91 99 0a 04 82 05 02 ......`......... +| 2304: 02 92 af 08 04 81 1c 02 02 94 b9 05 04 81 45 02 ..............E. +| 2320: 02 9c 8f 06 02 14 02 02 9e a2 07 04 83 78 02 02 .............x.. +| 2336: a0 83 06 04 83 08 02 02 a1 a8 01 02 78 02 02 ae ............x... +| 2352: ae 04 04 81 0e 02 02 b5 8c 06 04 84 6c 02 02 bf ............l... +| 2368: 80 09 04 83 1e 01 04 e8 83 93 73 0a 04 83 57 02 ..........s...W. +| 2384: 02 88 a8 0a 04 84 50 02 02 89 a0 09 04 83 70 02 ......P.......p. +| 2400: 02 8a ad 05 04 83 18 02 02 95 84 04 02 3b 03 01 .............;.. +| 2416: b5 06 04 81 26 02 08 a0 91 65 ce bb 6a d7 96 09 ....&....e..j... +| 2432: 04 82 05 02 02 a7 bf 02 04 84 47 02 02 b5 bb 05 ..........G..... +| 2448: 02 0a 02 04 bf 88 6a 6b 03 04 84 2a 01 04 e9 80 ......jk...*.... +| 2464: 97 65 09 04 83 26 02 02 9a 89 04 04 83 29 02 02 .e...&.......).. +| 2480: a4 b8 05 04 82 7f 02 05 ba 83 69 76 6c 04 02 3a ..........ivl..: +| 2496: 03 01 87 03 02 3c 02 02 be b6 06 04 84 25 01 06 .....<.......%.. +| 2512: ea 81 98 64 39 79 03 04 84 12 02 02 88 bc 09 04 ...d9y.......... +| 2528: 84 40 02 03 8c b5 39 01 04 83 11 02 02 91 ab 05 .@....9......... +| 2544: 04 83 01 02 03 9d 97 63 05 04 83 07 02 02 9e a3 .......c........ +| 2560: 04 04 82 04 02 02 ae a3 03 04 81 20 02 02 b2 a1 ........... .... +| 2576: 04 02 66 02 04 b4 9d 65 6b 02 02 29 02 03 ba bd ..f....ek..).... +| 2592: 72 08 04 83 75 02 03 bc a7 37 09 04 81 55 01 03 r...u....7...U.. +| 2608: eb 84 9d 07 02 60 02 06 8f aa 6f 69 6b 68 03 02 .....`....oikh.. +| 2624: 29 02 02 92 90 0a 04 81 30 02 04 99 ba 6e 65 02 ).......0....ne. +| 2640: 04 82 1e 02 02 b8 8f 05 04 81 52 02 02 bd 80 08 ..........R..... +| 2656: 04 84 23 02 02 be bc 08 02 2e 02 02 bf bc 01 04 ..#............. +| 2672: 82 6b 01 04 ec 85 b8 76 05 04 81 76 02 03 99 ad .k.....v...v.... +| 2688: 74 07 02 7e 02 02 a3 80 06 04 81 37 02 04 ac a9 t..~.......7.... +| 2704: 79 75 01 02 6f 02 02 b9 89 06 02 66 01 03 ed 8a yu..o......f.... +| 2720: 8a 05 02 3f 02 02 96 b6 05 02 63 01 04 ee 8d 91 ...?......c..... +| 2736: 7a 02 04 82 2a 02 02 92 af 03 04 84 06 02 03 98 z...*........... +| 2752: be 78 01 04 81 50 02 03 99 82 71 06 02 13 02 04 .x...P....q..... +| 2768: 9c 8d 68 61 09 04 83 1b 02 02 a4 bd 06 04 82 24 ..ha...........$ +| 2784: 02 02 ad a9 06 04 81 5d 02 02 b4 b9 09 04 81 75 .......].......u +| 2800: 02 02 bb 8d 05 04 84 39 01 03 ef af 90 03 04 82 .......9........ +| 2816: 3d 03 02 a3 61 03 04 83 02 02 02 b4 ae 0a 04 82 =...a........... +| 2832: 0e 02 02 b5 b1 06 02 47 02 02 b7 a9 02 04 84 42 .......G.......B +| 2848: 01 05 f0 90 8e bf 34 03 04 81 40 03 04 ad b0 33 ......4...@....3 +| 2864: 38 05 04 84 19 03 02 ae 89 07 04 81 68 03 03 af 8...........h... +| 2880: b7 61 0a 04 82 2f 02 03 91 81 a7 02 04 81 04 03 .a.../.......... +| 2896: 02 89 ac 08 04 83 62 03 02 97 a6 08 04 82 64 03 ......b.......d. +| 2912: 02 af bc 07 04 83 14 03 02 b1 b3 05 04 81 34 03 ..............4. +| 2928: 02 b7 ad 07 04 82 35 03 03 bb 9f 64 07 04 84 4c ......5....d...L +| 2944: 02 03 92 8d a5 05 04 82 7b 03 03 98 ab 62 09 04 .............b.. +| 2960: 81 2f 03 02 b7 99 05 04 82 43 02 04 93 85 98 6b ./.......C.....k +| 2976: 0a 02 1e 03 02 a9 90 01 04 84 62 03 05 b6 b0 74 ..........b....t +| 2992: c2 b3 07 04 82 55 02 03 95 8e a5 06 04 83 2a 03 .....U........*. +| 3008: 02 96 95 09 04 83 0c 03 02 97 8b 06 04 84 60 03 ..............`. +| 3024: 02 b5 bd 07 04 82 56 03 02 b7 9c 06 04 82 2d 02 ......V.......-. +| 3040: 04 96 a0 80 6e 01 04 83 7f 03 02 a5 ad 03 04 83 ....n........... +| 3056: 3c 03 02 ad ac 07 02 72 02 04 97 87 b5 35 07 02 <......r.....5.. +| 3072: 5f 04 01 bc 07 02 53 03 02 8f 90 02 02 70 02 06 _.....S......p.. +| 3088: 98 8f 88 73 7a 78 06 02 35 04 01 ba 02 02 23 03 ...szx..5.....#. +| 3104: 06 a2 92 ca 8a 36 6d 07 04 82 06 02 04 99 8b 93 .....6m......... +| 3120: 7a 09 04 84 6b 03 03 8c 82 72 07 04 81 7a 03 02 z...k....r...z.. +| 3136: 94 ba 04 04 82 3c 03 02 a1 92 0a 04 84 31 03 02 .....<.......1.. +| 3152: a8 b7 08 04 84 16 03 02 b1 ac 03 04 84 28 02 05 .............(.. +| 3168: 9a 99 8d d7 b5 01 04 84 65 03 03 9d 86 7a 09 04 ........e....z.. +| 3184: 84 07 03 03 ac 80 37 09 04 84 24 02 03 9b 87 93 ......7...$..... +| 3200: 04 04 82 60 03 02 91 8b 05 04 84 6c 03 02 a0 b9 ...`.......l.... +| 3216: 05 04 82 13 03 03 a5 81 70 06 04 83 6e 03 02 b6 ........p...n... +| 3232: 9a 03 04 82 00 02 03 9c 8a a3 04 04 83 7e 03 02 .............~.. +| 3248: 96 b7 01 04 83 5b 03 02 a8 88 02 04 83 7a 03 03 .....[.......z.. +| 3264: a9 98 77 02 04 81 38 03 02 b1 b2 04 02 46 03 03 ..w...8......F.. +| 3280: b6 bb 72 03 04 82 1a 03 02 be a9 04 04 83 73 02 ..r...........s. +| 3296: 04 9d 8a 9a 6a 01 02 32 03 02 96 9c 03 02 6f 03 ....j..2......o. +| 3312: 02 a6 b2 04 04 83 35 03 02 b4 a6 05 02 43 03 02 ......5......C.. +| 3328: bc 98 05 02 4a 02 03 9e 8f 98 0a 04 81 08 03 03 ....J........... +| 3344: 90 8c 6d 07 04 84 64 03 03 b4 9f 6e 03 04 84 35 ..m...d....n...5 +| 3360: 02 04 9f a1 a9 69 07 04 84 43 02 03 a7 85 b6 06 .....i...C...... +| 3376: 02 60 02 03 ad ba b7 06 02 2b 02 03 bc b6 b6 01 .`.......+...... +| 3392: 04 81 31 01 04 f1 92 a9 98 06 02 0c 03 03 b1 9d ..1............. +| 3408: 69 08 04 84 65 02 06 9e 91 a5 ea b0 98 07 04 82 i...e........... +| 3424: 43 02 03 a1 b7 be 02 04 81 13 02 03 b4 af 9e 0a C............... +| 3440: 04 84 1d 02 05 b5 92 84 69 6e 07 04 82 52 01 05 ........in...R.. +| 3456: f2 90 9a 83 73 03 04 82 45 02 03 91 ac 81 07 04 ....s...E....... +| 3472: 82 7a 03 02 be 94 02 04 81 46 02 03 94 89 ac 04 .z.......F...... +| 3488: 04 81 4a 02 04 95 b4 ae 39 08 04 82 2a 02 05 a0 ..J.....9...*... +| 3504: 90 b5 66 34 04 04 82 46 02 05 a6 91 b7 d5 bf 06 ..f4...F........ +| 3520: 04 81 7b 02 03 ab a0 93 05 02 2b 02 04 b2 8c 80 ..........+..... +| 3536: 6a 07 04 84 1a 03 02 9c a9 0a 04 81 36 03 02 aa j...........6... +| 3552: bf 04 04 84 34 01 05 f3 82 84 8d 66 0a 04 83 75 ....4......f...u +| 3568: 02 03 88 a7 b4 06 04 83 38 02 03 ab bf a6 02 04 ........8....... +| 3584: 81 65 02 05 b0 ae a8 73 70 0a 02 10 02 04 b8 a6 .e.....sp....... +| 3600: b6 75 05 04 83 4e 02 03 bd b2 84 08 04 83 50 01 .u...N........P. +| 3616: 04 f4 82 91 84 06 04 81 6a 02 03 8c b4 be 06 02 ........j....... +| 3632: 24 02 03 9b b2 83 01 04 84 4b 02 03 a0 87 b0 05 $........K...... +| 3648: 04 83 2f 03 03 ae ba 61 05 04 82 1b 04 08 07 07 ../....a........ +| 3664: 06 07 0b 08 06 09 07 07 06 07 0a 07 09 0b 07 06 ................ +| 3680: 08 07 0a 07 08 08 07 07 08 07 0b 07 08 08 07 0c ................ +| 3696: 09 07 07 09 0b 07 09 08 08 07 08 08 0a 0a 0a 07 ................ +| 3712: 0c 07 07 0b 07 08 07 06 06 07 08 09 0f 07 06 07 ................ +| 3728: 07 06 07 08 07 08 07 07 09 07 08 08 07 08 07 06 ................ +| 3744: 07 0a 07 07 06 06 08 09 08 07 07 07 06 08 07 0a ................ +| 3760: 07 07 07 09 07 07 06 07 07 06 09 06 07 09 0a 0b ................ +| 3776: 09 08 07 08 07 08 09 08 09 07 07 07 08 06 07 07 ................ +| 3792: 07 09 06 07 08 06 07 07 07 0b 0d 09 07 07 0a 08 ................ +| 3808: 07 0a 08 0a 09 07 06 08 08 07 06 07 08 0b 07 09 ................ +| 3824: 07 07 07 08 08 07 07 08 0a 06 07 07 0a 07 07 0b ................ +| 3840: 07 06 07 09 07 08 07 08 08 07 07 07 06 06 06 09 ................ +| 3856: 07 07 0a 09 08 07 06 0c 08 09 08 0a 06 08 09 0a ................ +| 3872: 08 07 08 08 08 0b 08 07 09 08 0c 08 09 08 08 08 ................ +| 3888: 08 08 08 0f 07 07 0b 08 0b 06 09 0b 08 08 08 0a ................ +| 3904: 08 09 0a 07 08 07 07 0e 08 07 0b 09 08 0b 08 08 ................ +| 3920: 0e 08 0a 08 0a 0a 08 08 0b 07 08 08 08 08 09 08 ................ +| 3936: 08 09 08 08 08 08 07 08 08 07 08 08 08 0a 08 08 ................ +| 3952: 08 07 07 0e 08 07 0a 0a 08 08 0a 06 08 0c 08 09 ................ +| 3968: 08 09 08 08 07 09 09 09 08 0b 08 0a 08 08 07 08 ................ +| 3984: 0a 08 08 09 07 08 07 0a 08 09 08 0a 08 08 08 08 ................ +| 4000: 09 08 08 07 08 0b 0a 08 09 09 08 08 08 08 08 09 ................ +| 4016: 09 09 08 09 08 0b 09 08 08 08 08 0a 08 07 09 06 ................ +| 4032: 07 0b 06 0c 0a 09 08 08 08 08 0b 09 09 09 08 08 ................ +| 4048: 09 08 09 08 08 09 07 09 08 09 07 08 07 07 09 09 ................ +| 4064: 09 0a 08 08 09 09 09 0c 09 09 0b 0b 09 08 09 0a ................ +| 4080: 0b 0b 08 0a 08 08 0b 09 09 0a 0a 09 0a 08 09 09 ................ +| page 26 offset 102400 +| 0: 0d 00 00 00 01 0e a7 00 0e a7 00 00 00 00 00 00 ................ +| 3744: 00 00 00 00 00 00 00 82 51 88 80 80 80 80 07 04 ........Q....... +| 3760: 00 85 26 00 00 01 2d 05 30 f4 a3 b5 8c 0a 02 1b ..&...-.0....... +| 3776: 02 03 b4 91 a9 02 04 82 1b 02 04 b7 b5 89 67 06 ..............g. +| 3792: 04 84 2f 01 04 f5 86 a9 bd 01 04 81 13 02 06 91 ../............. +| 3808: be ae ca 9e 78 04 04 81 1d 02 05 99 9b b2 6f 65 ....x.........oe +| 3824: 02 04 84 07 02 03 9e 99 8b 02 02 15 02 03 a1 ab ................ +| 3840: b6 02 04 83 0f 02 03 a3 96 8c 02 04 82 20 02 03 ............. .. +| 3856: ac a3 8d 08 02 55 02 03 b0 80 82 02 04 81 18 02 .....U.......... +| 3872: 03 b1 9c bb 08 02 67 02 03 b5 be 81 07 04 83 79 ......g........y +| 3888: 02 04 bf 81 9d 71 07 04 81 7c 01 04 f6 83 ab 80 .....q...|...... +| 3904: 01 04 81 2a 02 04 89 8c a8 61 01 04 82 30 02 03 ...*.....a...0.. +| 3920: 8e ae 81 07 04 84 05 02 04 8f bd b0 6a 09 04 82 ............j... +| 3936: 1e 02 04 92 95 9c 6c 07 04 82 64 03 02 b4 9f 09 ......l...d..... +| 3952: 04 83 65 02 04 9a a9 9f 34 06 04 83 70 02 03 a4 ..e.....4...p... +| 3968: ad af 01 02 02 02 03 b4 9a 90 07 04 83 5e 01 04 .............^.. +| 3984: f7 87 a5 9d 04 04 84 62 02 03 89 83 8e 0a 04 82 .......b........ +| 4000: 2e 02 03 98 bd 8e 01 04 84 06 02 03 99 a7 8e 02 ................ +| 4016: 04 83 7d 02 03 a0 a1 bb 01 04 81 4e 02 04 a8 86 ...........N.... +| 4032: b8 64 0a 02 34 02 03 b2 9a 9e 05 04 83 16 02 04 .d..4........... +| 4048: b4 a7 93 36 06 04 83 7d 03 02 ba 9c 02 04 83 07 ...6............ +| 4064: 04 09 09 0a 0a 0c 0b 08 09 09 08 09 08 09 0a 0a ................ +| 4080: 0a 09 0a 0a 08 0a 08 09 0a 09 09 09 09 09 09 0a ................ +| end x.db +}]} {} + +do_catchsql_test 74.0.5 { + SELECT matchinfo(1,2); +} {1 {unable to use function matchinfo in the requested context}} + +do_catchsql_test 74.1 { + SELECT rowid, quote(matchinfo(t1,'pxyb... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 82 d0 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 81 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f fc 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 00 f2 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 18 ea 00 ....is.......... +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 12 10 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 2f ff ff f0 11 02 01 ........./...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0d 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 01 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 e7 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 01 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 02 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 7f 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 03 .V.M.A.8./.&.... +| 64: 0f 0a 0f 00 9e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 18 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 0f 01 02 31 6e 12 0a ...1t.......1n.. +| 3888: d4 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 1e 61 30 66 74 04 05 04 09 0c 01 02 .....a0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 da 03 02 02 01 02 00 00 00 00 ................ +| end crash-53216796d7c3d8.db +}]} {} + +do_catchsql_test 77.1 { + UPDATE t1 SET B =quote(zeroblob(200)) WHERE b MATCH 'threa*thrad*tlrad*the�d'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 78.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a,b,c); + DELETE FROM t1_data; + INSERT INTO t1_data VALUES(1,X'245a2424'); + INSERT INTO t1_data VALUES(10,X'000000000101010001010101'); + INSERT INTO t1_data VALUES(137438953473,X'0000032b0230300102060102060102061f0203010203010203010832303136303630390102070102070102070101340102050102050102050101350102040102040102040207303030303030301c0204010204010204010662696e6172790306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020108636f6d70696c657201200102020201020201066462737ccccccccccccccccccccccccccccccccccccccccccccbccccccccccccccccccccccccccccccccccccccccccccccca2cccccccccc461740702030102030102030204656275670402020102020102020106656e61626c6507020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020523d6763632d352e342e30203230313630363039584e4f4341534526010500430f17434f4d50494c45523d6763632d352e342e3020323031363036303958525452494d0d000000240ee00004a810000fe80fe00fd80fd00fc80fc00fb80fb00fa80fa00f980f900f880f800f780f700f680f60945736502060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020201046f6d59741f0202010202010202010572747265651945030102030102030402696d010601020203060102020306010202030601120203060102020306010202030601020203060102020306010202030601020203060102020306010202010a7468726561647361666522020201020201020201047674616207020401020401020401407801060cb102010601010201060101020106010102010601010201060101020106010102010601010201060e0102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101021106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601011201060101020106010102010601010201060101020106010102041513020c124413110f47130f0c0e11100f0e100f440f1040150f'); +} + +do_execsql_test 78.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1','col'); +} + +do_execsql_test 78.2 { + SELECT count(rowid) FROM t3 WHERE term>='nsocse'; +} 2 + +#------------------------------------------------------------------------- +reset_db +do_test 79.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename sql053282.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 06 0d b6 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d b5 00 00 00 00 00 00 ...k............ +| 3504: 00 00 00 00 00 00 56 07 06 17 1f 1f 01 7d 74 61 ......V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 01 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 12 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 01 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 05 f1 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 01 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 00 00 00 00 00 .......h.O...... +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d ..%..THREADSAFE= +| 3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRTRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 2c 00000XRTRIM...., +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4c 45 20 4d 45 4d 5a 69 53 35 58 42 49 NABLE MEMZiS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 45 20 4d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 E MEMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 3f d8 .#..ENABLE FTS?. +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 55 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e UAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 41 42 58 4e 4f 43 41 53 45 1d TAT VTABXNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 46 CASE.......DEBUF +| 3968: e8 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d .RTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3d 67 ...C..COMPILER=g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 62 63 2d 35 2e 34 2e OMPILER=gbc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f bb 1f a8 0f a0 ................ +| 32: 0f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 00 00 00 00 .X.P.H.@.8.0.... +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 16 03 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 0a 83 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end sql053282.txt.db +}]} {} + +do_execsql_test 79.1 { + CREATE VIRTUAL TABLE t2 USING fts5vocab('t1','row'); +} + +do_catchsql_test 79.2 { + INSERT INTO t1(t1) SELECT 'merge' FROM t2; +} {1 {query aborted}} + +#------------------------------------------------------------------------- +reset_db +do_test 80.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-f928a9c1ec68dd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 01 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 00 00 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 12 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 00 00 00 00 ........vers.... +| end crash-f928a9c1ec68dd.db +}]} {} + +do_catchsql_test 80.1 { +SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 81.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-44e8035a976422.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42 .....h........jB +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69 h!i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-44e8035a976422.db +}]} {} + +do_catchsql_test 81.2 { + UPDATE t1 SET b=zeroblob(299); +} {1 {database disk image is malformed}} + + +sqlite3_fts5_may_be_corrupt 0 +finish_test + diff --git a/ext/fts5/test/fts5corrupt4.test b/ext/fts5/test/fts5corrupt4.test new file mode 100644 index 0000000000000000000000000000000000000000..0505250c2f857e5903c9238cd237b7c64fd1e822 --- /dev/null +++ b/ext/fts5/test/fts5corrupt4.test @@ -0,0 +1,63 @@ +# 2019 May 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ttt USING fts5(a, b); + INSERT INTO ttt + VALUES('e ee eee e ee eee e ee eee', 'eee ee e e e ee eee ee ee'); + INSERT INTO ttt SELECT a||a, b||b FROM ttt; + INSERT INTO ttt SELECT a||a, b||b FROM ttt; +} + +expr srand(1) + +proc mutate {blob i} { + set o [expr {$i % [string length $blob]}] + set a [string range $blob 0 $o-1] + set b [string range $blob $o+1 end] + set v [expr int(rand()*255) - 127] + return "$a[binary format c $v]$b" +} +db func mutate mutate + +for {set j 1000} {$j <= 5000} {incr j 1000} { + do_test 1.$j { + for {set i 0} {$i < 1000} {incr i} { + execsql { + BEGIN; + UPDATE ttt_data SET block = mutate(block, $i) WHERE id>10; + } + foreach sql { + {SELECT snippet(ttt, -1, '.', '..', '[', ']'), * FROM ttt('e*')} + {SELECT snippet(ttt, -1, '.', '..', '[', ']'), * FROM ttt('e* NOT ee*')} + } { + catch { execsql $sql } + } + execsql ROLLBACK + } + } {} +} + +sqlite3_fts5_may_be_corrupt 0 +finish_test + diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test new file mode 100644 index 0000000000000000000000000000000000000000..19f0538418d6d32a40c3d190927746ce0fa3dcc9 --- /dev/null +++ b/ext/fts5/test/fts5corrupt5.test @@ -0,0 +1,1456 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that FTS5 handles corrupt databases (i.e. internal +# inconsistencies in the backing tables) correctly. In this case +# "correctly" means without crashing. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt5 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +sqlite3_fts5_may_be_corrupt 1 +database_may_be_corrupt + +#------------------------------------------------------------------------- +# dbsqlfuzz crash-0f47112aa7520cf08c6a835a88fdff8c2a32a188 +# +reset_db +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-0f47112aa7520c.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 00 00 00 00 ...~.H.......... +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 11 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 00 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 03 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 04 f6 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: f6 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 0f 42 03 07 1c 8c 81 80 80 .0table.B....... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 13 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 9e 00 ............<... +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 08 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 80 72 65 01 02 05 01 ..<.....3.re.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 17 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 64 61 62 ............0dab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 13 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 14 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 01 64 09 10 01 32 69 14 07 04 09 10 01 32 66 ..d...2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 06 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 00 00 00 00 00 00 00 00 00 00 00 ................ +| page 4 offset 12288 +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 00 00 00 00 00 ................ +| end crash-0f47112aa7520c.db + }] +} {} + +do_catchsql_test 1.1 { + SELECT * FROM t1('R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_test 2.0 { + sqlite3 db {} + db deserialize [decode_hexdb { + +.open --hexdb +| size 24576 pagesize 4096 filename sql047467.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 00 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 01 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 00 00 00 00 ...~.H.......... +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 11 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3b ff f0 00 16 04 33 74 68 65 03 06 01 01 04 .;.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 00 6f 03 06 01 01 06 14 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 03 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 04 f6 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: f6 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 0f 42 03 07 1c 8c 81 80 80 .0table.B....... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 13 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 94 50 ............<..P +| 3200: 16 02 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 08 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 80 72 65 01 02 05 01 ..<.....3.re.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 17 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 64 61 62 ............0dab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 13 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 14 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 01 64 09 10 01 32 69 14 07 04 09 10 01 32 66 ..d...2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 06 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 00 00 00 00 00 00 00 00 00 00 00 ................ +| page 4 offset 12288 +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 00 00 00 00 00 ................ +| end sql047467.txt.db +}]} {} + +do_catchsql_test 2.1 { +SELECT * FROM t1('R*R*R*R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 3.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-c69fcaceff1e50.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 01 bb 02 30 30 01 02 06 01 02 06 01 02 06 .....00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6b b1 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 k.ble........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 05 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 1a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 01 ...omit......... +| 3744: ff ff ff ff ff ff ff ff ff ff ff ff ff ff f1 02 ................ +| 3760: 58 81 96 4d 01 06 01 02 02 03 06 01 02 02 03 06 X..M............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 22 02 02 01 ..threadsafe.... +| 3840: 02 02 01 02 02 01 04 76 74 61 62 07 02 04 01 02 .......vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 c6 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 00 f6 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-c69fcaceff1e50.db +}]} {} + +do_catchsql_test 3.1 { + UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thra*T'; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 4.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-ef6738247b1344.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 10 00 06 40 00 00 06 .....@ ....@... +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 00 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 6f 67 6e 6f 2c 20 50 52 49 4d term, ogno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 35 07 17 1b 1b 01 81 01 74 61 62 6c WIDU5.......tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 0b 0a ef 0a cc 0a 00 ...~.H.......... +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 02 9c 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 06 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 12 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 82 d0 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 03 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 80 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 81 80 80 06 03 00 36 00 00 00 13 04 12 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 65 80 80 04 03 00 30 00 00 00 11 01 01 06 ..e.....0....... +| 2816: 06 30 74 61 62 6c 65 03 02 03 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f fc 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 00 f2 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 03 02 01 02 69 73 03 06 04 0c 00 00 00 18 ea 00 ....is.......... +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 12 10 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 2f ff ff f0 11 02 01 ........./...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f cf 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0d 03 00 3c 00 00 00 16 00 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 03 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 00 .............6.. +| 3168: 00 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 04 05 07 1b 88 80 80 80 80 0a 03 00 3c 00 00 00 ............<... +| 3200: 16 12 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 78 80 80 80 80 08 03 00 34 10 ......x.......4. +| 3264: 01 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 80 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 e7 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 e5 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 01 00 00 12 02 30 31 02 06 01 01 04 01 01 32 4.....01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 01 01 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 02 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 0d 1a 00 00 00 15 04 33 66 6e 72 01 02 ..........3fnr.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 84 .....3eac....... +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 30 c9 6e 01 ......8.....0.n. +| 3776: 06 01 01 02 01 02 70 72 01 02 05 04 09 18 84 80 ......pr........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 0b e2 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 86 f0 08 03 00 3c 00 00 00 16 02 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 01 01 04 04 06 1b 84 80 80 80 ..f............. +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 7f 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 02 08 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 22 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 .V.............. +| 3392: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3408: 04 01 10 01 03 34 74 20 07 04 01 0e 01 03 34 1e .....4t ......4. +| 3424: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3440: 34 6e 1a 08 04 01 10 01 03 32 67 18 08 04 01 10 4n.......2g..... +| 3456: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3472: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3488: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3504: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3520: 30 74 08 09 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3536: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3552: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3568: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3584: 1c 18 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3600: 32 69 18 09 04 01 12 01 02 32 61 72 16 08 04 01 2i.......2ar.... +| 3616: 10 01 02 31 74 2a 08 04 01 0f 01 02 31 6e 12 0a ...1t*......1n.. +| 3632: d4 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3648: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3664: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3680: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3696: 04 06 03 f1 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3712: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3728: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 3744: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 3760: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 3776: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 3792: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 3808: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 3824: 06 08 04 09 1e 61 30 66 74 04 05 00 00 00 00 00 .....a0ft....... +| page 4 offset 12288 +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 73 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 gsz.....version. +| end crash-ef6738247b1344.db +}]} {} + + +do_catchsql_test 4.1 { + BEGIN; + REPLACE INTO t1(rowid,b,a,rowid) VALUES(200,1,2,3); +} {1 {database disk image is malformed}} + +do_catchsql_test 4.2 { + INSERT INTO t1(t1) VALUES('delete-all'); +} {1 {database disk image is malformed}} +do_catchsql_test 4.3 { + REPLACE INTO t1(rowid,b,rowid,a) VALUES(200,1,2,3); +} {1 {database disk image is malformed}} +do_catchsql_test 4.4 { + REPLACE INTO t1(rowid,b,a,rowid) VALUES(0,1,2,3); +} {1 {database disk image is malformed}} +do_catchsql_test 4.5 { + REPLACE INTO t1(rowid,a,b,rowid) VALUES(200,1,2,3); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 5.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-0c6d3451d11597.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 AR. KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 16: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 10 03 0f d6 00 0f f4 10 e1 0f d6 00 00 ................ +| 16: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00 heck....optim... +| end crash-0c6d3451d11597.db +}]} {} + +do_execsql_test 5.1 { + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); +} +do_catchsql_test 5.4 { + UPDATE t1 SET content=randomblob(500); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 6.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-42fa37b694d45a.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 49 20 4b 45 59 2c 20 63 R PRIMARI KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 AR. KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd f0 00 ................ +| 16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 10 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6c 01 02 02 04 02 66 74 02 5f 02 04 04 6e 64 cl.....ft._...nd +| 4064: 6f 6e 02 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f 00 01 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 10 03 0f d6 00 0f 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00 heck....optim... +| page 8 offset 28672 +| 0: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| end crash-42fa37b694d45a.db +}]} {} + +do_execsql_test 6.1 { + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); +} +do_catchsql_test 6.2 { + UPDATE t1 SET content=randomblob(500) WHERE t1; +} {1 {constraint failed}} + +#------------------------------------------------------------------------- +reset_db +do_test 7.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40 ..........b....@ +| 112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44 ...u.....f.....D +| 128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00 .....b.......... +| 2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74 ..C......utablet +| 2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41 4t4CREATE VIRTUA +| 2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47 L TABLE t4 USING +| 2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c fts5vocab('t2', +| 2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74 'row')C......ut +| 2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56 ablet3t3CREATE V +| 3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20 IRTUAL TABLE t3 +| 3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28 USING fts5vocab( +| 3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17 't1', 'row')V... +| 3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66 ....tablet2_conf +| 3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41 igt2_config.CREA +| 3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e TE TABLE 't2_con +| 3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62 OWID^...!!...tab +| 3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 let2_contentt2_c +| 3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41 ontent.CREATE TA +| 3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27 BLE 't2_content' +| 3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c ARY KEY, c0, c1, +| 3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62 c2)i.......-tab +| 3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43 let2_idxt2_idx.C +| 3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f REATE TABLE 't2_ +| 3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d idx'(segid, term +| 3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 , pgno, PRIMARY +| 3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 KEY(segid, term) +| 3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 ) WITHOUT ROWIDU +| 3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f ........tablet2_ +| 3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41 datat2_data.CREA +| 3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74 TE TABLE 't2_dat +| 3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 a'(id INTEGER PR +| 3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b IMARY KEY, block +| 3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74 BLOB)X........t +| 3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56 ablet2t2CREATE V +| 3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20 IRTUAL TABLE t2 +| 3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b USING fts5('a',[ +| 3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f b],.c.,detail=no +| 3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 ne,columnsize=0) +| 3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06 configt1_config. +| 3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81 UT ROWID[...!!.. +| 3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize +| 3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54 t1_docsize.CREAT +| 3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61 BLOB)^...!!...ta +| 3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f blet1_contentt1_ +| 3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54 content.CREATE T +| 3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 ABLE 't1_content +| 3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 '(id INTEGER PRI +| 3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 MARY KEY, c0, c1 +| 3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61 , c2)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20 USING fts5(a,b +| 4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65 unindexed,c,toke +| 4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63 nize=.porter asc +| 4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29 ii.,tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d8b4a99207c10b.db +}]} {} + +do_catchsql_test 7.1 { + SELECT snippet(t1, -1, '.', '..', '[', ']'), + highlight(t1, 2, '[', ']') + FROM t1('g + h') + WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40 ...............@ +| 112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00 ...h............ +| 3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05 configt1_config. +| 3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81 UT ROWIDk...!!.. +| 3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 !tablet1_docsize +| 3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54 t1_docsize.CREAT +| 3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54 BLOB, origin INT +| 3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61 EGER)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 USING fts5(a, b +| 4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f , content='', co +| 4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65 ntentless_delete +| 4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29 =1, tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00 .............1.. +| 16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 ...........6.... +| 3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00 ................ +| 3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01 ..).........V... +| 3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02 ..0aaa.......... +| 3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02 .............. . +| 3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01 ................ +| 3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04 @............... +| 3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62 ......bc.......b +| 3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01 bb.............. +| 3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06 .......... ..... +| 3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06 ............@... +| 3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 ................ +| 3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63 ...............c +| 3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04 d.......ccc..... +| 3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a ................ +| 3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a ... ............ +| 3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64 ......de.......d +| 3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01 dd.............. +| 3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01 ...........ef... +| 3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08 ....eee......... +| 4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66 ......fg.......f +| 4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01 Vf..!..!B..gh... +| 4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06 ....ggg..A..hi.. +| 4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c .......P........ +| 4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22 .........w...... +| 4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0 ................ +| 16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01 ................ +| 4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01 ......?......... +| 4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01 ................ +| 4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d57c01958e48ab.db +}]} {} + +do_catchsql_test 8.1 { + SELECT rowid FROM t1('a* NOT ý‘') ; +} {0 {1 2 3 4 5 6 7}} + +#------------------------------------------------------------------------- +reset_db +do_test 9.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-c76a16c24c8ba6.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56 ..threadsafe.W4V +| 3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02 .d.F...vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 10 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 31 46 45 3d ..%..THREADS1FE= +| 3152: 30 58 52 64 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRdRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4b 45 20 4d 45 4d 53 59 53 35 58 42 49 NABKE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 B`-EMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 39 0f 19 45 4e 41 42 4c NARY....9..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d TAT VT$.HNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3c 67 ...C..COMPILER 10; +} {X'0000001A04306162630102020101620202020101640206030303040806'} + +do_execsql_test 2.2 { + UPDATE t1_data SET + block=X'0000001A04306162630102025501620202020101640206030303040806' + WHERE id>10 +} + +do_catchsql_test 2.3 { + DELETE FROM t1 WHERE rowid = 1 +} {1 {database disk image is malformed}} + +finish_test diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test new file mode 100644 index 0000000000000000000000000000000000000000..d642920e45f83ce06985023b1f2d281a6c7d54bb --- /dev/null +++ b/ext/fts5/test/fts5corrupt8.test @@ -0,0 +1,94 @@ +# 2024 Aug 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt8 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} + +do_execsql_test 1.1 { + UPDATE t1_data SET block='hello world' WHERE id=10 +} + +db close +sqlite3 db test.db + +do_catchsql_test 1.2 { + SELECT * FROM t1 +} {1 {database disk image is malformed}} +do_catchsql_test 1.3 { + DROP TABLE t1 +} {0 {}} +do_execsql_test 1.4 { + SELECT * FROM sqlite_schema +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} +do_execsql_test 2.1 { + UPDATE t1_config SET v=555 WHERE k='version' +} +db close +sqlite3 db test.db +do_catchsql_test 2.2 { + SELECT * FROM t1 +} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}} +do_catchsql_test 2.3 { + DROP TABLE t1 +} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}} +do_test 2.4 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_execsql_test 2.5 { + SELECT * FROM sqlite_schema +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} +do_execsql_test 3.1 { + DELETE FROM t1_config; +} +db close +sqlite3 db test.db +do_catchsql_test 3.2 { + SELECT * FROM t1 +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} +do_catchsql_test 3.3 { + DROP TABLE t1 +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} + + +do_test 3.4 { + sqlite3_db_config db DEFENSIVE 1 +} {1} +do_test 3.5 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_test 3.6 { + sqlite3_db_config db DEFENSIVE -1 +} {1} +do_execsql_test 3.7 { + SELECT * FROM sqlite_schema +} + +finish_test + diff --git a/ext/fts5/test/fts5delete.test b/ext/fts5/test/fts5delete.test new file mode 100644 index 0000000000000000000000000000000000000000..024f89594c0ccbefb7a613b24c01accacebad5e7 --- /dev/null +++ b/ext/fts5/test/fts5delete.test @@ -0,0 +1,170 @@ +# 2017 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5delete + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000 + ) + INSERT INTO t1(rowid, x) SELECT i, (i/2)*2 FROM s; +} + +do_test 1.1 { + execsql BEGIN + for {set i 1} {$i<=5000} {incr i} { + if {$i % 2} { + execsql { INSERT INTO t1 VALUES($i) } + } else { + execsql { DELETE FROM t1 WHERE rowid = $i } + } + } + execsql COMMIT +} {} + +do_test 1.2 { + execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); } + for {set i 0} {$i < 5} {incr i} { + execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) } + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE test ( + id INTEGER PRIMARY KEY, + name TEXT, + value TEXT + ); + CREATE VIRTUAL TABLE test_idx USING fts5( + name, content=test, content_rowid=id + ); +} + +do_catchsql_test 2.1 { + INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 1, 'quick'); +} {1 {database disk image is malformed}} + +do_catchsql_test 2.2 { + INSERT INTO test_idx(rowid, name) VALUES(123, 'one one one'); + INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one'); + INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one'); +} {1 {database disk image is malformed}} + +do_execsql_test 2.3 { + DROP TABLE test_idx; + CREATE VIRTUAL TABLE test_idx USING fts5( + name, content=test, content_rowid=id + ); + + INSERT INTO test_idx(rowid, name) VALUES(123, 'one one one'); + INSERT INTO test_idx(rowid, name) VALUES(124, 'two two two'); + INSERT INTO test_idx(rowid, name) VALUES(125, 'two two two'); + INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one'); + INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one'); + INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one'); +} + +do_catchsql_test 2.4 { + SELECT rowid FROM test_idx WHERE test_idx MATCH 'two' ORDER BY rank; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE tx USING fts5(a, b, c, d, content=); + INSERT INTO tx(rowid, a, c) VALUES(1, 'abc def', 'a b c'); + INSERT INTO tx(rowid, a, c) VALUES(5, 'a b c', 'a b d def'); +} +do_execsql_test 3.1 { + INSERT INTO tx(tx, rowid, a, b, c, d) + VALUES('delete', 5, 'a b c', NULL, 'a b d def', NULL); +} +do_execsql_test 3.2 { + INSERT INTO tx(tx) VALUES('integrity-check'); +} +do_execsql_test 3.3 { + INSERT INTO tx(tx, rowid, a, b, c, d) + VALUES('delete', 1, 'abc def', NULL, 'a b c', NULL); +} +do_execsql_test 3.4 { + INSERT INTO tx(tx) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, + content='', contentless_unindexed=1 + ); + CREATE VIRTUAL TABLE ft2 USING fts5(a, b UNINDEXED, + content='', contentless_unindexed=1, contentless_delete=1 + ); + + INSERT INTO ft1(rowid, a, b) VALUES(1, 'one', 'i'); + INSERT INTO ft1(rowid, a, b) VALUES(2, 'two', 'ii'); + INSERT INTO ft1(rowid, a, b) VALUES(3, 'three', 'iii'); + INSERT INTO ft2(rowid, a, b) VALUES(1, 'one', 'i'); + INSERT INTO ft2(rowid, a, b) VALUES(2, 'two', 'ii'); + INSERT INTO ft2(rowid, a, b) VALUES(3, 'three', 'iii'); +} + +do_catchsql_test 4.1 { + DELETE FROM ft1 WHERE rowid=2 +} {1 {cannot DELETE from contentless fts5 table: ft1}} +do_catchsql_test 4.2 { + DELETE FROM ft2 WHERE rowid=2 +} {0 {}} + +do_catchsql_test 4.3 { + INSERT INTO ft1(ft1, rowid, a) VALUES('delete', 2, 'two'); +} {0 {}} +do_catchsql_test 4.2 { + INSERT INTO ft2(ft2, rowid, a) VALUES('delete', 2, 'two'); +} {1 {'delete' may not be used with a contentless_delete=1 table}} + +do_execsql_test 4.3 { + SELECT rowid, * FROM ft1; +} { + 1 {} i + 3 {} iii +} +do_execsql_test 4.4 { + SELECT rowid, * FROM ft2; +} { + 1 {} i + 3 {} iii +} + +do_execsql_test 4.5 { + SELECT * FROM ft1_content +} {1 i 3 iii} + +do_execsql_test 4.6 { + SELECT * FROM ft2_content +} {1 i 3 iii} + +finish_test + diff --git a/ext/fts5/test/fts5detail.test b/ext/fts5/test/fts5detail.test new file mode 100644 index 0000000000000000000000000000000000000000..267ce6187c514a23480cd5c1ccaf1a46299a39c4 --- /dev/null +++ b/ext/fts5/test/fts5detail.test @@ -0,0 +1,247 @@ +# 2015 December 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5detail + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +fts5_aux_test_functions db + +#-------------------------------------------------------------------------- +# Simple tests. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); + INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 + INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 + INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 + INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4 + INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5 + INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6 + INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7 + INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8 + INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9 + INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10 +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +foreach {tn match res} { + 1 "a:a" {9} + 2 "b:g" {1 4 6} + 3 "c:h" {1 3 6 8 9 10} +} { + do_execsql_test 1.2.$tn.1 { + SELECT rowid FROM t1($match); + } $res + + do_execsql_test 1.2.$tn.2 { + SELECT rowid FROM t1($match || '*'); + } $res +} + +do_catchsql_test 1.3.1 { + SELECT rowid FROM t1('h + d'); +} {1 {fts5: phrase queries are not supported (detail!=full)}} + +do_catchsql_test 1.3.2 { + SELECT rowid FROM t1('NEAR(h d)'); +} {1 {fts5: NEAR queries are not supported (detail!=full)}} + + +#------------------------------------------------------------------------- +# integrity-check with both detail= and prefix= options. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); + INSERT INTO t2(a) VALUES('aa ab'); +} + +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} + +do_execsql_test 2.1 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 2.2 { + SELECT fts5_test_poslist(t2) FROM t2('aa'); +} {0.0.0} + +do_execsql_test 2.3 { + SELECT fts5_test_collist(t2) FROM t2('aa'); +} {0.0} + +set ::pc 0 +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] +#exit + +#------------------------------------------------------------------------- +# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs +# work with detail=col tables. +# +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} + 2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc} + 3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca} + 4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca} + 5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb} + 6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb} + 7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc} + 8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb} + 9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb} + 10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa} + 11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac} + 12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc} + 13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc} + 14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa} + 15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc} + 16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc} + 17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca} + 18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc} + 19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc} + 20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc} + 21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb} + 22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc} + 23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc} + 24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb} + 25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc} + 26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa} + 27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac} + 28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa} + 29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac} + 30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac} + 31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba} + 32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc} + 33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa} + 34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac} + 35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb} + 36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa} + 37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb} + 38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac} + 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} + 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} +} + +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} +} + +proc matchdata {expr {bAsc 1}} { + + set tclexpr [db one { + SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') + }] + set res [list] + + #puts "$expr -> $tclexpr" + foreach {id x y z} $::data { + set cols [list $x $y $z] + set ::pc 0 + #set hits [lsort -command instcompare [eval $tclexpr]] + set hits [eval $tclexpr] + if {[llength $hits]>0} { + lappend res [list $id $hits] + } + } + + if {$bAsc} { + set res [lsort -integer -increasing -index 0 $res] + } else { + set res [lsort -integer -decreasing -index 0 $res] + } + + return [concat {*}$res] +} + +foreach {tn tbl} { + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } + 2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) } +} { + reset_db + fts5_aux_test_functions db + execsql $tbl + foreach {id x y z} $data { + execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) } + } + foreach {tn2 expr} { + 1 aaa 2 ccc 3 bab 4 aac + 5 aa* 6 cc* 7 ba* 8 aa* + 9 a* 10 b* 11 c* + } { + + set res [matchdata $expr] + + do_execsql_test 3.$tn.$tn2.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) + } $res + + do_execsql_test 3.$tn.$tn2.2 { + SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) + } $res + } +} + +#------------------------------------------------------------------------- +# Simple tests for detail=none tables. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); + INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); + INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); +} + +do_catchsql_test 4.1 { + SELECT * FROM t4('a:a') +} {1 {fts5: column queries are not supported (detail=none)}} + +do_catchsql_test 4.2 { + SELECT * FROM t4('a:a &') +} {1 {fts5: syntax error near "&"}} + +#------------------------------------------------------------------------- +# Test that for the same content detail=none uses less space than +# detail=col, and that detail=col uses less space than detail=full +# +reset_db +do_test 5.1 { + foreach {tbl detail} {t1 none t2 col t3 full} { + execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)" + foreach {rowid x y z} $::data { + execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)" + } + } +} {} + +do_execsql_test 5.2 { + SELECT + (SELECT sum(length(block)) from t1_data) < + (SELECT sum(length(block)) from t2_data) +} {1} + +do_execsql_test 5.3 { + SELECT + (SELECT sum(length(block)) from t2_data) < + (SELECT sum(length(block)) from t3_data) +} {1} + + + +finish_test diff --git a/ext/fts5/test/fts5determin.test b/ext/fts5/test/fts5determin.test new file mode 100644 index 0000000000000000000000000000000000000000..c9094edfa19cea9958fa1b5ab88dc4819e08f2fd --- /dev/null +++ b/ext/fts5/test/fts5determin.test @@ -0,0 +1,65 @@ +# 2016 March 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, that the fts5 module is deterministic. At one point, when +# segment ids were allocated using sqlite3_randomness(), this was not the +# case. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aa +return_if_no_fts5 + +proc do_determin_test {tn} { + uplevel [list + do_execsql_test $tn { + SELECT (SELECT md5sum(id, block) FROM t1_data)== + (SELECT md5sum(id, block) FROM t2_data), + (SELECT md5sum(id, block) FROM t1_data)== + (SELECT md5sum(id, block) FROM t3_data) + } {1 1} + ] +} + +foreach_detail_mode $::testprefix { + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + } + + do_test 1.1 { + foreach t {t1 t2 t3} { + execsql [string map [list TBL $t] { + INSERT INTO TBL VALUES('a b c', 'd e f'); + INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3'); + INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz'); + }] + } + } {} + + do_determin_test 1.2 + + do_test 1.3 { + foreach t {t1 t2 t3} { + execsql [string map [list TBL $t] { + INSERT INTO TBL(TBL) VALUES('optimize'); + }] + } + } {} + + do_determin_test 1.4 +} + + +finish_test diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test new file mode 100644 index 0000000000000000000000000000000000000000..db82db1c2b00eb58a1c85dbe7563dd97bc347182 --- /dev/null +++ b/ext/fts5/test/fts5dlidx.test @@ -0,0 +1,197 @@ +# 2015 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on uses of doclist-index records. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5dlidx + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if { $tcl_platform(wordSize)<8 } { + finish_test + return +} + +foreach_detail_mode $testprefix { + +proc do_fb_test {tn sql res} { + set res2 [lsort -integer -decr $res] + uplevel [list do_execsql_test $tn.1 $sql $res] + uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] +} + +# This test populates the FTS5 table with $nEntry entries. Rows are +# numbered from 0 to ($nEntry-1). The rowid for row $i is: +# +# ($iFirst + $i*$nStep) +# +# Each document is of the form "a b c a b c a b c...". If the row number ($i) +# is an integer multiple of $spc1, then an "x" token is appended to the +# document. If it is *also* a multiple of $spc2, a "y" token is also appended. +# +proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} { + + do_execsql_test $tn.0 { DELETE FROM t1 } + + set xdoc [list] + set ydoc [list] + + execsql BEGIN + for {set i 0} {$i < $nEntry} {incr i} { + set rowid [expr $i * $nStep] + set doc [string trim [string repeat "a b c " 100]] + if {($i % $spc1)==0} { + lappend xdoc $rowid + append doc " x" + if {($i % $spc2)==0} { + lappend ydoc $rowid + append doc " y" + } + } + execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) } + } + execsql COMMIT + + do_test $tn.1 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} + + do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc + do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc + + do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc + do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc + + if {[detail_is_full]} { + do_fb_test $tn.5.1 { + SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc + do_fb_test $tn.5.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc + } +} + + +foreach {tn pgsz} { + 1 32 + 2 200 +} { + do_execsql_test $tn.0 { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + } + + do_dlidx_test1 1.$tn.1 10 100 10000 0 1000 + do_dlidx_test1 1.$tn.2 10 10 10000 0 128 + do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970 + do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000 + do_dlidx_test1 1.$tn.5 10 10 200 0 [expr 1<<55] + do_dlidx_test1 1.$tn.6 10 10 30 0 [expr 1<<58] +} + +proc do_dlidx_test2 {tn nEntry iFirst nStep} { + set str [string repeat "a " 500] + execsql { + BEGIN; + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1 VALUES('b a'); + + WITH iii(ii, i) AS ( + SELECT 1, $iFirst UNION ALL + SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry + ) + INSERT INTO t1(rowid,x) SELECT i, $str FROM iii; + COMMIT; + } + + do_execsql_test $tn.1 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' + } {1} + do_execsql_test $tn.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC + } {1} +} + +do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] + +#-------------------------------------------------------------------- +# +reset_db + +set ::vocab [list \ + IteratorpItercurrentlypointstothefirstrowidofadoclist \ + Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ + pageIfthecurrenttermisthelasttermonthepageloadthe \ + doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \ + IteratorpItercurrentlypointstothefirstrowidofadoclist \ + Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ + pageIfthecurrenttermisthelasttermonthepageloadthe \ + doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \ +] +proc rnddoc {} { + global vocab + set nVocab [llength $vocab] + set ret [list] + for {set i 0} {$i < 64} {incr i} { + lappend ret [lindex $vocab [expr $i % $nVocab]] + } + set ret +} +db func rnddoc rnddoc + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%); + INSERT INTO abc(abc, rank) VALUES('pgsz', 32); + + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + + INSERT INTO abc SELECT rnddoc() FROM abc; + INSERT INTO abc SELECT rnddoc() FROM abc; +} + + + +do_execsql_test 3.2 { + SELECT rowid FROM abc WHERE abc + MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' + ORDER BY rowid DESC; +} {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} + +do_execsql_test 3.3 { + INSERT INTO abc(abc) VALUES('integrity-check'); + INSERT INTO abc(abc) VALUES('optimize'); + INSERT INTO abc(abc) VALUES('integrity-check'); +} + +set v [lindex $vocab 0] +set i 0 +foreach v $vocab { + do_execsql_test 3.4.[incr i] { + SELECT rowid FROM abc WHERE abc MATCH $v + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} +} + +} ;# foreach_detail_mode + + + +finish_test diff --git a/ext/fts5/test/fts5doclist.test b/ext/fts5/test/fts5doclist.test new file mode 100644 index 0000000000000000000000000000000000000000..5b1becb5147ca2939db6c8254753d633257dc13c --- /dev/null +++ b/ext/fts5/test/fts5doclist.test @@ -0,0 +1,67 @@ +# 2015 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on edge cases in the doclist format. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5doclist + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +#------------------------------------------------------------------------- +# Create a table with 1000 columns. Then add some large documents to it. +# All text is in the right most column of the table. +# +do_test 1.0 { + set cols [list] + for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" } + execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])" +} {} + +db func rnddoc fts5_rnddoc +do_execsql_test 1.1 { + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) + INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii; +} + +do_execsql_test 1.2 { + INSERT INTO ccc(ccc) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE tx USING fts5(x); +} + +set doc [string repeat "abc " 5000] +do_execsql_test 2.2 { + BEGIN; + INSERT INTO tx(rowid, x) VALUES(-9000000000000000000, $doc); + INSERT INTO tx(rowid, x) VALUES(9000000000000000000, $doc); + COMMIT; +} + +do_execsql_test 2.3 { + SELECT rowid FROM tx('abc'); +} { + -9000000000000000000 + 9000000000000000000 +} + +finish_test diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test new file mode 100644 index 0000000000000000000000000000000000000000..49c2f2753a189bb12cd08c98a2159fda558f304b --- /dev/null +++ b/ext/fts5/test/fts5ea.test @@ -0,0 +1,99 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Test the fts5 expression parser directly using the fts5_expr() SQL +# test function. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ea + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc do_syntax_error_test {tn expr err} { + set ::se_expr $expr + do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err] +} + +proc do_syntax_test {tn expr res} { + set ::se_expr $expr + do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res] +} + +foreach {tn expr res} { + 1 {abc} {"abc"} + 2 {abc def} {"abc" AND "def"} + 3 {abc*} {"abc" *} + 4 {"abc def ghi" *} {"abc" + "def" + "ghi" *} + 5 {one AND two} {"one" AND "two"} + 6 {one+two} {"one" + "two"} + 7 {one AND two OR three} {("one" AND "two") OR "three"} + 8 {one OR two AND three} {"one" OR ("two" AND "three")} + 9 {NEAR(one two)} {NEAR("one" "two", 10)} + 10 {NEAR("one three"* two, 5)} {NEAR("one" + "three" * "two", 5)} + 11 {a OR b NOT c} {"a" OR ("b" NOT "c")} + 12 "\x20one\x20two\x20three" {"one" AND "two" AND "three"} + 13 "\x09one\x0Atwo\x0Dthree" {"one" AND "two" AND "three"} + 14 {"abc""def"} {"abc" + "def"} +} { + do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] +} + +foreach {tn expr res} { + 1 {c1:abc} + {c1 : "abc"} + 2 {c2 : NEAR(one two) c1:"hello world"} + {c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"} +} { + do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res] +} + +foreach {tn expr err} { + 1 {AND} {fts5: syntax error near "AND"} + 2 {abc def AND} {fts5: syntax error near ""} + 3 {abc OR AND} {fts5: syntax error near "AND"} + 4 {(a OR b) abc} {fts5: syntax error near "abc"} + 5 {NEaR (a b)} {fts5: syntax error near "NEaR"} + 6 {NEa (a b)} {fts5: syntax error near "NEa"} + 7 {(a OR b) NOT c)} {fts5: syntax error near ")"} + 8 {nosuch: a nosuch2: b} {no such column: nosuch} + 9 {addr: a nosuch2: b} {no such column: nosuch2} + 10 {NOT} {fts5: syntax error near "NOT"} + 11 {a AND "abc} {unterminated string} + + 12 {NEAR(a b, xyz)} {expected integer, got "xyz"} + 13 {NEAR(a b, // )} {fts5: syntax error near "/"} + 14 {NEAR(a b, "xyz" )} {expected integer, got ""xyz""} +} { + do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err] +} + +#------------------------------------------------------------------------- +# Experiment with a tokenizer that considers " to be a token character. +# +do_execsql_test 4.0 { + SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"'); +} {{"a" AND """"}} + +#------------------------------------------------------------------------- +# Experiment with a tokenizer that considers " to be a token character. +# +do_catchsql_test 5.0 { + SELECT fts5_expr('abc | def'); +} {1 {fts5: syntax error near "|"}} + + + +finish_test diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test new file mode 100644 index 0000000000000000000000000000000000000000..bee9683c3cdf434e1196acc56b438902896ca2b6 --- /dev/null +++ b/ext/fts5/test/fts5eb.test @@ -0,0 +1,103 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5eb + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc do_syntax_error_test {tn expr err} { + set ::se_expr $expr + do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err] +} + +proc do_syntax_test {tn expr res} { + set ::se_expr $expr + do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res] +} + +foreach {tn expr res} { + 1 {abc} {"abc"} + 2 {abc ""} {"abc"} + 3 {""} {} + 4 {abc OR ""} {"abc" OR ""} + 5 {abc NOT ""} {"abc" NOT ""} + 6 {abc AND ""} {"abc" AND ""} + 7 {"" OR abc} {"" OR "abc"} + 8 {"" NOT abc} {"" NOT "abc"} + 9 {"" AND abc} {"" AND "abc"} + 10 {abc + "" + def} {"abc" + "def"} + 11 {abc "" def} {"abc" AND "def"} + 12 {r+e OR w} {"r" + "e" OR "w"} + + 13 {a AND b NOT c} {"a" AND ("b" NOT "c")} + 14 {a OR b NOT c} {"a" OR ("b" NOT "c")} + 15 {a NOT b AND c} {("a" NOT "b") AND "c"} + 16 {a NOT b OR c} {("a" NOT "b") OR "c"} + + 17 {a AND b OR c} {("a" AND "b") OR "c"} + 18 {a OR b AND c} {"a" OR ("b" AND "c")} + +} { + do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] +} + +do_catchsql_test 2.1 { + SELECT fts5_expr() +} {1 {wrong number of arguments to function fts5_expr}} + +do_catchsql_test 2.2 { + SELECT fts5_expr_tcl() +} {1 {wrong number of arguments to function fts5_expr_tcl}} + +do_catchsql_test 2.3 { + SELECT fts5_expr('') +} {1 {fts5: syntax error near ""}} + +do_catchsql_test 2.4 { + SELECT fts5_expr(NULL) +} {1 {fts5: syntax error near ""}} + +do_catchsql_test 2.5 { + SELECT fts5_expr(NULL, NULL) +} {1 {parse error in ""}} + +for {set i 0} {$i < 255} {incr i} { + do_test 2.6.$i { + lindex [catchsql {sELECT fts5_expr(NULL, char($i));}] 0 + } 1 +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61'); + INSERT INTO e1 VALUES ('just a few words with a / inside'); +} +do_execsql_test 3.1 { + SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; +} {1 -1e-06} +do_execsql_test 3.2 { + SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"' +} 1 +do_execsql_test 3.3 { + SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; +} {1 -1e-06} + +do_execsql_test 3.4 " + SELECT fts5_expr_tcl('e AND \" \"'); +" {{AND [nearset -- {e}] [{}]}} + + +finish_test diff --git a/ext/fts5/test/fts5expr.test b/ext/fts5/test/fts5expr.test new file mode 100644 index 0000000000000000000000000000000000000000..49be61d9c43cd91f4b842941fc7e0eb74ed3c370 --- /dev/null +++ b/ext/fts5/test/fts5expr.test @@ -0,0 +1,52 @@ +# 2024 August 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5expr + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test'); +} + +do_execsql_test 1.1 { + SELECT rowid FROM x1('expr'); +} {113} + +for {set ii 0} {$ii < 300} {incr ii} { + set expr "expr " + append expr [string repeat "NOT abcd " $ii] + + if {$ii<257} { + set res {0 113} + } else { + set res {1 {fts5 expression tree is too large (maximum depth 256)}} + } + do_catchsql_test 1.1.$ii { + SELECT rowid FROM x1($expr) + } $res +} + +do_execsql_test 1.2 { + SELECT rowid FROM x1 WHERE a MATCH '"..."' +} {} + +finish_test + diff --git a/ext/fts5/test/fts5fault1.test b/ext/fts5/test/fts5fault1.test new file mode 100644 index 0000000000000000000000000000000000000000..96025133e45789788ffd073200dfaf9678cc25c1 --- /dev/null +++ b/ext/fts5/test/fts5fault1.test @@ -0,0 +1,353 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault1 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Simple tests: +# +# 1: CREATE VIRTUAL TABLE +# 2: INSERT statement +# 3: DELETE statement +# 4: MATCH expressions +# +# + +faultsim_save_and_close +do_faultsim_test 1 -faults ioerr-t* -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3'); +} +faultsim_save_and_close +do_faultsim_test 2 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno'); + } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3'); + INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno'); +} +faultsim_save_and_close +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen +} -body { + execsql { DELETE FROM t1 } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b); + INSERT INTO t2 VALUES('m f a jj th q gi ar', 'hj n h h sg j i m'); + INSERT INTO t2 VALUES('nr s t g od j kf h', 'sb h aq rg op rb n nl'); + INSERT INTO t2 VALUES('do h h pb p p q fr', 'c rj qs or cr a l i'); + INSERT INTO t2 VALUES('lk gp t i lq mq qm p', 'h mr g f op ld aj h'); + INSERT INTO t2 VALUES('ct d sq kc qi k f j', 'sn gh c of g s qt q'); + INSERT INTO t2 VALUES('d ea d d om mp s ab', 'dm hg l df cm ft pa c'); + INSERT INTO t2 VALUES('tc dk c jn n t sr ge', 'a a kn bc n i af h'); + INSERT INTO t2 VALUES('ie ii d i b sa qo rf', 'a h m aq i b m fn'); + INSERT INTO t2 VALUES('gs r fo a er m h li', 'tm c p gl eb ml q r'); + INSERT INTO t2 VALUES('k fe fd rd a gi ho kk', 'ng m c r d ml rm r'); +} +faultsim_save_and_close + +foreach {tn expr res} { + 1 { dk } 7 + 2 { m f } 1 + 3 { f* } {1 3 4 5 6 8 9 10} + 4 { m OR f } {1 4 5 8 9 10} + 5 { sn + gh } {5} + 6 { "sn gh" } {5} + 7 { NEAR(r a, 5) } {9} + 8 { m* f* } {1 4 6 8 9 10} + 9 { m* + f* } {1 8} + 10 { c NOT p } {5 6 7 10} +} { + do_faultsim_test 4.$tn -prep { + faultsim_restore_and_reopen + } -body " + execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' } + " -test " + faultsim_test_result {[list 0 $res]} {1 {vtable constructor failed: t2}} + " +} + + +#------------------------------------------------------------------------- +# The following tests use a larger database populated with random data. +# +# The database page size is set to 512 bytes and the FTS5 page size left +# at the default 1000 bytes. This means that reading a node may require +# pulling an overflow page from disk, which is an extra opportunity for +# an error to occur. +# +reset_db +do_execsql_test 5.0.1 { + PRAGMA main.page_size = 512; + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + PRAGMA main.page_size; +} {512} + +proc rnddoc {n} { + set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]] + } + set doc +} +db func rnddoc rnddoc + +do_execsql_test 5.0.2 { + WITH r(a, b) AS ( + SELECT rnddoc(6), rnddoc(6) UNION ALL + SELECT rnddoc(6), rnddoc(6) FROM r + ) + INSERT INTO x1 SELECT * FROM r LIMIT 10000; +} + +set res [db one { + SELECT count(*) FROM x1 WHERE x1.a LIKE '%abc%' OR x1.b LIKE '%abc%'} +] + +do_faultsim_test 5.1 -faults oom* -body { + execsql { SELECT count(*) FROM x1 WHERE x1 MATCH 'abc' } +} -test { + faultsim_test_result [list 0 $::res] +} +do_faultsim_test 5.2 -faults oom* -body { + execsql { SELECT count(*) FROM x1 WHERE x1 MATCH 'abcd' } +} -test { + faultsim_test_result [list 0 0] +} + +proc test_astar {a b} { + return [expr { [regexp {a[^ ][^ ]} $a] || [regexp {a[^ ][^ ]} $b] }] +} +db func test_astar test_astar + +set res [db one { SELECT count(*) FROM x1 WHERE test_astar(a, b) } ] +do_faultsim_test 5.3 -faults oom* -body { + execsql { SELECT count(*) FROM x1 WHERE x1 MATCH 'a*' } +} -test { + faultsim_test_result [list 0 $::res] +} + +do_faultsim_test 5.4 -faults oom* -prep { + db close + sqlite3 db test.db +} -body { + execsql { INSERT INTO x1 VALUES('a b c d', 'e f g h') } +} -test { + faultsim_test_result [list 0 {}] +} + +do_faultsim_test 5.5.1 -faults oom* -body { + execsql { + SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=1 + } +} -test { + faultsim_test_result [list 0 1] +} +do_faultsim_test 5.5.2 -faults oom* -body { + execsql { + SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=10 + } +} -test { + faultsim_test_result [list 0 1] +} +do_faultsim_test 5.5.3 -faults oom* -body { + execsql { + SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = ( + SELECT min(rowid) FROM x1_data WHERE rowid>20 + ) + } +} -test { + faultsim_test_result [list 0 1] +} +do_faultsim_test 5.5.4 -faults oom* -body { + execsql { + SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = ( + SELECT max(rowid) FROM x1_data + ) + } +} -test { + faultsim_test_result [list 0 1] +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('automerge', 0); + + INSERT INTO x1 VALUES('a b c'); -- 1 + INSERT INTO x1 VALUES('a b c'); -- 2 + INSERT INTO x1 VALUES('a b c'); -- 3 + INSERT INTO x1 VALUES('a b c'); -- 4 + INSERT INTO x1 VALUES('a b c'); -- 5 + INSERT INTO x1 VALUES('a b c'); -- 6 + INSERT INTO x1 VALUES('a b c'); -- 7 + INSERT INTO x1 VALUES('a b c'); -- 8 + INSERT INTO x1 VALUES('a b c'); -- 9 + INSERT INTO x1 VALUES('a b c'); -- 10 + INSERT INTO x1 VALUES('a b c'); -- 11 + INSERT INTO x1 VALUES('a b c'); -- 12 + INSERT INTO x1 VALUES('a b c'); -- 13 + INSERT INTO x1 VALUES('a b c'); -- 14 + INSERT INTO x1 VALUES('a b c'); -- 15 + + SELECT count(*) FROM x1_data; +} {17} + +faultsim_save_and_close + +do_faultsim_test 6.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO x1 VALUES('d e f') } +} -test { + faultsim_test_result [list 0 {}] + if {$testrc==0} { + set nCnt [db one {SELECT count(*) FROM x1_data}] + if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" } + } +} + +do_faultsim_test 6.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO x1(x1, rank) VALUES('pgsz', 32) } +} -test { + faultsim_test_result [list 0 {}] +} + +do_faultsim_test 6.3 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO x1(x1) VALUES('integrity-check') } +} -test { + faultsim_test_result [list 0 {}] +} + +do_faultsim_test 6.4 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO x1(x1) VALUES('optimize') } +} -test { + faultsim_test_result [list 0 {}] +} + +#------------------------------------------------------------------------- +# +do_faultsim_test 7.0 -faults oom* -prep { + catch { db close } +} -body { + sqlite3 db test.db +} -test { + faultsim_test_result [list 0 {}] {1 {}} {1 {initialization of fts5 failed: }} +} + +#------------------------------------------------------------------------- +# A prefix query against a large document set. +# +proc rnddoc {n} { + set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]" + } + set doc +} + +reset_db +db func rnddoc rnddoc + +do_test 8.0 { + execsql { CREATE VIRTUAL TABLE x1 USING fts5(a) } + set ::res [list] + for {set i 1} {$i<100} {incr i 1} { + execsql { INSERT INTO x1 VALUES( rnddoc(50) ) } + lappend ::res $i + } +} {} + +do_faultsim_test 8.1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM x1 WHERE x1 MATCH 'x*' + } +} -test { + faultsim_test_result [list 0 $::res] +} + +#------------------------------------------------------------------------- +# Segment promotion. +# +do_test 9.0 { + reset_db + db func rnddoc fts5_rnddoc + execsql { + CREATE VIRTUAL TABLE s2 USING fts5(x); + INSERT INTO s2(s2, rank) VALUES('pgsz', 32); + INSERT INTO s2(s2, rank) VALUES('automerge', 0); + } + + for {set i 1} {$i <= 16} {incr i} { + execsql { INSERT INTO s2 VALUES(rnddoc(5)) } + } + fts5_level_segs s2 +} {0 1} +set insert_doc [db one {SELECT rnddoc(160)}] +faultsim_save_and_close + +do_faultsim_test 9.1 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO s2 VALUES($::insert_doc) } +} -test { + faultsim_test_result {0 {}} + if {$testrc==0} { + set ls [fts5_level_segs s2] + if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" } + } +} + + + +finish_test diff --git a/ext/fts5/test/fts5fault2.test b/ext/fts5/test/fts5fault2.test new file mode 100644 index 0000000000000000000000000000000000000000..aa21f7b4814e5ec3c539751c8342317d7f41bcc1 --- /dev/null +++ b/ext/fts5/test/fts5fault2.test @@ -0,0 +1,139 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set doc [string trim [string repeat "x y z " 200]] +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, x); + CREATE VIRTUAL TABLE x1 USING fts5(x, content='t1', content_rowid='a'); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + WITH input(a,b) AS ( + SELECT 1, $doc UNION ALL + SELECT a+1, ($doc || CASE WHEN (a+1)%100 THEN '' ELSE ' xyz' END) + FROM input WHERE a < 1000 + ) + INSERT INTO t1 SELECT * FROM input; + + INSERT INTO x1(x1) VALUES('rebuild'); +} + +do_faultsim_test 1.1 -faults oom-* -prep { +} -body { + execsql { SELECT rowid FROM x1 WHERE x1 MATCH 'z AND xyz' } +} -test { + faultsim_test_result {0 {100 200 300 400 500 600 700 800 900 1000}} +} + +do_faultsim_test 1.2 -faults oom-* -prep { +} -body { + execsql { SELECT rowid FROM x1 WHERE x1 MATCH 'z + xyz' ORDER BY 1 DESC} +} -test { + faultsim_test_result {0 {1000 900 800 700 600 500 400 300 200 100}} +} + +#------------------------------------------------------------------------- +# OOM within a query that accesses the in-memory hash table. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE "a b c" USING fts5(a, b, c); + INSERT INTO "a b c" VALUES('one two', 'x x x', 'three four'); + INSERT INTO "a b c" VALUES('nine ten', 'y y y', 'two two'); +} + +do_faultsim_test 2.1 -faults oom-trans* -prep { + execsql { + BEGIN; + INSERT INTO "a b c" VALUES('one one', 'z z z', 'nine ten'); + } +} -body { + execsql { SELECT rowid FROM "a b c" WHERE "a b c" MATCH 'one' } +} -test { + faultsim_test_result {0 {1 3}} + catchsql { ROLLBACK } +} + +#------------------------------------------------------------------------- +# OOM within an 'optimize' operation that writes multiple pages to disk. +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE zzz USING fts5(z); + INSERT INTO zzz(zzz, rank) VALUES('pgsz', 32); + INSERT INTO zzz VALUES('a b c d'); + INSERT INTO zzz SELECT 'c d e f' FROM zzz; + INSERT INTO zzz SELECT 'e f g h' FROM zzz; + INSERT INTO zzz SELECT 'i j k l' FROM zzz; + INSERT INTO zzz SELECT 'l k m n' FROM zzz; + INSERT INTO zzz SELECT 'o p q r' FROM zzz; +} +faultsim_save_and_close + +do_faultsim_test 3.1 -faults oom-trans* -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM zzz } +} -body { + execsql { INSERT INTO zzz(zzz) VALUES('optimize') } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM within an 'integrity-check' operation. +# +reset_db +db func rnddoc fts5_rnddoc +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE zzz USING fts5(z); + INSERT INTO zzz(zzz, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<10) + INSERT INTO zzz SELECT rnddoc(10) || ' xccc' FROM ii; +} + +do_faultsim_test 4.1 -faults oom-trans* -prep { +} -body { + execsql { INSERT INTO zzz(zzz) VALUES('integrity-check') } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM while parsing a tokenize=option +# +reset_db +faultsim_save_and_close +do_faultsim_test 5.0 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + CREATE VIRTUAL TABLE uio USING fts5(a, b, + tokenize="porter 'ascii'", + content="another table", + content_rowid="somecolumn" + ); + } +} -test { + faultsim_test_result {0 {}} +} + +finish_test diff --git a/ext/fts5/test/fts5fault3.test b/ext/fts5/test/fts5fault3.test new file mode 100644 index 0000000000000000000000000000000000000000..a294692dbec3e352fb644220cb2542289b4beb3a --- /dev/null +++ b/ext/fts5/test/fts5fault3.test @@ -0,0 +1,112 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault3 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# An OOM while resuming a partially completed segment merge. +# +db func rnddoc fts5_rnddoc +do_test 1.0 { + expr srand(0) + execsql { + CREATE VIRTUAL TABLE xx USING fts5(x); + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); + INSERT INTO xx(xx, rank) VALUES('automerge', 16); + } + for {set i 0} {$i < 10} {incr i} { + execsql { + BEGIN; + INSERT INTO xx(x) VALUES(rnddoc(20)); + INSERT INTO xx(x) VALUES(rnddoc(20)); + INSERT INTO xx(x) VALUES(rnddoc(20)); + COMMIT + } + } + + execsql { + INSERT INTO xx(xx, rank) VALUES('automerge', 2); + INSERT INTO xx(xx, rank) VALUES('merge', 50); + } +} {} +faultsim_save_and_close + +do_faultsim_test 1 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO xx(xx, rank) VALUES('merge', 1) } +} -test { + faultsim_test_result [list 0 {}] +} + +#------------------------------------------------------------------------- +# An OOM while flushing an unusually large term to disk. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xx USING fts5(x); + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); +} +faultsim_save_and_close + +set doc "a long term abcdefghijklmnopqrstuvwxyz " +append doc "and then abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz " +append doc [string repeat "abcdefghijklmnopqrstuvwxyz" 10] + +do_faultsim_test 2 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO xx(x) VALUES ($::doc) } +} -test { + faultsim_test_result [list 0 {}] +} + +#------------------------------------------------------------------------- +# An OOM while flushing an unusually large term to disk. +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE xx USING fts5(x); +} +faultsim_save_and_close + +set doc [fts5_rnddoc 1000] +do_faultsim_test 3.1 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO xx(x) VALUES ($::doc) } +} -test { + faultsim_test_result [list 0 {}] +} + +set doc [string repeat "abc " 100] +do_faultsim_test 3.2 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO xx(x) VALUES ($::doc) } +} -test { + faultsim_test_result [list 0 {}] +} + + + +finish_test diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test new file mode 100644 index 0000000000000000000000000000000000000000..2b4f6c4d2aaef69d166830731e25d5d2cd5b31c6 --- /dev/null +++ b/ext/fts5/test/fts5fault4.test @@ -0,0 +1,401 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::TMPDBERROR [list 1 \ + {unable to open a temporary database file for storing temporary tables} +] + +#------------------------------------------------------------------------- +# An OOM while dropping an fts5 table. +# +db func rnddoc fts5_rnddoc +do_test 1.0 { + execsql { CREATE VIRTUAL TABLE xx USING fts5(x) } +} {} +faultsim_save_and_close + +do_faultsim_test 1 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM xx } +} -body { + execsql { DROP TABLE xx } +} -test { + faultsim_test_result [list 0 {}] +} + +#------------------------------------------------------------------------- +# An OOM while "reseeking" an FTS cursor. +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE jj USING fts5(j); + INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s'); + INSERT INTO jj(rowid, j) VALUES(202, 't w t f s'); + INSERT INTO jj(rowid, j) VALUES(303, 'w t f'); + INSERT INTO jj(rowid, j) VALUES(404, 't'); +} +faultsim_save_and_close + +do_faultsim_test 3 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM jj } +} -body { + set res [list] + db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } { + lappend res $rowid + if {$rowid==303} { + execsql { DELETE FROM jj WHERE rowid=404 } + } + } + set res +} -test { + faultsim_test_result [list 0 {101 202 303}] +} + +#------------------------------------------------------------------------- +# An OOM within a special "*reads" query. +# +reset_db +db func rnddoc fts5_rnddoc +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + + WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 ) + INSERT INTO x1 SELECT rnddoc(5) FROM ii; +} + +set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] + +do_faultsim_test 4 -faults oom-* -body { + db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} +} -test { + faultsim_test_result {0 {0 {} 2}} +} + +#------------------------------------------------------------------------- +# An OOM within a query that uses a custom rank function. +# +reset_db +do_execsql_test 5.0 { + PRAGMA encoding='utf16'; + CREATE VIRTUAL TABLE x2 USING fts5(x); + INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3 + INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6 + INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2 + INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5 + INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1 +} + +proc rowidmod {cmd mod} { + set row [$cmd xRowid] + expr {$row % $mod} +} +sqlite3_fts5_create_function db rowidmod rowidmod + +do_faultsim_test 5.1 -faults oom-* -body { + db eval { + SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND + rank MATCH "rowidmod('7')" ORDER BY rank + } +} -test { + faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}} +} + +proc rowidprefix {cmd prefix} { + set row [$cmd xRowid] + set {} "${row}-${prefix}" +} +sqlite3_fts5_create_function db rowidprefix rowidprefix + +set str [string repeat abcdefghijklmnopqrstuvwxyz 10] +do_faultsim_test 5.2 -faults oom-* -body { + db eval " + SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND + rank MATCH 'rowidprefix(''$::str'')' + LIMIT 1 + " +} -test { + faultsim_test_result "0 {10-$::str {a b c}}" +} + + +#------------------------------------------------------------------------- +# OOM errors within auxiliary functions. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x3 USING fts5(xxx); + INSERT INTO x3 VALUES('a b c d c b a'); + INSERT INTO x3 VALUES('a a a a a a a'); + INSERT INTO x3 VALUES('a a a a a a a'); +} + +do_faultsim_test 6.1 -faults oom-t* -body { + db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' } +} -test { + faultsim_test_result {0 {{a b *c* d *c* b a}}} +} + +proc firstinst {cmd} { + foreach {p c o} [$cmd xInst 0] {} + expr $c*100 + $o +} +sqlite3_fts5_create_function db firstinst firstinst + +do_faultsim_test 6.2 -faults oom-t* -body { + db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' } +} -test { + faultsim_test_result {0 2} {1 SQLITE_NOMEM} +} + +proc previc {cmd} { + set res [$cmd xGetAuxdataInt 0] + $cmd xSetAuxdataInt [$cmd xInstCount] + return $res +} +sqlite3_fts5_create_function db previc previc + +do_faultsim_test 6.2 -faults oom-t* -body { + db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' } +} -test { + faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +# OOM error when querying for a phrase with many tokens. +# +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE tt USING fts5(x, y); + INSERT INTO tt VALUES('f b g b c b', 'f a d c c b'); -- 1 + INSERT INTO tt VALUES('d a e f e d', 'f b b d e e'); -- 2 + INSERT INTO tt VALUES('f b g a d c', 'e f c f a d'); -- 3 + INSERT INTO tt VALUES('f f c d g f', 'f a e b g b'); -- 4 + INSERT INTO tt VALUES('a g b d a g', 'e g a e a c'); -- 5 + INSERT INTO tt VALUES('c d b d e f', 'f g e g e e'); -- 6 + INSERT INTO tt VALUES('e g f f b c', 'f c e f g f'); -- 7 + INSERT INTO tt VALUES('e g c f c e', 'f e e a f g'); -- 8 + INSERT INTO tt VALUES('e a e b e e', 'd c c f f f'); -- 9 + INSERT INTO tt VALUES('f a g g c c', 'e g d g c e'); -- 10 + INSERT INTO tt VALUES('c d b a e f', 'f g e h e e'); -- 11 + + CREATE VIRTUAL TABLE tt2 USING fts5(o); + INSERT INTO tt2(rowid, o) SELECT rowid, x||' '||y FROM tt; + INSERT INTO tt2(rowid, o) VALUES(12, 'a b c d e f g h i j k l'); +} + +do_faultsim_test 7.2 -faults oom-* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'f+g+e+g+e+e' } +} -test { + faultsim_test_result {0 6} {1 SQLITE_NOMEM} +} + +do_faultsim_test 7.3 -faults oom-* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d e f)' } +} -test { + faultsim_test_result {0 11} {1 SQLITE_NOMEM} +} + +do_faultsim_test 7.4 -faults oom-t* -body { + db eval { SELECT rowid FROM tt2 WHERE tt2 MATCH '"g c f c e f e e a f"' } +} -test { + faultsim_test_result {0 8} {1 SQLITE_NOMEM} +} + +do_faultsim_test 7.5 -faults oom-* -body { + db eval {SELECT rowid FROM tt2 WHERE tt2 MATCH 'NEAR(a b c d e f g h i j k)'} +} -test { + faultsim_test_result {0 12} {1 SQLITE_NOMEM} +} + +do_faultsim_test 7.6 -faults oom-* -body { + db eval {SELECT rowid FROM tt WHERE tt MATCH 'y: "c c"'} +} -test { + faultsim_test_result {0 {1 9}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE tt USING fts5(x); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO tt(rowid, x) VALUES(1, 'a b c d x x'); + WITH ii(i) AS (SELECT 2 UNION ALL SELECT i+1 FROM ii WHERE i<99) + INSERT INTO tt(rowid, x) SELECT i, 'a b c x x d' FROM ii; + INSERT INTO tt(rowid, x) VALUES(100, 'a b c d x x'); + COMMIT; +} + +do_faultsim_test 8.1 -faults oom-t* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d, 2)' } +} -test { + faultsim_test_result {0 {1 100}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 8.2 -faults oom-t* -body { + db eval { SELECT count(*) FROM tt WHERE tt MATCH 'a OR d' } +} -test { + faultsim_test_result {0 100} {1 SQLITE_NOMEM} +} + + +#------------------------------------------------------------------------- +# Fault in NOT query. +# +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE tt USING fts5(x); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + BEGIN; + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<200) + INSERT INTO tt(rowid, x) + SELECT i, CASE WHEN (i%50)==0 THEN 'a a a a a a' ELSE 'a x a x a x' END + FROM ii; + COMMIT; +} + +do_faultsim_test 9.1 -faults oom-* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'a NOT x' } +} -test { + faultsim_test_result {0 {50 100 150 200}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +# OOM in fts5_expr() SQL function. +# +do_faultsim_test 10.1 -faults oom-t* -body { + db one { SELECT fts5_expr('a AND b NEAR(a b)') } +} -test { + faultsim_test_result {0 {"a" AND "b" AND NEAR("a" "b", 10)}} +} + +do_faultsim_test 10.2 -faults oom-t* -body { + db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') } +} -test { + set res {AND [ns -col 0 -- {a b c}] [ns -- {b}] [ns -near 10 -- {a} {b}]} + faultsim_test_result [list 0 $res] +} + +do_faultsim_test 10.3 -faults oom-t* -body { + db one { SELECT fts5_expr('x:a', 'x') } +} -test { + faultsim_test_result {0 {x : "a"}} +} + +#------------------------------------------------------------------------- +# OOM while configuring 'rank' option. +# +reset_db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); +} +do_faultsim_test 11.1 -faults oom-t* -body { + db eval { INSERT INTO ft(ft, rank) VALUES('rank', 'bm25(10.0, 5.0)') } +} -test { + faultsim_test_result {0 {}} {1 {disk I/O error}} +} + +#------------------------------------------------------------------------- +# OOM while creating an fts5vocab table. +# +reset_db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); +} +faultsim_save_and_close +do_faultsim_test 12.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM sqlite_master } +} -body { + db eval { CREATE VIRTUAL TABLE vv USING fts5vocab(ft, 'row') } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM while querying an fts5vocab table. +# +reset_db +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + INSERT INTO ft VALUES('a b'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, 'row'); +} +faultsim_save_and_close +do_faultsim_test 13.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM vv } +} -body { + db eval { SELECT * FROM vv } +} -test { + faultsim_test_result {0 {a 1 1 b 1 1}} +} + +#------------------------------------------------------------------------- +# OOM in multi-column token query. +# +reset_db +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, y, z); + INSERT INTO ft(ft, rank) VALUES('pgsz', 32); + INSERT INTO ft VALUES( + 'x x x x x x x x x x x x x x x x', + 'y y y y y y y y y y y y y y y y', + 'z z z z z z z z x x x x x x x x' + ); + INSERT INTO ft SELECT * FROM ft; + INSERT INTO ft SELECT * FROM ft; + INSERT INTO ft SELECT * FROM ft; + INSERT INTO ft SELECT * FROM ft; +} +faultsim_save_and_close +do_faultsim_test 13.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM ft } +} -body { + db eval { SELECT rowid FROM ft WHERE ft MATCH '{x z}: x' } +} -test { + faultsim_test_result {0 {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}} +} + +#------------------------------------------------------------------------- +# OOM in an "ALTER TABLE RENAME TO" +# +reset_db +do_execsql_test 14.0 { + CREATE VIRTUAL TABLE "tbl one" USING fts5(x, y, z); +} +faultsim_save_and_close +do_faultsim_test 14.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM "tbl one" } +} -body { + db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" } +} -test { + faultsim_test_result {0 {}} $::TMPDBERROR +} + +finish_test diff --git a/ext/fts5/test/fts5fault5.test b/ext/fts5/test/fts5fault5.test new file mode 100644 index 0000000000000000000000000000000000000000..6b8dade50466c731fe7dd94497c526b2b434b58a --- /dev/null +++ b/ext/fts5/test/fts5fault5.test @@ -0,0 +1,131 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault5 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# OOM while creating an FTS5 table. +# +do_faultsim_test 1.1 -faults oom-t* -prep { + db eval { DROP TABLE IF EXISTS abc } +} -body { + db eval { CREATE VIRTUAL TABLE abc USING fts5(x,y) } +} -test { + faultsim_test_result {0 {}} +} + + +#------------------------------------------------------------------------- +# OOM while writing a multi-tier doclist-index. And while running +# integrity-check on the same. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE tt USING fts5(x); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM tt } +} -body { + set str [string repeat "abc " 50] + db eval { + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) + INSERT INTO tt(rowid, x) SELECT i, $str FROM ii; + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 2.2 -faults oom-t* -body { + db eval { INSERT INTO tt(tt) VALUES('integrity-check') } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM while scanning fts5vocab tables. +# +reset_db +do_test 3.0 { + execsql { + CREATE VIRTUAL TABLE tt USING fts5(x); + CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row'); + + CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col); + CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col'); + + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32); + BEGIN; + } + + for {set i 0} {$i < 20} {incr i} { + set str [string repeat "$i " 50] + execsql { INSERT INTO tt VALUES($str) } + execsql { INSERT INTO tt2 VALUES($str) } + } + execsql COMMIT +} {} + +do_faultsim_test 3.1 -faults oom-t* -body { + db eval { + SELECT term FROM tv; + } +} -test { + faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}} +} + +do_faultsim_test 3.2 -faults oom-t* -body { + db eval { + SELECT term FROM tv WHERE term BETWEEN '1' AND '2'; + } +} -test { + faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}} +} + +do_execsql_test 3.3.0 { + SELECT * FROM tv2; +} { + 0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} + 14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} + 2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {} + 9 x 1 {} +} +do_faultsim_test 3.3 -faults oom-t* -body { + db eval { + SELECT * FROM tv2; + } +} -test { + faultsim_test_result [list 0 [list \ + 0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} \ + 14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} \ + 2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {} \ + 9 x 1 {} + ]] +} + + + +finish_test diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test new file mode 100644 index 0000000000000000000000000000000000000000..1aacddce9ff1ad7ed22b3938a0d408ba6bd6184a --- /dev/null +++ b/ext/fts5/test/fts5fault6.test @@ -0,0 +1,293 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault6 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +#------------------------------------------------------------------------- +# OOM while rebuilding an FTS5 table. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE tt USING fts5(a, b); + INSERT INTO tt VALUES('c d c g g f', 'a a a d g a'); + INSERT INTO tt VALUES('c d g b f d', 'b g e c g c'); + INSERT INTO tt VALUES('c c f d e d', 'c e g d b c'); + INSERT INTO tt VALUES('e a f c e f', 'g b a c d g'); + INSERT INTO tt VALUES('c g f b b d', 'g c d c f g'); + INSERT INTO tt VALUES('d a g a b b', 'g c g g c e'); + INSERT INTO tt VALUES('e f a b c e', 'f d c d c c'); + INSERT INTO tt VALUES('e c a g c d', 'b b g f f b'); + INSERT INTO tt VALUES('g b d d e b', 'f f b d a c'); + INSERT INTO tt VALUES('e a d a e d', 'c e a e f g'); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + db eval { INSERT INTO tt(tt) VALUES('rebuild') } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 1.2 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + db eval { REPLACE INTO tt(rowid, a, b) VALUES(6, 'x y z', 'l l l'); } +} -test { + faultsim_test_result {0 {}} +} + + +#------------------------------------------------------------------------- +# OOM within a special delete. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE tt USING fts5(a, content=""); + INSERT INTO tt VALUES('c d c g g f'); + INSERT INTO tt VALUES('c d g b f d'); + INSERT INTO tt VALUES('c c f d e d'); + INSERT INTO tt VALUES('e a f c e f'); + INSERT INTO tt VALUES('c g f b b d'); + INSERT INTO tt VALUES('d a g a b b'); + INSERT INTO tt VALUES('e f a b c e'); + INSERT INTO tt VALUES('e c a g c d'); + INSERT INTO tt VALUES('g b d d e b'); + INSERT INTO tt VALUES('e a d a e d'); +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + db eval { INSERT INTO tt(tt, rowid, a) VALUES('delete', 3, 'c d g b f d'); } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 2.2 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + db eval { INSERT INTO tt(tt) VALUES('delete-all') } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 2.3 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + db eval { INSERT INTO tt VALUES('x y z') } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM in the ASCII tokenizer with very large tokens. +# +# Also the unicode tokenizer. +# +set t1 [string repeat wxyz 20] +set t2 [string repeat wxyz 200] +set t3 [string repeat wxyz 2000] +set doc "$t1 $t2 $t3" +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE xyz USING fts5(c, tokenize=ascii, content=""); + CREATE VIRTUAL TABLE xyz2 USING fts5(c, content=""); +} +faultsim_save_and_close + +do_faultsim_test 3.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM xyz } +} -body { + db eval { INSERT INTO xyz VALUES($::doc) } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 3.2 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM xyz2 } +} -body { + db eval { INSERT INTO xyz2 VALUES($::doc) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM while initializing a unicode61 tokenizer. +# +reset_db +faultsim_save_and_close +do_faultsim_test 4.1 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + db eval { + CREATE VIRTUAL TABLE yu USING fts5(x, tokenize="unicode61 separators abc"); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +# 5.2.* OOM while running a query that includes synonyms and matchinfo(). +# +# 5.3.* OOM while running a query that returns a row containing instances +# of more than 4 synonyms for a single term. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i < 7} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} +proc tcl_create {args} { return "tcl_tokenize" } +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +db func mit mit +sqlite3_fts5_register_matchinfo db +do_test 5.0 { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } + foreach {rowid text} { + 1 {aaaa cc b aaaaa cc aa} + 2 {aa aa bb a bbb} + 3 {bb aaaaa aaaaa b aaaa aaaaa} + 4 {aa a b aaaa aa} + 5 {aa b ccc aaaaa cc} + 6 {aa aaaaa bbbb cc aaa} + 7 {aaaaa aa aa ccccc bb} + 8 {ccc bbbbb ccccc bbb c} + 9 {cccccc bbbb a aaa cccc c} + + 20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee} + 21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e} + 22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd} + 23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff} + 24 {eee dd ee dddd dddd eeeeee e eee fff ffff} + 25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f} + 26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee} + 27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d} + 28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd} + 29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee} + 30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee} + 31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd} + 32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff} + 33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd} + 34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff} + 35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff} + 36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff} + 37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee} + 38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee} + 39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd} + 40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f} + 41 {ddd ddd fff fffff ee fffff f fff ddddd fffff} + 42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee} + 43 {d dd fff fffff d f fff e dddd ee ee} + 44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee} + 45 {eeee f eeeee ee e ffff f ddd e fff} + 46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee} + 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} + 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} + 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} + + 50 {x e} + } { + execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } + } +} {} + +set res [list {*}{ + 1 {3 24 8 2 12 6} + 5 {2 24 8 2 12 6} + 6 {3 24 8 1 12 6} + 7 {3 24 8 1 12 6} + 9 {2 24 8 3 12 6} +}] +do_execsql_test 5.1.1 { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' +} $res +do_execsql_test 5.1.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f' +} 29 + +faultsim_save_and_close +do_faultsim_test 5.2 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create + sqlite3_fts5_register_matchinfo db + db func mit mit +} -body { + db eval { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' + } +} -test { + faultsim_test_result [list 0 $::res] {1 {SQL logic error}} +} + +do_faultsim_test 5.3 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f' + } +} -test { + faultsim_test_result {0 29} {1 {SQL logic error}} +} + +do_faultsim_test 5.4 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e' + } +} -test { + faultsim_test_result {0 1} {1 {SQL logic error}} +} + +#------------------------------------------------------------------------- +catch { db close } +do_faultsim_test 6 -faults oom* -prep { + sqlite_orig db test.db + sqlite3_db_config_lookaside db 0 0 0 +} -test { + faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} + if {$testrc==0} { + db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) } + } + db close +} +finish_test diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test new file mode 100644 index 0000000000000000000000000000000000000000..c93e099f82bdb9cd16b38eec03f57ce6e6c6afa9 --- /dev/null +++ b/ext/fts5/test/fts5fault7.test @@ -0,0 +1,118 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault7 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if 1 { + +#------------------------------------------------------------------------- +# Test fault-injection on a query that uses xColumnSize() on columnsize=0 +# table. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0); + INSERT INTO t1 VALUES('a b c d e f g'); + INSERT INTO t1 VALUES('a b c d'); + INSERT INTO t1 VALUES('a b c d e f g h i j'); +} + + +fts5_aux_test_functions db +do_faultsim_test 1 -faults oom* -body { + execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' } +} -test { + faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM} +} + +} + +#------------------------------------------------------------------------- +# Test fault-injection when a segment is promoted. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a); + INSERT INTO t2(t2, rank) VALUES('automerge', 0); + INSERT INTO t2(t2, rank) VALUES('crisismerge', 4); + INSERT INTO t2(t2, rank) VALUES('pgsz', 40); + + INSERT INTO t2 VALUES('a b c'); + INSERT INTO t2 VALUES('d e f'); + INSERT INTO t2 VALUES('f e d'); + INSERT INTO t2 VALUES('c b a'); + + INSERT INTO t2 VALUES('a b c'); + INSERT INTO t2 VALUES('d e f'); + INSERT INTO t2 VALUES('f e d'); + INSERT INTO t2 VALUES('c b a'); +} {} + +faultsim_save_and_close +do_faultsim_test 1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO t2 VALUES('c d c g g f'); + INSERT INTO t2 VALUES('c d g b f d'); + INSERT INTO t2 VALUES('c c f d e d'); + INSERT INTO t2 VALUES('e a f c e f'); + INSERT INTO t2 VALUES('c g f b b d'); + INSERT INTO t2 VALUES('d a g a b b'); + INSERT INTO t2 VALUES('e f a b c e'); + INSERT INTO t2 VALUES('e c a g c d'); + INSERT INTO t2 VALUES('g b d d e b'); + INSERT INTO t2 VALUES('e a d a e d'); + } +} -body { + db eval COMMIT +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# Test fault-injection when a segment is promoted. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xy USING fts5(x); + INSERT INTO xy(rowid, x) VALUES(1, '1 2 3'); + INSERT INTO xy(rowid, x) VALUES(2, '2 3 4'); + INSERT INTO xy(rowid, x) VALUES(3, '3 4 5'); +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + db eval { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2 } +} -test { + faultsim_test_result {0 {}} +} + +# Test fault-injection when an empty expression is parsed. +# +do_faultsim_test 2.2 -faults oom-* -body { + db eval { SELECT * FROM xy('""') } +} -test { + faultsim_test_result {0 {}} +} + +finish_test diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test new file mode 100644 index 0000000000000000000000000000000000000000..dc060a15928c324d0bb616a521ca2964ccc52a00 --- /dev/null +++ b/ext/fts5/test/fts5fault8.test @@ -0,0 +1,96 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault8 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + + fts5_aux_test_functions db + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); + INSERT INTO t1 VALUES('a b a b', NULL); + INSERT INTO t1 VALUES(NULL, '1 2 1 2'); + } + + do_faultsim_test 1 -faults oom-* -body { + execsql { + SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' + } + } -test { + faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ + {1 SQLITE_NOMEM} + } + + do_faultsim_test 2 -faults oom-* -body { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + } + + if {[detail_is_none]==0} { + do_faultsim_test 3 -faults oom-* -body { + execsql { SELECT rowid FROM t1('b:2') } + } -test { + faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM} + } + } + +} ;# foreach_detail_mode... + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x2 USING fts5(a); + INSERT INTO x2(x2, rank) VALUES('crisismerge', 2); + INSERT INTO x2(x2, rank) VALUES('pgsz', 32); + INSERT INTO x2 VALUES('a b c d'); + INSERT INTO x2 VALUES('e f g h'); + INSERT INTO x2 VALUES('i j k l'); + INSERT INTO x2 VALUES('m n o p'); + INSERT INTO x2 VALUES('q r s t'); + INSERT INTO x2 VALUES('u v w x'); + INSERT INTO x2 VALUES('y z a b'); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO x2(x2) VALUES('optimize') } +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} +} + +set TMPDBERROR {1 {unable to open a temporary database file for storing temporary tables}} + +do_faultsim_test 5 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA temp_store = memory } +} -body { + execsql { PRAGMA integrity_check } +} -test { + if {[string match {*error code=7*} $testresult]==0} { + faultsim_test_result {0 ok} {1 SQLITE_NOMEM} $::TMPDBERROR + } +} + + +finish_test diff --git a/ext/fts5/test/fts5fault9.test b/ext/fts5/test/fts5fault9.test new file mode 100644 index 0000000000000000000000000000000000000000..669b13efe70b4d32b1179913f1fa4a2b64cd6221 --- /dev/null +++ b/ext/fts5/test/fts5fault9.test @@ -0,0 +1,159 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault9 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +if {"%DETAIL%" != "none"} continue + +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50) + INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq; +} + +do_faultsim_test 1 -faults oom-* -body { + execsql { SELECT count(*) FROM t1('x AND y') } +} -test { + faultsim_test_result {0 50} +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t2(t2, rank) VALUES('pgsz', 32); + INSERT INTO t2 VALUES('abc cba', 'cba abc'); + INSERT INTO t2 VALUES('abc cba', 'cba abc'); + INSERT INTO t2 VALUES('abc cba', 'cba abc'); + + INSERT INTO t2 VALUES('axy cyx', 'cyx axy'); + INSERT INTO t2 VALUES('axy cyx', 'cyx axy'); + INSERT INTO t2 VALUES('axy cyx', 'cyx axy'); +} + +do_faultsim_test 2 -faults oom-* -body { + execsql { SELECT count(*) FROM t2('a* AND c*') } +} -test { + faultsim_test_result {0 6} +} + + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%); + INSERT INTO t3 VALUES('a x x a x a a a'); + INSERT INTO t3 VALUES('x a a x a x x x'); +} + +do_faultsim_test 3.1 -faults oom-* -body { + execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') } +} -test { + faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}} +} + +do_faultsim_test 3.2 -faults oom-t* -body { + execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') } +} -test { + faultsim_test_result \ + {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \ + {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer +# uses query synonyms. +# +fts5_tclnum_register db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum); + INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii'); + INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three'); + INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii'); + + INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9'); + INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9'); + INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9'); +} + +do_faultsim_test 4.1 -faults oom-t* -body { + execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') } +} -test { + faultsim_test_result \ + {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \ + {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}} +} + +do_faultsim_test 4.2 -faults oom-t* -body { + execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') } +} -test { + faultsim_test_result \ + {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \ + {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}} +} + + +#------------------------------------------------------------------------- +# An OOM within an "ORDER BY rank" query. +# +db func rnddoc fts5_rnddoc +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%); + INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10)); + INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9)); + INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8)); +} {} +faultsim_save_and_close + +do_faultsim_test 5 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM xx } +} -body { + execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank } +} -test { + faultsim_test_result [list 0 {3 2 1}] +} + +set doc [string repeat "xyz " 500] +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE yy USING fts5(y, detail=%DETAIL%); + INSERT INTO yy(yy, rank) VALUES('pgsz', 64); + INSERT INTO yy VALUES ($doc); + INSERT INTO yy VALUES ('1 2 3'); + INSERT INTO yy VALUES ('xyz'); + UPDATE yy SET y = y WHERE rowid = 1; + UPDATE yy SET y = y WHERE rowid = 1; + UPDATE yy SET y = y WHERE rowid = 1; + UPDATE yy SET y = y WHERE rowid = 1; +} {} + +do_faultsim_test 6 -faults oom-* -body { + execsql { SELECT rowid FROM yy('xyz') } +} -test { + faultsim_test_result [list 0 {1 3}] +} + + +} ;# foreach_detail_mode... + +finish_test diff --git a/ext/fts5/test/fts5faultA.test b/ext/fts5/test/fts5faultA.test new file mode 100644 index 0000000000000000000000000000000000000000..212401fe58116afe303f230eff789a5f2c21499f --- /dev/null +++ b/ext/fts5/test/fts5faultA.test @@ -0,0 +1,63 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultA + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE o1 USING fts5(a, detail=%DETAIL%); + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 ) + INSERT INTO o1 SELECT 'A B C' FROM s; + + INSERT INTO o1 VALUES('A X C'); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 ) + INSERT INTO o1 SELECT 'A B C' FROM s; + } + + do_faultsim_test 1 -faults oom* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT rowid FROM o1('a NOT b') } + } -test { + faultsim_test_result {0 301} + } +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE o2 USING fts5(a); + + INSERT INTO o2 VALUES('A B C'); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 ) + INSERT INTO o2 SELECT group_concat('A B C ') FROM s; +} + +do_faultsim_test 2 -faults oom* -prep { + sqlite3 db test.db +} -body { + execsql { SELECT rowid FROM o2('a+b+c NOT xyz') } +} -test { + faultsim_test_result {0 {1 2}} +} +finish_test diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test new file mode 100644 index 0000000000000000000000000000000000000000..e5fc514d096b338c152adb2de813968a0b3367d1 --- /dev/null +++ b/ext/fts5/test/fts5faultB.test @@ -0,0 +1,173 @@ +# 2016 February 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultB + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + + +#------------------------------------------------------------------------- +# Errors while registering the matchinfo() demo function. +# +do_faultsim_test 1 -faults oom* -prep { + sqlite3 db test.db +} -body { + sqlite3_fts5_register_matchinfo db +} -test { + faultsim_test_result {0 {}} {1 SQLITE_ERROR} {1 SQLITE_NOMEM} +} + + +#------------------------------------------------------------------------- +# Errors while executing the matchinfo() demo function. +# +reset_db +sqlite3_fts5_register_matchinfo db +db func mit mit +do_execsql_test 2 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('x y z', '1 2 3'); + INSERT INTO t1 VALUES('x', '1 2 3 4 5 6 7'); +} + +do_faultsim_test 2.1 -faults oom* -body { + execsql { SELECT mit(matchinfo(t1, 'a')) FROM t1('x') } +} -test { + faultsim_test_result {0 {{2 5} {2 5}}} +} + +do_faultsim_test 2.2 -faults oom* -body { + execsql { SELECT mit(matchinfo(t1, 'l')) FROM t1('x') } +} -test { + faultsim_test_result {0 {{3 3} {1 7}}} +} + +do_execsql_test 2.3 { + INSERT INTO t1 VALUES('a b c d e f', 'a b d e f c'); + INSERT INTO t1 VALUES('l m b c a', 'n o a b c z'); +} + +do_faultsim_test 2.4 -faults oom* -body { + execsql { SELECT mit(matchinfo(t1, 's')) FROM t1('a b c') } +} -test { + faultsim_test_result {0 {{3 2} {2 3}}} +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(z); +} + +do_faultsim_test 3.1 -faults oom* -body { + execsql { + SELECT rowid FROM x1('c') WHERE rowid>1; + } +} -test { + faultsim_test_result {0 {}} +} + +do_execsql_test 3.2 { + INSERT INTO x1 VALUES('a b c'); + INSERT INTO x1 VALUES('b c d'); + INSERT INTO x1 VALUES('c d e'); + INSERT INTO x1 VALUES('d e f'); +} +do_faultsim_test 3.3 -faults oom* -body { + execsql { + SELECT rowid FROM x1('c') WHERE rowid>1; + } +} -test { + faultsim_test_result {0 {2 3}} +} + +#------------------------------------------------------------------------- +# Test OOM injection with nested colsets. +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 +} +do_faultsim_test 4.1 -faults oom* -body { + execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); } +} -test { + faultsim_test_result {0 2} +} + +do_faultsim_test 4.2 -faults oom* -body { + execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') } +} -test { + faultsim_test_result {0 {2 3}} +} + +#------------------------------------------------------------------------- +# Test OOM injection while parsing a CARET expression +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('a b c d'); -- 1 + INSERT INTO t1 VALUES('d a b c'); -- 2 + INSERT INTO t1 VALUES('c d a b'); -- 3 + INSERT INTO t1 VALUES('b c d a'); -- 4 +} +do_faultsim_test 5.1 -faults oom* -body { + execsql { SELECT rowid FROM t1('^a OR ^b') } +} -test { + faultsim_test_result {0 {1 4}} +} + +#------------------------------------------------------------------------- +# Test OOM injection in a query with two MATCH expressions +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('a b c d'); -- 1 + INSERT INTO t1 VALUES('d a b c'); -- 2 + INSERT INTO t1 VALUES('c d a b'); -- 3 + INSERT INTO t1 VALUES('b c d a'); -- 4 +} +do_faultsim_test 6.1 -faults oom* -body { + execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'a' AND t1 MATCH 'b' } +} -test { + faultsim_test_result {0 {1 2 3 4}} +} +do_faultsim_test 6.2 -faults oom* -body { + execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'a OR b' AND t1 MATCH 'c OR d' } +} -test { + faultsim_test_result {0 {1 2 3 4}} +} + + +finish_test diff --git a/ext/fts5/test/fts5faultD.test b/ext/fts5/test/fts5faultD.test new file mode 100644 index 0000000000000000000000000000000000000000..33590645ce3519ea406a2c7e2b9969e2116503cf --- /dev/null +++ b/ext/fts5/test/fts5faultD.test @@ -0,0 +1,87 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultD + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + if {"%DETAIL%"=="none"} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s; + } + + do_faultsim_test 1 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('a') } + } -test { + faultsim_test_result {0 180} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 2 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 3 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 4 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('b:b OR a:a') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 5 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('c:b') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } +} + +finish_test diff --git a/ext/fts5/test/fts5faultE.test b/ext/fts5/test/fts5faultE.test new file mode 100644 index 0000000000000000000000000000000000000000..07a4542102f6263e3b62e1790257b319bcc3ab6f --- /dev/null +++ b/ext/fts5/test/fts5faultE.test @@ -0,0 +1,71 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultE + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(x, y, tokenize=trigram) } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, tokenize=trigram); +} + +faultsim_save_and_close +do_faultsim_test 2 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO t1 VALUES('abcdefghijklmnopqrstuvwxyz', NULL); + SELECT count(*) FROM t1 WHERE x LIKE '%mnop%' AND t1 MATCH 'jkl'; + } +} -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, tokenize=trigram, detail=none); + INSERT INTO t1 VALUES('abcdefghijklmnopqrstuvwxyz', NULL); +} + +faultsim_save_and_close +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT count(*) FROM t1 WHERE x LIKE '%mnopqrs%' AND t1 MATCH 'abc' + } +} -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: t1}} +} + + + +finish_test + diff --git a/ext/fts5/test/fts5faultF.test b/ext/fts5/test/fts5faultF.test new file mode 100644 index 0000000000000000000000000000000000000000..96cc2b083f5dcc30ea60def7b18502f4ae3ce118 --- /dev/null +++ b/ext/fts5/test/fts5faultF.test @@ -0,0 +1,111 @@ +# 2023 July 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. Particularly those that may occur +# when using contentless_delete=1 databases. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultF + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) + } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d'); + COMMIT; + DELETE FROM t1 WHERE rowid IN (2, 4); +} + +do_faultsim_test 2 -prep { + sqlite3 db test.db + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + SELECT rowid FROM t1('b c'); + } +} -test { + faultsim_test_result {0 {1 3}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d'); + COMMIT; +} + +faultsim_save_and_close +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d'); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s; +} + +do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 } + +faultsim_save_and_close +do_faultsim_test 4 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + DELETE FROM t1 WHERE rowid < 100 + } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test + diff --git a/ext/fts5/test/fts5faultG.test b/ext/fts5/test/fts5faultG.test new file mode 100644 index 0000000000000000000000000000000000000000..9110c6336d1707d825227404393b68e1cd275de0 --- /dev/null +++ b/ext/fts5/test/fts5faultG.test @@ -0,0 +1,76 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultG + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultG + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('test renaming the table'); + INSERT INTO t1 VALUES(' after it has been written'); + INSERT INTO t1 VALUES(' actually other stuff instead'); +} +faultsim_save_and_close +do_faultsim_test 1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + } +} -body { + execsql { + DELETE FROM t1; + } +} -test { + catchsql { COMMIT } + faultsim_integrity_check + faultsim_test_result {0 {}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1 VALUES('here''s some text'); + INSERT INTO t1 VALUES('useful stuff, text'); + INSERT INTO t1 VALUES('what would we do without text!'); + COMMIT; +} +faultsim_save_and_close +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + } +} -body { + execsql { + INSERT INTO t1(t1) VALUES('optimize'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {}} +} + + + +finish_test diff --git a/ext/fts5/test/fts5faultH.test b/ext/fts5/test/fts5faultH.test new file mode 100644 index 0000000000000000000000000000000000000000..0cbbf7f5eff5690139dd804efccada1d4496b63c --- /dev/null +++ b/ext/fts5/test/fts5faultH.test @@ -0,0 +1,150 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultG + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultH + +sqlite3_fts5_register_origintext db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + + BEGIN; + INSERT INTO t1 VALUES('oNe tWo thRee'); + INSERT INTO t1 VALUES('One Two Three'); + INSERT INTO t1 VALUES('onE twO threE'); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES('one two three'); + INSERT INTO t1 VALUES('one two three'); + INSERT INTO t1 VALUES('one two three'); + COMMIT; +} + +do_faultsim_test 1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('three'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {1 2 3 4 5 6}} +} + + +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 24}} +} + +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB'); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 3.1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 35}} +} +do_faultsim_test 3.2 -faults oom* -prep { +} -body { + execsql { + SELECT count(*) FROM t1('BBB'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 27} +} + + +finish_test diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test new file mode 100644 index 0000000000000000000000000000000000000000..72f25caee1b01d560e1c4f693db2a0d5633d4291 --- /dev/null +++ b/ext/fts5/test/fts5faultI.test @@ -0,0 +1,294 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultI + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultI + +do_execsql_test 1.0 { + PRAGMA encoding = utf16; + CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1); + INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram'); +} + +faultsim_save_and_close +faultsim_restore_and_reopen + +do_faultsim_test 1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1(fts5_locale('en_US', 'origintext')); + } +} -test { + faultsim_test_result {0 1} +} + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + SELECT * FROM t1('ascii'); + } +} -body { + execsql { + UPDATE t1 SET rowid=rowid+1; + } +} -test { + faultsim_test_result {0 {}} +} + +fts5_aux_test_functions db +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*'); + } +} -test { + faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('token token token'); + } +} -body { + execsql { + INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('one'); + SAVEPOINT one; + INSERT INTO w1 VALUES('two'); + ROLLBACK TO one; + } + +} -body { + execsql { + INSERT INTO w1 VALUES('string'); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db + +do_faultsim_test 5 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two'); + } +} -test { + faultsim_test_result {0 two} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db +faultsim_save_and_close + +do_faultsim_test 6 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO w1 VALUES('four five six'); + SAVEPOINT abc; + INSERT INTO w1 VALUES('seven eight nine'); + SAVEPOINT def; + INSERT INTO w1 VALUES('ten eleven twelve'); + } +} -body { + execsql { + RELEASE abc; + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); + INSERT INTO w1 VALUES('three two one'); + DELETE FROM w1_content WHERE rowid=1; +} + +faultsim_save_and_close + +do_faultsim_test 7 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM w1 } +} -body { + execsql { + PRAGMA integrity_check; + } +} -test { +} + +#------------------------------------------------------------------------- +reset_db +fts5_tclnum_register db +fts5_aux_test_functions db + +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize = "tclnum query", detail=columns + ); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_faultsim_test 8.1 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_collist (ft) FROM ft('one two'); + } +} -test { + faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 8.2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM ft('one two') ORDER BY rank; + } +} -test { + faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +faultsim_save_and_close + +do_faultsim_test 9.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE ft SET rowid=4 WHERE rowid=1 + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 9.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 1} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, locale=1); + INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii')); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_execsql_test 10.1 { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' +} {hello} + +faultsim_save_and_close +do_faultsim_test 10.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 hello} +} + +breakpoint +faultsim_save_and_close +do_faultsim_test 10.2 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO ft VALUES(zeroblob(10000)); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE f1 USING fts5(content); + CREATE TABLE g1(id, content); + INSERT INTO g1 VALUES(30000, 'a b c'); + INSERT INTO g1 VALUES(40000, 'd e f'); +} + +faultsim_save_and_close + +do_faultsim_test 11 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO f1(rowid, content) SELECT id, content FROM g1; + } +} -test { + faultsim_test_result {0 {}} +} + +finish_test + diff --git a/ext/fts5/test/fts5first.test b/ext/fts5/test/fts5first.test new file mode 100644 index 0000000000000000000000000000000000000000..492681eed780d92714029f41cf9bdc24254b0a43 --- /dev/null +++ b/ext/fts5/test/fts5first.test @@ -0,0 +1,96 @@ +# 2017 November 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5first + +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); +} + +unset -nocomplain res +foreach {tn expr ok} { + 1 {^abc} 1 + 2 {^abc + def} 1 + 3 {^ "abc def"} 1 + 4 {^"abc def"} 1 + 5 {abc ^def} 1 + 6 {abc + ^def} 0 + 7 {abc ^+ def} 0 + 8 {"^abc"} 1 + 9 {NEAR(^abc def)} 0 +} { + set res(0) {/1 {fts5: syntax error near .*}/} + set res(1) {0 {}} + + do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok) +} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + INSERT INTO x1 VALUES('a b c', 'b c a'); +} + +foreach {tn expr match} { + 1 {^a} 1 + 2 {^b} 1 + 3 {^c} 0 + 4 {^a + b} 1 + 5 {^b + c} 1 + 6 {^c + a} 0 + 7 {^"c a"} 0 + 8 {a:^a} 1 + 9 {a:^b} 0 + 10 {a:^"a b"} 1 +} { + do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + DELETE FROM x1; + INSERT INTO x1 VALUES('b a', 'c a'); + INSERT INTO x1 VALUES('a a', 'c c'); + INSERT INTO x1 VALUES('a b', 'a a'); +} +fts5_aux_test_functions db + +foreach {tn expr expect} { + 1 {^a} {{2 1}} + 2 {^c AND ^b} {{0 2} {1 0}} +} { + do_execsql_test 3.$tn { + SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1 + } [list $expect] +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column); +} + +do_catchsql_test 3.2 { + SELECT * FROM x2('a + b'); +} {1 {fts5: phrase queries are not supported (detail!=full)}} + +do_catchsql_test 3.3 { + SELECT * FROM x2('^a'); +} {1 {fts5: phrase queries are not supported (detail!=full)}} +finish_test diff --git a/ext/fts5/test/fts5full.test b/ext/fts5/test/fts5full.test new file mode 100644 index 0000000000000000000000000000000000000000..76fdc0288fbb752d1abeb0816fcc8e208651f167 --- /dev/null +++ b/ext/fts5/test/fts5full.test @@ -0,0 +1,42 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that SQLITE_FULL is returned if the FTS5 table cannot find a free +# segid to use. In practice this can only really happen when automerge and +# crisismerge are both disabled. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5full + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x8 USING fts5(i); + INSERT INTO x8(x8, rank) VALUES('automerge', 0); + INSERT INTO x8(x8, rank) VALUES('crisismerge', 100000); +} + +db func rnddoc fts5_rnddoc +do_test 1.1 { + list [catch { + for {set i 0} {$i < 2500} {incr i} { + execsql { INSERT INTO x8 VALUES( rnddoc(5) ); } + } + } msg] $msg +} {0 {}} + + +finish_test diff --git a/ext/fts5/test/fts5fuzz1.test b/ext/fts5/test/fts5fuzz1.test new file mode 100644 index 0000000000000000000000000000000000000000..52dc23d77ca037f4d55429e11c0a41fa30f9a7d7 --- /dev/null +++ b/ext/fts5/test/fts5fuzz1.test @@ -0,0 +1,92 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set testprefix fts5fuzz1 + + +#------------------------------------------------------------------------- +reset_db +do_catchsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a b); +} {/1 {parse error in.*}/} + + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1 VALUES('a b', 'c d'); + INSERT INTO f1 VALUES('e f', 'a b'); +} + +do_execsql_test 2.2.1 { + SELECT rowid FROM f1('""'); +} {} + +do_execsql_test 2.2.2 { + SELECT rowid FROM f1('"" AND a'); +} {} + + +do_execsql_test 2.2.3 { + SELECT rowid FROM f1('"" a'); +} {1 2} + +do_execsql_test 2.2.4 { + SELECT rowid FROM f1('"" OR a'); +} {1 2} + +do_execsql_test 2.3 { + SELECT a, b FROM f1('NEAR("")'); +} {} + +do_execsql_test 2.4 { + SELECT a, b FROM f1('NEAR("", 5)'); +} {} + +do_execsql_test 2.5 { + SELECT a, b FROM f1('NEAR("" c, 5)'); +} {{a b} {c d}} + +do_execsql_test 2.6 { + SELECT a, b FROM f1('NEAR("" c d, 5)'); +} {{a b} {c d}} + +do_execsql_test 2.7 { + SELECT a, b FROM f1('NEAR(c d, 5)'); +} {{a b} {c d}} + +do_execsql_test 2.8 { + SELECT rowid FROM f1('NEAR("a" "b", 5)'); +} {1 2} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.2 { + CREATE VIRTUAL TABLE f2 USING fts5(o, t, tokenize="ascii separators abc"); + SELECT * FROM f2('a+4'); +} {} + + + +#------------------------------------------------------------------------- +reset_db +do_catchsql_test 4.1 { + CREATE VIRTUAL TABLE f2 USING fts5(o, t); + SELECT * FROM f2('(8 AND 9)`AND 10'); +} {1 {fts5: syntax error near "`"}} + +finish_test diff --git a/ext/fts5/test/fts5hash.test b/ext/fts5/test/fts5hash.test new file mode 100644 index 0000000000000000000000000000000000000000..b3d8b562c8732e27fdc6bd82f330ffc751f660a1 --- /dev/null +++ b/ext/fts5/test/fts5hash.test @@ -0,0 +1,168 @@ +# 2015 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file are focused on the code in fts5_hash.c. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5hash + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Return a list of tokens (a vocabulary) that all share the same hash +# key value. This can be used to test hash collisions. +# +proc build_vocab1 {args} { + + set O(-nslot) 1024 + set O(-nword) 20 + set O(-hash) 88 + set O(-prefix) "" + + if {[llength $args] % 2} { error "bad args" } + array set O2 $args + foreach {k v} $args { + if {[info exists O($k)]==0} { error "bad option: $k" } + set O($k) $v + } + + set L [list] + while {[llength $L] < $O(-nword)} { + set t "$O(-prefix)[random_token]" + set h [sqlite3_fts5_token_hash $O(-nslot) $t] + if {$O(-hash)==$h} { lappend L $t } + } + return $L +} + +proc random_token {} { + set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] + set iVal [expr int(rand() * 2000000)] + return [string map $map $iVal] +} + +proc random_doc {vocab nWord} { + set doc "" + set nVocab [llength $vocab] + for {set i 0} {$i<$nWord} {incr i} { + set j [expr {int(rand() * $nVocab)}] + lappend doc [lindex $vocab $j] + } + return $doc +} + +foreach_detail_mode $testprefix { + + set vocab [build_vocab1] + db func r random_doc + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%); + BEGIN; + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) + INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii; + INSERT INTO eee(eee) VALUES('integrity-check'); + COMMIT; + INSERT INTO eee(eee) VALUES('integrity-check'); + } + + set hash [sqlite3_fts5_token_hash 1024 xyz] + set vocab [build_vocab1 -prefix xyz -hash $hash] + lappend vocab xyz + + do_execsql_test 1.1 { + CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); + BEGIN; + } + do_test 1.2 { + for {set i 1} {$i <= 100} {incr i} { + execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) } + } + } {} + + do_test 1.3 { + db eval { SELECT term, doc FROM vocab } { + set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}] + if {$nRow != $doc} { + error "term=$term fts5vocab=$doc cnt=$nRow" + } + } + set {} {} + } {} + + do_execsql_test 1.4 { + COMMIT; + INSERT INTO eee(eee) VALUES('integrity-check'); + } + + #----------------------------------------------------------------------- + # Add a small and very large token with the same hash value to an + # empty table. At one point this would provoke an asan error. + # + do_test 1.5 { + set big [string repeat 12345 40] + set hash [sqlite3_fts5_token_hash 1024 $big] + while {1} { + set small [random_token] + if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break + } + + execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) } + execsql { + INSERT INTO t2 VALUES($small || ' ' || $big); + } + } {} + +} ;# foreach_detail_mode + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('hashsize', 1024); + INSERT INTO t1(t1, rank) VALUES('automerge', 0); + INSERT INTO t1(t1, rank) VALUES('crisismerge', 1000); +} + +do_execsql_test 2.2 { + BEGIN; + INSERT INTO t1 VALUES('abc def ghi'); + SELECT count(*) FROM t1_data; +} {2} + +do_execsql_test 2.3 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1024 + ) + INSERT INTO t1 SELECT 'abc def ghi' FROM s; + SELECT (SELECT count(*) FROM t1_data) > 10; +} {1} + +do_execsql_test 2.4 { + COMMIT; + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('hashsize', 1024); + INSERT INTO t1(t1, rank) VALUES('automerge', 0); + INSERT INTO t1(t1, rank) VALUES('crisismerge', 1000); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1024 + ) + INSERT INTO t1 SELECT 'abc' || i || ' def' || i || ' ghi' || i FROM s; + SELECT (SELECT count(*) FROM t1_data) > 100; +} {1} + +finish_test diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test new file mode 100644 index 0000000000000000000000000000000000000000..5c40021803d6e160e4fb1f18ff85be8a98f4c9ea --- /dev/null +++ b/ext/fts5/test/fts5integrity.test @@ -0,0 +1,412 @@ +# 2015 Jan 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests focused on the integrity-check procedure. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5integrity + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xx USING fts5(x); + INSERT INTO xx VALUES('term'); +} +do_execsql_test 1.1 { + INSERT INTO xx(xx) VALUES('integrity-check'); +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, prefix=1); + INSERT INTO yy VALUES('term'); +} +do_execsql_test 2.1 { + INSERT INTO yy(yy) VALUES('integrity-check'); +} + +#-------------------------------------------------------------------- +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE zz USING fts5(z); + INSERT INTO zz(zz, rank) VALUES('pgsz', 32); + INSERT INTO zz VALUES('b b b b b b b b b b b b b b'); + INSERT INTO zz SELECT z FROM zz; + INSERT INTO zz SELECT z FROM zz; + INSERT INTO zz SELECT z FROM zz; + INSERT INTO zz SELECT z FROM zz; + INSERT INTO zz SELECT z FROM zz; + INSERT INTO zz SELECT z FROM zz; + INSERT INTO zz(zz) VALUES('optimize'); +} + +do_execsql_test 3.1 { INSERT INTO zz(zz) VALUES('integrity-check'); } + +#-------------------------------------------------------------------- +# Mess around with a docsize record. And the averages record. Then +# check that integrity-check picks it up. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE aa USING fts5(zz); + INSERT INTO aa(zz) VALUES('a b c d e'); + INSERT INTO aa(zz) VALUES('a b c d'); + INSERT INTO aa(zz) VALUES('a b c'); + INSERT INTO aa(zz) VALUES('a b'); + INSERT INTO aa(zz) VALUES('a'); + SELECT length(sz) FROM aa_docsize; +} {1 1 1 1 1} +do_execsql_test 4.1 { + INSERT INTO aa(aa) VALUES('integrity-check'); +} + +sqlite3_db_config db DEFENSIVE 0 +do_catchsql_test 4.2 { + BEGIN; + UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3; + INSERT INTO aa(aa) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 4.2.1 { + PRAGMA integrity_check(aa); +} {{malformed inverted index for FTS5 table main.aa}} + +do_catchsql_test 4.3 { + ROLLBACK; + BEGIN; + UPDATE aa_data SET block = X'44' WHERE rowid = 1; + INSERT INTO aa(aa) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +do_catchsql_test 4.4 { + ROLLBACK; + BEGIN; + INSERT INTO aa_docsize VALUES(23, X'04'); + INSERT INTO aa(aa) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +do_catchsql_test 4.5 { + ROLLBACK; + BEGIN; + INSERT INTO aa_docsize VALUES(23, X'00'); + INSERT INTO aa_content VALUES(23, ''); + INSERT INTO aa(aa) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r} +#exit + +execsql { ROLLBACK } + + +#------------------------------------------------------------------------- +# Test that integrity-check works on a reasonably large db with many +# different terms. + +# Document generator command. +proc rnddoc {n} { + set doc [list] + for {set i 0} {$i<$n} {incr i} { + lappend doc [format %.5d [expr int(rand()*10000)]] + } + return $doc +} +db func rnddoc rnddoc + +expr srand(0) +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE gg USING fts5(a, prefix="1,2,3"); + INSERT INTO gg(gg, rank) VALUES('pgsz', 256); + INSERT INTO gg VALUES(rnddoc(20)); + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; + INSERT INTO gg SELECT rnddoc(20) FROM gg; +} + +do_execsql_test 5.1 { + INSERT INTO gg(gg) VALUES('integrity-check'); +} + +do_execsql_test 5.2 { + INSERT INTO gg(gg) VALUES('optimize'); +} + +do_execsql_test 5.3 { + INSERT INTO gg(gg) VALUES('integrity-check'); +} + +unset -nocomplain res +do_test 5.4.1 { + set ok 0 + for {set i 0} {$i < 10000} {incr i} { + set T [format %.5d $i] + set res [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC }] + set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }] + if {$res == [lsort -integer $res2]} { incr ok } + } + set ok +} {10000} + +do_test 5.4.2 { + set ok 0 + for {set i 0} {$i < 100} {incr i} { + set T "[format %.3d $i]*" + set res [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC }] + set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }] + if {$res == [lsort -integer $res2]} { incr ok } + } + set ok +} {100} + +#------------------------------------------------------------------------- +# Similar to 5.*. +# +foreach {tn pgsz} { + 1 32 + 2 36 + 3 40 + 4 44 + 5 48 +} { + do_execsql_test 6.$tn.1 { + DROP TABLE IF EXISTS hh; + CREATE VIRTUAL TABLE hh USING fts5(y); + INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz); + + WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999) + INSERT INTO hh SELECT printf('%.3d%.3d%.3d %.3d%.3d%.3d',i,i,i,i+1,i+1,i+1) + FROM s; + + WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999) + INSERT INTO hh SELECT printf('%.3d%.3d%.3d %.3d%.3d%.3d',i,i,i,i+1,i+1,i+1) + FROM s; + + INSERT INTO hh(hh) VALUES('optimize'); + } + + do_test 6.$tn.2 { + set ok 0 + for {set i 0} {$i < 1000} {incr i} { + set T [format %.3d%.3d%.3d $i $i $i] + set res [db eval { SELECT rowid FROM hh($T) ORDER BY rowid ASC }] + set res2 [db eval { SELECT rowid FROM hh($T) ORDER BY rowid DESC }] + if {$res == [lsort -integer $res2]} { incr ok } + } + set ok + } {1000} +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.0 { + PRAGMA encoding = 'UTF-16'; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); + INSERT INTO vt0 VALUES (x'46f0'); + SELECT quote(c0) FROM vt0; +} {X'46F0'} +do_execsql_test 7.1 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} +do_execsql_test 7.2 { + INSERT INTO vt0(vt0) VALUES('rebuild'); +} +do_execsql_test 7.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} +do_execsql_test 7.4 { + UPDATE vt0 SET c0=''; +} +do_execsql_test 7.5 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Ticket 7a458c2a5f4 +# +reset_db +do_execsql_test 8.0 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA journal_mode = PERSIST; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} {exclusive persist} +do_execsql_test 8.1 { + PRAGMA data_version +} {1} +do_execsql_test 8.2 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); + PRAGMA data_version; +} {1} +do_execsql_test 8.1 { + INSERT INTO vt0(vt0, rank) VALUES('usermerge', 2); +} + +#------------------------------------------------------------------------- +# Ticket [771fe617] +# +reset_db +do_execsql_test 9.0 { + PRAGMA encoding = 'UTF16'; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} + +#explain_i { SELECT quote(SUBSTR(x'37', 0)); } +#execsql { PRAGMA vdbe_trace = 1 } +do_execsql_test 9.1.1 { + SELECT quote(SUBSTR(x'37', 0)); +} {X'37'} +do_execsql_test 9.1.2 { + SELECT quote(x'37'); +} {X'37'} + +do_execsql_test 9.2 { + INSERT INTO vt0 VALUES (SUBSTR(x'37', 0)); +-- INSERT INTO vt0 VALUES (x'37'); +} +do_execsql_test 9.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); + CREATE VIRTUAL TABLE vt0 USING fts5(a, b, content=t1); + INSERT INTO vt0(rowid, a, b) VALUES(1, 'abc', 'def'); +} +do_catchsql_test 10.1 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} {0 {}} +do_catchsql_test 10.2 { + INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 0); +} {0 {}} +do_catchsql_test 10.3 { + INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 1); +} {1 {database disk image is malformed}} +do_catchsql_test 10.3 { + INSERT INTO t1 VALUES(1, 'abc', 'def'); + INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 1); +} {0 {}} + +do_execsql_test 10.4 { + CREATE VIRTUAL TABLE vt1 USING fts5(a, b, content=); + INSERT INTO vt1(rowid, a, b) VALUES(1, 'abc', 'def'); +} + +do_catchsql_test 10.5.1 { + INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 0); +} {0 {}} +do_catchsql_test 10.5.2 { + INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 1); +} {0 {}} +do_catchsql_test 10.5.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} {0 {}} + +reset_db +proc slang {in} {return [string map {th d e eh} $in]} +db function slang -deterministic -innocuous slang +do_execsql_test 11.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b))); + INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog'); + SELECT c FROM t1; +} {{deh quick fox jumps ovehr deh lazy brown dog}} + +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE t2 USING fts5(content="t1", c); + INSERT INTO t2(t2) VALUES('rebuild'); + SELECT rowid FROM t2 WHERE t2 MATCH 'deh'; +} {1} + +do_execsql_test 11.2 { + PRAGMA integrity_check(t2); +} {ok} +db close +sqlite3 db test.db + +# FIX ME? +# +# FTS5 integrity-check does not care if the content table is unreadable or +# does not exist. It only looks for internal inconsistencies in the +# inverted index. +# +do_execsql_test 11.3 { + PRAGMA integrity_check(t2); +} {ok} +do_execsql_test 11.4 { + DROP TABLE t1; + PRAGMA integrity_check(t2); +} {ok} + +#------------------------------------------------------------------- +reset_db + +do_execsql_test 12.1 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('one', 'two'); + INSERT INTO x1 VALUES('three', 'four'); + INSERT INTO x1 VALUES('five', 'six'); +} + +do_execsql_test 12.2 { + PRAGMA integrity_check +} {ok} + +db close +sqlite3 db test.db -readonly 1 + +explain_i { + PRAGMA integrity_check + } +do_execsql_test 12.3 { + PRAGMA integrity_check +} {ok} + + +#------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +db close +sqlite3 db test.db +do_catchsql_test 13.2 { + PRAGMA integrity_check; +} {0 ok} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +breakpoint +do_catchsql_test 13.4 { + PRAGMA integrity_check; +} {1 {SQL logic error}} + + +finish_test diff --git a/ext/fts5/test/fts5integrity2.test b/ext/fts5/test/fts5integrity2.test new file mode 100644 index 0000000000000000000000000000000000000000..968be3bddfbc4fab69498c29cd4a82d1b2c316c1 --- /dev/null +++ b/ext/fts5/test/fts5integrity2.test @@ -0,0 +1,56 @@ +# 2024 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests focused on the integrity-check procedure. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5integrity2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, detail='none'); + BEGIN; + INSERT INTO t2(rowid, a) VALUES(-1, 'hello world'); + INSERT INTO t2(rowid, a) VALUES(9223372036854775807, 'hello world'); + COMMIT; +} + +do_execsql_test 2.1 { + SELECT rowid FROM t2('hello AND world'); +} {-1 9223372036854775807} + +#------------------------------------------------------------------------- +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none'); + CREATE TABLE r1(r); + + WITH c(x) AS (VALUES(1) UNION SELECT x<<1 FROM c) + INSERT INTO r1(r) SELECT -1-x FROM c; + + INSERT INTO t1(rowid, a) SELECT r, 'abc' FROM r1; +} + +do_execsql_test 2.1 { + PRAGMA integrity_check; +} {ok} + +do_execsql_test 2.2 { + SELECT rowid FROM t1('abc') ORDER BY +rowid; +} [db eval {SELECT r FROM r1 ORDER BY r}] + + +finish_test diff --git a/ext/fts5/test/fts5interrupt.test b/ext/fts5/test/fts5interrupt.test new file mode 100644 index 0000000000000000000000000000000000000000..67ef5f7e97f6b965ae04843f5a7d29dd584fed4a --- /dev/null +++ b/ext/fts5/test/fts5interrupt.test @@ -0,0 +1,67 @@ +# 2019 Jan 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5interrupt + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); +} +db_save_and_close + +proc progress_handler {args} { + incr ::progress_handler_delay -1 + if {$::progress_handler_delay<=0} { return 1 } + return 0 +} + +unset -nocomplain res +foreach {tn sql} { + 1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') } + 2 { COMMIT } +} { + set bDone 0 + for {set i 1} {$bDone==0} {incr i} { + do_test 1.$tn.$i { + db_restore_and_reopen + execsql { + BEGIN; + INSERT INTO t1(rowid, a) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, a) VALUES(2, 'd e f g'); + INSERT INTO t1(rowid, a) VALUES(3, 'h i j k'); + INSERT INTO t1(rowid, a) VALUES(4, 'l m n o'); + } + + set ::progress_handler_delay $i + db progress 1 progress_handler + set res [catchsql $sql] + db close + if {$res=="0 {}"} { + set bDone 1 + } else { + if {$res!="1 interrupted"} { error "got: $res" } + } + set {} {} + } {} + } +} + +finish_test diff --git a/ext/fts5/test/fts5lastrowid.test b/ext/fts5/test/fts5lastrowid.test new file mode 100644 index 0000000000000000000000000000000000000000..75866139d374337f4a8045708cfa5173ae6c6c3c --- /dev/null +++ b/ext/fts5/test/fts5lastrowid.test @@ -0,0 +1,72 @@ +# 2017 Feb 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the last_insert_rowid functionality with fts5. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5lastrowid + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(str); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + SELECT last_insert_rowid(); +} {3} + +do_execsql_test 1.2 { + BEGIN; + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + COMMIT; + SELECT last_insert_rowid(); +} {6} + +do_execsql_test 1.3 { + INSERT INTO t1(rowid, str) VALUES(-22, 'some more text'); + SELECT last_insert_rowid(); +} {-22} + +do_execsql_test 1.4 { + BEGIN; + INSERT INTO t1(rowid, str) VALUES(45, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(46, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(222, 'some more text'); + SELECT last_insert_rowid(); + COMMIT; + SELECT last_insert_rowid(); +} {222 222} + +do_execsql_test 1.5 { + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo'); + INSERT INTO t1 SELECT x FROM x1; + SELECT last_insert_rowid(); +} {226} + +do_execsql_test 1.6 { + INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; + SELECT last_insert_rowid(); +} {14} + + +finish_test diff --git a/ext/fts5/test/fts5leftjoin.test b/ext/fts5/test/fts5leftjoin.test new file mode 100644 index 0000000000000000000000000000000000000000..4ef6a8961b0577099fbb537d47ac5937d02310a1 --- /dev/null +++ b/ext/fts5/test/fts5leftjoin.test @@ -0,0 +1,43 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5leftjoin + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE vt USING fts5(x); + INSERT INTO vt VALUES('abc'); + INSERT INTO vt VALUES('xyz'); + + CREATE TABLE t1(a INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1), (2); +} + +do_execsql_test 1.1 { + SELECT * FROM t1 LEFT JOIN ( + SELECT rowid AS rrr, * FROM vt WHERE vt MATCH 'abc' + ) ON t1.a = rrr +} {1 1 abc 2 {} {}} + +do_execsql_test 1.2 { + SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc') +} {1 abc 2 abc} + +finish_test diff --git a/ext/fts5/test/fts5limits.test b/ext/fts5/test/fts5limits.test new file mode 100644 index 0000000000000000000000000000000000000000..90d175aa3108f1892a1eedc33c198c40241ba5c1 --- /dev/null +++ b/ext/fts5/test/fts5limits.test @@ -0,0 +1,47 @@ +# 2023 May 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5limits +return_if_no_fts5 + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); +} + +# Default limit for expression depth is 256 +# +foreach {tn nRepeat op bErr} { + 1 200 AND 0 + 2 200 NOT 0 + 3 200 OR 0 + + 4 260 AND 0 + 5 260 NOT 1 + 6 260 OR 0 +} { + set L [string repeat "abc " $nRepeat] + set Q [join $L " $op "] + + set res {0 {}} + if {$bErr} { + set res "1 {fts5 expression tree is too large (maximum depth 256)}" + } + + do_catchsql_test 1.$tn { + SELECT * FROM ft($Q) + } $res +} + +finish_test + diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test new file mode 100644 index 0000000000000000000000000000000000000000..e5799fb7fd78b354f2279be3d0a74cd600078169 --- /dev/null +++ b/ext/fts5/test/fts5locale.test @@ -0,0 +1,748 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5locale + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc transform_token {locale token} { + switch -- $locale { + reverse { + set ret "" + foreach c [split $token ""] { + set ret "$c$ret" + } + set token $ret + } + + default { + # no-op + } + } + + set token +} + +proc tcl_create {args} { return "tcl_tokenize" } +proc tcl_tokenize {tflags text} { + set iToken 1 + set bSkip 0 + if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 } + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + incr iToken + if {(($iToken) % ($bSkip + 1))} continue + + set w [transform_token [sqlite3_fts5_locale] $w] + sqlite3_fts5_token $w $iStart $iEnd + } +} + +#------------------------------------------------------------------------- +# Check that queries can have a locale attached to them. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('cba'); +} {} + +do_execsql_test 1.1 { + SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') ); +} {1 abc} + +do_execsql_test 1.2 { + SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); +} {2 cba} + + +#------------------------------------------------------------------------- +# Test that the locale= option exists and seems to accept values. And +# that fts5_locale() values may only be inserted into an internal-content +# table if the locale=1 option was specified. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl); + + CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance); +} + +do_catchsql_test 2.2.1 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2); +} {1 {malformed locale=... directive}} +do_catchsql_test 2.2.2 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111); +} {1 {malformed locale=... directive}} + +do_catchsql_test 2.3 { + INSERT INTO b1(b1, rank) VALUES('locale', 0); +} {1 {SQL logic error}} + +do_execsql_test 2.4.1 { + INSERT INTO b1 VALUES('abc', 'one two three'); +} + +do_execsql_test 2.4.2 { + INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); +} + +do_execsql_test 2.5 { + INSERT INTO b2 VALUES('abc', 'one two three'); +} + +do_catchsql_test 2.6 { + INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six')); +} {1 {fts5_locale() requires locale=1}} + +do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1} +do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {} +do_execsql_test 2.9 { SELECT rowid FROM b1('ruof') } 2 +do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2 + +do_execsql_test 2.11 { + SELECT x, quote(y) FROM b1 +} { + abc {'one two three'} + def {'four five six'} +} + +do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { + {'four five six'} +} + +do_execsql_test 2.13 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.14 { + INSERT INTO b1(b1) VALUES('rebuild'); +} +do_execsql_test 2.15 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.16 { + DELETE FROM b1 WHERE rowid=2 +} +do_execsql_test 2.17 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.18 { + INSERT INTO b1(rowid, x, y) VALUES( + test_setsubtype(45, 76), 'abc def', 'def abc' + ); +} + +#------------------------------------------------------------------------- +# Test the 'delete' command with contentless tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1); + CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance); + + INSERT INTO c1 VALUES('hello world'); + INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.2 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + eerht eno hello owt world +} + +do_execsql_test 3.3 { + INSERT INTO c1(c1, rowid, x) + VALUES('delete', 2, fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.4 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + hello world +} + +#------------------------------------------------------------------------- +# Test that an UPDATE that updates a subset of the columns does not +# magically discard the locale from those columns not updated. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance); + + INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def'); + INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello')); +} + +do_execsql_test 4.2 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def ghi olleh +} + +do_execsql_test 4.3 { + UPDATE d1 SET x='jkl' WHERE rowid=2; +} + +do_execsql_test 4.4 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def jkl olleh +} + +do_execsql_test 4.5 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 2 jkl hello +} + +do_execsql_test 4.6 { + UPDATE d1 SET rowid=4 WHERE rowid=2 +} + +do_execsql_test 4.7 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 4 jkl hello +} + +fts5_aux_test_functions db + +do_execsql_test 4.8.1 { + SELECT fts5_test_columntext(d1) FROM d1('jkl') +} {{jkl hello}} +do_execsql_test 4.8.2 { + SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{jkl hello}} + +do_execsql_test 4.9 { + SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{{} reverse}} + +do_execsql_test 4.10 { + SELECT fts5_test_columnlocale(d1) FROM d1 +} { + {{} {}} + {{} reverse} +} + +#------------------------------------------------------------------------- +# Test that if an fts5_locale() value is written to an UNINDEXED +# column it is stored as text. This is so that blobs and other values +# can also be stored as is. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, y UNINDEXED, locale=1, tokenize=tcl + ); + + INSERT INTO t1(rowid, x, y) VALUES(111, + fts5_locale('reverse', 'one two three'), + fts5_locale('reverse', 'four five six') + ); +} + +do_execsql_test 5.2 { + SELECT rowid, x, y FROM t1 +} { + 111 {one two three} {four five six} +} + +do_execsql_test 5.3 { + SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content +} { + text text text +} + +#------------------------------------------------------------------------- + +foreach {tn opt} { + 1 {} + 2 {, columnsize=0} +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 6.$tn.1 " + CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt); + " + + do_execsql_test 6.$tn.2 { + INSERT INTO y1(rowid, t) VALUES + (1, fts5_locale('second', 'the city of London')), + (2, fts5_locale('second', 'shall have all the old')), + (3, fts5_locale('second', 'Liberties and Customs')), + (4, fts5_locale('second', 'which it hath been used to have')); + } + + fts5_aux_test_functions db + + do_execsql_test 6.$tn.3 { + SELECT fts5_test_columnsize(y1) FROM y1 + } { + 2 3 2 4 + } + + do_execsql_test 6.$tn.4 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 6.$tn.5 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 6.$tn.6 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); + } { + 4 4 + } + + do_execsql_test 6.$tn.7 { + SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); + } { + 4 {which it hath been used to [have]} + } + + do_execsql_test 6.$tn.8 { + SELECT rowid, + highlight(y1, 0, '[', ']'), + snippet(y1, 0, '[', ']', '...', 10) + FROM y1('Liberties + Customs'); + } { + 3 {[Liberties and Customs]} + {[Liberties and Customs]} + } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} +do_catchsql_test 6.1 { + INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world')); +} {1 {fts5_locale() requires locale=1}} + +do_execsql_test 6.2 { + SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') ); +} {text text} + +#-------------------------------------------------------------------------- +# Test that fts5_locale() works with external-content tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 7.1 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT); + CREATE VIEW v1 AS + SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1; + + CREATE VIRTUAL TABLE ft USING fts5( + bb, tt, locale=1, tokenize=tcl, content=v1 + ); + + INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL); + INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse'); + INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second'); +} + +do_execsql_test 7.2 { + INSERT INTO ft(ft) VALUES('rebuild'); + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 7.3 { + SELECT rowid, quote(bb), quote(tt) FROM ft +} { + 1 NULL {'one two three'} + 2 '7800616263' {'four five six'} + 3 '000000007800616263' {'seven eight nine'} +} + +do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } +do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2 + +fts5_aux_test_functions db + +do_execsql_test 7.6 { + SELECT fts5_test_columnlocale(ft) FROM ft; +} { + {{} {}} {{} reverse} {{} second} +} + +#------------------------------------------------------------------------- +# Test that the porter tokenizer works with locales. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl"); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + + INSERT INTO ft(rowid, tt) VALUES + (111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')), + (222, fts5_locale('reverse', 'This value may also be set')); +} + +do_execsql_test 8.1 { + SELECT DISTINCT term FROM vocab ORDER BY 1 +} { + a eb eulav osla sihT te the token yam +} + +#------------------------------------------------------------------------- +# Test that position-lists (used by xInst, xPhraseFirst etc.) work with +# locales and modes other than detail=full. +# +foreach {tn detail} { + 1 detail=full + 2 detail=none + 3 detail=column +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 9.$tn.0 " + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail); + " + do_execsql_test 9.$tn.1 { + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + INSERT INTO ft(rowid, tt) VALUES + (-1, fts5_locale('second', 'it is an ancient mariner')); + } + + do_execsql_test 9.$tn.2 { + SELECT DISTINCT term FROM vocab + } {an it mariner} + + do_execsql_test 9.$tn.3 { + SELECT highlight(ft, 0, '[', ']') FROM ft('mariner') + } {{it is an ancient [mariner]}} +} + +#------------------------------------------------------------------------- +# Check some corrupt fts5_locale() blob formats are detected. +# +foreach_detail_mode $::testprefix { + + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + fts5_aux_test_functions db + do_execsql_test 10.1 { + CREATE TABLE x1(ii INTEGER PRIMARY KEY, x); + CREATE VIRTUAL TABLE ft USING fts5(x, + content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0 + ); + + CREATE VIRTUAL TABLE ft2 USING fts5( + x, locale=1, detail=%DETAIL%, columnsize=0 + ); + } + + foreach {tn v} { + 1 X'001152' + 2 X'0011223344' + 3 X'00E0B2EB68656c6c6f' + 4 X'00E0B2EB0068656c6c6f' + } { + do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') } + do_execsql_test 10.2.$tn.1 { DELETE FROM x1; } + do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) " + + do_catchsql_test 10.2.$tn.3 { + INSERT INTO ft(ft) VALUES('rebuild'); + } {0 {}} + + do_catchsql_test 10.2.$tn.4 " + SELECT * FROM ft( test_setsubtype($v, 76) ); + " {1 {fts5: syntax error near ""}} + + do_execsql_test 10.2.$tn.5 { + INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); + } + + if {"%DETAIL%"=="full"} { + do_execsql_test 10.2.$tn.6 { + SELECT fts5_test_poslist(ft) FROM ft('world'); + } {0.0.1} + + do_execsql_test 10.2.$tn.7.1 { + SELECT fts5_test_columnsize(ft) FROM ft('world'); + } {1} + + do_execsql_test 10.2.$tn.7.2 { + SELECT fts5_test_columnlocale(ft) FROM ft('world'); + } {{{}}} + } + + do_catchsql_test 10.2.$tn.8 { + SELECT count(*) FROM ft('hello') + } {0 1} + + do_catchsql_test 10.2.$tn.9 { + PRAGMA integrity_check; + } {0 ok} + + do_execsql_test 10.2.$tn.10 { + DELETE FROM x1; + INSERT INTO x1(ii, x) VALUES(1, 'hello world'); + } + + do_catchsql_test 10.2.$tn.11 " + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) + " {0 {}} + + do_catchsql_test 10.2.$tn.12 " + INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) + " {0 {}} + + do_execsql_test 10.2.$tn.13 { + INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); + } + do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v" + + do_catchsql_test 10.2.$tn.15 { + PRAGMA integrity_check; + } {0 {{malformed inverted index for FTS5 table main.ft2}}} + + do_execsql_test 10.2.$tn.16 { + DELETE FROM ft2_content; + INSERT INTO ft2(ft2) VALUES('rebuild'); + } + } + +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1); + INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three')); +} + +do_catchsql_test 11.1 { + SELECT fts5_columnlocale(x1, -1) FROM x1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 11.2 { + SELECT fts5_columnlocale(x1, 1) FROM x1('two'); +} {1 SQLITE_RANGE} + +#------------------------------------------------------------------------- +# +reset_db +do_test 12.0 { + list [catch { + sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create + } msg] $msg +} {1 {error in fts5_api.xCreateTokenizer_v2()}} + +#------------------------------------------------------------------------- +# Tests for auxiliary function fts5_get_locale(). +# +reset_db + +# Check that if the table does not support locale=1, fts5_get_locale() +# always returns NULL. +do_execsql_test 13.1.0 { + CREATE VIRTUAL TABLE nolocale USING fts5(a, b); + INSERT INTO nolocale VALUES('one two three', 'four five six'); + INSERT INTO nolocale VALUES('three two one', 'seven eight nine'); +} +do_execsql_test 13.1.1 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale; +} {1 1} +do_execsql_test 13.1.2 { + SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two'); +} {1} +do_execsql_test 13.1.3 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two'); +} {1 1} +do_execsql_test 13.1.4 { + SELECT + fts5_get_locale(nolocale, 1) IS NULL + FROM nolocale('three AND two') ORDER BY rank +} {1 1} +do_catchsql_test 13.1.5 { + SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.6 { + SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.7 { + SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.8 { + SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.9 { + SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two'); +} {1 {non-integer argument passed to function fts5_get_locale()}} + + +# Check that if the table does support locale=1, fts5_get_locale() +# returns the locale of the identified row/column. +do_execsql_test 13.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); + INSERT INTO ft VALUES( + fts5_locale('th_TH', 'one two three'), 'four five six seven' + ); + INSERT INTO ft VALUES( + 'three two one', fts5_locale('en_AU', 'seven eight nine') + ); +} + +do_execsql_test 13.2.1 { + SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.2 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.3 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rank +} { NULL 'en_AU' 'th_TH' NULL } +do_execsql_test 13.2.4 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_execsql_test 13.2.5 { + SELECT + quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_catchsql_test 13.2.6 { + SELECT + quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} +do_catchsql_test 13.2.7 { + SELECT + quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} + +#------------------------------------------------------------------------- +# Check that UPDATE statements that may affect more than one row work. +# +reset_db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); +} + +do_execsql_test 14.2 { + INSERT INTO ft VALUES('hello', 'world'); +} + +do_execsql_test 14.3 { + UPDATE ft SET b = fts5_locale('en_AU', 'world'); +} + +do_execsql_test 14.4 { + INSERT INTO ft VALUES(X'abcd', X'1234'); +} {} + +do_execsql_test 14.5 { + SELECT quote(a), quote(b) FROM ft +} {'hello' 'world' X'ABCD' X'1234'} + +do_execsql_test 14.6 { + DELETE FROM ft; + INSERT INTO ft VALUES(NULL, 'null'); + INSERT INTO ft VALUES(123, 'int'); + INSERT INTO ft VALUES(345.0, 'real'); + INSERT INTO ft VALUES('abc', 'text'); + INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text'); + + SELECT a, typeof(a), b FROM ft +} { + {} null null + 123 integer int + 345.0 real real + abc text text + def text text +} + +do_execsql_test 14.7 { + SELECT quote(c0), typeof(c0) FROM ft_content +} { + NULL null + 123 integer + 345.0 real + 'abc' text + 'def' text +} + +#------------------------------------------------------------------------- +# Check that inserting UNINDEXED columns between indexed columns of a +# locale=1 table does not cause a problem. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db + +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl); +} + +do_execsql_test 15.2 { + INSERT INTO ft VALUES('one', 'two', 'three'); + INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three')); +} + +do_execsql_test 15.3 { + SELECT c2, l2 FROM ft_content +} {three {} three loc} + +do_execsql_test 15.4 { + SELECT c, fts5_columnlocale(ft, 2) FROM ft +} {three {} three loc} + + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test new file mode 100644 index 0000000000000000000000000000000000000000..93361a5fe74a5fff2bc1bf3700b8a1cae8f4e049 --- /dev/null +++ b/ext/fts5/test/fts5matchinfo.test @@ -0,0 +1,523 @@ +# 2015 August 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5matchinfo + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { finish_test ; return } + +foreach_detail_mode $testprefix { + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +sqlite3_fts5_register_matchinfo db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%); +} + +do_execsql_test 1.1 { + INSERT INTO t1(content) VALUES('I wandered lonely as a cloud'); + INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,'); + INSERT INTO t1(content) VALUES('When all at once I saw a crowd,'); + INSERT INTO t1(content) VALUES('A host, of golden daffodils,'); + SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I'; +} {{1 1 1 2 2} {1 1 1 2 2}} + +# Now create an FTS4 table that does not specify matchinfo=fts3. +# +do_execsql_test 1.2 { + CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%); + INSERT INTO t2 SELECT * FROM t1; + SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I'; +} {{1 1 1 2 2} {1 1 1 2 2}} + + +#-------------------------------------------------------------------------- +# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function. +# +# The first argument - $tn - is a test identifier. This may be either a +# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix +# is set, just the numeric component (i.e. "1.1"). +# +# The second argument is the name of an FTSX table. The third is the +# full text of a WHERE/MATCH expression to query the table for +# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a +# key-value list (serialized array) with matchinfo() format specifiers +# as keys, and the results of executing the statement: +# +# SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr +# +# For example: +# +# CREATE VIRTUAL TABLE t1 USING fts4; +# INSERT INTO t1 VALUES('abc'); +# INSERT INTO t1 VALUES('def'); +# INSERT INTO t1 VALUES('abc abc'); +# +# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" { +# n {3 3} +# p {1 1} +# c {1 1} +# x {{1 3 2} {2 3 2}} +# } +# +# If the $results list contains keys mapped to "-" instead of a matchinfo() +# result, then this command computes the expected results based on other +# mappings to test the matchinfo() function. For example, the command above +# could be changed to: +# +# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" { +# n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}} +# pcx - +# } +# +# And this command would compute the expected results for matchinfo(t1, 'pcx') +# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and +# matchinfo(t1, 'x') in order to test 'pcx'. +# +proc do_matchinfo_test {tn tbl expr results} { + + foreach {fmt res} $results { + if {$res == "-"} continue + set resarray($fmt) $res + } + + set nRow 0 + foreach {fmt res} [array get resarray] { + if {[llength $res]>$nRow} { set nRow [llength $res] } + } + + # Construct expected results for any formats for which the caller + # supplied result is "-". + # + foreach {fmt res} $results { + if {$res == "-"} { + set res [list] + for {set iRow 0} {$iRow<$nRow} {incr iRow} { + set rowres [list] + foreach c [split $fmt ""] { + set rowres [concat $rowres [lindex $resarray($c) $iRow]] + } + lappend res $rowres + } + set resarray($fmt) $res + } + } + + # Test each matchinfo() request individually. + # + foreach {fmt res} [array get resarray] { + set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr" + do_execsql_test $tn.$fmt $sql [normalize2 $res] + } + + # Test them all executed together (multiple invocations of matchinfo()). + # + set exprlist [list] + foreach {format res} [array get resarray] { + lappend exprlist "mit(matchinfo($tbl, '$format'))" + } + set allres [list] + for {set iRow 0} {$iRow<$nRow} {incr iRow} { + foreach {format res} [array get resarray] { + lappend allres [lindex $res $iRow] + } + } + set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr" + do_execsql_test $tn.multi $sql [normalize2 $allres] +} +proc normalize2 {list_of_lists} { + set res [list] + foreach elem $list_of_lists { + lappend res [list {*}$elem] + } + return $res +} + +# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5 +# mode is not detail=full. +# +proc do_matchinfo_p_test {tn tbl expr results} { + if {[detail_is_full]} { + uplevel [list do_matchinfo_test $tn $tbl $expr $results] + } +} + +do_execsql_test 4.1.0 { + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); + INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); +} + +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + s {{3 0} {0 3}} +} + +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + p {3 3} + x { + {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1} + {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1} + } +} + +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + p {3 3} + c {2 2} + x { + {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1} + {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1} + } + n {2 2} + l {{5 5} {5 5}} + a {{5 5} {5 5}} + + s {{3 0} {0 3}} + + xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - + xpxsscplax - +} + +do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} { + p {1 1} + c {2 2} + x { + {0 1 1 1 1 1} + {1 1 1 0 1 1} + } + n {2 2} + l {{5 5} {5 5}} + a {{5 5} {5 5}} + + s {{0 1} {1 0}} + + xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - + sxsxs - +} + +do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } +do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } +do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } +do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } +do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { + x { + {0 1 1 1 1 1 0 0 0 0 0 0} + {1 1 1 0 1 1 0 0 0 0 0 0} + } +} +do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} { + x { + {0 1 1 1 1 1 0 0 0 0 0 0} + {1 1 1 0 1 1 0 0 0 0 0 0} + } +} + +do_execsql_test 4.2.0 { + CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%); + INSERT INTO t5 VALUES('a a a a a'); + INSERT INTO t5 VALUES('a b a b a'); + INSERT INTO t5 VALUES('c b c b c'); + INSERT INTO t5 VALUES('x x x x x'); +} +do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 8 2} {3 8 2 3 8 2}} + s {2 1} +} +do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} } +do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} } +do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } + +do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; + +# It used to be that the second 'a' token would be deferred. That doesn't +# work any longer. +if 0 { + do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 5 5} {3 8 2 3 5 5}} + s {2 1} + } +} + +do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } +do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } +do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } + +do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } + +do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } +do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } +do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} } +do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} } +do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } + +do_execsql_test 4.5.0 { + CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO t6 VALUES('a', 'b', 'c'); +} +do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } + + +#------------------------------------------------------------------------- +# Test the outcome of matchinfo() when used within a query that does not +# use the full-text index (i.e. lookup by rowid or full-table scan). +# +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%); + INSERT INTO t10 VALUES('first record'); + INSERT INTO t10 VALUES('second record'); +} +do_execsql_test 7.2 { + SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10; +} {blob 8 blob 8} +do_execsql_test 7.3 { + SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE rowid=1; +} {blob 8} +do_execsql_test 7.4 { + SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) + FROM t10 WHERE t10 MATCH 'record' +} {blob 20 blob 20} + +#------------------------------------------------------------------------- +# Test a special case - matchinfo('nxa') with many zero length documents. +# Special because "x" internally uses a statement used by both "n" and "a". +# This was causing a problem at one point in the obscure case where the +# total number of bytes of data stored in an fts3 table was greater than +# the number of rows. i.e. when the following query returns true: +# +# SELECT sum(length(content)) < count(*) FROM fts4table; +# +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%); + INSERT INTO t11(t11, rank) VALUES('pgsz', 32); + INSERT INTO t11 VALUES('quitealongstringoftext'); + INSERT INTO t11 VALUES('anotherquitealongstringoftext'); + INSERT INTO t11 VALUES('athirdlongstringoftext'); + INSERT INTO t11 VALUES('andonemoreforgoodluck'); +} +do_test 8.2 { + for {set i 0} {$i < 200} {incr i} { + execsql { INSERT INTO t11 VALUES('') } + } + execsql { INSERT INTO t11(t11) VALUES('optimize') } +} {} +do_execsql_test 8.3 { + SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*' +} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}} + +#------------------------------------------------------------------------- + +if {[detail_is_full]} { + do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%); + INSERT INTO t12 VALUES('a b c d'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } {{0 1 1 0 1 1 1 1 1}} + do_execsql_test 9.2 { + INSERT INTO t12 VALUES('a d c d'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } { + {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} + } + do_execsql_test 9.3 { + INSERT INTO t12 VALUES('a d d a'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } { + {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} + } +} + +#--------------------------------------------------------------------------- +# Test for a memory leak +# +do_execsql_test 10.1 { + DROP TABLE t10; + CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%); + INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); + SELECT t10.rowid, t10.* + FROM t10 + JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x + WHERE t10 MATCH x.idx + AND matchinfo(t10) not null + GROUP BY t10.rowid + ORDER BY 1; +} {1 1 one 2 2 two 3 3 three} + +#--------------------------------------------------------------------------- +# Test the 'y' matchinfo flag +# +reset_db +sqlite3_fts5_register_matchinfo db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%); + INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 + INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 + INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 + INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4 + INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5 + INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6 + INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7 + INSERT INTO tt VALUES('a c b b g f', 'a b a e d f'); -- 8 + INSERT INTO tt VALUES('b a f e c c', 'f d b b a b'); -- 9 + INSERT INTO tt VALUES('f d c e a c', 'f a f a a f'); -- 10 +} + +db func mit mit +foreach {tn expr res} { + 1 "a" { + 1 {1 2} 2 {1 0} 3 {0 1} 4 {1 0} 5 {1 0} + 6 {1 0} 7 {2 1} 8 {1 2} 9 {1 1} 10 {1 3} + } + + 2 "b" { + 1 {0 1} 2 {1 0} 3 {1 2} 4 {0 1} 5 {0 1} + 6 {2 2} 8 {2 1} 9 {1 3} + } + + 3 "y:a" { + 1 {0 2} 3 {0 1} + 7 {0 1} 8 {0 2} 9 {0 1} 10 {0 3} + } + + 4 "x:a" { + 1 {1 0} 2 {1 0} 4 {1 0} 5 {1 0} + 6 {1 0} 7 {2 0} 8 {1 0} 9 {1 0} 10 {1 0} + } + + 5 "a OR b" { + 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1} + 6 {1 0 2 2} 7 {2 1 0 0} 8 {1 2 2 1} 9 {1 1 1 3} 10 {1 3 0 0} + } + + 6 "a AND b" { + 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1} + 6 {1 0 2 2} 8 {1 2 2 1} 9 {1 1 1 3} + } + + 7 "a OR (a AND b)" { + 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} + 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} + 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} + } + +} { + + if {[string match *:* $expr] && [detail_is_none]} continue + do_execsql_test 11.1.$tn.1 { + SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr + } $res + + set r2 [list] + foreach {rowid L} $res { + lappend r2 $rowid + set M [list] + foreach {a b} $L { + lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)] + } + lappend r2 $M + } + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 + + do_execsql_test 11.1.$tn.2 { + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr + } $r2 +} + +#--------------------------------------------------------------------------- +# Test the 'b' matchinfo flag +# +reset_db +sqlite3_fts5_register_matchinfo db +db func mit mit + +do_test 12.0 { + set cols [list] + for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)" +} {} + +do_execsql_test 12.1 { + INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); + SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; +} [list [list [expr 1<<4] [expr 1<<(45-32)]]] + +} ;# foreach_detail_mode + +#------------------------------------------------------------------------- +# Test that a bad fts5() return is detected +# +reset_db +proc xyz {} {} +db func fts5 -argcount 1 xyz +do_test 13.1 { + list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +# Test that an invalid matchinfo() flag is detected +# +reset_db +sqlite3_fts5_register_matchinfo db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE x1 USING fts5(z); + INSERT INTO x1 VALUES('a b c a b c a b c'); +} {} + +do_catchsql_test 14.2 { + SELECT matchinfo(x1, 'd') FROM x1('a b c'); +} {1 {unrecognized matchinfo flag: d}} + +#------------------------------------------------------------------------- +# Test using matchinfo() and similar on a non-full-text query +# +do_execsql_test 15.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y); + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); +} + +if {$tcl_platform(byteOrder)=="littleEndian"} { + set res {X'02000000'} +} else { + set res {X'00000002'} +} +do_execsql_test 15.1 { + SELECT quote(matchinfo(t1, 'n')) FROM t1 LIMIT 1; +} $res +do_execsql_test 15.2 { + DELETE FROM t1_content WHERE rowid=1; + SELECT quote(matchinfo(t1, 'n')) FROM t1 LIMIT 1; +} $res + +fts5_aux_test_functions db +do_execsql_test 15.3 { + SELECT fts5_test_all(t1) FROM t1 LIMIT 1; +} { + {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2} +} + +finish_test diff --git a/ext/fts5/test/fts5merge.test b/ext/fts5/test/fts5merge.test new file mode 100644 index 0000000000000000000000000000000000000000..c57c21ded3cea93dcb1f295a5b098c3de9188949 --- /dev/null +++ b/ext/fts5/test/fts5merge.test @@ -0,0 +1,243 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that focus on incremental merges of segments. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5merge + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +db func repeat [list string repeat] + +#------------------------------------------------------------------------- +# Create an fts index so that: +# +# * the index consists of two top-level segments +# * each segment contains records related to $nRowPerSeg rows +# * all rows consist of tokens "x" and "y" only. +# +# Then run ('merge', 1) until everything is completely merged. +# +proc do_merge1_test {testname nRowPerSeg} { + set ::nRowPerSeg [expr $nRowPerSeg] + do_execsql_test $testname.0 { + DROP TABLE IF EXISTS x8; + CREATE VIRTUAL TABLE x8 USING fts5(i); + INSERT INTO x8(x8, rank) VALUES('pgsz', 32); + + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg) + INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii; + + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg) + INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii; + + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); + } + + for {set tn 1} {[lindex [fts5_level_segs x8] 0]>0} {incr tn} { + do_execsql_test $testname.$tn { + INSERT INTO x8(x8, rank) VALUES('merge', 1); + INSERT INTO x8(x8) VALUES('integrity-check'); + } + if {$tn>5} break + } + + do_test $testname.x [list expr "$tn < 5"] 1 +} + +do_merge1_test 1.1 1 +do_merge1_test 1.2 2 +do_merge1_test 1.3 3 +do_merge1_test 1.4 4 +do_merge1_test 1.5 10 +do_merge1_test 1.6 20 +do_merge1_test 1.7 100 + +#------------------------------------------------------------------------- +# +proc do_merge2_test {testname nRow} { + db func rnddoc fts5_rnddoc + + do_execsql_test $testname.0 { + DROP TABLE IF EXISTS x8; + CREATE VIRTUAL TABLE x8 USING fts5(i); + INSERT INTO x8(x8, rank) VALUES('pgsz', 32); + } + + set ::nRow $nRow + do_test $testname.1 { + for {set i 0} {$i < $::nRow} {incr i} { + execsql { INSERT INTO x8 VALUES( rnddoc(($i%16) + 5) ) } + while {[not_merged x8]} { + execsql { + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); + INSERT INTO x8(x8, rank) VALUES('merge', 1); + INSERT INTO x8(x8, rank) VALUES('usermerge', 16); + INSERT INTO x8(x8) VALUES('integrity-check'); + } + } + } + } {} +} +proc not_merged {tbl} { + set segs [fts5_level_segs $tbl] + foreach s $segs { if {$s>1} { return 1 } } + return 0 +} + +do_merge2_test 2.1 5 +do_merge2_test 2.2 10 +do_merge2_test 2.3 20 + +#------------------------------------------------------------------------- +# Test that a merge will complete any merge that has already been +# started, even if the number of input segments is less than the current +# value of the 'usermerge' configuration parameter. +# +db func rnddoc fts5_rnddoc + +do_execsql_test 3.1 { + DROP TABLE IF EXISTS x8; + CREATE VIRTUAL TABLE x8 USING fts5(i); + INSERT INTO x8(x8, rank) VALUES('pgsz', 32); + INSERT INTO x8 VALUES(rnddoc(100)); + INSERT INTO x8 VALUES(rnddoc(100)); +} +do_test 3.2 { + execsql { + INSERT INTO x8(x8, rank) VALUES('usermerge', 4); + INSERT INTO x8(x8, rank) VALUES('merge', 1); + } + fts5_level_segs x8 +} {2} + +do_test 3.3 { + execsql { + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); + INSERT INTO x8(x8, rank) VALUES('merge', 1); + } + fts5_level_segs x8 +} {2 1} + +do_test 3.4 { + execsql { INSERT INTO x8(x8, rank) VALUES('usermerge', 4) } + while {[not_merged x8]} { + execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) } + } + fts5_level_segs x8 +} {0 1} + +#------------------------------------------------------------------------- +# +proc mydoc {} { + set x [lindex {a b c d e f g h i j} [expr int(rand()*10)]] + return [string repeat "$x " 30] +} +db func mydoc mydoc + +proc mycount {} { + set res [list] + foreach x {a b c d e f g h i j} { + lappend res [db one {SELECT count(*) FROM x8 WHERE x8 MATCH $x}] + } + set res +} + + #1 32 +foreach {tn pgsz} { + 2 1000 +} { + do_execsql_test 4.$tn.1 { + DROP TABLE IF EXISTS x8; + CREATE VIRTUAL TABLE x8 USING fts5(i); + INSERT INTO x8(x8, rank) VALUES('pgsz', $pgsz); + } + + do_execsql_test 4.$tn.2 { + INSERT INTO x8(x8, rank) VALUES('merge', 1); + } + + do_execsql_test 4.$tn.3 { + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) + INSERT INTO x8 SELECT mydoc() FROM ii; + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) + INSERT INTO x8 SELECT mydoc() FROM ii; + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); + } + + set expect [mycount] + for {set i 0} {$i < 20} {incr i} { + do_test 4.$tn.4.$i { + execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1); } + mycount + } $expect + break + } +# db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r } +} + +#------------------------------------------------------------------------- +# Test that the 'merge' command does not modify the database if there is +# no work to do. + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x9 USING fts5(one, two); + INSERT INTO x9(x9, rank) VALUES('pgsz', 32); + INSERT INTO x9(x9, rank) VALUES('automerge', 2); + INSERT INTO x9(x9, rank) VALUES('usermerge', 2); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); +} + +do_test 5.2 { + while 1 { + set nChange [db total_changes] + execsql { INSERT INTO x9(x9, rank) VALUES('merge', 1); } + set nChange [expr [db total_changes] - $nChange] + #puts $nChange + if {$nChange<2} break + } +} {} + + +#-------------------------------------------------------------------------- +# Test that running 'merge' on an empty database does not cause a +# problem. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE g1 USING fts5(a, b); +} +do_execsql_test 6.1 { + INSERT INTO g1(g1, rank) VALUES('merge', 10); +} +do_execsql_test 6.2 { + INSERT INTO g1(g1, rank) VALUES('merge', -10); +} +do_execsql_test 6.3 { + INSERT INTO g1(g1) VALUES('integrity-check'); +} + + + +finish_test diff --git a/ext/fts5/test/fts5merge2.test b/ext/fts5/test/fts5merge2.test new file mode 100644 index 0000000000000000000000000000000000000000..e8f5bb161cc2b6d1fe4466be81ef961a70dae077 --- /dev/null +++ b/ext/fts5/test/fts5merge2.test @@ -0,0 +1,57 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that focus on incremental merges of segments. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5merge2 +return_if_no_fts5 + +proc dump_structure {} { + db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} { + foreach lvl [lrange $t 1 end] { + set seg [string repeat . [expr [llength $lvl]-2]] + puts "[lrange $lvl 0 1] $seg" + } + } +} + +foreach_detail_mode $testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('crisismerge', 2); + INSERT INTO t1 VALUES('1 2 3 4'); +} + +expr srand(0) +db func rnddoc fts5_rnddoc +do_test 1.1 { + for {set i 0} {$i < 100} {incr i} { + execsql { + BEGIN; + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1(rowid, x) VALUES(1, '1 2 3 4'); + INSERT INTO t1 VALUES(rnddoc(10)); + COMMIT; + } + } +} {} + +do_execsql_test 1.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +} + +finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test new file mode 100644 index 0000000000000000000000000000000000000000..c2e580c5640912a5d4fc1aa714fea3ce8eaf0369 --- /dev/null +++ b/ext/fts5/test/fts5misc.test @@ -0,0 +1,669 @@ +# 2019 September 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5misc + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); +} + +do_catchsql_test 1.1.1 { + SELECT highlight(t1, 4, '', '') FROM t1('*'); +} {1 {unknown special query: }} +do_catchsql_test 1.1.2 { + SELECT a FROM t1 + WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*')); +} {1 {unknown special query: }} + +do_catchsql_test 1.2.1 { + SELECT highlight(t1, 4, '', '') FROM t1('*id'); +} {1 {no such cursor: 4}} + +do_catchsql_test 1.2.2 { + SELECT a FROM t1 + WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*id')); +} {1 {no such cursor: 6}} + +do_catchsql_test 1.3.1 { + SELECT highlight(t1, 4, '', '') FROM t1('*reads'); +} {1 {no such cursor: 0}} + +do_catchsql_test 1.3.2 { + SELECT a FROM t1 + WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); +} {1 {no such cursor: 0}} + +db close +sqlite3 db test.db + +do_catchsql_test 1.3.3 { + SELECT a FROM t1 + WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); +} {1 {no such cursor: 0}} + +fts5_aux_test_functions db +do_catchsql_test 1.3.4 { + SELECT fts5_columntext(t1) FROM t1('*reads'); +} {1 {no such cursor: 0}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t0(c0); + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} +do_execsql_test 2.1.1 { + BEGIN TRANSACTION; + INSERT INTO vt0(c0) VALUES ('xyz'); +} +do_execsql_test 2.1.2 { + ALTER TABLE t0 ADD COLUMN c5; +} +do_execsql_test 2.1.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} +do_execsql_test 2.1.4 { + INSERT INTO vt0(c0) VALUES ('abc'); + COMMIT +} +do_execsql_test 2.1.5 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +reset_db +do_execsql_test 2.2.1 { + CREATE TABLE t0(c0); + CREATE VIRTUAL TABLE vt0 USING fts5(c0); + BEGIN TRANSACTION; + INSERT INTO vt0(c0) VALUES ('xyz'); +} + +do_execsql_test 2.2.2 { + ALTER TABLE t0 RENAME TO t1; +} +do_execsql_test 2.2.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} +do_execsql_test 2.2.4 { + INSERT INTO vt0(c0) VALUES ('abc'); + COMMIT; +} +do_execsql_test 2.2.5 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE vt0 USING fts5(a); + PRAGMA reverse_unordered_selects = true; + INSERT INTO vt0 VALUES('365062398'), (0), (0); + INSERT INTO vt0(vt0, rank) VALUES('pgsz', '38'); +} +do_execsql_test 3.1 { + UPDATE vt0 SET a = 399905135; -- unexpected: database disk image is malformed +} +do_execsql_test 3.2 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE vt0 USING fts5(c0); + INSERT INTO vt0(c0) VALUES ('xyz'); +} + +do_execsql_test 4.1 { + BEGIN; + INSERT INTO vt0(c0) VALUES ('abc'); + INSERT INTO vt0(vt0) VALUES('rebuild'); + COMMIT; +} + +do_execsql_test 4.2 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +do_execsql_test 4.3 { + BEGIN; + INSERT INTO vt0(vt0) VALUES('rebuild'); + INSERT INTO vt0(vt0) VALUES('rebuild'); + COMMIT; +} + +do_execsql_test 4.4 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Ticket [81a7f7b9]. +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE vt0 USING fts5(c0, c1); + INSERT INTO vt0(vt0, rank) VALUES('pgsz', '65536'); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1236 + ) + INSERT INTO vt0(c0) SELECT '0' FROM s; +} {} + +do_execsql_test 5.1 { + UPDATE vt0 SET c1 = 'T,D&p^y/7#3*v100; } {101} + +do_execsql_test 1.9 { + DELETE FROM ft; + INSERT INTO ft(ft) VALUES('optimize'); + SELECT count(*) FROM ft_data; +} {2} +do_execsql_test 1.10 { + BEGIN; + INSERT INTO ft VALUES('Hello'); + INSERT INTO ft VALUES('hello'); + INSERT INTO ft VALUES('HELLO'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('World'); + INSERT INTO ft VALUES('world'); + INSERT INTO ft VALUES('WORLD'); +} + +do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3} +do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6} +do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9} +do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3} + +#------------------------------------------------------------------------ +reset_db +sqlite3_fts5_register_origintext db +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING fts5( + v, tokenize="origintext unicode61", tokendata=1, detail=none + ); + + INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy'); + INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy'); +} + +do_execsql_test 2.1 { + SELECT tokens(x1) FROM x1('xxx'); +} { + {xxx xxx.Xxx xxx.XXX} {xxx xxx} +} + +do_execsql_test 2.2 { + UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1; +} + +do_execsql_test 2.3 { + SELECT tokens(x1) FROM x1('xxx'); +} { + {xxx {} xxx} {xxx xxx} +} + +finish_test + diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test new file mode 100644 index 0000000000000000000000000000000000000000..351ab1f6172fec67c0aa6a6c3ca0cf26669389a1 --- /dev/null +++ b/ext/fts5/test/fts5origintext3.test @@ -0,0 +1,141 @@ +# 2023 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + foreach {tn insttoken} { + 1 0 + 2 1 + } { + + reset_db + + sqlite3_fts5_register_origintext db + fts5_aux_test_functions db + proc insttoken {cmd iIdx iToken} { + set txt [$cmd xInstToken $iIdx $iToken] + string map [list "\0" "."] $txt + } + sqlite3_fts5_create_function db insttoken insttoken + + do_execsql_test $tn.1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + } + + do_execsql_test $tn.1.0.1 { + INSERT INTO ft(ft, rank) VALUES('insttoken', 1); + } + + do_execsql_test $tn.1.1 { + INSERT INTO ft VALUES('Hello world HELLO WORLD hello'); + } + + do_execsql_test $tn.1.2 { + SELECT fts5_test_poslist(ft) FROM ft('hello'); + } {{0.0.0 0.0.2 0.0.4}} + + do_execsql_test $tn.1.3 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hello'); + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.3.1 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hel*'); + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.4 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hello') ORDER BY rank; + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.5 { + CREATE VIRTUAL TABLE ft2 USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE'); + INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO'); + INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE'); + } + + do_execsql_test $tn.1.6 { + SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank; + } {three.THREE 3 three 1 three 2} + + do_execsql_test $tn.1.7 { + INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb'); + INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB'); + } + + do_execsql_test $tn.1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24} + do_execsql_test $tn.1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24} + + do_execsql_test $tn.2.0 { + CREATE VIRTUAL TABLE ft3 USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + prefix=2 + ); + } + do_execsql_test $tn.2.1 { + INSERT INTO ft3(rowid, x) VALUES(1, 'one'); + INSERT INTO ft3(rowid, x) VALUES(2, 'ONE'); + INSERT INTO ft3(rowid, x) VALUES(3, 'ONT'); + INSERT INTO ft3(rowid, x) VALUES(4, 'on'); + INSERT INTO ft3(rowid, x) VALUES(5, 'On'); + } + + do_execsql_test $tn.2.2 { + SELECT rowid FROM ft3('on*'); + } {1 2 3 4 5} + + do_execsql_test $tn.2.3 { + SELECT rowid, insttoken(ft3, 0, 0) FROM ft3('on*'); + } {1 one 2 one.ONE 3 ont.ONT 4 on 5 on.On} + + } +} + +finish_test + diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test new file mode 100644 index 0000000000000000000000000000000000000000..3b907ba2cc40459321f7dae2dd967d52986f8507 --- /dev/null +++ b/ext/fts5/test/fts5origintext4.test @@ -0,0 +1,80 @@ +# 2023 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# The tests below verify that a doclist-index is used to limit the number +# of pages loaded into the cache. It does this by querying sqlite3_db_status() +# for the amount of memory used by the pager cache. +# +# memsubsys1 effectively limits the page-cache to 24 pages. Which masks +# the effect tested by the tests in this file. And "mmap" prevents the +# cache from being used, also preventing these tests from working. +# +if {[permutation]=="memsubsys1" || [permutation]=="mmap"} { + finish_test + return +} + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + PRAGMA page_size = 4096; + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft SELECT 'the first thing'; + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000 + ) + INSERT INTO ft SELECT 'The second thing' FROM s; + + INSERT INTO ft SELECT 'the first thing'; + COMMIT; + INSERT INTO ft(ft) VALUES('optimize'); +} + +foreach {tn sql expr} { + 1 { SELECT rowid FROM ft('the') } {$mem > 250000} + 2 { SELECT rowid FROM ft('first') } {$mem < 50000} + 3 { SELECT rowid FROM ft('the first') } {$mem < 50000} +} { + db close + sqlite3 db test.db + sqlite3_fts5_register_origintext db + + execsql $sql + do_test 1.2.$tn { + set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1] + expr $expr + } 1 +} + +proc b {x} { string map [list "\0" "."] $x } +db func b b +# execsql_pp { SELECT segid, b(term), pgno from ft_idx } + +finish_test + diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test new file mode 100644 index 0000000000000000000000000000000000000000..848cc15b5cda0a1396ae3b1c9f07562d57612bb5 --- /dev/null +++ b/ext/fts5/test/fts5origintext5.test @@ -0,0 +1,273 @@ +# 2023 Dec 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for tables that use both tokendata=1 and contentless_delete=1. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Return a random integer between 0 and n-1. +# +proc random {n} { expr {abs(int(rand()*$n))} } + +# Select an element of the list passed as the only argument at random and +# return it. +# +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} + +# Given a term that consists entirely of alphabet characters, return all +# permutations of the term using upper and lower case characters. e.g. +# +# "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba} +# +proc casify {term {lRet {{}}}} { + if {$term==""} { return $lRet } + set t [string range $term 1 end] + set f1 [string toupper [string range $term 0 0]] + set f2 [string tolower [string range $term 0 0]] + set ret [list] + foreach x $lRet { + lappend ret "$x$f1" + lappend ret "$x$f2" + } + return [casify $t $ret] +} + +proc vocab {} { + list abc def ghi jkl mno pqr stu vwx yza +} + +# Return a random 3 letter term. +# +proc term {} { + if {[info exists ::expanded_vocab]==0} { + foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] } + } + + select_one $::expanded_vocab +} + +# Return a document - between 3 and 10 terms. +# +proc document {} { + set nTerm [expr [random 3] + 7] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +#------------------------------------------------------------------------- + +expr srand(6) + +set NDOC 200 +set NLOOP 50 + +sqlite3_fts5_register_origintext db + +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +proc rankfunc {cmd} { + $cmd xRowid +} +sqlite3_fts5_create_function db rankfunc rankfunc + +proc ctrl_tokens {term args} { + set ret [list] + set term [string tolower $term] + foreach doc $args { + foreach a $doc { + if {[string tolower $a]==$term} { + if {$a==$term} { + lappend ret $a + } else { + lappend ret [string tolower $a].$a + } + } + } + } + set ret +} +db func ctrl_tokens ctrl_tokens + +proc do_all_vocab_test {tn} { + foreach ::v [concat [vocab] nnn] { + set answer [execsql { + SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%' + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft) FROM ft($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank + } $answer + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + + CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()'); +} +do_test 1.1 { + for {set ii 0} {$ii < $NDOC} {incr ii} { + set doc [document] + execsql { + INSERT INTO ft(rowid, x) VALUES($ii, $doc); + INSERT INTO ctrl(id, x) VALUES($ii, $doc); + } + } +} {} + +#execsql_pp { SELECT * FROM ctrl } +#execsql_pp { SELECT * FROM ft } +#fts5_aux_test_functions db +#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); } + +do_all_vocab_test 1.2 + +for {set ii 0} {$ii < $NLOOP} {incr ii} { + set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }] + foreach r $lRowid { + execsql { DELETE FROM ft WHERE rowid = $r } + execsql { DELETE FROM ctrl WHERE rowid = $r } + + set doc [document] + execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) } + execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) } + } + do_all_vocab_test 1.3.$ii +} + +#------------------------------------------------------------------------- + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts5( + x, y, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + + CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT); + INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64); + INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()'); +} +do_test 2.1 { + for {set ii 0} {$ii < $NDOC} {incr ii} { + set doc1 [document] + set doc2 [document] + execsql { + INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2); + INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2); + } + } +} {} + +proc do_all_vocab_test2 {tn} { + foreach ::v [vocab] { + set answer [execsql { + SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2 + WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%'; + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft2) FROM ft2($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank + } $answer + } +} + +do_all_vocab_test2 2.2 + +for {set ii 0} {$ii < $NLOOP} {incr ii} { + set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }] + foreach r $lRowid { + execsql { DELETE FROM ft2 WHERE rowid = $r } + execsql { DELETE FROM ctrl2 WHERE rowid = $r } + + set doc1 [document] + set doc2 [document] + execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) } + execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) } + } + do_all_vocab_test 2.3.$ii +} + +#------------------------------------------------------------------------- + +unset -nocomplain ::expanded_vocab +proc vocab {} { + list abcde fghij klmno +} + +proc do_all_vocab_test3 {tn} { + foreach ::v [concat [vocab] nnn] { + set answer [execsql { + SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%' + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft3) FROM ft3($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank + } $answer + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft3 USING fts5( + w, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()'); + CREATE TABLE ctrl3(w); +} + +do_execsql_test 3.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2 + ) + INSERT INTO ctrl3 SELECT document() FROM s; + INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3; +} + +do_all_vocab_test3 3.2 + + +finish_test + diff --git a/ext/fts5/test/fts5origintext6.test b/ext/fts5/test/fts5origintext6.test new file mode 100644 index 0000000000000000000000000000000000000000..7b27e310b9e1f2b7f6b59e2af1e4db0ef49ba757 --- /dev/null +++ b/ext/fts5/test/fts5origintext6.test @@ -0,0 +1,209 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext6 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc insert_data {tbl} { + db eval " + INSERT INTO $tbl (rowid, x, y) VALUES + (1, 'ChH BDd HhG efc BjJ BGi GBG FdD','ciJ AFf ADf fBJ fhC GFI JEH fcA'), + (2, 'deg AIG Fie jII cCd Hbf igF fEE','GeA Ija gJg EDc HFi DDI dCf aDd'), + (3, 'IJC hga deC Jfa Aeg hfh CcH dfb','ajD hgC Jaf IfH CHe jIG AjD adF'), + (4, 'FiH GJH IDA AiG bBc CGG Eih bIH','hHg JaH aii IHE Ggd gcH gji CGc'), + (5, 'ceg CAd jFI GAB BGg EeC IdH acG','bBC eIG ifH eDE Adj bjb GCj ebA'), + (6, 'Eac Fbh aFF Eea jeG EIj HCc JJH','hbd giE Gfe eiI dEF abE cJf cAb'), + (7, 'dic hAc jEC AiG FEF jHc HiD HBI','aEd ebE Gfi AJG EBA faj GiG jjE'), + (8, 'Fca iEe EgE jjJ gce ijf EGc EBi','gaI dhH bFg CFc HeC CjI Jfg ccH'), + (9, 'cfd iaa HCf iHJ HjG ffh ABb ibi','CfG bia Dai eii Ejg Jeg fCg hDb'), + (10, 'Jjf hJC IID HJj bGB EbJ cgg eBj','jci jhi JAF jIg Bei Bcd cAC AJd'), + (11, 'egG Cdi bFf fEB hfH jDH jia Efd','FAd eCg fAi aiC baC eJG acF iGE'), + (12, 'Ada Gde CJI ADG gJA Cbb ccF iAB','eAE ajC FBB ccd Jgh fJg ieg hGE'), + (13, 'gBb fDG Jdd HdD fiJ Bed Cig iGg','heC FeI iaj gdg ebB giC HaD FIe'), + (14, 'FiI iDd Ffe igI bgB EJf FHG hDF','cjC AeI abf Fah cbJ ffH jEb aib'), + (15, 'jaF hBI jIH Gdh FEc Fij hgj jFh','dGA ADH feh AAI AfJ DbC gBi hGH'), + (16, 'gjH BGg iGj aFE CAH edI idf HEH','hIf DDg fjB hGi cHF BCH FjG Bgd'), + (17, 'iaI JGH hji gcj Dda eeG jDd CBi','cHg jeh caG gIc feF ihG hgJ Abj'), + (18, 'jHI iDB eFf AiH EFB CDb IAj GbC','Ghe dEI gdI jai gib dAG BIa djb'), + (19, 'abI fHG Ccf aAc FDa fiC agF bdB','afi hde IgE bGF cfg DHD diE aca'), + (20, 'IFh eDJ jfh cDg dde JGJ GAf fIJ','IBa EfH faE aeI FIF baJ FGj EIH'), + (21, 'Dee bFC bBA dEI CEj aJI ghA dCH','hBA ddA HJh dfj egI Dij dFE bGE'), + (22, 'JFE BCj FgA afc Jda FGD iHJ HDh','eAI jHe BHD Gah bbD Bgj gbh eGB'), + (23, 'edE CJE FjG aFI edA Cea FId iFe','ABG jcA ddj EEc Dcg hAI agA biA'), + (24, 'AgE cfc eef cGh aFB DcH efJ hcH','eGF HaB diG fgi bdc iGJ FGJ fFB'), + (25, 'aCa AgI GhC DDI hGJ Hgc Gcg bbG','iID Fga jHa jIj idj DFD bAC AFJ'), + (26, 'gjC JGh Fge faa eCA iGG gHE Gai','bDi hFE BbI DHD Adb Fgi hCa Hij'), + (27, 'Eji jEI jhF DFC afH cDh AGc dHA','IDe GcA ChF DIb Bif HfH agD DGh'), + (28, 'gDD AEE Dfg ICf Cbi JdE jgH eEi','eEb dBG FDE jgf cAI FaJ jaA cDd'), + (29, 'cbe Gec hgB Egi bca dHg bAJ jBf','EFB DgD GJc fDb EeE bBA GFC Hbe'), + (30, 'Adc eHB afI hDc Bhh baE hcJ BBd','JAH deg bcF Dab Bgj Gbb JHi FIB'), + (31, 'agF dIj AJJ Hfg cCG hED Igc fHC','JEf eia dHf Ggc Agj geD bEE Gei'), + (32, 'DAd cCe cbJ FjG gJe gba dJA GCf','eAf hFc bGE ABI hHA IcE abF CCE'), + (33, 'fFh jJe DhJ cDJ EBi AfD eFI IhG','fEG GCc Bjd EFF ggg CFe EHd ciB'), + (34, 'Ejb BjI eAF HaD eEJ FaG Eda AHC','Iah hgD EJG fdD cIE Daj IFf eJh'), + (35, 'aHG eCe FjA djJ dAJ jiJ IaE GGB','Acg iEF JfB FIC Eei ggj dic Iii'), + (36, 'Fdb EDF GaF JjB ehH IgC hgi DCG','cag DHI Fah hAJ bbh egG Hia hgJ'), + (37, 'HGg icC JEC AFJ Ddh dhi hfC Ich','fEg bED Bff hCJ EiA cIf bfG cGA'), + (38, 'aEJ jGI BCi FaA ebA BHj cIJ GcC','dCH ADd bGB cFE AgF geD cbG jIc'), + (39, 'JFB bBi heA BFA hgB Ahj EIE CgI','EIJ JFG FJE GeA Hdg HeH ACh GiA'), + (40, 'agB DDC CED igC Dfc DhI eiC fHi','dAB dcg iJF cej Fcc cAc AfB Fdd'), + (41, 'BdF DHj Ege hcG DEd eFa dCf gBb','FBG ChB cej iGd Hbh fCc Ibe Abh'), + (42, 'Bgc DjI cbC jGD bdb hHB IJA IJH','heg cii abb IGf eDe hJc dii fcE'), + (43, 'fhf ECa FiA aDh Jbf CiB Jhe ajD','GFE bIF aeD gDE BIE Jea DfC BEc'), + (44, 'GjE dBj DbJ ICF aDh EEH Ejb jFb','dJj aEc IBg bEG Faf fjA hjf FAF'), + (45, 'BfA efd IIJ AHG dDF eGg dIJ Gcb','Bfj jeb Ahc dAE ACH Dfb ieb dhC'), + (46, 'Ibj ege geC dJh CIi hbD EAG fGA','DEb BFe Bjg FId Fhg HeF JAc BbE'), + (47, 'dhB afC hgG bEJ aIe Cbe iEE JCD','bdg Ajc FGA jbh Jge iAj fIA jbE'), + (48, 'egH iDi bfH iiI hGC jFF Hfd AHB','bjE Beb iCc haB gIH Dea bga dfd'), + (49, 'jgf chc jGc Baj HBb jdE hgh heI','FFB aBd iEB EIG HGf Bbj EIi JbI'), + (50, 'jhe EGi ajA fbH geh EHe FdC bij','jDE bBC gbH HeE dcH iBH IFE AHi'), + (51, 'aCb JiD cgJ Bjj iAI Hbe IAF FhH','ijf bhE Jdf FED dCH bbG HcJ ebH'); + " +} + +foreach_detail_mode $testprefix { +foreach external {0 1 2} { + reset_db + + proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret + } + sqlite3_fts5_create_function db tokens tokens + sqlite3_fts5_register_origintext db + + set E(0) internal + set E(1) external + set E(2) contentless + set e $E($external) + + db eval { CREATE TABLE ex(x, y) } + switch -- $external { + 0 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + } + } + + 1 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + content=ex + ); + } + } + + 2 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + content= + ); + } + } + } + insert_data ex + insert_data ft + + proc prefixquery {prefix bInst bYOnly} { + set ret [list] + db eval { SELECT rowid, x, y FROM ex ORDER BY rowid } { + set row [list] + set bSeen 0 + + set T [concat $x $y] + if {$bYOnly} { set T $y } + + foreach w $T { + if {[string match -nocase $prefix $w]} { + set bSeen 1 + if {$bInst} { + set v [string tolower $w] + if {$w != $v} { append v ".$w" } + lappend row $v + } + } + } + + if {$bSeen} { + lappend ret $rowid + lappend ret $row + } + } + + set ret + } + + proc do_prefixquery_test {tn prefix} { + set bInst [expr {$::e!="contentless" || "%DETAIL%"=="full"}] + set expect [prefixquery $prefix $bInst 0] + set expect2 [prefixquery $prefix $bInst 1] + + uplevel [list do_execsql_test $tn.1 " + SELECT rowid, tokens(ft) FROM ft('$prefix') + " $expect] + uplevel [list do_execsql_test $tn.2 " + SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('$prefix')) + " $expect] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) } + uplevel [list do_execsql_test $tn.3 " + SELECT rowid, tokens(ft) FROM ft('$prefix') + " $expect] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) } + + if {"%DETAIL%"!="none"} { + uplevel [list do_execsql_test $tn.4 " + SELECT rowid, tokens(ft) FROM ft('y: $prefix') + " $expect2] + uplevel [list do_execsql_test $tn.5 " + SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('y: $prefix')) + " $expect2] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) } + uplevel [list do_execsql_test $tn.6 " + SELECT rowid, tokens(ft) FROM ft('y: $prefix') + " $expect2] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) } + } + } + + do_prefixquery_test 1.$e.1 a* + do_prefixquery_test 1.$e.2 b* + do_prefixquery_test 1.$e.3 c* + do_prefixquery_test 1.$e.4 d* + do_prefixquery_test 1.$e.5 e* + do_prefixquery_test 1.$e.6 f* + do_prefixquery_test 1.$e.7 g* + do_prefixquery_test 1.$e.8 h* + do_prefixquery_test 1.$e.9 i* + do_prefixquery_test 1.$e.10 j* +}} + + + +finish_test + diff --git a/ext/fts5/test/fts5phrase.test b/ext/fts5/test/fts5phrase.test new file mode 100644 index 0000000000000000000000000000000000000000..708cdfd83e87bc7ce39d44df4ca0f89e7ac81d31 --- /dev/null +++ b/ext/fts5/test/fts5phrase.test @@ -0,0 +1,124 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5phrase + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, b, c); + INSERT INTO t3 VALUES('d e a', 'd i j j f', 'i j i e b f h'); -- 1 + INSERT INTO t3 VALUES('g a e', 'f g i g a', 'h d g i g h c'); -- 2 + INSERT INTO t3 VALUES('e a d', 'e i h a f', 'c e h i f b i'); -- 3 + INSERT INTO t3 VALUES('a g c', 'd j d j c', 'c d f j i g j'); -- 4 + INSERT INTO t3 VALUES('b c b', 'j g c d f', 'j c j d g f b'); -- 5 + INSERT INTO t3 VALUES('j a d', 'e b i h h', 'c c f g d i d'); -- 6 + INSERT INTO t3 VALUES('a d f', 'h g i i i', 'e a g c i f b'); -- 7 + INSERT INTO t3 VALUES('g f d', 'f c g b j', 'b b h h h j j'); -- 8 + INSERT INTO t3 VALUES('f h g', 'c j f g j', 'd h d f e b h'); -- 9 + INSERT INTO t3 VALUES('f h d', 'c i a d b', 'g b j b a d e'); -- 10 + INSERT INTO t3 VALUES('j h h', 'j i h a g', 'd e i e a g j'); -- 11 + INSERT INTO t3 VALUES('a b e', 'h g a g c', 'h c a a d e g'); -- 12 + INSERT INTO t3 VALUES('a j g', 'i h i f i', 'a g h j g i b'); -- 13 + INSERT INTO t3 VALUES('j h e', 'f e d i e', 'i d c f e d c'); -- 14 + INSERT INTO t3 VALUES('d j d', 'd b i a c', 'g d h i d b e'); -- 15 + INSERT INTO t3 VALUES('h j e', 'e b b c f', 'j a f g h d j'); -- 16 + INSERT INTO t3 VALUES('c b j', 'c a b a i', 'h f i d a d c'); -- 17 + INSERT INTO t3 VALUES('e e d', 'i d f c c', 'g i d a f e a'); -- 18 + INSERT INTO t3 VALUES('e i g', 'e a b i h', 'i f d d a d f'); -- 19 + INSERT INTO t3 VALUES('h g f', 'b h h j d', 'i f d e g j a'); -- 20 + INSERT INTO t3 VALUES('e h f', 'j c b c f', 'j a j g h a c'); -- 21 + INSERT INTO t3 VALUES('d c h', 'b g i c e', 'i i c d e h i'); -- 22 + INSERT INTO t3 VALUES('a h i', 'a g d f f', 'e f i i b b h'); -- 23 + INSERT INTO t3 VALUES('d d g', 'c c b c g', 'g c h e b c e'); -- 24 + INSERT INTO t3 VALUES('a b b', 'b f a d i', 'd a h a b c i'); -- 25 + INSERT INTO t3 VALUES('a f d', 'a j e a h', 'j i h j a i f'); -- 26 + INSERT INTO t3 VALUES('d j d', 'h a d i a', 'h h f j h g a'); -- 27 + INSERT INTO t3 VALUES('g a e', 'd g f a g', 'i d b c g g j'); -- 28 + INSERT INTO t3 VALUES('j e h', 'g h j h g', 'd a e j a a h'); -- 29 + INSERT INTO t3 VALUES('e j e', 'g e j g c', 'f c e b e e a'); -- 30 + INSERT INTO t3 VALUES('h f f', 'i j g e c', 'j j f c a i j'); -- 31 + INSERT INTO t3 VALUES('a g c', 'c g d b i', 'g h c b a a f'); -- 32 + INSERT INTO t3 VALUES('c h i', 'j d h e e', 'a h i d c c j'); -- 33 + INSERT INTO t3 VALUES('d a c', 'e d d b j', 'c e b b h i h'); -- 34 + INSERT INTO t3 VALUES('d f h', 'c a f c c', 'j b b c c j f'); -- 35 + INSERT INTO t3 VALUES('b g h', 'g c c c f', 'c g c f h e e'); -- 36 + INSERT INTO t3 VALUES('f e a', 'b h f j h', 'j g h f d g f'); -- 37 + INSERT INTO t3 VALUES('h f a', 'a e i j g', 'f d a f d f c'); -- 38 + INSERT INTO t3 VALUES('f i c', 'f i i i i', 'e c f d h j f'); -- 39 + INSERT INTO t3 VALUES('h h d', 'd i e d i', 'd f e i a h a'); -- 40 + INSERT INTO t3 VALUES('f g c', 'd a f c h', 'b b g j c e g'); -- 41 + INSERT INTO t3 VALUES('h i h', 'h d j d e', 'e d b b i e g'); -- 42 + INSERT INTO t3 VALUES('b h i', 'j e i d a', 'j j h e e c a'); -- 43 + INSERT INTO t3 VALUES('g i g', 'f c c f d', 'a c i c a d a'); -- 44 + INSERT INTO t3 VALUES('c c f', 'a b j d b', 'c a e g f e c'); -- 45 + INSERT INTO t3 VALUES('d h j', 'g c b j d', 'e a h f h j g'); -- 46 + INSERT INTO t3 VALUES('a a d', 'j e j a i', 'i d c f f f b'); -- 47 + INSERT INTO t3 VALUES('b g j', 'e c i h f', 'd d h b g a d'); -- 48 + INSERT INTO t3 VALUES('c i a', 'a c c c c', 'e h i e h i e'); -- 49 + INSERT INTO t3 VALUES('f f c', 'f f b i i', 'f f a j e c i'); -- 50 +} + +proc pmatch {col expr} { + return [expr {[string first $expr $col]>=0}] +} +db func pmatch pmatch + +foreach {tn cols tokens} { + 1 a "c c" + 2 b "c c" + 3 c "c c" + 4 {a b c} "c c" + 5 {a b c} "b h" + 6 {a b} "b h" + 7 {a c} "b h" + 8 {c a} "b h" + 9 {c} "i e" + 10 {b} "i e" + 11 {a} "i e" +} { + set where [list] + foreach c $cols { lappend where "pmatch($c, '$tokens')" } + set where [join $where " OR "] + + foreach fts [list \ + "{$cols}:[join $tokens +]" \ + "{$cols}:NEAR([join $tokens +])" \ + "{$cols}:NEAR([join $tokens +],1)" \ + "{$cols}:NEAR([join $tokens +],111)" \ + ] { + set res [db eval "SELECT rowid FROM t3 WHERE $where"] + do_execsql_test "1.$tn.$fts->([llength $res] rows)" { + SELECT rowid FROM t3($fts) + } $res + } +} + +do_execsql_test 2.0 { + SELECT rowid, + highlight(t3, 0, '*', '*'), + highlight(t3, 1, '*', '*'), + highlight(t3, 2, '*', '*') + FROM t3('a:f+f') +} { + 31 {h *f f*} {i j g e c} {j j f c a i j} + 50 {*f f* c} {f f b i i} {f f a j e c i} +} + +finish_test diff --git a/ext/fts5/test/fts5plan.test b/ext/fts5/test/fts5plan.test new file mode 100644 index 0000000000000000000000000000000000000000..57d5254a35f080a56f8f210d16ebba7cf4d9d488 --- /dev/null +++ b/ext/fts5/test/fts5plan.test @@ -0,0 +1,65 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing the planner (xBestIndex function). +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5plan + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + CREATE VIRTUAL TABLE f1 USING fts5(ff); +} + +do_eqp_test 1.1 { + SELECT * FROM t1, f1 WHERE f1 MATCH t1.x +} { + QUERY PLAN + |--SCAN t1 + `--SCAN f1 VIRTUAL TABLE INDEX 0:M1 +} + +do_eqp_test 1.2 { + SELECT * FROM t1, f1 WHERE f1 > t1.x +} { + QUERY PLAN + |--SCAN f1 VIRTUAL TABLE INDEX 0: + `--SCAN t1 +} + +do_eqp_test 1.3 { + SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff +} { + QUERY PLAN + |--SCAN f1 VIRTUAL TABLE INDEX 0:M1 + `--USE TEMP B-TREE FOR ORDER BY +} + +do_eqp_test 1.4 { + SELECT * FROM f1 ORDER BY rank +} { + QUERY PLAN + |--SCAN f1 VIRTUAL TABLE INDEX 0: + `--USE TEMP B-TREE FOR ORDER BY +} + +do_eqp_test 1.5 { + SELECT * FROM f1 WHERE rank MATCH ? +} {SCAN f1 VIRTUAL TABLE INDEX 0:r} + +finish_test diff --git a/ext/fts5/test/fts5porter.test b/ext/fts5/test/fts5porter.test new file mode 100644 index 0000000000000000000000000000000000000000..de1c3e15a31de5ebe463dd180be90116e9cc00ec --- /dev/null +++ b/ext/fts5/test/fts5porter.test @@ -0,0 +1,11805 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the fts5 porter stemmer implementation. +# +# http://tartarus.org/martin/PorterStemmer/ +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5porter + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set test_vocab { + a a aaron aaron + abaissiez abaissiez abandon abandon + abandoned abandon abase abas + abash abash abate abat + abated abat abatement abat + abatements abat abates abat + abbess abbess abbey abbei + abbeys abbei abbominable abbomin + abbot abbot abbots abbot + abbreviated abbrevi abed ab + abel abel aberga aberga + abergavenny abergavenni abet abet + abetting abet abhominable abhomin + abhor abhor abhorr abhorr + abhorred abhor abhorring abhor + abhors abhor abhorson abhorson + abide abid abides abid + abilities abil ability abil + abject abject abjectly abjectli + abjects abject abjur abjur + abjure abjur able abl + abler abler aboard aboard + abode abod aboded abod + abodements abod aboding abod + abominable abomin abominably abomin + abominations abomin abortive abort + abortives abort abound abound + abounding abound about about + above abov abr abr + abraham abraham abram abram + abreast abreast abridg abridg + abridge abridg abridged abridg + abridgment abridg abroach abroach + abroad abroad abrogate abrog + abrook abrook abrupt abrupt + abruption abrupt abruptly abruptli + absence absenc absent absent + absey absei absolute absolut + absolutely absolut absolv absolv + absolver absolv abstains abstain + abstemious abstemi abstinence abstin + abstract abstract absurd absurd + absyrtus absyrtu abundance abund + abundant abund abundantly abundantli + abus abu abuse abus + abused abus abuser abus + abuses abus abusing abus + abutting abut aby abi + abysm abysm ac ac + academe academ academes academ + accent accent accents accent + accept accept acceptable accept + acceptance accept accepted accept + accepts accept access access + accessary accessari accessible access + accidence accid accident accid + accidental accident accidentally accident + accidents accid accite accit + accited accit accites accit + acclamations acclam accommodate accommod + accommodated accommod accommodation accommod + accommodations accommod accommodo accommodo + accompanied accompani accompany accompani + accompanying accompani accomplices accomplic + accomplish accomplish accomplished accomplish + accomplishing accomplish accomplishment accomplish + accompt accompt accord accord + accordant accord accorded accord + accordeth accordeth according accord + accordingly accordingli accords accord + accost accost accosted accost + account account accountant account + accounted account accounts account + accoutred accoutr accoutrement accoutr + accoutrements accoutr accrue accru + accumulate accumul accumulated accumul + accumulation accumul accurs accur + accursed accurs accurst accurst + accus accu accusation accus + accusations accus accusative accus + accusativo accusativo accuse accus + accused accus accuser accus + accusers accus accuses accus + accuseth accuseth accusing accus + accustom accustom accustomed accustom + ace ac acerb acerb + ache ach acheron acheron + aches ach achiev achiev + achieve achiev achieved achiev + achievement achiev achievements achiev + achiever achiev achieves achiev + achieving achiev achilles achil + aching ach achitophel achitophel + acknowledg acknowledg acknowledge acknowledg + acknowledged acknowledg acknowledgment acknowledg + acknown acknown acold acold + aconitum aconitum acordo acordo + acorn acorn acquaint acquaint + acquaintance acquaint acquainted acquaint + acquaints acquaint acquir acquir + acquire acquir acquisition acquisit + acquit acquit acquittance acquitt + acquittances acquitt acquitted acquit + acre acr acres acr + across across act act + actaeon actaeon acted act + acting act action action + actions action actium actium + active activ actively activ + activity activ actor actor + actors actor acts act + actual actual acture actur + acute acut acutely acut + ad ad adage adag + adallas adalla adam adam + adamant adam add add + added ad adder adder + adders adder addeth addeth + addict addict addicted addict + addiction addict adding ad + addition addit additions addit + addle addl address address + addressing address addrest addrest + adds add adhere adher + adheres adher adieu adieu + adieus adieu adjacent adjac + adjoin adjoin adjoining adjoin + adjourn adjourn adjudg adjudg + adjudged adjudg adjunct adjunct + administer administ administration administr + admir admir admirable admir + admiral admir admiration admir + admire admir admired admir + admirer admir admiring admir + admiringly admiringli admission admiss + admit admit admits admit + admittance admitt admitted admit + admitting admit admonish admonish + admonishing admonish admonishment admonish + admonishments admonish admonition admonit + ado ado adonis adoni + adopt adopt adopted adopt + adoptedly adoptedli adoption adopt + adoptious adopti adopts adopt + ador ador adoration ador + adorations ador adore ador + adorer ador adores ador + adorest adorest adoreth adoreth + adoring ador adorn adorn + adorned adorn adornings adorn + adornment adorn adorns adorn + adown adown adramadio adramadio + adrian adrian adriana adriana + adriano adriano adriatic adriat + adsum adsum adulation adul + adulterate adulter adulterates adulter + adulterers adulter adulteress adulteress + adulteries adulteri adulterous adulter + adultery adulteri adultress adultress + advanc advanc advance advanc + advanced advanc advancement advanc + advancements advanc advances advanc + advancing advanc advantage advantag + advantageable advantag advantaged advantag + advantageous advantag advantages advantag + advantaging advantag advent advent + adventur adventur adventure adventur + adventures adventur adventuring adventur + adventurous adventur adventurously adventur + adversaries adversari adversary adversari + adverse advers adversely advers + adversities advers adversity advers + advertis adverti advertise advertis + advertised advertis advertisement advertis + advertising advertis advice advic + advis advi advise advis + advised advis advisedly advisedli + advises advis advisings advis + advocate advoc advocation advoc + aeacida aeacida aeacides aeacid + aedile aedil aediles aedil + aegeon aegeon aegion aegion + aegles aegl aemelia aemelia + aemilia aemilia aemilius aemiliu + aeneas aenea aeolus aeolu + aer aer aerial aerial + aery aeri aesculapius aesculapiu + aeson aeson aesop aesop + aetna aetna afar afar + afear afear afeard afeard + affability affabl affable affabl + affair affair affaire affair + affairs affair affect affect + affectation affect affectations affect + affected affect affectedly affectedli + affecteth affecteth affecting affect + affection affect affectionate affection + affectionately affection affections affect + affects affect affeer affeer + affianc affianc affiance affianc + affianced affianc affied affi + affin affin affined affin + affinity affin affirm affirm + affirmation affirm affirmatives affirm + afflict afflict afflicted afflict + affliction afflict afflictions afflict + afflicts afflict afford afford + affordeth affordeth affords afford + affray affrai affright affright + affrighted affright affrights affright + affront affront affronted affront + affy affi afield afield + afire afir afloat afloat + afoot afoot afore afor + aforehand aforehand aforesaid aforesaid + afraid afraid afresh afresh + afric afric africa africa + african african afront afront + after after afternoon afternoon + afterward afterward afterwards afterward + ag ag again again + against against agamemmon agamemmon + agamemnon agamemnon agate agat + agaz agaz age ag + aged ag agenor agenor + agent agent agents agent + ages ag aggravate aggrav + aggrief aggrief agile agil + agincourt agincourt agitation agit + aglet aglet agnize agniz + ago ago agone agon + agony agoni agree agre + agreed agre agreeing agre + agreement agreement agrees agre + agrippa agrippa aground aground + ague agu aguecheek aguecheek + agued agu agueface aguefac + agues agu ah ah + aha aha ahungry ahungri + ai ai aialvolio aialvolio + aiaria aiaria aid aid + aidance aidanc aidant aidant + aided aid aiding aid + aidless aidless aids aid + ail ail aim aim + aimed aim aimest aimest + aiming aim aims aim + ainsi ainsi aio aio + air air aired air + airless airless airs air + airy airi ajax ajax + akilling akil al al + alabaster alabast alack alack + alacrity alacr alarbus alarbu + alarm alarm alarms alarm + alarum alarum alarums alarum + alas ala alb alb + alban alban albans alban + albany albani albeit albeit + albion albion alchemist alchemist + alchemy alchemi alcibiades alcibiad + alcides alcid alder alder + alderman alderman aldermen aldermen + ale al alecto alecto + alehouse alehous alehouses alehous + alencon alencon alengon alengon + aleppo aleppo ales al + alewife alewif alexander alexand + alexanders alexand alexandria alexandria + alexandrian alexandrian alexas alexa + alias alia alice alic + alien alien aliena aliena + alight alight alighted alight + alights alight aliis alii + alike alik alisander alisand + alive aliv all all + alla alla allay allai + allayed allai allaying allai + allayment allay allayments allay + allays allai allegation alleg + allegations alleg allege alleg + alleged alleg allegiance allegi + allegiant allegi alley allei + alleys allei allhallowmas allhallowma + alliance allianc allicholy allicholi + allied alli allies alli + alligant allig alligator allig + allons allon allot allot + allots allot allotted allot + allottery allotteri allow allow + allowance allow allowed allow + allowing allow allows allow + allur allur allure allur + allurement allur alluring allur + allusion allus ally alli + allycholly allycholli almain almain + almanac almanac almanack almanack + almanacs almanac almighty almighti + almond almond almost almost + alms alm almsman almsman + aloes alo aloft aloft + alone alon along along + alonso alonso aloof aloof + aloud aloud alphabet alphabet + alphabetical alphabet alphonso alphonso + alps alp already alreadi + also also alt alt + altar altar altars altar + alter alter alteration alter + altered alter alters alter + althaea althaea although although + altitude altitud altogether altogeth + alton alton alway alwai + always alwai am am + amaimon amaimon amain amain + amaking amak amamon amamon + amaz amaz amaze amaz + amazed amaz amazedly amazedli + amazedness amazed amazement amaz + amazes amaz amazeth amazeth + amazing amaz amazon amazon + amazonian amazonian amazons amazon + ambassador ambassador ambassadors ambassador + amber amber ambiguides ambiguid + ambiguities ambigu ambiguous ambigu + ambition ambit ambitions ambit + ambitious ambiti ambitiously ambiti + amble ambl ambled ambl + ambles ambl ambling ambl + ambo ambo ambuscadoes ambuscado + ambush ambush amen amen + amend amend amended amend + amendment amend amends amend + amerce amerc america america + ames am amiable amiabl + amid amid amidst amidst + amiens amien amis ami + amiss amiss amities amiti + amity amiti amnipotent amnipot + among among amongst amongst + amorous amor amorously amor + amort amort amount amount + amounts amount amour amour + amphimacus amphimacu ample ampl + ampler ampler amplest amplest + amplified amplifi amplify amplifi + amply ampli ampthill ampthil + amurath amurath amyntas amynta + an an anatomiz anatomiz + anatomize anatom anatomy anatomi + ancestor ancestor ancestors ancestor + ancestry ancestri anchises anchis + anchor anchor anchorage anchorag + anchored anchor anchoring anchor + anchors anchor anchovies anchovi + ancient ancient ancientry ancientri + ancients ancient ancus ancu + and and andirons andiron + andpholus andpholu andren andren + andrew andrew andromache andromach + andronici andronici andronicus andronicu + anew anew ang ang + angel angel angelica angelica + angelical angel angelo angelo + angels angel anger anger + angerly angerli angers anger + anges ang angiers angier + angl angl anglais anglai + angle angl angler angler + angleterre angleterr angliae anglia + angling angl anglish anglish + angrily angrili angry angri + anguish anguish angus angu + animal anim animals anim + animis animi anjou anjou + ankle ankl anna anna + annals annal anne ann + annex annex annexed annex + annexions annexion annexment annex + annothanize annothan announces announc + annoy annoi annoyance annoy + annoying annoi annual annual + anoint anoint anointed anoint + anon anon another anoth + anselmo anselmo answer answer + answerable answer answered answer + answerest answerest answering answer + answers answer ant ant + ante ant antenor antenor + antenorides antenorid anteroom anteroom + anthem anthem anthems anthem + anthony anthoni anthropophagi anthropophagi + anthropophaginian anthropophaginian antiates antiat + antic antic anticipate anticip + anticipates anticip anticipatest anticipatest + anticipating anticip anticipation anticip + antick antick anticly anticli + antics antic antidote antidot + antidotes antidot antigonus antigonu + antiopa antiopa antipathy antipathi + antipholus antipholu antipholuses antipholus + antipodes antipod antiquary antiquari + antique antiqu antiquity antiqu + antium antium antoniad antoniad + antonio antonio antonius antoniu + antony antoni antres antr + anvil anvil any ani + anybody anybodi anyone anyon + anything anyth anywhere anywher + ap ap apace apac + apart apart apartment apart + apartments apart ape ap + apemantus apemantu apennines apennin + apes ap apiece apiec + apish apish apollinem apollinem + apollo apollo apollodorus apollodoru + apology apolog apoplex apoplex + apoplexy apoplexi apostle apostl + apostles apostl apostrophas apostropha + apoth apoth apothecary apothecari + appal appal appall appal + appalled appal appals appal + apparel apparel apparell apparel + apparelled apparel apparent appar + apparently appar apparition apparit + apparitions apparit appeach appeach + appeal appeal appeals appeal + appear appear appearance appear + appeared appear appeareth appeareth + appearing appear appears appear + appeas appea appease appeas + appeased appeas appelant appel + appele appel appelee appele + appeles appel appelez appelez + appellant appel appellants appel + appelons appelon appendix appendix + apperil apperil appertain appertain + appertaining appertain appertainings appertain + appertains appertain appertinent appertin + appertinents appertin appetite appetit + appetites appetit applaud applaud + applauded applaud applauding applaud + applause applaus applauses applaus + apple appl apples appl + appletart appletart appliance applianc + appliances applianc applications applic + applied appli applies appli + apply appli applying appli + appoint appoint appointed appoint + appointment appoint appointments appoint + appoints appoint apprehend apprehend + apprehended apprehend apprehends apprehend + apprehension apprehens apprehensions apprehens + apprehensive apprehens apprendre apprendr + apprenne apprenn apprenticehood apprenticehood + appris appri approach approach + approachers approach approaches approach + approacheth approacheth approaching approach + approbation approb approof approof + appropriation appropri approv approv + approve approv approved approv + approvers approv approves approv + appurtenance appurten appurtenances appurten + apricocks apricock april april + apron apron aprons apron + apt apt apter apter + aptest aptest aptly aptli + aptness apt aqua aqua + aquilon aquilon aquitaine aquitain + arabia arabia arabian arabian + araise arais arbitrate arbitr + arbitrating arbitr arbitrator arbitr + arbitrement arbitr arbors arbor + arbour arbour arc arc + arch arch archbishop archbishop + archbishopric archbishopr archdeacon archdeacon + arched arch archelaus archelau + archer archer archers archer + archery archeri archibald archibald + archidamus archidamu architect architect + arcu arcu arde ard + arden arden ardent ardent + ardour ardour are ar + argal argal argier argier + argo argo argosies argosi + argosy argosi argu argu + argue argu argued argu + argues argu arguing argu + argument argument arguments argument + argus argu ariachne ariachn + ariadne ariadn ariel ariel + aries ari aright aright + arinado arinado arinies arini + arion arion arise aris + arises aris ariseth ariseth + arising aris aristode aristod + aristotle aristotl arithmetic arithmet + arithmetician arithmetician ark ark + arm arm arma arma + armado armado armadoes armado + armagnac armagnac arme arm + armed arm armenia armenia + armies armi armigero armigero + arming arm armipotent armipot + armor armor armour armour + armourer armour armourers armour + armours armour armoury armouri + arms arm army armi + arn arn aroint aroint + arose aros arouse arous + aroused arous arragon arragon + arraign arraign arraigned arraign + arraigning arraign arraignment arraign + arrant arrant arras arra + array arrai arrearages arrearag + arrest arrest arrested arrest + arrests arrest arriv arriv + arrival arriv arrivance arriv + arrive arriv arrived arriv + arrives arriv arriving arriv + arrogance arrog arrogancy arrog + arrogant arrog arrow arrow + arrows arrow art art + artemidorus artemidoru arteries arteri + arthur arthur article articl + articles articl articulate articul + artificer artific artificial artifici + artillery artilleri artire artir + artist artist artists artist + artless artless artois artoi + arts art artus artu + arviragus arviragu as as + asaph asaph ascanius ascaniu + ascend ascend ascended ascend + ascendeth ascendeth ascends ascend + ascension ascens ascent ascent + ascribe ascrib ascribes ascrib + ash ash asham asham + ashamed asham asher asher + ashes ash ashford ashford + ashore ashor ashouting ashout + ashy ashi asia asia + aside asid ask ask + askance askanc asked ask + asker asker asketh asketh + asking ask asks ask + aslant aslant asleep asleep + asmath asmath asp asp + aspect aspect aspects aspect + aspen aspen aspersion aspers + aspic aspic aspicious aspici + aspics aspic aspir aspir + aspiration aspir aspire aspir + aspiring aspir asquint asquint + ass ass assail assail + assailable assail assailant assail + assailants assail assailed assail + assaileth assaileth assailing assail + assails assail assassination assassin + assault assault assaulted assault + assaults assault assay assai + assaying assai assays assai + assemblance assembl assemble assembl + assembled assembl assemblies assembl + assembly assembl assent assent + asses ass assez assez + assign assign assigned assign + assigns assign assinico assinico + assist assist assistance assist + assistances assist assistant assist + assistants assist assisted assist + assisting assist associate associ + associated associ associates associ + assuage assuag assubjugate assubjug + assum assum assume assum + assumes assum assumption assumpt + assur assur assurance assur + assure assur assured assur + assuredly assuredli assures assur + assyrian assyrian astonish astonish + astonished astonish astraea astraea + astray astrai astrea astrea + astronomer astronom astronomers astronom + astronomical astronom astronomy astronomi + asunder asund at at + atalanta atalanta ate at + ates at athenian athenian + athenians athenian athens athen + athol athol athversary athversari + athwart athwart atlas atla + atomies atomi atomy atomi + atone aton atonement aton + atonements aton atropos atropo + attach attach attached attach + attachment attach attain attain + attainder attaind attains attain + attaint attaint attainted attaint + attainture attaintur attempt attempt + attemptable attempt attempted attempt + attempting attempt attempts attempt + attend attend attendance attend + attendant attend attendants attend + attended attend attendents attend + attendeth attendeth attending attend + attends attend attent attent + attention attent attentive attent + attentivenes attentiven attest attest + attested attest attir attir + attire attir attired attir + attires attir attorney attornei + attorneyed attornei attorneys attornei + attorneyship attorneyship attract attract + attraction attract attractive attract + attracts attract attribute attribut + attributed attribut attributes attribut + attribution attribut attributive attribut + atwain atwain au au + aubrey aubrei auburn auburn + aucun aucun audacious audaci + audaciously audaci audacity audac + audible audibl audience audienc + audis audi audit audit + auditor auditor auditors auditor + auditory auditori audre audr + audrey audrei aufidius aufidiu + aufidiuses aufidius auger auger + aught aught augment augment + augmentation augment augmented augment + augmenting augment augurer augur + augurers augur augures augur + auguring augur augurs augur + augury auguri august august + augustus augustu auld auld + aumerle aumerl aunchient aunchient + aunt aunt aunts aunt + auricular auricular aurora aurora + auspicious auspici aussi aussi + austere auster austerely auster + austereness auster austerity auster + austria austria aut aut + authentic authent author author + authorities author authority author + authorized author authorizing author + authors author autolycus autolycu + autre autr autumn autumn + auvergne auvergn avail avail + avails avail avarice avaric + avaricious avarici avaunt avaunt + ave av aveng aveng + avenge aveng avenged aveng + averring aver avert avert + aves av avez avez + avis avi avoid avoid + avoided avoid avoiding avoid + avoids avoid avoirdupois avoirdupoi + avouch avouch avouched avouch + avouches avouch avouchment avouch + avow avow aw aw + await await awaits await + awak awak awake awak + awaked awak awaken awaken + awakened awaken awakens awaken + awakes awak awaking awak + award award awards award + awasy awasi away awai + awe aw aweary aweari + aweless aweless awful aw + awhile awhil awkward awkward + awl awl awooing awoo + awork awork awry awri + axe ax axle axl + axletree axletre ay ay + aye ay ayez ayez + ayli ayli azur azur + azure azur b b + ba ba baa baa + babbl babbl babble babbl + babbling babbl babe babe + babes babe babies babi + baboon baboon baboons baboon + baby babi babylon babylon + bacare bacar bacchanals bacchan + bacchus bacchu bach bach + bachelor bachelor bachelors bachelor + back back backbite backbit + backbitten backbitten backing back + backs back backward backward + backwardly backwardli backwards backward + bacon bacon bacons bacon + bad bad bade bade + badge badg badged badg + badges badg badly badli + badness bad baes bae + baffl baffl baffle baffl + baffled baffl bag bag + baggage baggag bagot bagot + bagpipe bagpip bags bag + bail bail bailiff bailiff + baillez baillez baily baili + baisant baisant baisees baise + baiser baiser bait bait + baited bait baiting bait + baitings bait baits bait + bajazet bajazet bak bak + bake bake baked bake + baker baker bakers baker + bakes bake baking bake + bal bal balanc balanc + balance balanc balcony balconi + bald bald baldrick baldrick + bale bale baleful bale + balk balk ball ball + ballad ballad ballads ballad + ballast ballast ballasting ballast + ballet ballet ballow ballow + balls ball balm balm + balms balm balmy balmi + balsam balsam balsamum balsamum + balth balth balthasar balthasar + balthazar balthazar bames bame + ban ban banbury banburi + band band bandied bandi + banding band bandit bandit + banditti banditti banditto banditto + bands band bandy bandi + bandying bandi bane bane + banes bane bang bang + bangor bangor banish banish + banished banish banishers banish + banishment banish banister banist + bank bank bankrout bankrout + bankrupt bankrupt bankrupts bankrupt + banks bank banner banner + bannerets banneret banners banner + banning ban banns bann + banquet banquet banqueted banquet + banqueting banquet banquets banquet + banquo banquo bans ban + baptism baptism baptista baptista + baptiz baptiz bar bar + barbarian barbarian barbarians barbarian + barbarism barbar barbarous barbar + barbary barbari barbason barbason + barbed barb barber barber + barbermonger barbermong bard bard + bardolph bardolph bards bard + bare bare bared bare + barefac barefac barefaced barefac + barefoot barefoot bareheaded barehead + barely bare bareness bare + barful bar bargain bargain + bargains bargain barge barg + bargulus bargulu baring bare + bark bark barking bark + barkloughly barkloughli barks bark + barky barki barley barlei + barm barm barn barn + barnacles barnacl barnardine barnardin + barne barn barnes barn + barnet barnet barns barn + baron baron barons baron + barony baroni barr barr + barrabas barraba barrel barrel + barrels barrel barren barren + barrenly barrenli barrenness barren + barricado barricado barricadoes barricado + barrow barrow bars bar + barson barson barter barter + bartholomew bartholomew bas ba + basan basan base base + baseless baseless basely base + baseness base baser baser + bases base basest basest + bashful bash bashfulness bash + basilisco basilisco basilisk basilisk + basilisks basilisk basimecu basimecu + basin basin basingstoke basingstok + basins basin basis basi + bask bask basket basket + baskets basket bass bass + bassanio bassanio basset basset + bassianus bassianu basta basta + bastard bastard bastardizing bastard + bastardly bastardli bastards bastard + bastardy bastardi basted bast + bastes bast bastinado bastinado + basting bast bat bat + batailles batail batch batch + bate bate bated bate + bates bate bath bath + bathe bath bathed bath + bathing bath baths bath + bating bate batler batler + bats bat batt batt + battalia battalia battalions battalion + batten batten batter batter + battering batter batters batter + battery batteri battle battl + battled battl battlefield battlefield + battlements battlement battles battl + batty batti bauble baubl + baubles baubl baubling baubl + baulk baulk bavin bavin + bawcock bawcock bawd bawd + bawdry bawdri bawds bawd + bawdy bawdi bawl bawl + bawling bawl bay bai + baying bai baynard baynard + bayonne bayonn bays bai + be be beach beach + beached beach beachy beachi + beacon beacon bead bead + beaded bead beadle beadl + beadles beadl beads bead + beadsmen beadsmen beagle beagl + beagles beagl beak beak + beaks beak beam beam + beamed beam beams beam + bean bean beans bean + bear bear beard beard + bearded beard beardless beardless + beards beard bearer bearer + bearers bearer bearest bearest + beareth beareth bearing bear + bears bear beast beast + beastliest beastliest beastliness beastli + beastly beastli beasts beast + beat beat beated beat + beaten beaten beating beat + beatrice beatric beats beat + beau beau beaufort beaufort + beaumond beaumond beaumont beaumont + beauteous beauteou beautied beauti + beauties beauti beautified beautifi + beautiful beauti beautify beautifi + beauty beauti beaver beaver + beavers beaver became becam + because becaus bechanc bechanc + bechance bechanc bechanced bechanc + beck beck beckon beckon + beckons beckon becks beck + becom becom become becom + becomed becom becomes becom + becoming becom becomings becom + bed bed bedabbled bedabbl + bedash bedash bedaub bedaub + bedazzled bedazzl bedchamber bedchamb + bedclothes bedcloth bedded bed + bedeck bedeck bedecking bedeck + bedew bedew bedfellow bedfellow + bedfellows bedfellow bedford bedford + bedlam bedlam bedrench bedrench + bedrid bedrid beds bed + bedtime bedtim bedward bedward + bee bee beef beef + beefs beef beehives beehiv + been been beer beer + bees bee beest beest + beetle beetl beetles beetl + beeves beev befall befal + befallen befallen befalls befal + befell befel befits befit + befitted befit befitting befit + befor befor before befor + beforehand beforehand befortune befortun + befriend befriend befriended befriend + befriends befriend beg beg + began began beget beget + begets beget begetting beget + begg begg beggar beggar + beggared beggar beggarly beggarli + beggarman beggarman beggars beggar + beggary beggari begging beg + begin begin beginners beginn + beginning begin beginnings begin + begins begin begnawn begnawn + begone begon begot begot + begotten begotten begrimed begrim + begs beg beguil beguil + beguile beguil beguiled beguil + beguiles beguil beguiling beguil + begun begun behalf behalf + behalfs behalf behav behav + behaved behav behavedst behavedst + behavior behavior behaviors behavior + behaviour behaviour behaviours behaviour + behead behead beheaded behead + beheld beheld behest behest + behests behest behind behind + behold behold beholder behold + beholders behold beholdest beholdest + beholding behold beholds behold + behoof behoof behooffull behoofful + behooves behoov behove behov + behoves behov behowls behowl + being be bel bel + belarius belariu belch belch + belching belch beldam beldam + beldame beldam beldams beldam + belee bele belgia belgia + belie beli belied beli + belief belief beliest beliest + believ believ believe believ + believed believ believes believ + believest believest believing believ + belike belik bell bell + bellario bellario belle bell + bellied belli bellies belli + bellman bellman bellona bellona + bellow bellow bellowed bellow + bellowing bellow bellows bellow + bells bell belly belli + bellyful belly belman belman + belmont belmont belock belock + belong belong belonging belong + belongings belong belongs belong + belov belov beloved belov + beloving belov below below + belt belt belzebub belzebub + bemadding bemad bemet bemet + bemete bemet bemoan bemoan + bemoaned bemoan bemock bemock + bemoil bemoil bemonster bemonst + ben ben bench bench + bencher bencher benches bench + bend bend bended bend + bending bend bends bend + bene bene beneath beneath + benedicite benedicit benedick benedick + benediction benedict benedictus benedictu + benefactors benefactor benefice benefic + beneficial benefici benefit benefit + benefited benefit benefits benefit + benetted benet benevolence benevol + benevolences benevol benied beni + benison benison bennet bennet + bent bent bentii bentii + bentivolii bentivolii bents bent + benumbed benumb benvolio benvolio + bepaint bepaint bepray beprai + bequeath bequeath bequeathed bequeath + bequeathing bequeath bequest bequest + ber ber berard berard + berattle berattl beray berai + bere bere bereave bereav + bereaved bereav bereaves bereav + bereft bereft bergamo bergamo + bergomask bergomask berhym berhym + berhyme berhym berkeley berkelei + bermoothes bermooth bernardo bernardo + berod berod berowne berown + berri berri berries berri + berrord berrord berry berri + bertram bertram berwick berwick + bescreen bescreen beseech beseech + beseeched beseech beseechers beseech + beseeching beseech beseek beseek + beseem beseem beseemeth beseemeth + beseeming beseem beseems beseem + beset beset beshrew beshrew + beside besid besides besid + besieg besieg besiege besieg + besieged besieg beslubber beslubb + besmear besmear besmeared besmear + besmirch besmirch besom besom + besort besort besotted besot + bespake bespak bespeak bespeak + bespice bespic bespoke bespok + bespotted bespot bess bess + bessy bessi best best + bestained bestain bested best + bestial bestial bestir bestir + bestirr bestirr bestow bestow + bestowed bestow bestowing bestow + bestows bestow bestraught bestraught + bestrew bestrew bestrid bestrid + bestride bestrid bestrides bestrid + bet bet betake betak + beteem beteem bethink bethink + bethought bethought bethrothed bethroth + bethump bethump betid betid + betide betid betideth betideth + betime betim betimes betim + betoken betoken betook betook + betossed betoss betray betrai + betrayed betrai betraying betrai + betrays betrai betrims betrim + betroth betroth betrothed betroth + betroths betroth bett bett + betted bet better better + bettered better bettering better + betters better betting bet + bettre bettr between between + betwixt betwixt bevel bevel + beverage beverag bevis bevi + bevy bevi bewail bewail + bewailed bewail bewailing bewail + bewails bewail beware bewar + bewasted bewast beweep beweep + bewept bewept bewet bewet + bewhored bewhor bewitch bewitch + bewitched bewitch bewitchment bewitch + bewray bewrai beyond beyond + bezonian bezonian bezonians bezonian + bianca bianca bianco bianco + bias bia bibble bibbl + bickerings bicker bid bid + bidden bidden bidding bid + biddings bid biddy biddi + bide bide bides bide + biding bide bids bid + bien bien bier bier + bifold bifold big big + bigamy bigami biggen biggen + bigger bigger bigness big + bigot bigot bilberry bilberri + bilbo bilbo bilboes bilbo + bilbow bilbow bill bill + billeted billet billets billet + billiards billiard billing bill + billow billow billows billow + bills bill bin bin + bind bind bindeth bindeth + binding bind binds bind + biondello biondello birch birch + bird bird birding bird + birdlime birdlim birds bird + birnam birnam birth birth + birthday birthdai birthdom birthdom + birthplace birthplac birthright birthright + birthrights birthright births birth + bis bi biscuit biscuit + bishop bishop bishops bishop + bisson bisson bit bit + bitch bitch bite bite + biter biter bites bite + biting bite bits bit + bitt bitt bitten bitten + bitter bitter bitterest bitterest + bitterly bitterli bitterness bitter + blab blab blabb blabb + blabbing blab blabs blab + black black blackamoor blackamoor + blackamoors blackamoor blackberries blackberri + blackberry blackberri blacker blacker + blackest blackest blackfriars blackfriar + blackheath blackheath blackmere blackmer + blackness black blacks black + bladder bladder bladders bladder + blade blade bladed blade + blades blade blains blain + blam blam blame blame + blamed blame blameful blame + blameless blameless blames blame + blanc blanc blanca blanca + blanch blanch blank blank + blanket blanket blanks blank + blaspheme blasphem blaspheming blasphem + blasphemous blasphem blasphemy blasphemi + blast blast blasted blast + blasting blast blastments blastment + blasts blast blaz blaz + blaze blaze blazes blaze + blazing blaze blazon blazon + blazoned blazon blazoning blazon + bleach bleach bleaching bleach + bleak bleak blear blear + bleared blear bleat bleat + bleated bleat bleats bleat + bled bled bleed bleed + bleedest bleedest bleedeth bleedeth + bleeding bleed bleeds bleed + blemish blemish blemishes blemish + blench blench blenches blench + blend blend blended blend + blent blent bless bless + blessed bless blessedly blessedli + blessedness blessed blesses bless + blesseth blesseth blessing bless + blessings bless blest blest + blew blew blind blind + blinded blind blindfold blindfold + blinding blind blindly blindli + blindness blind blinds blind + blink blink blinking blink + bliss bliss blist blist + blister blister blisters blister + blithe blith blithild blithild + bloat bloat block block + blockish blockish blocks block + blois bloi blood blood + blooded blood bloodhound bloodhound + bloodied bloodi bloodier bloodier + bloodiest bloodiest bloodily bloodili + bloodless bloodless bloods blood + bloodshed bloodsh bloodshedding bloodshed + bloodstained bloodstain bloody bloodi + bloom bloom blooms bloom + blossom blossom blossoming blossom + blossoms blossom blot blot + blots blot blotted blot + blotting blot blount blount + blow blow blowed blow + blowers blower blowest blowest + blowing blow blown blown + blows blow blowse blows + blubb blubb blubber blubber + blubbering blubber blue blue + bluecaps bluecap bluest bluest + blunt blunt blunted blunt + blunter blunter bluntest bluntest + blunting blunt bluntly bluntli + bluntness blunt blunts blunt + blur blur blurr blurr + blurs blur blush blush + blushes blush blushest blushest + blushing blush blust blust + bluster bluster blusterer bluster + blusters bluster bo bo + boar boar board board + boarded board boarding board + boards board boarish boarish + boars boar boast boast + boasted boast boastful boast + boasting boast boasts boast + boat boat boats boat + boatswain boatswain bob bob + bobb bobb boblibindo boblibindo + bobtail bobtail bocchus bocchu + bode bode boded bode + bodements bodement bodes bode + bodg bodg bodied bodi + bodies bodi bodiless bodiless + bodily bodili boding bode + bodkin bodkin body bodi + bodykins bodykin bog bog + boggle boggl boggler boggler + bogs bog bohemia bohemia + bohemian bohemian bohun bohun + boil boil boiling boil + boils boil boist boist + boisterous boister boisterously boister + boitier boitier bold bold + bolden bolden bolder bolder + boldest boldest boldly boldli + boldness bold bolds bold + bolingbroke bolingbrok bolster bolster + bolt bolt bolted bolt + bolter bolter bolters bolter + bolting bolt bolts bolt + bombard bombard bombards bombard + bombast bombast bon bon + bona bona bond bond + bondage bondag bonded bond + bondmaid bondmaid bondman bondman + bondmen bondmen bonds bond + bondslave bondslav bone bone + boneless boneless bones bone + bonfire bonfir bonfires bonfir + bonjour bonjour bonne bonn + bonnet bonnet bonneted bonnet + bonny bonni bonos bono + bonto bonto bonville bonvil + bood bood book book + bookish bookish books book + boon boon boor boor + boorish boorish boors boor + boot boot booted boot + booties booti bootless bootless + boots boot booty booti + bor bor bora bora + borachio borachio bordeaux bordeaux + border border bordered border + borderers border borders border + bore bore boreas borea + bores bore boring bore + born born borne born + borough borough boroughs borough + borrow borrow borrowed borrow + borrower borrow borrowing borrow + borrows borrow bosko bosko + boskos bosko bosky boski + bosom bosom bosoms bosom + boson boson boss boss + bosworth bosworth botch botch + botcher botcher botches botch + botchy botchi both both + bots bot bottle bottl + bottled bottl bottles bottl + bottom bottom bottomless bottomless + bottoms bottom bouciqualt bouciqualt + bouge boug bough bough + boughs bough bought bought + bounce bounc bouncing bounc + bound bound bounded bound + bounden bounden boundeth boundeth + bounding bound boundless boundless + bounds bound bounteous bounteou + bounteously bounteous bounties bounti + bountiful bounti bountifully bountifulli + bounty bounti bourbier bourbier + bourbon bourbon bourchier bourchier + bourdeaux bourdeaux bourn bourn + bout bout bouts bout + bove bove bow bow + bowcase bowcas bowed bow + bowels bowel bower bower + bowing bow bowl bowl + bowler bowler bowling bowl + bowls bowl bows bow + bowsprit bowsprit bowstring bowstr + box box boxes box + boy boi boyet boyet + boyish boyish boys boi + brabant brabant brabantio brabantio + brabble brabbl brabbler brabbler + brac brac brace brace + bracelet bracelet bracelets bracelet + brach brach bracy braci + brag brag bragg bragg + braggardism braggard braggards braggard + braggart braggart braggarts braggart + bragged brag bragging brag + bragless bragless brags brag + braid braid braided braid + brain brain brained brain + brainford brainford brainish brainish + brainless brainless brains brain + brainsick brainsick brainsickly brainsickli + brake brake brakenbury brakenburi + brakes brake brambles brambl + bran bran branch branch + branches branch branchless branchless + brand brand branded brand + brandish brandish brandon brandon + brands brand bras bra + brass brass brassy brassi + brat brat brats brat + brav brav brave brave + braved brave bravely brave + braver braver bravery braveri + braves brave bravest bravest + braving brave brawl brawl + brawler brawler brawling brawl + brawls brawl brawn brawn + brawns brawn bray brai + braying brai braz braz + brazen brazen brazier brazier + breach breach breaches breach + bread bread breadth breadth + break break breaker breaker + breakfast breakfast breaking break + breaks break breast breast + breasted breast breasting breast + breastplate breastplat breasts breast + breath breath breathe breath + breathed breath breather breather + breathers breather breathes breath + breathest breathest breathing breath + breathless breathless breaths breath + brecknock brecknock bred bred + breech breech breeches breech + breeching breech breed breed + breeder breeder breeders breeder + breeding breed breeds breed + breese brees breeze breez + breff breff bretagne bretagn + brethen brethen bretheren bretheren + brethren brethren brevis brevi + brevity breviti brew brew + brewage brewag brewer brewer + brewers brewer brewing brew + brews brew briareus briareu + briars briar brib brib + bribe bribe briber briber + bribes bribe brick brick + bricklayer bricklay bricks brick + bridal bridal bride bride + bridegroom bridegroom bridegrooms bridegroom + brides bride bridge bridg + bridgenorth bridgenorth bridges bridg + bridget bridget bridle bridl + bridled bridl brief brief + briefer briefer briefest briefest + briefly briefli briefness brief + brier brier briers brier + brigandine brigandin bright bright + brighten brighten brightest brightest + brightly brightli brightness bright + brim brim brimful brim + brims brim brimstone brimston + brinded brind brine brine + bring bring bringer bringer + bringeth bringeth bringing bring + bringings bring brings bring + brinish brinish brink brink + brisk brisk brisky briski + bristle bristl bristled bristl + bristly bristli bristol bristol + bristow bristow britain britain + britaine britain britaines britain + british british briton briton + britons briton brittany brittani + brittle brittl broach broach + broached broach broad broad + broader broader broadsides broadsid + brocas broca brock brock + brogues brogu broil broil + broiling broil broils broil + broke broke broken broken + brokenly brokenli broker broker + brokers broker brokes broke + broking broke brooch brooch + brooches brooch brood brood + brooded brood brooding brood + brook brook brooks brook + broom broom broomstaff broomstaff + broth broth brothel brothel + brother brother brotherhood brotherhood + brotherhoods brotherhood brotherly brotherli + brothers brother broths broth + brought brought brow brow + brown brown browner browner + brownist brownist browny browni + brows brow browse brows + browsing brows bruis brui + bruise bruis bruised bruis + bruises bruis bruising bruis + bruit bruit bruited bruit + brundusium brundusium brunt brunt + brush brush brushes brush + brute brute brutish brutish + brutus brutu bubble bubbl + bubbles bubbl bubbling bubbl + bubukles bubukl buck buck + bucket bucket buckets bucket + bucking buck buckingham buckingham + buckle buckl buckled buckl + buckler buckler bucklers buckler + bucklersbury bucklersburi buckles buckl + buckram buckram bucks buck + bud bud budded bud + budding bud budge budg + budger budger budget budget + buds bud buff buff + buffet buffet buffeting buffet + buffets buffet bug bug + bugbear bugbear bugle bugl + bugs bug build build + builded build buildeth buildeth + building build buildings build + builds build built built + bulk bulk bulks bulk + bull bull bullcalf bullcalf + bullen bullen bullens bullen + bullet bullet bullets bullet + bullocks bullock bulls bull + bully bulli bulmer bulmer + bulwark bulwark bulwarks bulwark + bum bum bumbast bumbast + bump bump bumper bumper + bums bum bunch bunch + bunches bunch bundle bundl + bung bung bunghole bunghol + bungle bungl bunting bunt + buoy buoi bur bur + burbolt burbolt burd burd + burden burden burdened burden + burdening burden burdenous burden + burdens burden burgh burgh + burgher burgher burghers burgher + burglary burglari burgomasters burgomast + burgonet burgonet burgundy burgundi + burial burial buried buri + burier burier buriest buriest + burly burli burn burn + burned burn burnet burnet + burneth burneth burning burn + burnish burnish burns burn + burnt burnt burr burr + burrows burrow burs bur + burst burst bursting burst + bursts burst burthen burthen + burthens burthen burton burton + bury buri burying buri + bush bush bushels bushel + bushes bush bushy bushi + busied busi busily busili + busines busin business busi + businesses busi buskin buskin + busky buski buss buss + busses buss bussing buss + bustle bustl bustling bustl + busy busi but but + butcheed butche butcher butcher + butchered butcher butcheries butcheri + butcherly butcherli butchers butcher + butchery butcheri butler butler + butt butt butter butter + buttered butter butterflies butterfli + butterfly butterfli butterwoman butterwoman + buttery butteri buttock buttock + buttocks buttock button button + buttonhole buttonhol buttons button + buttress buttress buttry buttri + butts butt buxom buxom + buy bui buyer buyer + buying bui buys bui + buzz buzz buzzard buzzard + buzzards buzzard buzzers buzzer + buzzing buzz by by + bye bye byzantium byzantium + c c ca ca + cabbage cabbag cabileros cabilero + cabin cabin cabins cabin + cable cabl cables cabl + cackling cackl cacodemon cacodemon + caddis caddi caddisses caddiss + cade cade cadence cadenc + cadent cadent cades cade + cadmus cadmu caduceus caduceu + cadwal cadwal cadwallader cadwallad + caelius caeliu caelo caelo + caesar caesar caesarion caesarion + caesars caesar cage cage + caged cage cagion cagion + cain cain caithness caith + caitiff caitiff caitiffs caitiff + caius caiu cak cak + cake cake cakes cake + calaber calab calais calai + calamities calam calamity calam + calchas calcha calculate calcul + calen calen calendar calendar + calendars calendar calf calf + caliban caliban calibans caliban + calipolis calipoli cality caliti + caliver caliv call call + callat callat called call + callet callet calling call + calls call calm calm + calmest calmest calmly calmli + calmness calm calms calm + calpurnia calpurnia calumniate calumni + calumniating calumni calumnious calumni + calumny calumni calve calv + calved calv calves calv + calveskins calveskin calydon calydon + cam cam cambio cambio + cambria cambria cambric cambric + cambrics cambric cambridge cambridg + cambyses cambys came came + camel camel camelot camelot + camels camel camest camest + camillo camillo camlet camlet + camomile camomil camp camp + campeius campeiu camping camp + camps camp can can + canakin canakin canaries canari + canary canari cancel cancel + cancell cancel cancelled cancel + cancelling cancel cancels cancel + cancer cancer candidatus candidatu + candied candi candle candl + candles candl candlesticks candlestick + candy candi canidius canidiu + cank cank canker canker + cankerblossom cankerblossom cankers canker + cannibally cannib cannibals cannib + cannon cannon cannoneer cannon + cannons cannon cannot cannot + canon canon canoniz canoniz + canonize canon canonized canon + canons canon canopied canopi + canopies canopi canopy canopi + canst canst canstick canstick + canterbury canterburi cantle cantl + cantons canton canus canu + canvas canva canvass canvass + canzonet canzonet cap cap + capability capabl capable capabl + capacities capac capacity capac + caparison caparison capdv capdv + cape cape capel capel + capels capel caper caper + capers caper capet capet + caphis caphi capilet capilet + capitaine capitain capital capit + capite capit capitol capitol + capitulate capitul capocchia capocchia + capon capon capons capon + capp capp cappadocia cappadocia + capriccio capriccio capricious caprici + caps cap capt capt + captain captain captains captain + captainship captainship captious captiou + captivate captiv captivated captiv + captivates captiv captive captiv + captives captiv captivity captiv + captum captum capucius capuciu + capulet capulet capulets capulet + car car carack carack + caracks carack carat carat + caraways carawai carbonado carbonado + carbuncle carbuncl carbuncled carbuncl + carbuncles carbuncl carcanet carcanet + carcase carcas carcases carcas + carcass carcass carcasses carcass + card card cardecue cardecu + carded card carders carder + cardinal cardin cardinally cardin + cardinals cardin cardmaker cardmak + cards card carduus carduu + care care cared care + career career careers career + careful care carefully carefulli + careless careless carelessly carelessli + carelessness careless cares care + caret caret cargo cargo + carl carl carlisle carlisl + carlot carlot carman carman + carmen carmen carnal carnal + carnally carnal carnarvonshire carnarvonshir + carnation carnat carnations carnat + carol carol carous carou + carouse carous caroused carous + carouses carous carousing carous + carp carp carpenter carpent + carper carper carpet carpet + carpets carpet carping carp + carriage carriag carriages carriag + carried carri carrier carrier + carriers carrier carries carri + carrion carrion carrions carrion + carry carri carrying carri + cars car cart cart + carters carter carthage carthag + carts cart carv carv + carve carv carved carv + carver carver carves carv + carving carv cas ca + casa casa casaer casaer + casca casca case case + casement casement casements casement + cases case cash cash + cashier cashier casing case + cask cask casket casket + casketed casket caskets casket + casque casqu casques casqu + cassado cassado cassandra cassandra + cassibelan cassibelan cassio cassio + cassius cassiu cassocks cassock + cast cast castalion castalion + castaway castawai castaways castawai + casted cast caster caster + castigate castig castigation castig + castile castil castiliano castiliano + casting cast castle castl + castles castl casts cast + casual casual casually casual + casualties casualti casualty casualti + cat cat cataian cataian + catalogue catalogu cataplasm cataplasm + cataracts cataract catarrhs catarrh + catastrophe catastroph catch catch + catcher catcher catches catch + catching catch cate cate + catechising catechis catechism catech + catechize catech cater cater + caterpillars caterpillar caters cater + caterwauling caterwaul cates cate + catesby catesbi cathedral cathedr + catlike catlik catling catl + catlings catl cato cato + cats cat cattle cattl + caucasus caucasu caudle caudl + cauf cauf caught caught + cauldron cauldron caus cau + cause caus caused caus + causeless causeless causer causer + causes caus causest causest + causeth causeth cautel cautel + cautelous cautel cautels cautel + cauterizing cauter caution caution + cautions caution cavaleiro cavaleiro + cavalery cavaleri cavaliers cavali + cave cave cavern cavern + caverns cavern caves cave + caveto caveto caviary caviari + cavil cavil cavilling cavil + cawdor cawdor cawdron cawdron + cawing caw ce ce + ceas cea cease ceas + ceases ceas ceaseth ceaseth + cedar cedar cedars cedar + cedius cediu celebrate celebr + celebrated celebr celebrates celebr + celebration celebr celerity celer + celestial celesti celia celia + cell cell cellar cellar + cellarage cellarag celsa celsa + cement cement censer censer + censor censor censorinus censorinu + censur censur censure censur + censured censur censurers censur + censures censur censuring censur + centaur centaur centaurs centaur + centre centr cents cent + centuries centuri centurion centurion + centurions centurion century centuri + cerberus cerberu cerecloth cerecloth + cerements cerement ceremonial ceremoni + ceremonies ceremoni ceremonious ceremoni + ceremoniously ceremoni ceremony ceremoni + ceres cere cerns cern + certain certain certainer certain + certainly certainli certainties certainti + certainty certainti certes cert + certificate certif certified certifi + certifies certifi certify certifi + ces ce cesario cesario + cess cess cesse cess + cestern cestern cetera cetera + cette cett chaces chace + chaf chaf chafe chafe + chafed chafe chafes chafe + chaff chaff chaffless chaffless + chafing chafe chain chain + chains chain chair chair + chairs chair chalic chalic + chalice chalic chalices chalic + chalk chalk chalks chalk + chalky chalki challeng challeng + challenge challeng challenged challeng + challenger challeng challengers challeng + challenges challeng cham cham + chamber chamber chamberers chamber + chamberlain chamberlain chamberlains chamberlain + chambermaid chambermaid chambermaids chambermaid + chambers chamber chameleon chameleon + champ champ champagne champagn + champain champain champains champain + champion champion champions champion + chanc chanc chance chanc + chanced chanc chancellor chancellor + chances chanc chandler chandler + chang chang change chang + changeable changeabl changed chang + changeful chang changeling changel + changelings changel changer changer + changes chang changest changest + changing chang channel channel + channels channel chanson chanson + chant chant chanticleer chanticl + chanting chant chantries chantri + chantry chantri chants chant + chaos chao chap chap + chape chape chapel chapel + chapeless chapeless chapels chapel + chaplain chaplain chaplains chaplain + chapless chapless chaplet chaplet + chapmen chapmen chaps chap + chapter chapter character charact + charactered charact characterless characterless + characters charact charactery characteri + characts charact charbon charbon + chare chare chares chare + charg charg charge charg + charged charg chargeful charg + charges charg chargeth chargeth + charging charg chariest chariest + chariness chari charing chare + chariot chariot chariots chariot + charitable charit charitably charit + charities chariti charity chariti + charlemain charlemain charles charl + charm charm charmed charm + charmer charmer charmeth charmeth + charmian charmian charming charm + charmingly charmingli charms charm + charneco charneco charnel charnel + charolois charoloi charon charon + charter charter charters charter + chartreux chartreux chary chari + charybdis charybdi chas cha + chase chase chased chase + chaser chaser chaseth chaseth + chasing chase chaste chast + chastely chast chastis chasti + chastise chastis chastised chastis + chastisement chastis chastity chastiti + chat chat chatham chatham + chatillon chatillon chats chat + chatt chatt chattels chattel + chatter chatter chattering chatter + chattles chattl chaud chaud + chaunted chaunt chaw chaw + chawdron chawdron che che + cheap cheap cheapen cheapen + cheaper cheaper cheapest cheapest + cheaply cheapli cheapside cheapsid + cheat cheat cheated cheat + cheater cheater cheaters cheater + cheating cheat cheats cheat + check check checked check + checker checker checking check + checks check cheek cheek + cheeks cheek cheer cheer + cheered cheer cheerer cheerer + cheerful cheer cheerfully cheerfulli + cheering cheer cheerless cheerless + cheerly cheerli cheers cheer + cheese chees chequer chequer + cher cher cherish cherish + cherished cherish cherisher cherish + cherishes cherish cherishing cherish + cherries cherri cherry cherri + cherrypit cherrypit chertsey chertsei + cherub cherub cherubims cherubim + cherubin cherubin cherubins cherubin + cheshu cheshu chess chess + chest chest chester chester + chestnut chestnut chestnuts chestnut + chests chest chetas cheta + chev chev cheval cheval + chevalier chevali chevaliers chevali + cheveril cheveril chew chew + chewed chew chewet chewet + chewing chew chez chez + chi chi chick chick + chicken chicken chickens chicken + chicurmurco chicurmurco chid chid + chidden chidden chide chide + chiders chider chides chide + chiding chide chief chief + chiefest chiefest chiefly chiefli + chien chien child child + childed child childeric childer + childhood childhood childhoods childhood + childing child childish childish + childishness childish childlike childlik + childness child children children + chill chill chilling chill + chime chime chimes chime + chimney chimnei chimneypiece chimneypiec + chimneys chimnei chimurcho chimurcho + chin chin china china + chine chine chines chine + chink chink chinks chink + chins chin chipp chipp + chipper chipper chips chip + chiron chiron chirping chirp + chirrah chirrah chirurgeonly chirurgeonli + chisel chisel chitopher chitoph + chivalrous chivalr chivalry chivalri + choice choic choicely choic + choicest choicest choir choir + choirs choir chok chok + choke choke choked choke + chokes choke choking choke + choler choler choleric choler + cholers choler chollors chollor + choose choos chooser chooser + chooses choos chooseth chooseth + choosing choos chop chop + chopine chopin choplogic choplog + chopp chopp chopped chop + chopping chop choppy choppi + chops chop chopt chopt + chor chor choristers chorist + chorus choru chose chose + chosen chosen chough chough + choughs chough chrish chrish + christ christ christen christen + christendom christendom christendoms christendom + christening christen christenings christen + christian christian christianlike christianlik + christians christian christmas christma + christom christom christopher christoph + christophero christophero chronicle chronicl + chronicled chronicl chronicler chronicl + chroniclers chronicl chronicles chronicl + chrysolite chrysolit chuck chuck + chucks chuck chud chud + chuffs chuff church church + churches church churchman churchman + churchmen churchmen churchyard churchyard + churchyards churchyard churl churl + churlish churlish churlishly churlishli + churls churl churn churn + chus chu cicatrice cicatric + cicatrices cicatric cicely cice + cicero cicero ciceter cicet + ciel ciel ciitzens ciitzen + cilicia cilicia cimber cimber + cimmerian cimmerian cinable cinabl + cincture cinctur cinders cinder + cine cine cinna cinna + cinque cinqu cipher cipher + ciphers cipher circa circa + circe circ circle circl + circled circl circlets circlet + circling circl circuit circuit + circum circum circumcised circumcis + circumference circumfer circummur circummur + circumscrib circumscrib circumscribed circumscrib + circumscription circumscript circumspect circumspect + circumstance circumst circumstanced circumstanc + circumstances circumst circumstantial circumstanti + circumvent circumv circumvention circumvent + cistern cistern citadel citadel + cital cital cite cite + cited cite cites cite + cities citi citing cite + citizen citizen citizens citizen + cittern cittern city citi + civet civet civil civil + civility civil civilly civilli + clack clack clad clad + claim claim claiming claim + claims claim clamb clamb + clamber clamber clammer clammer + clamor clamor clamorous clamor + clamors clamor clamour clamour + clamours clamour clang clang + clangor clangor clap clap + clapp clapp clapped clap + clapper clapper clapping clap + claps clap clare clare + clarence clarenc claret claret + claribel claribel clasp clasp + clasps clasp clatter clatter + claud claud claudio claudio + claudius claudiu clause claus + claw claw clawed claw + clawing claw claws claw + clay clai clays clai + clean clean cleanliest cleanliest + cleanly cleanli cleans clean + cleanse cleans cleansing cleans + clear clear clearer clearer + clearest clearest clearly clearli + clearness clear clears clear + cleave cleav cleaving cleav + clef clef cleft cleft + cleitus cleitu clemency clemenc + clement clement cleomenes cleomen + cleopatpa cleopatpa cleopatra cleopatra + clepeth clepeth clept clept + clerestories clerestori clergy clergi + clergyman clergyman clergymen clergymen + clerk clerk clerkly clerkli + clerks clerk clew clew + client client clients client + cliff cliff clifford clifford + cliffords clifford cliffs cliff + clifton clifton climate climat + climature climatur climb climb + climbed climb climber climber + climbeth climbeth climbing climb + climbs climb clime clime + cling cling clink clink + clinking clink clinquant clinquant + clip clip clipp clipp + clipper clipper clippeth clippeth + clipping clip clipt clipt + clitus clitu clo clo + cloak cloak cloakbag cloakbag + cloaks cloak clock clock + clocks clock clod clod + cloddy cloddi clodpole clodpol + clog clog clogging clog + clogs clog cloister cloister + cloistress cloistress cloquence cloquenc + clos clo close close + closed close closely close + closeness close closer closer + closes close closest closest + closet closet closing close + closure closur cloten cloten + clotens cloten cloth cloth + clothair clothair clotharius clothariu + clothe cloth clothes cloth + clothier clothier clothiers clothier + clothing cloth cloths cloth + clotpoles clotpol clotpoll clotpol + cloud cloud clouded cloud + cloudiness cloudi clouds cloud + cloudy cloudi clout clout + clouted clout clouts clout + cloven cloven clover clover + cloves clove clovest clovest + clowder clowder clown clown + clownish clownish clowns clown + cloy cloi cloyed cloi + cloying cloi cloyless cloyless + cloyment cloyment cloys cloi + club club clubs club + cluck cluck clung clung + clust clust clusters cluster + clutch clutch clyster clyster + cneius cneiu cnemies cnemi + co co coach coach + coaches coach coachmakers coachmak + coact coact coactive coactiv + coagulate coagul coal coal + coals coal coarse coars + coarsely coars coast coast + coasting coast coasts coast + coat coat coated coat + coats coat cobble cobbl + cobbled cobbl cobbler cobbler + cobham cobham cobloaf cobloaf + cobweb cobweb cobwebs cobweb + cock cock cockatrice cockatric + cockatrices cockatric cockle cockl + cockled cockl cockney cocknei + cockpit cockpit cocks cock + cocksure cocksur coctus coctu + cocytus cocytu cod cod + codding cod codling codl + codpiece codpiec codpieces codpiec + cods cod coelestibus coelestibu + coesar coesar coeur coeur + coffer coffer coffers coffer + coffin coffin coffins coffin + cog cog cogging cog + cogitation cogit cogitations cogit + cognition cognit cognizance cogniz + cogscomb cogscomb cohabitants cohabit + coher coher cohere coher + coherence coher coherent coher + cohorts cohort coif coif + coign coign coil coil + coin coin coinage coinag + coiner coiner coining coin + coins coin col col + colbrand colbrand colchos colcho + cold cold colder colder + coldest coldest coldly coldli + coldness cold coldspur coldspur + colebrook colebrook colic colic + collar collar collars collar + collateral collater colleagued colleagu + collect collect collected collect + collection collect college colleg + colleges colleg collied colli + collier collier colliers collier + collop collop collusion collus + colme colm colmekill colmekil + coloquintida coloquintida color color + colors color colossus colossu + colour colour colourable colour + coloured colour colouring colour + colours colour colt colt + colted colt colts colt + columbine columbin columbines columbin + colville colvil com com + comagene comagen comart comart + comb comb combat combat + combatant combat combatants combat + combated combat combating combat + combin combin combinate combin + combination combin combine combin + combined combin combless combless + combustion combust come come + comedian comedian comedians comedian + comedy comedi comeliness comeli + comely come comer comer + comers comer comes come + comest comest comet comet + cometh cometh comets comet + comfect comfect comfit comfit + comfits comfit comfort comfort + comfortable comfort comforted comfort + comforter comfort comforting comfort + comfortless comfortless comforts comfort + comic comic comical comic + coming come comings come + cominius cominiu comma comma + command command commande command + commanded command commander command + commanders command commanding command + commandment command commandments command + commands command comme comm + commenc commenc commence commenc + commenced commenc commencement commenc + commences commenc commencing commenc + commend commend commendable commend + commendation commend commendations commend + commended commend commending commend + commends commend comment comment + commentaries commentari commenting comment + comments comment commerce commerc + commingled commingl commiseration commiser + commission commiss commissioners commission + commissions commiss commit commit + commits commit committ committ + committed commit committing commit + commix commix commixed commix + commixtion commixt commixture commixtur + commodious commodi commodities commod + commodity commod common common + commonalty commonalti commoner common + commoners common commonly commonli + commons common commonweal commonw + commonwealth commonwealth commotion commot + commotions commot commune commun + communicat communicat communicate commun + communication commun communities commun + community commun comonty comonti + compact compact companies compani + companion companion companions companion + companionship companionship company compani + compar compar comparative compar + compare compar compared compar + comparing compar comparison comparison + comparisons comparison compartner compartn + compass compass compasses compass + compassing compass compassion compass + compassionate compassion compeers compeer + compel compel compell compel + compelled compel compelling compel + compels compel compensation compens + competence compet competency compet + competent compet competitor competitor + competitors competitor compil compil + compile compil compiled compil + complain complain complainer complain + complainest complainest complaining complain + complainings complain complains complain + complaint complaint complaints complaint + complement complement complements complement + complete complet complexion complexion + complexioned complexion complexions complexion + complices complic complies compli + compliment compliment complimental compliment + compliments compliment complot complot + complots complot complotted complot + comply compli compos compo + compose compos composed compos + composition composit compost compost + composture compostur composure composur + compound compound compounded compound + compounds compound comprehend comprehend + comprehended comprehend comprehends comprehend + compremises compremis compris compri + comprising compris compromis compromi + compromise compromis compt compt + comptible comptibl comptrollers comptrol + compulsatory compulsatori compulsion compuls + compulsive compuls compunctious compuncti + computation comput comrade comrad + comrades comrad comutual comutu + con con concave concav + concavities concav conceal conceal + concealed conceal concealing conceal + concealment conceal concealments conceal + conceals conceal conceit conceit + conceited conceit conceitless conceitless + conceits conceit conceiv conceiv + conceive conceiv conceived conceiv + conceives conceiv conceiving conceiv + conception concept conceptions concept + conceptious concepti concern concern + concernancy concern concerneth concerneth + concerning concern concernings concern + concerns concern conclave conclav + conclud conclud conclude conclud + concluded conclud concludes conclud + concluding conclud conclusion conclus + conclusions conclus concolinel concolinel + concord concord concubine concubin + concupiscible concupisc concupy concupi + concur concur concurring concur + concurs concur condemn condemn + condemnation condemn condemned condemn + condemning condemn condemns condemn + condescend condescend condign condign + condition condit conditionally condition + conditions condit condole condol + condolement condol condoling condol + conduce conduc conduct conduct + conducted conduct conducting conduct + conductor conductor conduit conduit + conduits conduit conected conect + coney conei confection confect + confectionary confectionari confections confect + confederacy confederaci confederate confeder + confederates confeder confer confer + conference confer conferr conferr + conferring confer confess confess + confessed confess confesses confess + confesseth confesseth confessing confess + confession confess confessions confess + confessor confessor confidence confid + confident confid confidently confid + confin confin confine confin + confined confin confineless confineless + confiners confin confines confin + confining confin confirm confirm + confirmation confirm confirmations confirm + confirmed confirm confirmer confirm + confirmers confirm confirming confirm + confirmities confirm confirms confirm + confiscate confisc confiscated confisc + confiscation confisc confixed confix + conflict conflict conflicting conflict + conflicts conflict confluence confluenc + conflux conflux conform conform + conformable conform confound confound + confounded confound confounding confound + confounds confound confront confront + confronted confront confus confu + confused confus confusedly confusedli + confusion confus confusions confus + confutation confut confutes confut + congeal congeal congealed congeal + congealment congeal congee conge + conger conger congest congest + congied congi congratulate congratul + congreeing congre congreeted congreet + congregate congreg congregated congreg + congregation congreg congregations congreg + congruent congruent congruing congru + conies coni conjectural conjectur + conjecture conjectur conjectures conjectur + conjoin conjoin conjoined conjoin + conjoins conjoin conjointly conjointli + conjunct conjunct conjunction conjunct + conjunctive conjunct conjur conjur + conjuration conjur conjurations conjur + conjure conjur conjured conjur + conjurer conjur conjurers conjur + conjures conjur conjuring conjur + conjuro conjuro conn conn + connected connect connive conniv + conqu conqu conquer conquer + conquered conquer conquering conquer + conqueror conqueror conquerors conqueror + conquers conquer conquest conquest + conquests conquest conquring conqur + conrade conrad cons con + consanguineous consanguin consanguinity consanguin + conscienc conscienc conscience conscienc + consciences conscienc conscionable conscion + consecrate consecr consecrated consecr + consecrations consecr consent consent + consented consent consenting consent + consents consent consequence consequ + consequences consequ consequently consequ + conserve conserv conserved conserv + conserves conserv consider consid + considerance consider considerate consider + consideration consider considerations consider + considered consid considering consid + considerings consid considers consid + consign consign consigning consign + consist consist consisteth consisteth + consisting consist consistory consistori + consists consist consolate consol + consolation consol consonancy conson + consonant conson consort consort + consorted consort consortest consortest + conspectuities conspectu conspir conspir + conspiracy conspiraci conspirant conspir + conspirator conspir conspirators conspir + conspire conspir conspired conspir + conspirers conspir conspires conspir + conspiring conspir constable constabl + constables constabl constance constanc + constancies constanc constancy constanc + constant constant constantine constantin + constantinople constantinopl constantly constantli + constellation constel constitution constitut + constrain constrain constrained constrain + constraineth constraineth constrains constrain + constraint constraint constring constr + construction construct construe constru + consul consul consuls consul + consulship consulship consulships consulship + consult consult consulting consult + consults consult consum consum + consume consum consumed consum + consumes consum consuming consum + consummate consumm consummation consumm + consumption consumpt consumptions consumpt + contagion contagion contagious contagi + contain contain containing contain + contains contain contaminate contamin + contaminated contamin contemn contemn + contemned contemn contemning contemn + contemns contemn contemplate contempl + contemplation contempl contemplative contempl + contempt contempt contemptible contempt + contempts contempt contemptuous contemptu + contemptuously contemptu contend contend + contended contend contending contend + contendon contendon content content + contenta contenta contented content + contenteth contenteth contention content + contentious contenti contentless contentless + contento contento contents content + contest contest contestation contest + continence contin continency contin + continent contin continents contin + continu continu continual continu + continually continu continuance continu + continuantly continuantli continuate continu + continue continu continued continu + continuer continu continues continu + continuing continu contract contract + contracted contract contracting contract + contraction contract contradict contradict + contradicted contradict contradiction contradict + contradicts contradict contraries contrari + contrarieties contrarieti contrariety contrarieti + contrarious contrari contrariously contrari + contrary contrari contre contr + contribution contribut contributors contributor + contrite contrit contriv contriv + contrive contriv contrived contriv + contriver contriv contrives contriv + contriving contriv control control + controll control controller control + controlling control controlment control + controls control controversy controversi + contumelious contumeli contumeliously contumeli + contumely contum contusions contus + convenience conveni conveniences conveni + conveniency conveni convenient conveni + conveniently conveni convented convent + conventicles conventicl convents convent + convers conver conversant convers + conversation convers conversations convers + converse convers conversed convers + converses convers conversing convers + conversion convers convert convert + converted convert convertest convertest + converting convert convertite convertit + convertites convertit converts convert + convey convei conveyance convey + conveyances convey conveyers convey + conveying convei convict convict + convicted convict convince convinc + convinced convinc convinces convinc + convive conviv convocation convoc + convoy convoi convulsions convuls + cony coni cook cook + cookery cookeri cooks cook + cool cool cooled cool + cooling cool cools cool + coop coop coops coop + cop cop copatain copatain + cope cope cophetua cophetua + copied copi copies copi + copious copiou copper copper + copperspur copperspur coppice coppic + copulation copul copulatives copul + copy copi cor cor + coragio coragio coral coral + coram coram corambus corambu + coranto coranto corantos coranto + corbo corbo cord cord + corded cord cordelia cordelia + cordial cordial cordis cordi + cords cord core core + corin corin corinth corinth + corinthian corinthian coriolanus coriolanu + corioli corioli cork cork + corky corki cormorant cormor + corn corn cornelia cornelia + cornelius corneliu corner corner + corners corner cornerstone cornerston + cornets cornet cornish cornish + corns corn cornuto cornuto + cornwall cornwal corollary corollari + coronal coron coronation coron + coronet coronet coronets coronet + corporal corpor corporals corpor + corporate corpor corpse corps + corpulent corpul correct correct + corrected correct correcting correct + correction correct correctioner correction + corrects correct correspondence correspond + correspondent correspond corresponding correspond + corresponsive correspons corrigible corrig + corrival corriv corrivals corriv + corroborate corrobor corrosive corros + corrupt corrupt corrupted corrupt + corrupter corrupt corrupters corrupt + corruptible corrupt corruptibly corrupt + corrupting corrupt corruption corrupt + corruptly corruptli corrupts corrupt + corse cors corses cors + corslet corslet cosmo cosmo + cost cost costard costard + costermongers costermong costlier costlier + costly costli costs cost + cot cot cote cote + coted cote cotsall cotsal + cotsole cotsol cotswold cotswold + cottage cottag cottages cottag + cotus cotu couch couch + couched couch couching couch + couchings couch coude coud + cough cough coughing cough + could could couldst couldst + coulter coulter council council + councillor councillor councils council + counsel counsel counsell counsel + counsellor counsellor counsellors counsellor + counselor counselor counselors counselor + counsels counsel count count + counted count countenanc countenanc + countenance counten countenances counten + counter counter counterchange counterchang + countercheck countercheck counterfeit counterfeit + counterfeited counterfeit counterfeiting counterfeit + counterfeitly counterfeitli counterfeits counterfeit + countermand countermand countermands countermand + countermines countermin counterpart counterpart + counterpoints counterpoint counterpois counterpoi + counterpoise counterpois counters counter + countervail countervail countess countess + countesses countess counties counti + counting count countless countless + countries countri countrv countrv + country countri countryman countryman + countrymen countrymen counts count + county counti couper couper + couple coupl coupled coupl + couplement couplement couples coupl + couplet couplet couplets couplet + cour cour courage courag + courageous courag courageously courag + courages courag courier courier + couriers courier couronne couronn + cours cour course cours + coursed cours courser courser + coursers courser courses cours + coursing cours court court + courted court courteous courteou + courteously courteous courtesan courtesan + courtesies courtesi courtesy courtesi + courtezan courtezan courtezans courtezan + courtier courtier courtiers courtier + courtlike courtlik courtly courtli + courtney courtnei courts court + courtship courtship cousin cousin + cousins cousin couterfeit couterfeit + coutume coutum covenant coven + covenants coven covent covent + coventry coventri cover cover + covered cover covering cover + coverlet coverlet covers cover + covert covert covertly covertli + coverture covertur covet covet + coveted covet coveting covet + covetings covet covetous covet + covetously covet covetousness covet + covets covet cow cow + coward coward cowarded coward + cowardice cowardic cowardly cowardli + cowards coward cowardship cowardship + cowish cowish cowl cowl + cowslip cowslip cowslips cowslip + cox cox coxcomb coxcomb + coxcombs coxcomb coy coi + coystrill coystril coz coz + cozen cozen cozenage cozenag + cozened cozen cozener cozen + cozeners cozen cozening cozen + coziers cozier crab crab + crabbed crab crabs crab + crack crack cracked crack + cracker cracker crackers cracker + cracking crack cracks crack + cradle cradl cradled cradl + cradles cradl craft craft + crafted craft craftied crafti + craftier craftier craftily craftili + crafts craft craftsmen craftsmen + crafty crafti cram cram + cramm cramm cramp cramp + cramps cramp crams cram + cranking crank cranks crank + cranmer cranmer crannied cranni + crannies cranni cranny cranni + crants crant crare crare + crash crash crassus crassu + crav crav crave crave + craved crave craven craven + cravens craven craves crave + craveth craveth craving crave + crawl crawl crawling crawl + crawls crawl craz craz + crazed craze crazy crazi + creaking creak cream cream + create creat created creat + creates creat creating creat + creation creation creator creator + creature creatur creatures creatur + credence credenc credent credent + credible credibl credit credit + creditor creditor creditors creditor + credo credo credulity credul + credulous credul creed creed + creek creek creeks creek + creep creep creeping creep + creeps creep crept crept + crescent crescent crescive cresciv + cressets cresset cressid cressid + cressida cressida cressids cressid + cressy cressi crest crest + crested crest crestfall crestfal + crestless crestless crests crest + cretan cretan crete crete + crevice crevic crew crew + crews crew crib crib + cribb cribb cribs crib + cricket cricket crickets cricket + cried cri criedst criedst + crier crier cries cri + criest criest crieth crieth + crime crime crimeful crime + crimeless crimeless crimes crime + criminal crimin crimson crimson + cringe cring cripple crippl + crisp crisp crisped crisp + crispian crispian crispianus crispianu + crispin crispin critic critic + critical critic critics critic + croak croak croaking croak + croaks croak crocodile crocodil + cromer cromer cromwell cromwel + crone crone crook crook + crookback crookback crooked crook + crooking crook crop crop + cropp cropp crosby crosbi + cross cross crossed cross + crosses cross crossest crossest + crossing cross crossings cross + crossly crossli crossness cross + crost crost crotchets crotchet + crouch crouch crouching crouch + crow crow crowd crowd + crowded crowd crowding crowd + crowds crowd crowflowers crowflow + crowing crow crowkeeper crowkeep + crown crown crowned crown + crowner crowner crownet crownet + crownets crownet crowning crown + crowns crown crows crow + crudy crudi cruel cruel + cruell cruell crueller crueller + cruelly cruelli cruels cruel + cruelty cruelti crum crum + crumble crumbl crumbs crumb + crupper crupper crusadoes crusado + crush crush crushed crush + crushest crushest crushing crush + crust crust crusts crust + crusty crusti crutch crutch + crutches crutch cry cry + crying cry crystal crystal + crystalline crystallin crystals crystal + cub cub cubbert cubbert + cubiculo cubiculo cubit cubit + cubs cub cuckold cuckold + cuckoldly cuckoldli cuckolds cuckold + cuckoo cuckoo cucullus cucullu + cudgel cudgel cudgeled cudgel + cudgell cudgel cudgelling cudgel + cudgels cudgel cue cue + cues cue cuff cuff + cuffs cuff cuique cuiqu + cull cull culling cull + cullion cullion cullionly cullionli + cullions cullion culpable culpabl + culverin culverin cum cum + cumber cumber cumberland cumberland + cunning cun cunningly cunningli + cunnings cun cuore cuor + cup cup cupbearer cupbear + cupboarding cupboard cupid cupid + cupids cupid cuppele cuppel + cups cup cur cur + curan curan curate curat + curb curb curbed curb + curbing curb curbs curb + curd curd curdied curdi + curds curd cure cure + cured cure cureless cureless + curer curer cures cure + curfew curfew curing cure + curio curio curiosity curios + curious curiou curiously curious + curl curl curled curl + curling curl curls curl + currance curranc currants currant + current current currents current + currish currish curry curri + curs cur curse curs + cursed curs curses curs + cursies cursi cursing curs + cursorary cursorari curst curst + curster curster curstest curstest + curstness curst cursy cursi + curtail curtail curtain curtain + curtains curtain curtal curtal + curtis curti curtle curtl + curtsied curtsi curtsies curtsi + curtsy curtsi curvet curvet + curvets curvet cushes cush + cushion cushion cushions cushion + custalorum custalorum custard custard + custody custodi custom custom + customary customari customed custom + customer custom customers custom + customs custom custure custur + cut cut cutler cutler + cutpurse cutpurs cutpurses cutpurs + cuts cut cutter cutter + cutting cut cuttle cuttl + cxsar cxsar cyclops cyclop + cydnus cydnu cygnet cygnet + cygnets cygnet cym cym + cymbals cymbal cymbeline cymbelin + cyme cyme cynic cynic + cynthia cynthia cypress cypress + cypriot cypriot cyprus cypru + cyrus cyru cytherea cytherea + d d dabbled dabbl + dace dace dad dad + daedalus daedalu daemon daemon + daff daff daffed daf + daffest daffest daffodils daffodil + dagger dagger daggers dagger + dagonet dagonet daily daili + daintier daintier dainties dainti + daintiest daintiest daintily daintili + daintiness dainti daintry daintri + dainty dainti daisied daisi + daisies daisi daisy daisi + dale dale dalliance dallianc + dallied dalli dallies dalli + dally dalli dallying dalli + dalmatians dalmatian dam dam + damage damag damascus damascu + damask damask damasked damask + dame dame dames dame + damm damm damn damn + damnable damnabl damnably damnabl + damnation damnat damned damn + damns damn damoiselle damoisel + damon damon damosella damosella + damp damp dams dam + damsel damsel damsons damson + dan dan danc danc + dance danc dancer dancer + dances danc dancing danc + dandle dandl dandy dandi + dane dane dang dang + danger danger dangerous danger + dangerously danger dangers danger + dangling dangl daniel daniel + danish danish dank dank + dankish dankish danskers dansker + daphne daphn dappled dappl + dapples dappl dar dar + dardan dardan dardanian dardanian + dardanius dardaniu dare dare + dared dare dareful dare + dares dare darest darest + daring dare darius dariu + dark dark darken darken + darkening darken darkens darken + darker darker darkest darkest + darkling darkl darkly darkli + darkness dark darling darl + darlings darl darnel darnel + darraign darraign dart dart + darted dart darter darter + dartford dartford darting dart + darts dart dash dash + dashes dash dashing dash + dastard dastard dastards dastard + dat dat datchet datchet + date date dated date + dateless dateless dates date + daub daub daughter daughter + daughters daughter daunt daunt + daunted daunt dauntless dauntless + dauphin dauphin daventry daventri + davy davi daw daw + dawn dawn dawning dawn + daws daw day dai + daylight daylight days dai + dazzle dazzl dazzled dazzl + dazzling dazzl de de + dead dead deadly deadli + deaf deaf deafing deaf + deafness deaf deafs deaf + deal deal dealer dealer + dealers dealer dealest dealest + dealing deal dealings deal + deals deal dealt dealt + dean dean deanery deaneri + dear dear dearer dearer + dearest dearest dearly dearli + dearness dear dears dear + dearth dearth dearths dearth + death death deathbed deathb + deathful death deaths death + deathsman deathsman deathsmen deathsmen + debarred debar debase debas + debate debat debated debat + debatement debat debateth debateth + debating debat debauch debauch + debile debil debility debil + debitor debitor debonair debonair + deborah deborah debosh debosh + debt debt debted debt + debtor debtor debtors debtor + debts debt debuty debuti + decay decai decayed decai + decayer decay decaying decai + decays decai deceas decea + decease deceas deceased deceas + deceit deceit deceitful deceit + deceits deceit deceiv deceiv + deceivable deceiv deceive deceiv + deceived deceiv deceiver deceiv + deceivers deceiv deceives deceiv + deceivest deceivest deceiveth deceiveth + deceiving deceiv december decemb + decent decent deceptious decepti + decerns decern decide decid + decides decid decimation decim + decipher deciph deciphers deciph + decision decis decius deciu + deck deck decking deck + decks deck deckt deckt + declare declar declares declar + declension declens declensions declens + declin declin decline declin + declined declin declines declin + declining declin decoct decoct + decorum decorum decreas decrea + decrease decreas decreasing decreas + decree decre decreed decre + decrees decre decrepit decrepit + dedicate dedic dedicated dedic + dedicates dedic dedication dedic + deed deed deedless deedless + deeds deed deem deem + deemed deem deep deep + deeper deeper deepest deepest + deeply deepli deeps deep + deepvow deepvow deer deer + deesse deess defac defac + deface defac defaced defac + defacer defac defacers defac + defacing defac defam defam + default default defeat defeat + defeated defeat defeats defeat + defeatures defeatur defect defect + defective defect defects defect + defence defenc defences defenc + defend defend defendant defend + defended defend defender defend + defenders defend defending defend + defends defend defense defens + defensible defens defensive defens + defer defer deferr deferr + defiance defianc deficient defici + defied defi defies defi + defil defil defile defil + defiler defil defiles defil + defiling defil define defin + definement defin definite definit + definitive definit definitively definit + deflow deflow deflower deflow + deflowered deflow deform deform + deformed deform deformities deform + deformity deform deftly deftli + defunct defunct defunction defunct + defuse defus defy defi + defying defi degenerate degener + degraded degrad degree degre + degrees degre deified deifi + deifying deifi deign deign + deigned deign deiphobus deiphobu + deities deiti deity deiti + deja deja deject deject + dejected deject delabreth delabreth + delay delai delayed delai + delaying delai delays delai + delectable delect deliberate deliber + delicate delic delicates delic + delicious delici deliciousness delici + delight delight delighted delight + delightful delight delights delight + delinquents delinqu deliv deliv + deliver deliv deliverance deliver + delivered deliv delivering deliv + delivers deliv delivery deliveri + delphos delpho deluded delud + deluding delud deluge delug + delve delv delver delver + delves delv demand demand + demanded demand demanding demand + demands demand demean demean + demeanor demeanor demeanour demeanour + demerits demerit demesnes demesn + demetrius demetriu demi demi + demigod demigod demise demis + demoiselles demoisel demon demon + demonstrable demonstr demonstrate demonstr + demonstrated demonstr demonstrating demonstr + demonstration demonstr demonstrative demonstr + demure demur demurely demur + demuring demur den den + denay denai deni deni + denial denial denials denial + denied deni denier denier + denies deni deniest deniest + denis deni denmark denmark + dennis denni denny denni + denote denot denoted denot + denotement denot denounc denounc + denounce denounc denouncing denounc + dens den denunciation denunci + deny deni denying deni + deo deo depart depart + departed depart departest departest + departing depart departure departur + depeche depech depend depend + dependant depend dependants depend + depended depend dependence depend + dependences depend dependency depend + dependent depend dependents depend + depender depend depending depend + depends depend deplore deplor + deploring deplor depopulate depopul + depos depo depose depos + deposed depos deposing depos + depositaries depositari deprav deprav + depravation deprav deprave deprav + depraved deprav depraves deprav + depress depress depriv depriv + deprive depriv depth depth + depths depth deputation deput + depute deput deputed deput + deputies deputi deputing deput + deputy deputi deracinate deracin + derby derbi dercetas derceta + dere dere derides derid + derision deris deriv deriv + derivation deriv derivative deriv + derive deriv derived deriv + derives deriv derogate derog + derogately derog derogation derog + des de desartless desartless + descant descant descend descend + descended descend descending descend + descends descend descension descens + descent descent descents descent + describe describ described describ + describes describ descried descri + description descript descriptions descript + descry descri desdemon desdemon + desdemona desdemona desert desert + deserts desert deserv deserv + deserve deserv deserved deserv + deservedly deservedli deserver deserv + deservers deserv deserves deserv + deservest deservest deserving deserv + deservings deserv design design + designment design designments design + designs design desir desir + desire desir desired desir + desirers desir desires desir + desirest desirest desiring desir + desirous desir desist desist + desk desk desolate desol + desolation desol desp desp + despair despair despairing despair + despairs despair despatch despatch + desperate desper desperately desper + desperation desper despis despi + despise despis despised despis + despiser despis despiseth despiseth + despising despis despite despit + despiteful despit despoiled despoil + dest dest destin destin + destined destin destinies destini + destiny destini destitute destitut + destroy destroi destroyed destroi + destroyer destroy destroyers destroy + destroying destroi destroys destroi + destruction destruct destructions destruct + det det detain detain + detains detain detect detect + detected detect detecting detect + detection detect detector detector + detects detect detention detent + determin determin determinate determin + determination determin determinations determin + determine determin determined determin + determines determin detest detest + detestable detest detested detest + detesting detest detests detest + detract detract detraction detract + detractions detract deucalion deucalion + deuce deuc deum deum + deux deux devant devant + devesting devest device devic + devices devic devil devil + devilish devilish devils devil + devis devi devise devis + devised devis devises devis + devising devis devoid devoid + devonshire devonshir devote devot + devoted devot devotion devot + devour devour devoured devour + devourers devour devouring devour + devours devour devout devout + devoutly devoutli dew dew + dewberries dewberri dewdrops dewdrop + dewlap dewlap dewlapp dewlapp + dews dew dewy dewi + dexter dexter dexteriously dexteri + dexterity dexter di di + diable diabl diablo diablo + diadem diadem dial dial + dialect dialect dialogue dialogu + dialogued dialogu dials dial + diameter diamet diamond diamond + diamonds diamond dian dian + diana diana diaper diaper + dibble dibbl dic dic + dice dice dicers dicer + dich dich dick dick + dickens dicken dickon dickon + dicky dicki dictator dictat + diction diction dictynna dictynna + did did diddle diddl + didest didest dido dido + didst didst die die + died di diedst diedst + dies di diest diest + diet diet dieted diet + dieter dieter dieu dieu + diff diff differ differ + difference differ differences differ + differency differ different differ + differing differ differs differ + difficile difficil difficult difficult + difficulties difficulti difficulty difficulti + diffidence diffid diffidences diffid + diffus diffu diffused diffus + diffusest diffusest dig dig + digest digest digested digest + digestion digest digestions digest + digg digg digging dig + dighton dighton dignified dignifi + dignifies dignifi dignify dignifi + dignities digniti dignity digniti + digress digress digressing digress + digression digress digs dig + digt digt dilate dilat + dilated dilat dilations dilat + dilatory dilatori dild dild + dildos dildo dilemma dilemma + dilemmas dilemma diligence dilig + diligent dilig diluculo diluculo + dim dim dimension dimens + dimensions dimens diminish diminish + diminishing diminish diminution diminut + diminutive diminut diminutives diminut + dimm dimm dimmed dim + dimming dim dimpled dimpl + dimples dimpl dims dim + din din dine dine + dined dine diner diner + dines dine ding ding + dining dine dinner dinner + dinners dinner dinnertime dinnertim + dint dint diomed diom + diomede diomed diomedes diomed + dion dion dip dip + dipp dipp dipping dip + dips dip dir dir + dire dire direct direct + directed direct directing direct + direction direct directions direct + directitude directitud directive direct + directly directli directs direct + direful dire direness dire + direst direst dirge dirg + dirges dirg dirt dirt + dirty dirti dis di + disability disabl disable disabl + disabled disabl disabling disabl + disadvantage disadvantag disagree disagre + disallow disallow disanimates disanim + disannul disannul disannuls disannul + disappointed disappoint disarm disarm + disarmed disarm disarmeth disarmeth + disarms disarm disaster disast + disasters disast disastrous disastr + disbench disbench disbranch disbranch + disburdened disburden disburs disbur + disburse disburs disbursed disburs + discandy discandi discandying discandi + discard discard discarded discard + discase discas discased discas + discern discern discerner discern + discerning discern discernings discern + discerns discern discharg discharg + discharge discharg discharged discharg + discharging discharg discipled discipl + disciples discipl disciplin disciplin + discipline disciplin disciplined disciplin + disciplines disciplin disclaim disclaim + disclaiming disclaim disclaims disclaim + disclos disclo disclose disclos + disclosed disclos discloses disclos + discolour discolour discoloured discolour + discolours discolour discomfit discomfit + discomfited discomfit discomfiture discomfitur + discomfort discomfort discomfortable discomfort + discommend discommend disconsolate disconsol + discontent discont discontented discont + discontentedly discontentedli discontenting discont + discontents discont discontinue discontinu + discontinued discontinu discord discord + discordant discord discords discord + discourse discours discoursed discours + discourser discours discourses discours + discoursive discours discourtesy discourtesi + discov discov discover discov + discovered discov discoverers discover + discoveries discoveri discovering discov + discovers discov discovery discoveri + discredit discredit discredited discredit + discredits discredit discreet discreet + discreetly discreetli discretion discret + discretions discret discuss discuss + disdain disdain disdained disdain + disdaineth disdaineth disdainful disdain + disdainfully disdainfulli disdaining disdain + disdains disdain disdnguish disdnguish + diseas disea disease diseas + diseased diseas diseases diseas + disedg disedg disembark disembark + disfigure disfigur disfigured disfigur + disfurnish disfurnish disgorge disgorg + disgrac disgrac disgrace disgrac + disgraced disgrac disgraceful disgrac + disgraces disgrac disgracing disgrac + disgracious disgraci disguis disgui + disguise disguis disguised disguis + disguiser disguis disguises disguis + disguising disguis dish dish + dishabited dishabit dishclout dishclout + dishearten dishearten disheartens dishearten + dishes dish dishonest dishonest + dishonestly dishonestli dishonesty dishonesti + dishonor dishonor dishonorable dishonor + dishonors dishonor dishonour dishonour + dishonourable dishonour dishonoured dishonour + dishonours dishonour disinherit disinherit + disinherited disinherit disjoin disjoin + disjoining disjoin disjoins disjoin + disjoint disjoint disjunction disjunct + dislik dislik dislike dislik + disliken disliken dislikes dislik + dislimns dislimn dislocate disloc + dislodg dislodg disloyal disloy + disloyalty disloyalti dismal dismal + dismantle dismantl dismantled dismantl + dismask dismask dismay dismai + dismayed dismai dismemb dismemb + dismember dismemb dismes dism + dismiss dismiss dismissed dismiss + dismissing dismiss dismission dismiss + dismount dismount dismounted dismount + disnatur disnatur disobedience disobedi + disobedient disobedi disobey disobei + disobeys disobei disorb disorb + disorder disord disordered disord + disorderly disorderli disorders disord + disparage disparag disparagement disparag + disparagements disparag dispark dispark + dispatch dispatch dispensation dispens + dispense dispens dispenses dispens + dispers disper disperse dispers + dispersed dispers dispersedly dispersedli + dispersing dispers dispiteous dispit + displac displac displace displac + displaced displac displant displant + displanting displant display displai + displayed displai displeas displea + displease displeas displeased displeas + displeasing displeas displeasure displeasur + displeasures displeasur disponge dispong + disport disport disports disport + dispos dispo dispose dispos + disposed dispos disposer dispos + disposing dispos disposition disposit + dispositions disposit dispossess dispossess + dispossessing dispossess disprais disprai + dispraise disprais dispraising disprais + dispraisingly dispraisingli dispropertied disproperti + disproportion disproport disproportioned disproport + disprov disprov disprove disprov + disproved disprov dispursed dispurs + disputable disput disputation disput + disputations disput dispute disput + disputed disput disputes disput + disputing disput disquantity disquant + disquiet disquiet disquietly disquietli + disrelish disrelish disrobe disrob + disseat disseat dissemble dissembl + dissembled dissembl dissembler dissembl + dissemblers dissembl dissembling dissembl + dissembly dissembl dissension dissens + dissensions dissens dissentious dissenti + dissever dissev dissipation dissip + dissolute dissolut dissolutely dissolut + dissolution dissolut dissolutions dissolut + dissolv dissolv dissolve dissolv + dissolved dissolv dissolves dissolv + dissuade dissuad dissuaded dissuad + distaff distaff distaffs distaff + distain distain distains distain + distance distanc distant distant + distaste distast distasted distast + distasteful distast distemp distemp + distemper distemp distemperature distemperatur + distemperatures distemperatur distempered distemp + distempering distemp distil distil + distill distil distillation distil + distilled distil distills distil + distilment distil distinct distinct + distinction distinct distinctly distinctli + distingue distingu distinguish distinguish + distinguishes distinguish distinguishment distinguish + distract distract distracted distract + distractedly distractedli distraction distract + distractions distract distracts distract + distrain distrain distraught distraught + distress distress distressed distress + distresses distress distressful distress + distribute distribut distributed distribut + distribution distribut distrust distrust + distrustful distrust disturb disturb + disturbed disturb disturbers disturb + disturbing disturb disunite disunit + disvalued disvalu disvouch disvouch + dit dit ditch ditch + ditchers ditcher ditches ditch + dites dite ditties ditti + ditty ditti diurnal diurnal + div div dive dive + diver diver divers diver + diversely divers diversity divers + divert divert diverted divert + diverts divert dives dive + divest divest dividable divid + dividant divid divide divid + divided divid divides divid + divideth divideth divin divin + divination divin divine divin + divinely divin divineness divin + diviner divin divines divin + divinest divinest divining divin + divinity divin division divis + divisions divis divorc divorc + divorce divorc divorced divorc + divorcement divorc divorcing divorc + divulg divulg divulge divulg + divulged divulg divulging divulg + dizy dizi dizzy dizzi + do do doating doat + dobbin dobbin dock dock + docks dock doct doct + doctor doctor doctors doctor + doctrine doctrin document document + dodge dodg doe doe + doer doer doers doer + does doe doest doest + doff doff dog dog + dogberry dogberri dogfish dogfish + dogg dogg dogged dog + dogs dog doigts doigt + doing do doings do + doit doit doits doit + dolabella dolabella dole dole + doleful dole doll doll + dollar dollar dollars dollar + dolor dolor dolorous dolor + dolour dolour dolours dolour + dolphin dolphin dolt dolt + dolts dolt domestic domest + domestics domest dominance domin + dominations domin dominator domin + domine domin domineer domin + domineering domin dominical domin + dominion dominion dominions dominion + domitius domitiu dommelton dommelton + don don donalbain donalbain + donation donat donc donc + doncaster doncast done done + dong dong donn donn + donne donn donner donner + donnerai donnerai doom doom + doomsday doomsdai door door + doorkeeper doorkeep doors door + dorcas dorca doreus doreu + doricles doricl dormouse dormous + dorothy dorothi dorset dorset + dorsetshire dorsetshir dost dost + dotage dotag dotant dotant + dotard dotard dotards dotard + dote dote doted dote + doters doter dotes dote + doteth doteth doth doth + doting dote double doubl + doubled doubl doubleness doubl + doubler doubler doublet doublet + doublets doublet doubling doubl + doubly doubli doubt doubt + doubted doubt doubtful doubt + doubtfully doubtfulli doubting doubt + doubtless doubtless doubts doubt + doug doug dough dough + doughty doughti doughy doughi + douglas dougla dout dout + doute dout douts dout + dove dove dovehouse dovehous + dover dover doves dove + dow dow dowager dowag + dowdy dowdi dower dower + dowerless dowerless dowers dower + dowlas dowla dowle dowl + down down downfall downfal + downright downright downs down + downstairs downstair downtrod downtrod + downward downward downwards downward + downy downi dowries dowri + dowry dowri dowsabel dowsabel + doxy doxi dozed doze + dozen dozen dozens dozen + dozy dozi drab drab + drabbing drab drabs drab + drachma drachma drachmas drachma + draff draff drag drag + dragg dragg dragged drag + dragging drag dragon dragon + dragonish dragonish dragons dragon + drain drain drained drain + drains drain drake drake + dram dram dramatis dramati + drank drank draught draught + draughts draught drave drave + draw draw drawbridge drawbridg + drawer drawer drawers drawer + draweth draweth drawing draw + drawling drawl drawn drawn + draws draw drayman drayman + draymen draymen dread dread + dreaded dread dreadful dread + dreadfully dreadfulli dreading dread + dreads dread dream dream + dreamer dreamer dreamers dreamer + dreaming dream dreams dream + dreamt dreamt drearning drearn + dreary dreari dreg dreg + dregs dreg drench drench + drenched drench dress dress + dressed dress dresser dresser + dressing dress dressings dress + drest drest drew drew + dribbling dribbl dried dri + drier drier dries dri + drift drift drily drili + drink drink drinketh drinketh + drinking drink drinkings drink + drinks drink driv driv + drive drive drivelling drivel + driven driven drives drive + driveth driveth driving drive + drizzle drizzl drizzled drizzl + drizzles drizzl droit droit + drollery drolleri dromio dromio + dromios dromio drone drone + drones drone droop droop + droopeth droopeth drooping droop + droops droop drop drop + dropheir dropheir droplets droplet + dropp dropp dropper dropper + droppeth droppeth dropping drop + droppings drop drops drop + dropsied dropsi dropsies dropsi + dropsy dropsi dropt dropt + dross dross drossy drossi + drought drought drove drove + droven droven drovier drovier + drown drown drowned drown + drowning drown drowns drown + drows drow drowse drows + drowsily drowsili drowsiness drowsi + drowsy drowsi drudge drudg + drudgery drudgeri drudges drudg + drug drug drugg drugg + drugs drug drum drum + drumble drumbl drummer drummer + drumming drum drums drum + drunk drunk drunkard drunkard + drunkards drunkard drunken drunken + drunkenly drunkenli drunkenness drunken + dry dry dryness dryness + dst dst du du + dub dub dubb dubb + ducat ducat ducats ducat + ducdame ducdam duchess duchess + duchies duchi duchy duchi + duck duck ducking duck + ducks duck dudgeon dudgeon + due due duellist duellist + duello duello duer duer + dues due duff duff + dug dug dugs dug + duke duke dukedom dukedom + dukedoms dukedom dukes duke + dulcet dulcet dulche dulch + dull dull dullard dullard + duller duller dullest dullest + dulling dull dullness dull + dulls dull dully dulli + dulness dul duly duli + dumain dumain dumb dumb + dumbe dumb dumbly dumbl + dumbness dumb dump dump + dumps dump dun dun + duncan duncan dung dung + dungeon dungeon dungeons dungeon + dunghill dunghil dunghills dunghil + dungy dungi dunnest dunnest + dunsinane dunsinan dunsmore dunsmor + dunstable dunstabl dupp dupp + durance duranc during dure + durst durst dusky duski + dust dust dusted dust + dusty dusti dutch dutch + dutchman dutchman duteous duteou + duties duti dutiful duti + duty duti dwarf dwarf + dwarfish dwarfish dwell dwell + dwellers dweller dwelling dwell + dwells dwell dwelt dwelt + dwindle dwindl dy dy + dye dye dyed dy + dyer dyer dying dy + e e each each + eager eager eagerly eagerli + eagerness eager eagle eagl + eagles eagl eaning ean + eanlings eanl ear ear + earing ear earl earl + earldom earldom earlier earlier + earliest earliest earliness earli + earls earl early earli + earn earn earned earn + earnest earnest earnestly earnestli + earnestness earnest earns earn + ears ear earth earth + earthen earthen earthlier earthlier + earthly earthli earthquake earthquak + earthquakes earthquak earthy earthi + eas ea ease eas + eased eas easeful eas + eases eas easier easier + easiest easiest easiliest easiliest + easily easili easiness easi + easing eas east east + eastcheap eastcheap easter easter + eastern eastern eastward eastward + easy easi eat eat + eaten eaten eater eater + eaters eater eating eat + eats eat eaux eaux + eaves eav ebb ebb + ebbing eb ebbs ebb + ebon ebon ebony eboni + ebrew ebrew ecce ecc + echapper echapp echo echo + echoes echo eclips eclip + eclipse eclips eclipses eclips + ecolier ecoli ecoutez ecoutez + ecstacy ecstaci ecstasies ecstasi + ecstasy ecstasi ecus ecu + eden eden edg edg + edgar edgar edge edg + edged edg edgeless edgeless + edges edg edict edict + edicts edict edifice edific + edifices edific edified edifi + edifies edifi edition edit + edm edm edmund edmund + edmunds edmund edmundsbury edmundsburi + educate educ educated educ + education educ edward edward + eel eel eels eel + effect effect effected effect + effectless effectless effects effect + effectual effectu effectually effectu + effeminate effemin effigies effigi + effus effu effuse effus + effusion effus eftest eftest + egal egal egally egal + eget eget egeus egeu + egg egg eggs egg + eggshell eggshel eglamour eglamour + eglantine eglantin egma egma + ego ego egregious egregi + egregiously egregi egress egress + egypt egypt egyptian egyptian + egyptians egyptian eie eie + eight eight eighteen eighteen + eighth eighth eightpenny eightpenni + eighty eighti eisel eisel + either either eject eject + eke ek el el + elbe elb elbow elbow + elbows elbow eld eld + elder elder elders elder + eldest eldest eleanor eleanor + elect elect elected elect + election elect elegancy eleg + elegies elegi element element + elements element elephant eleph + elephants eleph elevated elev + eleven eleven eleventh eleventh + elf elf elflocks elflock + eliads eliad elinor elinor + elizabeth elizabeth ell ell + elle ell ellen ellen + elm elm eloquence eloqu + eloquent eloqu else els + elsewhere elsewher elsinore elsinor + eltham eltham elves elv + elvish elvish ely eli + elysium elysium em em + emballing embal embalm embalm + embalms embalm embark embark + embarked embark embarquements embarqu + embassade embassad embassage embassag + embassies embassi embassy embassi + embattailed embattail embattl embattl + embattle embattl embay embai + embellished embellish embers ember + emblaze emblaz emblem emblem + emblems emblem embodied embodi + embold embold emboldens embolden + emboss emboss embossed emboss + embounded embound embowel embowel + embowell embowel embrac embrac + embrace embrac embraced embrac + embracement embrac embracements embrac + embraces embrac embracing embrac + embrasures embrasur embroider embroid + embroidery embroideri emhracing emhrac + emilia emilia eminence emin + eminent emin eminently emin + emmanuel emmanuel emnity emniti + empale empal emperal emper + emperess emperess emperial emperi + emperor emperor empery emperi + emphasis emphasi empire empir + empirics empir empiricutic empiricut + empleached empleach employ emploi + employed emploi employer employ + employment employ employments employ + empoison empoison empress empress + emptied empti emptier emptier + empties empti emptiness empti + empty empti emptying empti + emulate emul emulation emul + emulations emul emulator emul + emulous emul en en + enact enact enacted enact + enacts enact enactures enactur + enamell enamel enamelled enamel + enamour enamour enamoured enamour + enanmour enanmour encamp encamp + encamped encamp encave encav + enceladus enceladu enchaf enchaf + enchafed enchaf enchant enchant + enchanted enchant enchanting enchant + enchantingly enchantingli enchantment enchant + enchantress enchantress enchants enchant + enchas encha encircle encircl + encircled encircl enclos enclo + enclose enclos enclosed enclos + encloses enclos encloseth encloseth + enclosing enclos enclouded encloud + encompass encompass encompassed encompass + encompasseth encompasseth encompassment encompass + encore encor encorporal encorpor + encount encount encounter encount + encountered encount encounters encount + encourage encourag encouraged encourag + encouragement encourag encrimsoned encrimson + encroaching encroach encumb encumb + end end endamage endamag + endamagement endamag endanger endang + endart endart endear endear + endeared endear endeavour endeavour + endeavours endeavour ended end + ender ender ending end + endings end endite endit + endless endless endow endow + endowed endow endowments endow + endows endow ends end + endu endu endue endu + endur endur endurance endur + endure endur endured endur + endures endur enduring endur + endymion endymion eneas enea + enemies enemi enemy enemi + enernies enerni enew enew + enfeebled enfeebl enfeebles enfeebl + enfeoff enfeoff enfetter enfett + enfoldings enfold enforc enforc + enforce enforc enforced enforc + enforcedly enforcedli enforcement enforc + enforces enforc enforcest enforcest + enfranched enfranch enfranchis enfranchi + enfranchise enfranchis enfranchised enfranchis + enfranchisement enfranchis enfreed enfre + enfreedoming enfreedom engag engag + engage engag engaged engag + engagements engag engaging engag + engaol engaol engend engend + engender engend engenders engend + engilds engild engine engin + engineer engin enginer engin + engines engin engirt engirt + england england english english + englishman englishman englishmen englishmen + engluts englut englutted englut + engraffed engraf engraft engraft + engrafted engraft engrav engrav + engrave engrav engross engross + engrossed engross engrossest engrossest + engrossing engross engrossments engross + enguard enguard enigma enigma + enigmatical enigmat enjoin enjoin + enjoined enjoin enjoy enjoi + enjoyed enjoi enjoyer enjoy + enjoying enjoi enjoys enjoi + enkindle enkindl enkindled enkindl + enlard enlard enlarg enlarg + enlarge enlarg enlarged enlarg + enlargement enlarg enlargeth enlargeth + enlighten enlighten enlink enlink + enmesh enmesh enmities enmiti + enmity enmiti ennoble ennobl + ennobled ennobl enobarb enobarb + enobarbus enobarbu enon enon + enormity enorm enormous enorm + enough enough enow enow + enpatron enpatron enpierced enpierc + enquir enquir enquire enquir + enquired enquir enrag enrag + enrage enrag enraged enrag + enrages enrag enrank enrank + enrapt enrapt enrich enrich + enriched enrich enriches enrich + enridged enridg enrings enr + enrob enrob enrobe enrob + enroll enrol enrolled enrol + enrooted enroot enrounded enround + enschedul enschedul ensconce ensconc + ensconcing ensconc enseamed enseam + ensear ensear enseigne enseign + enseignez enseignez ensemble ensembl + enshelter enshelt enshielded enshield + enshrines enshrin ensign ensign + ensigns ensign enskied enski + ensman ensman ensnare ensnar + ensnared ensnar ensnareth ensnareth + ensteep ensteep ensu ensu + ensue ensu ensued ensu + ensues ensu ensuing ensu + enswathed enswath ent ent + entail entail entame entam + entangled entangl entangles entangl + entendre entendr enter enter + entered enter entering enter + enterprise enterpris enterprises enterpris + enters enter entertain entertain + entertained entertain entertainer entertain + entertaining entertain entertainment entertain + entertainments entertain enthrall enthral + enthralled enthral enthron enthron + enthroned enthron entice entic + enticements entic enticing entic + entire entir entirely entir + entitle entitl entitled entitl + entitling entitl entomb entomb + entombed entomb entrails entrail + entrance entranc entrances entranc + entrap entrap entrapp entrapp + entre entr entreat entreat + entreated entreat entreaties entreati + entreating entreat entreatments entreat + entreats entreat entreaty entreati + entrench entrench entry entri + entwist entwist envelop envelop + envenom envenom envenomed envenom + envenoms envenom envied envi + envies envi envious enviou + enviously envious environ environ + environed environ envoy envoi + envy envi envying envi + enwheel enwheel enwombed enwomb + enwraps enwrap ephesian ephesian + ephesians ephesian ephesus ephesu + epicure epicur epicurean epicurean + epicures epicur epicurism epicur + epicurus epicuru epidamnum epidamnum + epidaurus epidauru epigram epigram + epilepsy epilepsi epileptic epilept + epilogue epilogu epilogues epilogu + epistles epistl epistrophus epistrophu + epitaph epitaph epitaphs epitaph + epithet epithet epitheton epitheton + epithets epithet epitome epitom + equal equal equalities equal + equality equal equall equal + equally equal equalness equal + equals equal equinoctial equinocti + equinox equinox equipage equipag + equity equiti equivocal equivoc + equivocate equivoc equivocates equivoc + equivocation equivoc equivocator equivoc + er er erbear erbear + erbearing erbear erbears erbear + erbeat erbeat erblows erblow + erboard erboard erborne erborn + ercame ercam ercast ercast + ercharg ercharg ercharged ercharg + ercharging ercharg ercles ercl + ercome ercom ercover ercov + ercrows ercrow erdoing erdo + ere er erebus erebu + erect erect erected erect + erecting erect erection erect + erects erect erewhile erewhil + erflourish erflourish erflow erflow + erflowing erflow erflows erflow + erfraught erfraught erga erga + ergalled ergal erglanced erglanc + ergo ergo ergone ergon + ergrow ergrow ergrown ergrown + ergrowth ergrowth erhang erhang + erhanging erhang erhasty erhasti + erhear erhear erheard erheard + eringoes eringo erjoy erjoi + erleap erleap erleaps erleap + erleavens erleaven erlook erlook + erlooking erlook ermaster ermast + ermengare ermengar ermount ermount + ern ern ernight ernight + eros ero erpaid erpaid + erparted erpart erpast erpast + erpays erpai erpeer erpeer + erperch erperch erpicturing erpictur + erpingham erpingham erposting erpost + erpow erpow erpress erpress + erpressed erpress err err + errand errand errands errand + errant errant errate errat + erraught erraught erreaches erreach + erred er errest errest + erring er erroneous erron + error error errors error + errs err errule errul + errun errun erset erset + ershade ershad ershades ershad + ershine ershin ershot ershot + ersized ersiz erskip erskip + erslips erslip erspreads erspread + erst erst erstare erstar + erstep erstep erstunk erstunk + ersway erswai ersways erswai + erswell erswel erta erta + ertake ertak erteemed erteem + erthrow erthrow erthrown erthrown + erthrows erthrow ertook ertook + ertop ertop ertopping ertop + ertrip ertrip erturn erturn + erudition erudit eruption erupt + eruptions erupt ervalues ervalu + erwalk erwalk erwatch erwatch + erween erween erweens erween + erweigh erweigh erweighs erweigh + erwhelm erwhelm erwhelmed erwhelm + erworn erworn es es + escalus escalu escap escap + escape escap escaped escap + escapes escap eschew eschew + escoted escot esill esil + especial especi especially especi + esperance esper espials espial + espied espi espies espi + espous espou espouse espous + espy espi esquire esquir + esquires esquir essay essai + essays essai essence essenc + essential essenti essentially essenti + esses ess essex essex + est est establish establish + established establish estate estat + estates estat esteem esteem + esteemed esteem esteemeth esteemeth + esteeming esteem esteems esteem + estimable estim estimate estim + estimation estim estimations estim + estime estim estranged estrang + estridge estridg estridges estridg + et et etc etc + etceteras etcetera ete et + eternal etern eternally etern + eterne etern eternity etern + eterniz eterniz etes et + ethiop ethiop ethiope ethiop + ethiopes ethiop ethiopian ethiopian + etna etna eton eton + etre etr eunuch eunuch + eunuchs eunuch euphrates euphrat + euphronius euphroniu euriphile euriphil + europa europa europe europ + ev ev evade evad + evades evad evans evan + evasion evas evasions evas + eve ev even even + evening even evenly evenli + event event eventful event + events event ever ever + everlasting everlast everlastingly everlastingli + evermore evermor every everi + everyone everyon everything everyth + everywhere everywher evidence evid + evidences evid evident evid + evil evil evilly evilli + evils evil evitate evit + ewe ew ewer ewer + ewers ewer ewes ew + exact exact exacted exact + exactest exactest exacting exact + exaction exact exactions exact + exactly exactli exacts exact + exalt exalt exalted exalt + examin examin examination examin + examinations examin examine examin + examined examin examines examin + exampl exampl example exampl + exampled exampl examples exampl + exasperate exasper exasperates exasper + exceed exce exceeded exceed + exceedeth exceedeth exceeding exceed + exceedingly exceedingli exceeds exce + excel excel excelled excel + excellence excel excellencies excel + excellency excel excellent excel + excellently excel excelling excel + excels excel except except + excepted except excepting except + exception except exceptions except + exceptless exceptless excess excess + excessive excess exchang exchang + exchange exchang exchanged exchang + exchequer exchequ exchequers exchequ + excite excit excited excit + excitements excit excites excit + exclaim exclaim exclaims exclaim + exclamation exclam exclamations exclam + excludes exclud excommunicate excommun + excommunication excommun excrement excrement + excrements excrement excursion excurs + excursions excurs excus excu + excusable excus excuse excus + excused excus excuses excus + excusez excusez excusing excus + execrable execr execrations execr + execute execut executed execut + executing execut execution execut + executioner execution executioners execution + executor executor executors executor + exempt exempt exempted exempt + exequies exequi exercise exercis + exercises exercis exeter exet + exeunt exeunt exhal exhal + exhalation exhal exhalations exhal + exhale exhal exhales exhal + exhaust exhaust exhibit exhibit + exhibiters exhibit exhibition exhibit + exhort exhort exhortation exhort + exigent exig exil exil + exile exil exiled exil + exion exion exist exist + exists exist exit exit + exits exit exorciser exorcis + exorcisms exorc exorcist exorcist + expect expect expectance expect + expectancy expect expectation expect + expectations expect expected expect + expecters expect expecting expect + expects expect expedience expedi + expedient expedi expediently expedi + expedition expedit expeditious expediti + expel expel expell expel + expelling expel expels expel + expend expend expense expens + expenses expens experienc experienc + experience experi experiences experi + experiment experi experimental experiment + experiments experi expert expert + expertness expert expiate expiat + expiation expiat expir expir + expiration expir expire expir + expired expir expires expir + expiring expir explication explic + exploit exploit exploits exploit + expos expo expose expos + exposing expos exposition exposit + expositor expositor expostulate expostul + expostulation expostul exposture expostur + exposure exposur expound expound + expounded expound express express + expressed express expresseth expresseth + expressing express expressive express + expressly expressli expressure expressur + expuls expul expulsion expuls + exquisite exquisit exsufflicate exsuffl + extant extant extemporal extempor + extemporally extempor extempore extempor + extend extend extended extend + extends extend extent extent + extenuate extenu extenuated extenu + extenuates extenu extenuation extenu + exterior exterior exteriorly exteriorli + exteriors exterior extermin extermin + extern extern external extern + extinct extinct extincted extinct + extincture extinctur extinguish extinguish + extirp extirp extirpate extirp + extirped extirp extol extol + extoll extol extolment extol + exton exton extort extort + extorted extort extortion extort + extortions extort extra extra + extract extract extracted extract + extracting extract extraordinarily extraordinarili + extraordinary extraordinari extraught extraught + extravagancy extravag extravagant extravag + extreme extrem extremely extrem + extremes extrem extremest extremest + extremities extrem extremity extrem + exuent exuent exult exult + exultation exult ey ey + eyas eya eyases eyas + eye ey eyeball eyebal + eyeballs eyebal eyebrow eyebrow + eyebrows eyebrow eyed ei + eyeless eyeless eyelid eyelid + eyelids eyelid eyes ey + eyesight eyesight eyestrings eyestr + eying ei eyne eyn + eyrie eyri fa fa + fabian fabian fable fabl + fables fabl fabric fabric + fabulous fabul fac fac + face face faced face + facere facer faces face + faciant faciant facile facil + facility facil facinerious facineri + facing face facit facit + fact fact faction faction + factionary factionari factions faction + factious factiou factor factor + factors factor faculties faculti + faculty faculti fade fade + faded fade fadeth fadeth + fadge fadg fading fade + fadings fade fadom fadom + fadoms fadom fagot fagot + fagots fagot fail fail + failing fail fails fail + fain fain faint faint + fainted faint fainter fainter + fainting faint faintly faintli + faintness faint faints faint + fair fair fairer fairer + fairest fairest fairies fairi + fairing fair fairings fair + fairly fairli fairness fair + fairs fair fairwell fairwel + fairy fairi fais fai + fait fait faites fait + faith faith faithful faith + faithfull faithful faithfully faithfulli + faithless faithless faiths faith + faitors faitor fal fal + falchion falchion falcon falcon + falconbridge falconbridg falconer falcon + falconers falcon fall fall + fallacy fallaci fallen fallen + falleth falleth falliable falliabl + fallible fallibl falling fall + fallow fallow fallows fallow + falls fall fally falli + falorous falor false fals + falsehood falsehood falsely fals + falseness fals falser falser + falsify falsifi falsing fals + falstaff falstaff falstaffs falstaff + falter falter fam fam + fame fame famed fame + familiar familiar familiarity familiar + familiarly familiarli familiars familiar + family famili famine famin + famish famish famished famish + famous famou famoused famous + famously famous fan fan + fanatical fanat fancies fanci + fancy fanci fane fane + fanes fane fang fang + fangled fangl fangless fangless + fangs fang fann fann + fanning fan fans fan + fantasied fantasi fantasies fantasi + fantastic fantast fantastical fantast + fantastically fantast fantasticoes fantastico + fantasy fantasi fap fap + far far farborough farborough + farced farc fardel fardel + fardels fardel fare fare + fares fare farewell farewel + farewells farewel fariner farin + faring fare farm farm + farmer farmer farmhouse farmhous + farms farm farre farr + farrow farrow farther farther + farthest farthest farthing farth + farthingale farthingal farthingales farthingal + farthings farth fartuous fartuou + fas fa fashion fashion + fashionable fashion fashioning fashion + fashions fashion fast fast + fasted fast fasten fasten + fastened fasten faster faster + fastest fastest fasting fast + fastly fastli fastolfe fastolf + fasts fast fat fat + fatal fatal fatally fatal + fate fate fated fate + fates fate father father + fathered father fatherless fatherless + fatherly fatherli fathers father + fathom fathom fathomless fathomless + fathoms fathom fatigate fatig + fatness fat fats fat + fatted fat fatter fatter + fattest fattest fatting fat + fatuus fatuu fauconbridge fauconbridg + faulconbridge faulconbridg fault fault + faultiness faulti faultless faultless + faults fault faulty faulti + fausse fauss fauste faust + faustuses faustus faut faut + favor favor favorable favor + favorably favor favors favor + favour favour favourable favour + favoured favour favouredly favouredli + favourer favour favourers favour + favouring favour favourite favourit + favourites favourit favours favour + favout favout fawn fawn + fawneth fawneth fawning fawn + fawns fawn fay fai + fe fe fealty fealti + fear fear feared fear + fearest fearest fearful fear + fearfull fearful fearfully fearfulli + fearfulness fear fearing fear + fearless fearless fears fear + feast feast feasted feast + feasting feast feasts feast + feat feat feated feat + feater feater feather feather + feathered feather feathers feather + featly featli feats feat + featur featur feature featur + featured featur featureless featureless + features featur february februari + fecks feck fed fed + fedary fedari federary federari + fee fee feeble feebl + feebled feebl feebleness feebl + feebling feebl feebly feebli + feed feed feeder feeder + feeders feeder feedeth feedeth + feeding feed feeds feed + feel feel feeler feeler + feeling feel feelingly feelingli + feels feel fees fee + feet feet fehemently fehement + feign feign feigned feign + feigning feign feil feil + feith feith felicitate felicit + felicity felic fell fell + fellest fellest fellies felli + fellow fellow fellowly fellowli + fellows fellow fellowship fellowship + fellowships fellowship fells fell + felon felon felonious feloni + felony feloni felt felt + female femal females femal + feminine feminin fen fen + fenc fenc fence fenc + fencer fencer fencing fenc + fends fend fennel fennel + fenny fenni fens fen + fenton fenton fer fer + ferdinand ferdinand fere fere + fernseed fernse ferrara ferrara + ferrers ferrer ferret ferret + ferry ferri ferryman ferryman + fertile fertil fertility fertil + fervency fervenc fervour fervour + fery feri fest fest + feste fest fester fester + festinate festin festinately festin + festival festiv festivals festiv + fet fet fetch fetch + fetches fetch fetching fetch + fetlock fetlock fetlocks fetlock + fett fett fetter fetter + fettering fetter fetters fetter + fettle fettl feu feu + feud feud fever fever + feverous fever fevers fever + few few fewer fewer + fewest fewest fewness few + fickle fickl fickleness fickl + fico fico fiction fiction + fiddle fiddl fiddler fiddler + fiddlestick fiddlestick fidele fidel + fidelicet fidelicet fidelity fidel + fidius fidiu fie fie + field field fielded field + fields field fiend fiend + fiends fiend fierce fierc + fiercely fierc fierceness fierc + fiery fieri fife fife + fifes fife fifteen fifteen + fifteens fifteen fifteenth fifteenth + fifth fifth fifty fifti + fiftyfold fiftyfold fig fig + fight fight fighter fighter + fightest fightest fighteth fighteth + fighting fight fights fight + figo figo figs fig + figur figur figure figur + figured figur figures figur + figuring figur fike fike + fil fil filberts filbert + filch filch filches filch + filching filch file file + filed file files file + filial filial filius filiu + fill fill filled fill + fillet fillet filling fill + fillip fillip fills fill + filly filli film film + fils fil filth filth + filths filth filthy filthi + fin fin finally final + finch finch find find + finder finder findeth findeth + finding find findings find + finds find fine fine + fineless fineless finely fine + finem finem fineness fine + finer finer fines fine + finest finest fing fing + finger finger fingering finger + fingers finger fingre fingr + fingres fingr finical finic + finish finish finished finish + finisher finish finless finless + finn finn fins fin + finsbury finsburi fir fir + firago firago fire fire + firebrand firebrand firebrands firebrand + fired fire fires fire + firework firework fireworks firework + firing fire firk firk + firm firm firmament firmament + firmly firmli firmness firm + first first firstlings firstl + fish fish fisher fisher + fishermen fishermen fishers fisher + fishes fish fishified fishifi + fishmonger fishmong fishpond fishpond + fisnomy fisnomi fist fist + fisting fist fists fist + fistula fistula fit fit + fitchew fitchew fitful fit + fitly fitli fitment fitment + fitness fit fits fit + fitted fit fitter fitter + fittest fittest fitteth fitteth + fitting fit fitzwater fitzwat + five five fivepence fivep + fives five fix fix + fixed fix fixes fix + fixeth fixeth fixing fix + fixture fixtur fl fl + flag flag flagging flag + flagon flagon flagons flagon + flags flag flail flail + flakes flake flaky flaki + flam flam flame flame + flamen flamen flamens flamen + flames flame flaming flame + flaminius flaminiu flanders flander + flannel flannel flap flap + flaring flare flash flash + flashes flash flashing flash + flask flask flat flat + flatly flatli flatness flat + flats flat flatt flatt + flatter flatter flattered flatter + flatterer flatter flatterers flatter + flatterest flatterest flatteries flatteri + flattering flatter flatters flatter + flattery flatteri flaunts flaunt + flavio flavio flavius flaviu + flaw flaw flaws flaw + flax flax flaxen flaxen + flay flai flaying flai + flea flea fleance fleanc + fleas flea flecked fleck + fled fled fledge fledg + flee flee fleec fleec + fleece fleec fleeces fleec + fleer fleer fleering fleer + fleers fleer fleet fleet + fleeter fleeter fleeting fleet + fleming fleme flemish flemish + flesh flesh fleshes flesh + fleshly fleshli fleshment fleshment + fleshmonger fleshmong flew flew + flexible flexibl flexure flexur + flibbertigibbet flibbertigibbet flickering flicker + flidge flidg fliers flier + flies fli flieth flieth + flight flight flights flight + flighty flighti flinch flinch + fling fling flint flint + flints flint flinty flinti + flirt flirt float float + floated float floating float + flock flock flocks flock + flood flood floodgates floodgat + floods flood floor floor + flora flora florence florenc + florentine florentin florentines florentin + florentius florentiu florizel florizel + flote flote floulish floulish + flour flour flourish flourish + flourishes flourish flourisheth flourisheth + flourishing flourish flout flout + flouted flout flouting flout + flouts flout flow flow + flowed flow flower flower + flowerets floweret flowers flower + flowing flow flown flown + flows flow fluellen fluellen + fluent fluent flung flung + flush flush flushing flush + fluster fluster flute flute + flutes flute flutter flutter + flux flux fluxive fluxiv + fly fly flying fly + fo fo foal foal + foals foal foam foam + foamed foam foaming foam + foams foam foamy foami + fob fob focative foc + fodder fodder foe foe + foeman foeman foemen foemen + foes foe fog fog + foggy foggi fogs fog + foh foh foi foi + foil foil foiled foil + foils foil foin foin + foining foin foins foin + fois foi foison foison + foisons foison foist foist + foix foix fold fold + folded fold folds fold + folio folio folk folk + folks folk follies folli + follow follow followed follow + follower follow followers follow + followest followest following follow + follows follow folly folli + fond fond fonder fonder + fondly fondli fondness fond + font font fontibell fontibel + food food fool fool + fooleries fooleri foolery fooleri + foolhardy foolhardi fooling fool + foolish foolish foolishly foolishli + foolishness foolish fools fool + foot foot football footbal + footboy footboi footboys footboi + footed foot footfall footfal + footing foot footman footman + footmen footmen footpath footpath + footsteps footstep footstool footstool + fopp fopp fopped fop + foppery fopperi foppish foppish + fops fop for for + forage forag foragers forag + forbade forbad forbear forbear + forbearance forbear forbears forbear + forbid forbid forbidden forbidden + forbiddenly forbiddenli forbids forbid + forbod forbod forborne forborn + forc forc force forc + forced forc forceful forc + forceless forceless forces forc + forcible forcibl forcibly forcibl + forcing forc ford ford + fordid fordid fordo fordo + fordoes fordo fordone fordon + fore fore forecast forecast + forefather forefath forefathers forefath + forefinger forefing forego forego + foregone foregon forehand forehand + forehead forehead foreheads forehead + forehorse forehors foreign foreign + foreigner foreign foreigners foreign + foreknowing foreknow foreknowledge foreknowledg + foremost foremost forenamed forenam + forenoon forenoon forerun forerun + forerunner forerunn forerunning forerun + foreruns forerun foresaid foresaid + foresaw foresaw foresay foresai + foresee forese foreseeing forese + foresees forese foreshow foreshow + foreskirt foreskirt forespent foresp + forest forest forestall forestal + forestalled forestal forester forest + foresters forest forests forest + foretell foretel foretelling foretel + foretells foretel forethink forethink + forethought forethought foretold foretold + forever forev foreward foreward + forewarn forewarn forewarned forewarn + forewarning forewarn forfeit forfeit + forfeited forfeit forfeiters forfeit + forfeiting forfeit forfeits forfeit + forfeiture forfeitur forfeitures forfeitur + forfend forfend forfended forfend + forg forg forgave forgav + forge forg forged forg + forgeries forgeri forgery forgeri + forges forg forget forget + forgetful forget forgetfulness forget + forgetive forget forgets forget + forgetting forget forgive forgiv + forgiven forgiven forgiveness forgiv + forgo forgo forgoing forgo + forgone forgon forgot forgot + forgotten forgotten fork fork + forked fork forks fork + forlorn forlorn form form + formal formal formally formal + formed form former former + formerly formerli formless formless + forms form fornication fornic + fornications fornic fornicatress fornicatress + forres forr forrest forrest + forsake forsak forsaken forsaken + forsaketh forsaketh forslow forslow + forsook forsook forsooth forsooth + forspent forspent forspoke forspok + forswear forswear forswearing forswear + forswore forswor forsworn forsworn + fort fort forted fort + forth forth forthcoming forthcom + forthlight forthlight forthright forthright + forthwith forthwith fortification fortif + fortifications fortif fortified fortifi + fortifies fortifi fortify fortifi + fortinbras fortinbra fortitude fortitud + fortnight fortnight fortress fortress + fortresses fortress forts fort + fortun fortun fortuna fortuna + fortunate fortun fortunately fortun + fortune fortun fortuned fortun + fortunes fortun fortward fortward + forty forti forum forum + forward forward forwarding forward + forwardness forward forwards forward + forwearied forweari fosset fosset + fost fost foster foster + fostered foster fought fought + foughten foughten foul foul + fouler fouler foulest foulest + foully foulli foulness foul + found found foundation foundat + foundations foundat founded found + founder founder fount fount + fountain fountain fountains fountain + founts fount four four + fourscore fourscor fourteen fourteen + fourth fourth foutra foutra + fowl fowl fowler fowler + fowling fowl fowls fowl + fox fox foxes fox + foxship foxship fracted fract + fraction fraction fractions fraction + fragile fragil fragment fragment + fragments fragment fragrant fragrant + frail frail frailer frailer + frailties frailti frailty frailti + fram fram frame frame + framed frame frames frame + frampold frampold fran fran + francais francai france franc + frances franc franchise franchis + franchised franchis franchisement franchis + franchises franchis franciae francia + francis franci francisca francisca + franciscan franciscan francisco francisco + frank frank franker franker + frankfort frankfort franklin franklin + franklins franklin frankly frankli + frankness frank frantic frantic + franticly franticli frateretto frateretto + fratrum fratrum fraud fraud + fraudful fraud fraught fraught + fraughtage fraughtag fraughting fraught + fray frai frays frai + freckl freckl freckled freckl + freckles freckl frederick frederick + free free freed freed + freedom freedom freedoms freedom + freehearted freeheart freelier freelier + freely freeli freeman freeman + freemen freemen freeness freeness + freer freer frees free + freestone freeston freetown freetown + freeze freez freezes freez + freezing freez freezings freez + french french frenchman frenchman + frenchmen frenchmen frenchwoman frenchwoman + frenzy frenzi frequent frequent + frequents frequent fresh fresh + fresher fresher freshes fresh + freshest freshest freshly freshli + freshness fresh fret fret + fretful fret frets fret + fretted fret fretten fretten + fretting fret friar friar + friars friar friday fridai + fridays fridai friend friend + friended friend friending friend + friendless friendless friendliness friendli + friendly friendli friends friend + friendship friendship friendships friendship + frieze friez fright fright + frighted fright frightened frighten + frightful fright frighting fright + frights fright fringe fring + fringed fring frippery fripperi + frisk frisk fritters fritter + frivolous frivol fro fro + frock frock frog frog + frogmore frogmor froissart froissart + frolic frolic from from + front front fronted front + frontier frontier frontiers frontier + fronting front frontlet frontlet + fronts front frost frost + frosts frost frosty frosti + froth froth froward froward + frown frown frowning frown + frowningly frowningli frowns frown + froze froze frozen frozen + fructify fructifi frugal frugal + fruit fruit fruiterer fruiter + fruitful fruit fruitfully fruitfulli + fruitfulness fruit fruition fruition + fruitless fruitless fruits fruit + frush frush frustrate frustrat + frutify frutifi fry fry + fubb fubb fuel fuel + fugitive fugit fulfil fulfil + fulfill fulfil fulfilling fulfil + fulfils fulfil full full + fullam fullam fuller fuller + fullers fuller fullest fullest + fullness full fully fulli + fulness ful fulsome fulsom + fulvia fulvia fum fum + fumble fumbl fumbles fumbl + fumblest fumblest fumbling fumbl + fume fume fumes fume + fuming fume fumiter fumit + fumitory fumitori fun fun + function function functions function + fundamental fundament funeral funer + funerals funer fur fur + furbish furbish furies furi + furious furiou furlongs furlong + furnace furnac furnaces furnac + furnish furnish furnished furnish + furnishings furnish furniture furnitur + furnival furniv furor furor + furr furr furrow furrow + furrowed furrow furrows furrow + furth furth further further + furtherance further furtherer further + furthermore furthermor furthest furthest + fury furi furze furz + furzes furz fust fust + fustian fustian fustilarian fustilarian + fusty fusti fut fut + future futur futurity futur + g g gabble gabbl + gaberdine gaberdin gabriel gabriel + gad gad gadding gad + gads gad gadshill gadshil + gag gag gage gage + gaged gage gagg gagg + gaging gage gagne gagn + gain gain gained gain + gainer gainer gaingiving gaingiv + gains gain gainsaid gainsaid + gainsay gainsai gainsaying gainsai + gainsays gainsai gainst gainst + gait gait gaited gait + galathe galath gale gale + galen galen gales gale + gall gall gallant gallant + gallantly gallantli gallantry gallantri + gallants gallant galled gall + gallery galleri galley gallei + galleys gallei gallia gallia + gallian gallian galliard galliard + galliasses galliass gallimaufry gallimaufri + galling gall gallons gallon + gallop gallop galloping gallop + gallops gallop gallow gallow + galloway gallowai gallowglasses gallowglass + gallows gallow gallowses gallows + galls gall gallus gallu + gam gam gambol gambol + gambold gambold gambols gambol + gamboys gamboi game game + gamers gamer games game + gamesome gamesom gamester gamest + gaming game gammon gammon + gamut gamut gan gan + gangren gangren ganymede ganymed + gaol gaol gaoler gaoler + gaolers gaoler gaols gaol + gap gap gape gape + gapes gape gaping gape + gar gar garb garb + garbage garbag garboils garboil + garcon garcon gard gard + garde gard garden garden + gardener garden gardeners garden + gardens garden gardez gardez + gardiner gardin gardon gardon + gargantua gargantua gargrave gargrav + garish garish garland garland + garlands garland garlic garlic + garment garment garments garment + garmet garmet garner garner + garners garner garnish garnish + garnished garnish garret garret + garrison garrison garrisons garrison + gart gart garter garter + garterd garterd gartering garter + garters garter gascony gasconi + gash gash gashes gash + gaskins gaskin gasp gasp + gasping gasp gasted gast + gastness gast gat gat + gate gate gated gate + gates gate gath gath + gather gather gathered gather + gathering gather gathers gather + gatories gatori gatory gatori + gaud gaud gaudeo gaudeo + gaudy gaudi gauge gaug + gaul gaul gaultree gaultre + gaunt gaunt gauntlet gauntlet + gauntlets gauntlet gav gav + gave gave gavest gavest + gawded gawd gawds gawd + gawsey gawsei gay gai + gayness gay gaz gaz + gaze gaze gazed gaze + gazer gazer gazers gazer + gazes gaze gazeth gazeth + gazing gaze gear gear + geck geck geese gees + geffrey geffrei geld geld + gelded geld gelding geld + gelida gelida gelidus gelidu + gelt gelt gem gem + geminy gemini gems gem + gen gen gender gender + genders gender general gener + generally gener generals gener + generation gener generations gener + generative gener generosity generos + generous gener genitive genit + genitivo genitivo genius geniu + gennets gennet genoa genoa + genoux genoux gens gen + gent gent gentilhomme gentilhomm + gentility gentil gentle gentl + gentlefolks gentlefolk gentleman gentleman + gentlemanlike gentlemanlik gentlemen gentlemen + gentleness gentl gentler gentler + gentles gentl gentlest gentlest + gentlewoman gentlewoman gentlewomen gentlewomen + gently gentli gentry gentri + george georg gerard gerard + germaines germain germains germain + german german germane german + germans german germany germani + gertrude gertrud gest gest + gests gest gesture gestur + gestures gestur get get + getrude getrud gets get + getter getter getting get + ghastly ghastli ghost ghost + ghosted ghost ghostly ghostli + ghosts ghost gi gi + giant giant giantess giantess + giantlike giantlik giants giant + gib gib gibber gibber + gibbet gibbet gibbets gibbet + gibe gibe giber giber + gibes gibe gibing gibe + gibingly gibingli giddily giddili + giddiness giddi giddy giddi + gift gift gifts gift + gig gig giglets giglet + giglot giglot gilbert gilbert + gild gild gilded gild + gilding gild gilliams gilliam + gillian gillian gills gill + gillyvors gillyvor gilt gilt + gimmal gimmal gimmers gimmer + gin gin ging ging + ginger ginger gingerbread gingerbread + gingerly gingerli ginn ginn + gins gin gioucestershire gioucestershir + gipes gipe gipsies gipsi + gipsy gipsi gird gird + girded gird girdle girdl + girdled girdl girdles girdl + girdling girdl girl girl + girls girl girt girt + girth girth gis gi + giv giv give give + given given giver giver + givers giver gives give + givest givest giveth giveth + giving give givings give + glad glad gladded glad + gladding glad gladly gladli + gladness glad glamis glami + glanc glanc glance glanc + glanced glanc glances glanc + glancing glanc glanders glander + glansdale glansdal glare glare + glares glare glass glass + glasses glass glassy glassi + glaz glaz glazed glaze + gleams gleam glean glean + gleaned glean gleaning glean + gleeful gleeful gleek gleek + gleeking gleek gleeks gleek + glend glend glendower glendow + glib glib glide glide + glided glide glides glide + glideth glideth gliding glide + glimmer glimmer glimmering glimmer + glimmers glimmer glimpse glimps + glimpses glimps glist glist + glistening glisten glister glister + glistering glister glisters glister + glitt glitt glittering glitter + globe globe globes globe + glooming gloom gloomy gloomi + glories glori glorified glorifi + glorify glorifi glorious gloriou + gloriously glorious glory glori + glose glose gloss gloss + glosses gloss glou glou + glouceste gloucest gloucester gloucest + gloucestershire gloucestershir glove glove + glover glover gloves glove + glow glow glowed glow + glowing glow glowworm glowworm + gloz gloz gloze gloze + glozes gloze glu glu + glue glue glued glu + glues glue glut glut + glutt glutt glutted glut + glutton glutton gluttoning glutton + gluttony gluttoni gnarled gnarl + gnarling gnarl gnat gnat + gnats gnat gnaw gnaw + gnawing gnaw gnawn gnawn + gnaws gnaw go go + goad goad goaded goad + goads goad goal goal + goat goat goatish goatish + goats goat gobbets gobbet + gobbo gobbo goblet goblet + goblets goblet goblin goblin + goblins goblin god god + godded god godden godden + goddess goddess goddesses goddess + goddild goddild godfather godfath + godfathers godfath godhead godhead + godlike godlik godliness godli + godly godli godmother godmoth + gods god godson godson + goer goer goers goer + goes goe goest goest + goeth goeth goffe goff + gogs gog going go + gold gold golden golden + goldenly goldenli goldsmith goldsmith + goldsmiths goldsmith golgotha golgotha + goliases golias goliath goliath + gon gon gondola gondola + gondolier gondoli gone gone + goneril goneril gong gong + gonzago gonzago gonzalo gonzalo + good good goodfellow goodfellow + goodlier goodlier goodliest goodliest + goodly goodli goodman goodman + goodness good goodnight goodnight + goodrig goodrig goods good + goodwife goodwif goodwill goodwil + goodwin goodwin goodwins goodwin + goodyear goodyear goodyears goodyear + goose goos gooseberry gooseberri + goosequills goosequil goot goot + gor gor gorbellied gorbelli + gorboduc gorboduc gordian gordian + gore gore gored gore + gorg gorg gorge gorg + gorgeous gorgeou gorget gorget + gorging gorg gorgon gorgon + gormandize gormand gormandizing gormand + gory gori gosling gosl + gospel gospel gospels gospel + goss goss gossamer gossam + gossip gossip gossiping gossip + gossiplike gossiplik gossips gossip + got got goth goth + goths goth gotten gotten + gourd gourd gout gout + gouts gout gouty gouti + govern govern governance govern + governed govern governess gover + government govern governor governor + governors governor governs govern + gower gower gown gown + gowns gown grac grac + grace grace graced grace + graceful grace gracefully gracefulli + graceless graceless graces grace + gracing grace gracious graciou + graciously gracious gradation gradat + graff graff graffing graf + graft graft grafted graft + grafters grafter grain grain + grained grain grains grain + gramercies gramerci gramercy gramerci + grammar grammar grand grand + grandam grandam grandame grandam + grandchild grandchild grande grand + grandeur grandeur grandfather grandfath + grandjurors grandjuror grandmother grandmoth + grandpre grandpr grandsir grandsir + grandsire grandsir grandsires grandsir + grange grang grant grant + granted grant granting grant + grants grant grape grape + grapes grape grapple grappl + grapples grappl grappling grappl + grasp grasp grasped grasp + grasps grasp grass grass + grasshoppers grasshopp grassy grassi + grate grate grated grate + grateful grate grates grate + gratiano gratiano gratify gratifi + gratii gratii gratillity gratil + grating grate gratis grati + gratitude gratitud gratulate gratul + grav grav grave grave + gravediggers gravedigg gravel gravel + graveless graveless gravell gravel + gravely grave graven graven + graveness grave graver graver + graves grave gravest gravest + gravestone graveston gravities graviti + gravity graviti gravy gravi + gray grai graymalkin graymalkin + graz graz graze graze + grazed graze grazing graze + grease greas greases greas + greasily greasili greasy greasi + great great greater greater + greatest greatest greatly greatli + greatness great grecian grecian + grecians grecian gree gree + greece greec greed greed + greedily greedili greediness greedi + greedy greedi greeing gree + greek greek greekish greekish + greeks greek green green + greener greener greenly greenli + greens green greensleeves greensleev + greenwich greenwich greenwood greenwood + greet greet greeted greet + greeting greet greetings greet + greets greet greg greg + gregory gregori gremio gremio + grew grew grey grei + greybeard greybeard greybeards greybeard + greyhound greyhound greyhounds greyhound + grief grief griefs grief + griev griev grievance grievanc + grievances grievanc grieve griev + grieved griev grieves griev + grievest grievest grieving griev + grievingly grievingli grievous grievou + grievously grievous griffin griffin + griffith griffith grim grim + grime grime grimly grimli + grin grin grind grind + grinding grind grindstone grindston + grinning grin grip grip + gripe gripe gripes gripe + griping gripe grise grise + grisly grisli grissel grissel + grize grize grizzle grizzl + grizzled grizzl groan groan + groaning groan groans groan + groat groat groats groat + groin groin groom groom + grooms groom grop grop + groping grope gros gro + gross gross grosser grosser + grossly grossli grossness gross + ground ground grounded ground + groundlings groundl grounds ground + grove grove grovel grovel + grovelling grovel groves grove + grow grow groweth groweth + growing grow grown grown + grows grow growth growth + grub grub grubb grubb + grubs grub grudge grudg + grudged grudg grudges grudg + grudging grudg gruel gruel + grumble grumbl grumblest grumblest + grumbling grumbl grumblings grumbl + grumio grumio grund grund + grunt grunt gualtier gualtier + guard guard guardage guardag + guardant guardant guarded guard + guardian guardian guardians guardian + guards guard guardsman guardsman + gud gud gudgeon gudgeon + guerdon guerdon guerra guerra + guess guess guesses guess + guessingly guessingli guest guest + guests guest guiana guiana + guichard guichard guide guid + guided guid guider guider + guiderius guideriu guides guid + guiding guid guidon guidon + guienne guienn guil guil + guildenstern guildenstern guilders guilder + guildford guildford guildhall guildhal + guile guil guiled guil + guileful guil guilfords guilford + guilt guilt guiltian guiltian + guiltier guiltier guiltily guiltili + guiltiness guilti guiltless guiltless + guilts guilt guilty guilti + guinea guinea guinever guinev + guise guis gul gul + gules gule gulf gulf + gulfs gulf gull gull + gulls gull gum gum + gumm gumm gums gum + gun gun gunner gunner + gunpowder gunpowd guns gun + gurnet gurnet gurney gurnei + gust gust gusts gust + gusty gusti guts gut + gutter gutter guy gui + guynes guyn guysors guysor + gypsy gypsi gyve gyve + gyved gyve gyves gyve + h h ha ha + haberdasher haberdash habiliment habili + habiliments habili habit habit + habitation habit habited habit + habits habit habitude habitud + hack hack hacket hacket + hackney hacknei hacks hack + had had hadst hadst + haec haec haeres haer + hag hag hagar hagar + haggard haggard haggards haggard + haggish haggish haggled haggl + hags hag hail hail + hailed hail hailstone hailston + hailstones hailston hair hair + hairless hairless hairs hair + hairy hairi hal hal + halberd halberd halberds halberd + halcyon halcyon hale hale + haled hale hales hale + half half halfcan halfcan + halfpence halfpenc halfpenny halfpenni + halfpennyworth halfpennyworth halfway halfwai + halidom halidom hall hall + halloa halloa halloing hallo + hallond hallond halloo halloo + hallooing halloo hallow hallow + hallowed hallow hallowmas hallowma + hallown hallown hals hal + halt halt halter halter + halters halter halting halt + halts halt halves halv + ham ham hames hame + hamlet hamlet hammer hammer + hammered hammer hammering hammer + hammers hammer hamper hamper + hampton hampton hams ham + hamstring hamstr hand hand + handed hand handful hand + handicraft handicraft handicraftsmen handicraftsmen + handing hand handiwork handiwork + handkercher handkerch handkerchers handkerch + handkerchief handkerchief handle handl + handled handl handles handl + handless handless handlest handlest + handling handl handmaid handmaid + handmaids handmaid hands hand + handsaw handsaw handsome handsom + handsomely handsom handsomeness handsom + handwriting handwrit handy handi + hang hang hanged hang + hangers hanger hangeth hangeth + hanging hang hangings hang + hangman hangman hangmen hangmen + hangs hang hannibal hannib + hap hap hapless hapless + haply hapli happ happ + happen happen happened happen + happier happier happies happi + happiest happiest happily happili + happiness happi happy happi + haps hap harbinger harbing + harbingers harbing harbor harbor + harbour harbour harbourage harbourag + harbouring harbour harbours harbour + harcourt harcourt hard hard + harder harder hardest hardest + hardiest hardiest hardiment hardiment + hardiness hardi hardly hardli + hardness hard hardocks hardock + hardy hardi hare hare + harelip harelip hares hare + harfleur harfleur hark hark + harlot harlot harlotry harlotri + harlots harlot harm harm + harmed harm harmful harm + harming harm harmless harmless + harmonious harmoni harmony harmoni + harms harm harness har + harp harp harper harper + harpier harpier harping harp + harpy harpi harried harri + harrow harrow harrows harrow + harry harri harsh harsh + harshly harshli harshness harsh + hart hart harts hart + harum harum harvest harvest + has ha hast hast + haste hast hasted hast + hasten hasten hastes hast + hastily hastili hasting hast + hastings hast hasty hasti + hat hat hatch hatch + hatches hatch hatchet hatchet + hatching hatch hatchment hatchment + hate hate hated hate + hateful hate hater hater + haters hater hates hate + hateth hateth hatfield hatfield + hath hath hating hate + hatred hatr hats hat + haud haud hauf hauf + haught haught haughtiness haughti + haughty haughti haunch haunch + haunches haunch haunt haunt + haunted haunt haunting haunt + haunts haunt hautboy hautboi + hautboys hautboi have have + haven haven havens haven + haver haver having have + havings have havior havior + haviour haviour havoc havoc + hawk hawk hawking hawk + hawks hawk hawthorn hawthorn + hawthorns hawthorn hay hai + hazard hazard hazarded hazard + hazards hazard hazel hazel + hazelnut hazelnut he he + head head headborough headborough + headed head headier headier + heading head headland headland + headless headless headlong headlong + heads head headsman headsman + headstrong headstrong heady headi + heal heal healed heal + healing heal heals heal + health health healthful health + healths health healthsome healthsom + healthy healthi heap heap + heaping heap heaps heap + hear hear heard heard + hearer hearer hearers hearer + hearest hearest heareth heareth + hearing hear hearings hear + heark heark hearken hearken + hearkens hearken hears hear + hearsay hearsai hearse hears + hearsed hears hearst hearst + heart heart heartache heartach + heartbreak heartbreak heartbreaking heartbreak + hearted heart hearten hearten + hearth hearth hearths hearth + heartily heartili heartiness hearti + heartless heartless heartlings heartl + heartly heartli hearts heart + heartsick heartsick heartstrings heartstr + hearty hearti heat heat + heated heat heath heath + heathen heathen heathenish heathenish + heating heat heats heat + heauties heauti heav heav + heave heav heaved heav + heaven heaven heavenly heavenli + heavens heaven heaves heav + heavier heavier heaviest heaviest + heavily heavili heaviness heavi + heaving heav heavings heav + heavy heavi hebona hebona + hebrew hebrew hecate hecat + hectic hectic hector hector + hectors hector hecuba hecuba + hedg hedg hedge hedg + hedgehog hedgehog hedgehogs hedgehog + hedges hedg heed heed + heeded heed heedful heed + heedfull heedful heedfully heedfulli + heedless heedless heel heel + heels heel hefted heft + hefts heft heifer heifer + heifers heifer heigh heigh + height height heighten heighten + heinous heinou heinously heinous + heir heir heiress heiress + heirless heirless heirs heir + held held helen helen + helena helena helenus helenu + helias helia helicons helicon + hell hell hellespont hellespont + hellfire hellfir hellish hellish + helm helm helmed helm + helmet helmet helmets helmet + helms helm help help + helper helper helpers helper + helpful help helping help + helpless helpless helps help + helter helter hem hem + heme heme hemlock hemlock + hemm hemm hemp hemp + hempen hempen hems hem + hen hen hence henc + henceforth henceforth henceforward henceforward + henchman henchman henri henri + henricus henricu henry henri + hens hen hent hent + henton henton her her + herald herald heraldry heraldri + heralds herald herb herb + herbert herbert herblets herblet + herbs herb herculean herculean + hercules hercul herd herd + herds herd herdsman herdsman + herdsmen herdsmen here here + hereabout hereabout hereabouts hereabout + hereafter hereaft hereby herebi + hereditary hereditari hereford hereford + herefordshire herefordshir herein herein + hereof hereof heresies heresi + heresy heresi heretic heret + heretics heret hereto hereto + hereupon hereupon heritage heritag + heritier heriti hermes herm + hermia hermia hermione hermion + hermit hermit hermitage hermitag + hermits hermit herne hern + hero hero herod herod + herods herod heroes hero + heroic heroic heroical heroic + herring her herrings her + hers her herself herself + hesperides hesperid hesperus hesperu + hest hest hests hest + heure heur heureux heureux + hew hew hewgh hewgh + hewing hew hewn hewn + hews hew hey hei + heyday heydai hibocrates hibocr + hic hic hiccups hiccup + hick hick hid hid + hidden hidden hide hide + hideous hideou hideously hideous + hideousness hideous hides hide + hidest hidest hiding hide + hie hie hied hi + hiems hiem hies hi + hig hig high high + higher higher highest highest + highly highli highmost highmost + highness high hight hight + highway highwai highways highwai + hilding hild hildings hild + hill hill hillo hillo + hilloa hilloa hills hill + hilt hilt hilts hilt + hily hili him him + himself himself hinc hinc + hinckley hincklei hind hind + hinder hinder hindered hinder + hinders hinder hindmost hindmost + hinds hind hing hing + hinge hing hinges hing + hint hint hip hip + hipp hipp hipparchus hipparchu + hippolyta hippolyta hips hip + hir hir hire hire + hired hire hiren hiren + hirtius hirtiu his hi + hisperia hisperia hiss hiss + hisses hiss hissing hiss + hist hist historical histor + history histori hit hit + hither hither hitherto hitherto + hitherward hitherward hitherwards hitherward + hits hit hitting hit + hive hive hives hive + hizzing hizz ho ho + hoa hoa hoar hoar + hoard hoard hoarded hoard + hoarding hoard hoars hoar + hoarse hoars hoary hoari + hob hob hobbididence hobbidid + hobby hobbi hobbyhorse hobbyhors + hobgoblin hobgoblin hobnails hobnail + hoc hoc hod hod + hodge hodg hog hog + hogs hog hogshead hogshead + hogsheads hogshead hois hoi + hoise hois hoist hoist + hoisted hoist hoists hoist + holborn holborn hold hold + holden holden holder holder + holdeth holdeth holdfast holdfast + holding hold holds hold + hole hole holes hole + holidam holidam holidame holidam + holiday holidai holidays holidai + holier holier holiest holiest + holily holili holiness holi + holla holla holland holland + hollander holland hollanders holland + holloa holloa holloaing holloa + hollow hollow hollowly hollowli + hollowness hollow holly holli + holmedon holmedon holofernes holofern + holp holp holy holi + homage homag homager homag + home home homely home + homes home homespuns homespun + homeward homeward homewards homeward + homicide homicid homicides homicid + homily homili hominem hominem + hommes homm homo homo + honest honest honester honest + honestest honestest honestly honestli + honesty honesti honey honei + honeycomb honeycomb honeying honei + honeyless honeyless honeysuckle honeysuckl + honeysuckles honeysuckl honi honi + honneur honneur honor honor + honorable honor honorably honor + honorato honorato honorificabilitudinitatibus honorificabilitudinitatibu + honors honor honour honour + honourable honour honourably honour + honoured honour honourest honourest + honourible honour honouring honour + honours honour hoo hoo + hood hood hooded hood + hoodman hoodman hoods hood + hoodwink hoodwink hoof hoof + hoofs hoof hook hook + hooking hook hooks hook + hoop hoop hoops hoop + hoot hoot hooted hoot + hooting hoot hoots hoot + hop hop hope hope + hopeful hope hopeless hopeless + hopes hope hopest hopest + hoping hope hopkins hopkin + hoppedance hopped hor hor + horace horac horatio horatio + horizon horizon horn horn + hornbook hornbook horned horn + horner horner horning horn + hornpipes hornpip horns horn + horologe horolog horrible horribl + horribly horribl horrid horrid + horrider horrid horridly horridli + horror horror horrors horror + hors hor horse hors + horseback horseback horsed hors + horsehairs horsehair horseman horseman + horsemanship horsemanship horsemen horsemen + horses hors horseway horsewai + horsing hors hortensio hortensio + hortensius hortensiu horum horum + hose hose hospitable hospit + hospital hospit hospitality hospit + host host hostage hostag + hostages hostag hostess hostess + hostile hostil hostility hostil + hostilius hostiliu hosts host + hot hot hotly hotli + hotspur hotspur hotter hotter + hottest hottest hound hound + hounds hound hour hour + hourly hourli hours hour + hous hou house hous + household household householder household + householders household households household + housekeeper housekeep housekeepers housekeep + housekeeping housekeep houseless houseless + houses hous housewife housewif + housewifery housewiferi housewives housew + hovel hovel hover hover + hovered hover hovering hover + hovers hover how how + howbeit howbeit howe how + howeer howeer however howev + howl howl howled howl + howlet howlet howling howl + howls howl howsoe howso + howsoever howsoev howsome howsom + hoxes hox hoy hoi + hoyday hoydai hubert hubert + huddled huddl huddling huddl + hue hue hued hu + hues hue hug hug + huge huge hugely huge + hugeness huge hugg hugg + hugger hugger hugh hugh + hugs hug hujus huju + hulk hulk hulks hulk + hull hull hulling hull + hullo hullo hum hum + human human humane human + humanely human humanity human + humble humbl humbled humbl + humbleness humbl humbler humbler + humbles humbl humblest humblest + humbling humbl humbly humbl + hume hume humh humh + humidity humid humility humil + humming hum humor humor + humorous humor humors humor + humour humour humourists humourist + humours humour humphrey humphrei + humphry humphri hums hum + hundred hundr hundreds hundr + hundredth hundredth hung hung + hungarian hungarian hungary hungari + hunger hunger hungerford hungerford + hungerly hungerli hungry hungri + hunt hunt hunted hunt + hunter hunter hunters hunter + hunteth hunteth hunting hunt + huntington huntington huntress huntress + hunts hunt huntsman huntsman + huntsmen huntsmen hurdle hurdl + hurl hurl hurling hurl + hurls hurl hurly hurli + hurlyburly hurlyburli hurricano hurricano + hurricanoes hurricano hurried hurri + hurries hurri hurry hurri + hurt hurt hurting hurt + hurtled hurtl hurtless hurtless + hurtling hurtl hurts hurt + husband husband husbanded husband + husbandless husbandless husbandry husbandri + husbands husband hush hush + hushes hush husht husht + husks husk huswife huswif + huswifes huswif hutch hutch + hybla hybla hydra hydra + hyen hyen hymen hymen + hymenaeus hymenaeu hymn hymn + hymns hymn hyperboles hyperbol + hyperbolical hyperbol hyperion hyperion + hypocrisy hypocrisi hypocrite hypocrit + hypocrites hypocrit hyrcan hyrcan + hyrcania hyrcania hyrcanian hyrcanian + hyssop hyssop hysterica hysterica + i i iachimo iachimo + iaculis iaculi iago iago + iament iament ibat ibat + icarus icaru ice ic + iceland iceland ici ici + icicle icicl icicles icicl + icy ici idea idea + ideas idea idem idem + iden iden ides id + idiot idiot idiots idiot + idle idl idleness idl + idles idl idly idli + idol idol idolatrous idolatr + idolatry idolatri ield ield + if if ifs if + ignis igni ignoble ignobl + ignobly ignobl ignominious ignomini + ignominy ignomini ignomy ignomi + ignorance ignor ignorant ignor + ii ii iii iii + iiii iiii il il + ilbow ilbow ild ild + ilion ilion ilium ilium + ill ill illegitimate illegitim + illiterate illiter illness ill + illo illo ills ill + illume illum illumin illumin + illuminate illumin illumineth illumineth + illusion illus illusions illus + illustrate illustr illustrated illustr + illustrious illustri illyria illyria + illyrian illyrian ils il + im im image imag + imagery imageri images imag + imagin imagin imaginary imaginari + imagination imagin imaginations imagin + imagine imagin imagining imagin + imaginings imagin imbar imbar + imbecility imbecil imbrue imbru + imitari imitari imitate imit + imitated imit imitation imit + imitations imit immaculate immacul + immanity imman immask immask + immaterial immateri immediacy immediaci + immediate immedi immediately immedi + imminence immin imminent immin + immoderate immoder immoderately immoder + immodest immodest immoment immoment + immortal immort immortaliz immortaliz + immortally immort immur immur + immured immur immures immur + imogen imogen imp imp + impaint impaint impair impair + impairing impair impale impal + impaled impal impanelled impanel + impart impart imparted impart + impartial imparti impartment impart + imparts impart impasted impast + impatience impati impatient impati + impatiently impati impawn impawn + impeach impeach impeached impeach + impeachment impeach impeachments impeach + impedes imped impediment impedi + impediments impedi impenetrable impenetr + imperator imper imperceiverant imperceiver + imperfect imperfect imperfection imperfect + imperfections imperfect imperfectly imperfectli + imperial imperi imperious imperi + imperiously imperi impertinency impertin + impertinent impertin impeticos impetico + impetuosity impetuos impetuous impetu + impieties impieti impiety impieti + impious impiou implacable implac + implements implement implies impli + implor implor implorators implor + implore implor implored implor + imploring implor impon impon + import import importance import + importancy import important import + importantly importantli imported import + importeth importeth importing import + importless importless imports import + importun importun importunacy importunaci + importunate importun importune importun + importunes importun importunity importun + impos impo impose impos + imposed impos imposition imposit + impositions imposit impossibilities imposs + impossibility imposs impossible imposs + imposthume imposthum impostor impostor + impostors impostor impotence impot + impotent impot impounded impound + impregnable impregn imprese impres + impress impress impressed impress + impressest impressest impression impress + impressure impressur imprimendum imprimendum + imprimis imprimi imprint imprint + imprinted imprint imprison imprison + imprisoned imprison imprisoning imprison + imprisonment imprison improbable improb + improper improp improve improv + improvident improvid impudence impud + impudency impud impudent impud + impudently impud impudique impudiqu + impugn impugn impugns impugn + impure impur imputation imput + impute imput in in + inaccessible inaccess inaidable inaid + inaudible inaud inauspicious inauspici + incaged incag incantations incant + incapable incap incardinate incardin + incarnadine incarnadin incarnate incarn + incarnation incarn incens incen + incense incens incensed incens + incensement incens incenses incens + incensing incens incertain incertain + incertainties incertainti incertainty incertainti + incessant incess incessantly incessantli + incest incest incestuous incestu + inch inch incharitable incharit + inches inch incidency incid + incident incid incision incis + incite incit incites incit + incivil incivil incivility incivil + inclin inclin inclinable inclin + inclination inclin incline inclin + inclined inclin inclines inclin + inclining inclin inclips inclip + include includ included includ + includes includ inclusive inclus + incomparable incompar incomprehensible incomprehens + inconsiderate inconsider inconstancy inconst + inconstant inconst incontinency incontin + incontinent incontin incontinently incontin + inconvenience inconveni inconveniences inconveni + inconvenient inconveni incony inconi + incorporate incorpor incorps incorp + incorrect incorrect increas increa + increase increas increases increas + increaseth increaseth increasing increas + incredible incred incredulous incredul + incur incur incurable incur + incurr incurr incurred incur + incursions incurs ind ind + inde ind indebted indebt + indeed inde indent indent + indented indent indenture indentur + indentures indentur index index + indexes index india india + indian indian indict indict + indicted indict indictment indict + indies indi indifferency indiffer + indifferent indiffer indifferently indiffer + indigent indig indigest indigest + indigested indigest indign indign + indignation indign indignations indign + indigne indign indignities indign + indignity indign indirect indirect + indirection indirect indirections indirect + indirectly indirectli indiscreet indiscreet + indiscretion indiscret indispos indispo + indisposition indisposit indissoluble indissolubl + indistinct indistinct indistinguish indistinguish + indistinguishable indistinguish indited indit + individable individ indrench indrench + indu indu indubitate indubit + induc induc induce induc + induced induc inducement induc + induction induct inductions induct + indue indu indued indu + indues indu indulgence indulg + indulgences indulg indulgent indulg + indurance indur industrious industri + industriously industri industry industri + inequality inequ inestimable inestim + inevitable inevit inexecrable inexecr + inexorable inexor inexplicable inexplic + infallible infal infallibly infal + infamonize infamon infamous infam + infamy infami infancy infanc + infant infant infants infant + infect infect infected infect + infecting infect infection infect + infections infect infectious infecti + infectiously infecti infects infect + infer infer inference infer + inferior inferior inferiors inferior + infernal infern inferr inferr + inferreth inferreth inferring infer + infest infest infidel infidel + infidels infidel infinite infinit + infinitely infinit infinitive infinit + infirm infirm infirmities infirm + infirmity infirm infixed infix + infixing infix inflam inflam + inflame inflam inflaming inflam + inflammation inflamm inflict inflict + infliction inflict influence influenc + influences influenc infold infold + inform inform informal inform + information inform informations inform + informed inform informer inform + informs inform infortunate infortun + infring infr infringe infring + infringed infring infus infu + infuse infus infused infus + infusing infus infusion infus + ingener ingen ingenious ingeni + ingeniously ingeni inglorious inglori + ingots ingot ingraffed ingraf + ingraft ingraft ingrate ingrat + ingrated ingrat ingrateful ingrat + ingratitude ingratitud ingratitudes ingratitud + ingredient ingredi ingredients ingredi + ingross ingross inhabit inhabit + inhabitable inhabit inhabitants inhabit + inhabited inhabit inhabits inhabit + inhearse inhears inhearsed inhears + inherent inher inherit inherit + inheritance inherit inherited inherit + inheriting inherit inheritor inheritor + inheritors inheritor inheritrix inheritrix + inherits inherit inhibited inhibit + inhibition inhibit inhoop inhoop + inhuman inhuman iniquities iniqu + iniquity iniqu initiate initi + injointed injoint injunction injunct + injunctions injunct injur injur + injure injur injurer injur + injuries injuri injurious injuri + injury injuri injustice injustic + ink ink inkhorn inkhorn + inkle inkl inkles inkl + inkling inkl inky inki + inlaid inlaid inland inland + inlay inlai inly inli + inmost inmost inn inn + inner inner innkeeper innkeep + innocence innoc innocency innoc + innocent innoc innocents innoc + innovation innov innovator innov + inns inn innumerable innumer + inoculate inocul inordinate inordin + inprimis inprimi inquir inquir + inquire inquir inquiry inquiri + inquisition inquisit inquisitive inquisit + inroads inroad insane insan + insanie insani insatiate insati + insconce insconc inscrib inscrib + inscription inscript inscriptions inscript + inscroll inscrol inscrutable inscrut + insculp insculp insculpture insculptur + insensible insens inseparable insepar + inseparate insepar insert insert + inserted insert inset inset + inshell inshel inshipp inshipp + inside insid insinewed insinew + insinuate insinu insinuateth insinuateth + insinuating insinu insinuation insinu + insisted insist insisting insist + insisture insistur insociable insoci + insolence insol insolent insol + insomuch insomuch inspir inspir + inspiration inspir inspirations inspir + inspire inspir inspired inspir + install instal installed instal + instalment instal instance instanc + instances instanc instant instant + instantly instantli instate instat + instead instead insteeped insteep + instigate instig instigated instig + instigation instig instigations instig + instigator instig instinct instinct + instinctively instinct institute institut + institutions institut instruct instruct + instructed instruct instruction instruct + instructions instruct instructs instruct + instrument instrument instrumental instrument + instruments instrument insubstantial insubstanti + insufficience insuffici insufficiency insuffici + insult insult insulted insult + insulting insult insultment insult + insults insult insupportable insupport + insuppressive insuppress insurrection insurrect + insurrections insurrect int int + integer integ integritas integrita + integrity integr intellect intellect + intellects intellect intellectual intellectu + intelligence intellig intelligencer intelligenc + intelligencing intelligenc intelligent intellig + intelligis intelligi intelligo intelligo + intemperance intemper intemperate intemper + intend intend intended intend + intendeth intendeth intending intend + intendment intend intends intend + intenible inten intent intent + intention intent intentively intent + intents intent inter inter + intercept intercept intercepted intercept + intercepter intercept interception intercept + intercepts intercept intercession intercess + intercessors intercessor interchained interchain + interchang interchang interchange interchang + interchangeably interchang interchangement interchang + interchanging interchang interdiction interdict + interest interest interim interim + interims interim interior interior + interjections interject interjoin interjoin + interlude interlud intermingle intermingl + intermission intermiss intermissive intermiss + intermit intermit intermix intermix + intermixed intermix interpose interpos + interposer interpos interposes interpos + interpret interpret interpretation interpret + interpreted interpret interpreter interpret + interpreters interpret interprets interpret + interr interr interred inter + interrogatories interrogatori interrupt interrupt + interrupted interrupt interrupter interrupt + interruptest interruptest interruption interrupt + interrupts interrupt intertissued intertissu + intervallums intervallum interview interview + intestate intest intestine intestin + intil intil intimate intim + intimation intim intitled intitl + intituled intitul into into + intolerable intoler intoxicates intox + intreasured intreasur intreat intreat + intrench intrench intrenchant intrench + intricate intric intrinse intrins + intrinsicate intrins intrude intrud + intruder intrud intruding intrud + intrusion intrus inundation inund + inure inur inurn inurn + invade invad invades invad + invasion invas invasive invas + invectively invect invectives invect + inveigled inveigl invent invent + invented invent invention invent + inventions invent inventor inventor + inventorially inventori inventoried inventori + inventors inventor inventory inventori + inverness inver invert invert + invest invest invested invest + investing invest investments invest + inveterate inveter invincible invinc + inviolable inviol invised invis + invisible invis invitation invit + invite invit invited invit + invites invit inviting invit + invitis inviti invocate invoc + invocation invoc invoke invok + invoked invok invulnerable invulner + inward inward inwardly inwardli + inwardness inward inwards inward + ionia ionia ionian ionian + ipse ips ipswich ipswich + ira ira irae ira + iras ira ire ir + ireful ir ireland ireland + iris iri irish irish + irishman irishman irishmen irishmen + irks irk irksome irksom + iron iron irons iron + irreconcil irreconcil irrecoverable irrecover + irregular irregular irregulous irregul + irreligious irreligi irremovable irremov + irreparable irrepar irresolute irresolut + irrevocable irrevoc is is + isabel isabel isabella isabella + isbel isbel isbels isbel + iscariot iscariot ise is + ish ish isidore isidor + isis isi island island + islander island islanders island + islands island isle isl + isles isl israel israel + issu issu issue issu + issued issu issueless issueless + issues issu issuing issu + ist ist ista ista + it it italian italian + italy itali itch itch + itches itch itching itch + item item items item + iteration iter ithaca ithaca + its it itself itself + itshall itshal iv iv + ivory ivori ivy ivi + iwis iwi ix ix + j j jacet jacet + jack jack jackanapes jackanap + jacks jack jacksauce jacksauc + jackslave jackslav jacob jacob + jade jade jaded jade + jades jade jail jail + jakes jake jamany jamani + james jame jamy jami + jane jane jangled jangl + jangling jangl january januari + janus janu japhet japhet + jaquenetta jaquenetta jaques jaqu + jar jar jarring jar + jars jar jarteer jarteer + jasons jason jaunce jaunc + jauncing jaunc jaundice jaundic + jaundies jaundi jaw jaw + jawbone jawbon jaws jaw + jay jai jays jai + jc jc je je + jealous jealou jealousies jealousi + jealousy jealousi jeer jeer + jeering jeer jelly jelli + jenny jenni jeopardy jeopardi + jephtha jephtha jephthah jephthah + jerkin jerkin jerkins jerkin + jerks jerk jeronimy jeronimi + jerusalem jerusalem jeshu jeshu + jesses jess jessica jessica + jest jest jested jest + jester jester jesters jester + jesting jest jests jest + jesu jesu jesus jesu + jet jet jets jet + jew jew jewel jewel + jeweller jewel jewels jewel + jewess jewess jewish jewish + jewry jewri jews jew + jezebel jezebel jig jig + jigging jig jill jill + jills jill jingling jingl + joan joan job job + jockey jockei jocund jocund + jog jog jogging jog + john john johns john + join join joinder joinder + joined join joiner joiner + joineth joineth joins join + joint joint jointed joint + jointing joint jointly jointli + jointress jointress joints joint + jointure jointur jollity jolliti + jolly jolli jolt jolt + joltheads jolthead jordan jordan + joseph joseph joshua joshua + jot jot jour jour + jourdain jourdain journal journal + journey journei journeying journei + journeyman journeyman journeymen journeymen + journeys journei jove jove + jovem jovem jovial jovial + jowl jowl jowls jowl + joy joi joyed joi + joyful joy joyfully joyfulli + joyless joyless joyous joyou + joys joi juan juan + jud jud judas juda + judases judas jude jude + judg judg judge judg + judged judg judgement judgement + judges judg judgest judgest + judging judg judgment judgment + judgments judgment judicious judici + jug jug juggle juggl + juggled juggl juggler juggler + jugglers juggler juggling juggl + jugs jug juice juic + juiced juic jul jul + jule jule julia julia + juliet juliet julietta julietta + julio julio julius juliu + july juli jump jump + jumpeth jumpeth jumping jump + jumps jump june june + junes june junior junior + junius juniu junkets junket + juno juno jupiter jupit + jure jure jurement jurement + jurisdiction jurisdict juror juror + jurors juror jury juri + jurymen jurymen just just + justeius justeiu justest justest + justice justic justicer justic + justicers justic justices justic + justification justif justified justifi + justify justifi justle justl + justled justl justles justl + justling justl justly justli + justness just justs just + jutting jut jutty jutti + juvenal juven kam kam + kate kate kated kate + kates kate katharine katharin + katherina katherina katherine katherin + kecksies kecksi keech keech + keel keel keels keel + keen keen keenness keen + keep keep keepdown keepdown + keeper keeper keepers keeper + keepest keepest keeping keep + keeps keep keiser keiser + ken ken kendal kendal + kennel kennel kent kent + kentish kentish kentishman kentishman + kentishmen kentishmen kept kept + kerchief kerchief kerely kere + kern kern kernal kernal + kernel kernel kernels kernel + kerns kern kersey kersei + kettle kettl kettledrum kettledrum + kettledrums kettledrum key kei + keys kei kibe kibe + kibes kibe kick kick + kicked kick kickshaws kickshaw + kickshawses kickshaws kicky kicki + kid kid kidney kidnei + kikely kike kildare kildar + kill kill killed kill + killer killer killeth killeth + killing kill killingworth killingworth + kills kill kiln kiln + kimbolton kimbolton kin kin + kind kind kinder kinder + kindest kindest kindle kindl + kindled kindl kindless kindless + kindlier kindlier kindling kindl + kindly kindli kindness kind + kindnesses kind kindred kindr + kindreds kindr kinds kind + kine kine king king + kingdom kingdom kingdoms kingdom + kingly kingli kings king + kinred kinr kins kin + kinsman kinsman kinsmen kinsmen + kinswoman kinswoman kirtle kirtl + kirtles kirtl kiss kiss + kissed kiss kisses kiss + kissing kiss kitchen kitchen + kitchens kitchen kite kite + kites kite kitten kitten + kj kj kl kl + klll klll knack knack + knacks knack knapp knapp + knav knav knave knave + knaveries knaveri knavery knaveri + knaves knave knavish knavish + knead knead kneaded knead + kneading knead knee knee + kneel kneel kneeling kneel + kneels kneel knees knee + knell knell knew knew + knewest knewest knife knife + knight knight knighted knight + knighthood knighthood knighthoods knighthood + knightly knightli knights knight + knit knit knits knit + knitters knitter knitteth knitteth + knives knive knobs knob + knock knock knocking knock + knocks knock knog knog + knoll knoll knot knot + knots knot knotted knot + knotty knotti know know + knower knower knowest knowest + knowing know knowingly knowingli + knowings know knowledge knowledg + known known knows know + l l la la + laban laban label label + labell label labienus labienu + labio labio labor labor + laboring labor labors labor + labour labour laboured labour + labourer labour labourers labour + labouring labour labours labour + laboursome laboursom labras labra + labyrinth labyrinth lac lac + lace lace laced lace + lacedaemon lacedaemon laces lace + lacies laci lack lack + lackbeard lackbeard lacked lack + lackey lackei lackeying lackei + lackeys lackei lacking lack + lacks lack lad lad + ladder ladder ladders ladder + lade lade laden laden + ladies ladi lading lade + lads lad lady ladi + ladybird ladybird ladyship ladyship + ladyships ladyship laer laer + laertes laert lafeu lafeu + lag lag lagging lag + laid laid lain lain + laissez laissez lake lake + lakes lake lakin lakin + lam lam lamb lamb + lambert lambert lambkin lambkin + lambkins lambkin lambs lamb + lame lame lamely lame + lameness lame lament lament + lamentable lament lamentably lament + lamentation lament lamentations lament + lamented lament lamenting lament + lamentings lament laments lament + lames lame laming lame + lammas lamma lammastide lammastid + lamound lamound lamp lamp + lampass lampass lamps lamp + lanc lanc lancaster lancast + lance lanc lances lanc + lanceth lanceth lanch lanch + land land landed land + landing land landless landless + landlord landlord landmen landmen + lands land lane lane + lanes lane langage langag + langley langlei langton langton + language languag languageless languageless + languages languag langues langu + languish languish languished languish + languishes languish languishing languish + languishings languish languishment languish + languor languor lank lank + lantern lantern lanterns lantern + lanthorn lanthorn lap lap + lapis lapi lapland lapland + lapp lapp laps lap + lapse laps lapsed laps + lapsing laps lapwing lapw + laquais laquai larded lard + larder larder larding lard + lards lard large larg + largely larg largeness larg + larger larger largess largess + largest largest lark lark + larks lark larron larron + lartius lartiu larum larum + larums larum las la + lascivious lascivi lash lash + lass lass lasses lass + last last lasted last + lasting last lastly lastli + lasts last latch latch + latches latch late late + lated late lately late + later later latest latest + lath lath latin latin + latten latten latter latter + lattice lattic laud laud + laudable laudabl laudis laudi + laugh laugh laughable laughabl + laughed laugh laugher laugher + laughest laughest laughing laugh + laughs laugh laughter laughter + launce launc launcelot launcelot + launces launc launch launch + laund laund laundress laundress + laundry laundri laur laur + laura laura laurel laurel + laurels laurel laurence laurenc + laus lau lavache lavach + lave lave lavee lave + lavender lavend lavina lavina + lavinia lavinia lavish lavish + lavishly lavishli lavolt lavolt + lavoltas lavolta law law + lawful law lawfully lawfulli + lawless lawless lawlessly lawlessli + lawn lawn lawns lawn + lawrence lawrenc laws law + lawyer lawyer lawyers lawyer + lay lai layer layer + layest layest laying lai + lays lai lazar lazar + lazars lazar lazarus lazaru + lazy lazi lc lc + ld ld ldst ldst + le le lead lead + leaden leaden leader leader + leaders leader leadest leadest + leading lead leads lead + leaf leaf leagu leagu + league leagu leagued leagu + leaguer leaguer leagues leagu + leah leah leak leak + leaky leaki lean lean + leander leander leaner leaner + leaning lean leanness lean + leans lean leap leap + leaped leap leaping leap + leaps leap leapt leapt + lear lear learn learn + learned learn learnedly learnedli + learning learn learnings learn + learns learn learnt learnt + leas lea lease leas + leases leas leash leash + leasing leas least least + leather leather leathern leathern + leav leav leave leav + leaven leaven leavening leaven + leaver leaver leaves leav + leaving leav leavy leavi + lecher lecher lecherous lecher + lechers lecher lechery lecheri + lecon lecon lecture lectur + lectures lectur led led + leda leda leech leech + leeches leech leek leek + leeks leek leer leer + leers leer lees lee + leese lees leet leet + leets leet left left + leg leg legacies legaci + legacy legaci legate legat + legatine legatin lege lege + legerity leger leges lege + legg legg legion legion + legions legion legitimate legitim + legitimation legitim legs leg + leicester leicest leicestershire leicestershir + leiger leiger leigers leiger + leisure leisur leisurely leisur + leisures leisur leman leman + lemon lemon lena lena + lend lend lender lender + lending lend lendings lend + lends lend length length + lengthen lengthen lengthens lengthen + lengths length lenity leniti + lennox lennox lent lent + lenten lenten lentus lentu + leo leo leon leon + leonardo leonardo leonati leonati + leonato leonato leonatus leonatu + leontes leont leopard leopard + leopards leopard leper leper + leperous leper lepidus lepidu + leprosy leprosi lequel lequel + lers ler les le + less less lessen lessen + lessens lessen lesser lesser + lesson lesson lessoned lesson + lessons lesson lest lest + lestrake lestrak let let + lethargied lethargi lethargies lethargi + lethargy lethargi lethe leth + lets let lett lett + letter letter letters letter + letting let lettuce lettuc + leur leur leve leve + level level levell level + levelled level levels level + leven leven levers lever + leviathan leviathan leviathans leviathan + levied levi levies levi + levity leviti levy levi + levying levi lewd lewd + lewdly lewdli lewdness lewd + lewdsters lewdster lewis lewi + liable liabl liar liar + liars liar libbard libbard + libelling libel libels libel + liberal liber liberality liber + liberte libert liberties liberti + libertine libertin libertines libertin + liberty liberti library librari + libya libya licence licenc + licens licen license licens + licentious licenti lichas licha + licio licio lick lick + licked lick licker licker + lictors lictor lid lid + lids lid lie lie + lied li lief lief + liefest liefest liege lieg + liegeman liegeman liegemen liegemen + lien lien lies li + liest liest lieth lieth + lieu lieu lieutenant lieuten + lieutenantry lieutenantri lieutenants lieuten + lieve liev life life + lifeblood lifeblood lifeless lifeless + lifelings lifel lift lift + lifted lift lifter lifter + lifteth lifteth lifting lift + lifts lift lig lig + ligarius ligariu liggens liggen + light light lighted light + lighten lighten lightens lighten + lighter lighter lightest lightest + lightly lightli lightness light + lightning lightn lightnings lightn + lights light lik lik + like like liked like + likeliest likeliest likelihood likelihood + likelihoods likelihood likely like + likeness like liker liker + likes like likest likest + likewise likewis liking like + likings like lilies lili + lily lili lim lim + limander limand limb limb + limbeck limbeck limbecks limbeck + limber limber limbo limbo + limbs limb lime lime + limed lime limehouse limehous + limekilns limekiln limit limit + limitation limit limited limit + limits limit limn limn + limp limp limping limp + limps limp lin lin + lincoln lincoln lincolnshire lincolnshir + line line lineal lineal + lineally lineal lineament lineament + lineaments lineament lined line + linen linen linens linen + lines line ling ling + lingare lingar linger linger + lingered linger lingers linger + linguist linguist lining line + link link links link + linsey linsei linstock linstock + linta linta lion lion + lionel lionel lioness lioness + lions lion lip lip + lipp lipp lips lip + lipsbury lipsburi liquid liquid + liquor liquor liquorish liquorish + liquors liquor lirra lirra + lisbon lisbon lisp lisp + lisping lisp list list + listen listen listening listen + lists list literatured literatur + lither lither litter litter + little littl littlest littlest + liv liv live live + lived live livelier liveli + livelihood livelihood livelong livelong + lively live liver liver + liveries liveri livers liver + livery liveri lives live + livest livest liveth liveth + livia livia living live + livings live lizard lizard + lizards lizard ll ll + lll lll llous llou + lnd lnd lo lo + loa loa loach loach + load load loaden loaden + loading load loads load + loaf loaf loam loam + loan loan loath loath + loathe loath loathed loath + loather loather loathes loath + loathing loath loathly loathli + loathness loath loathsome loathsom + loathsomeness loathsom loathsomest loathsomest + loaves loav lob lob + lobbies lobbi lobby lobbi + local local lochaber lochab + lock lock locked lock + locking lock lockram lockram + locks lock locusts locust + lode lode lodg lodg + lodge lodg lodged lodg + lodgers lodger lodges lodg + lodging lodg lodgings lodg + lodovico lodovico lodowick lodowick + lofty lofti log log + logger logger loggerhead loggerhead + loggerheads loggerhead loggets logget + logic logic logs log + loins loin loiter loiter + loiterer loiter loiterers loiter + loitering loiter lolling loll + lolls loll lombardy lombardi + london london londoners london + lone lone loneliness loneli + lonely lone long long + longaville longavil longboat longboat + longed long longer longer + longest longest longeth longeth + longing long longings long + longly longli longs long + longtail longtail loo loo + loof loof look look + looked look looker looker + lookers looker lookest lookest + looking look looks look + loon loon loop loop + loos loo loose loos + loosed loos loosely loos + loosen loosen loosing loos + lop lop lopp lopp + loquitur loquitur lord lord + lorded lord lording lord + lordings lord lordliness lordli + lordly lordli lords lord + lordship lordship lordships lordship + lorenzo lorenzo lorn lorn + lorraine lorrain lorship lorship + los lo lose lose + loser loser losers loser + loses lose losest losest + loseth loseth losing lose + loss loss losses loss + lost lost lot lot + lots lot lott lott + lottery lotteri loud loud + louder louder loudly loudli + lour lour loureth loureth + louring lour louse lous + louses lous lousy lousi + lout lout louted lout + louts lout louvre louvr + lov lov love love + loved love lovedst lovedst + lovel lovel lovelier loveli + loveliness loveli lovell lovel + lovely love lover lover + lovered lover lovers lover + loves love lovest lovest + loveth loveth loving love + lovingly lovingli low low + lowe low lower lower + lowest lowest lowing low + lowliness lowli lowly lowli + lown lown lowness low + loyal loyal loyally loyal + loyalties loyalti loyalty loyalti + lozel lozel lt lt + lubber lubber lubberly lubberli + luc luc luccicos luccico + luce luce lucentio lucentio + luces luce lucetta lucetta + luciana luciana lucianus lucianu + lucifer lucif lucifier lucifi + lucilius luciliu lucina lucina + lucio lucio lucius luciu + luck luck luckier luckier + luckiest luckiest luckily luckili + luckless luckless lucky lucki + lucre lucr lucrece lucrec + lucretia lucretia lucullius luculliu + lucullus lucullu lucy luci + lud lud ludlow ludlow + lug lug lugg lugg + luggage luggag luke luke + lukewarm lukewarm lull lull + lulla lulla lullaby lullabi + lulls lull lumbert lumbert + lump lump lumpish lumpish + luna luna lunacies lunaci + lunacy lunaci lunatic lunat + lunatics lunat lunes lune + lungs lung lupercal luperc + lurch lurch lure lure + lurk lurk lurketh lurketh + lurking lurk lurks lurk + luscious lusciou lush lush + lust lust lusted lust + luster luster lustful lust + lustier lustier lustiest lustiest + lustig lustig lustihood lustihood + lustily lustili lustre lustr + lustrous lustrou lusts lust + lusty lusti lute lute + lutes lute lutestring lutestr + lutheran lutheran luxurious luxuri + luxuriously luxuri luxury luxuri + ly ly lycaonia lycaonia + lycurguses lycurgus lydia lydia + lye lye lyen lyen + lying ly lym lym + lymoges lymog lynn lynn + lysander lysand m m + ma ma maan maan + mab mab macbeth macbeth + maccabaeus maccabaeu macdonwald macdonwald + macduff macduff mace mace + macedon macedon maces mace + machiavel machiavel machination machin + machinations machin machine machin + mack mack macmorris macmorri + maculate macul maculation macul + mad mad madam madam + madame madam madams madam + madcap madcap madded mad + madding mad made made + madeira madeira madly madli + madman madman madmen madmen + madness mad madonna madonna + madrigals madrig mads mad + maecenas maecena maggot maggot + maggots maggot magic magic + magical magic magician magician + magistrate magistr magistrates magistr + magnanimity magnanim magnanimous magnanim + magni magni magnifi magnifi + magnificence magnific magnificent magnific + magnifico magnifico magnificoes magnifico + magnus magnu mahomet mahomet + mahu mahu maid maid + maiden maiden maidenhead maidenhead + maidenheads maidenhead maidenhood maidenhood + maidenhoods maidenhood maidenliest maidenliest + maidenly maidenli maidens maiden + maidhood maidhood maids maid + mail mail mailed mail + mails mail maim maim + maimed maim maims maim + main main maincourse maincours + maine main mainly mainli + mainmast mainmast mains main + maintain maintain maintained maintain + maintains maintain maintenance mainten + mais mai maison maison + majestas majesta majestee majeste + majestic majest majestical majest + majestically majest majesties majesti + majesty majesti major major + majority major mak mak + make make makeless makeless + maker maker makers maker + makes make makest makest + maketh maketh making make + makings make mal mal + mala mala maladies maladi + malady maladi malapert malapert + malcolm malcolm malcontent malcont + malcontents malcont male male + maledictions maledict malefactions malefact + malefactor malefactor malefactors malefactor + males male malevolence malevol + malevolent malevol malhecho malhecho + malice malic malicious malici + maliciously malici malign malign + malignancy malign malignant malign + malignantly malignantli malkin malkin + mall mall mallard mallard + mallet mallet mallows mallow + malmsey malmsei malt malt + maltworms maltworm malvolio malvolio + mamillius mamilliu mammering mammer + mammet mammet mammets mammet + mammock mammock man man + manacle manacl manacles manacl + manage manag managed manag + manager manag managing manag + manakin manakin manchus manchu + mandate mandat mandragora mandragora + mandrake mandrak mandrakes mandrak + mane mane manent manent + manes mane manet manet + manfully manfulli mangle mangl + mangled mangl mangles mangl + mangling mangl mangy mangi + manhood manhood manhoods manhood + manifest manifest manifested manifest + manifests manifest manifold manifold + manifoldly manifoldli manka manka + mankind mankind manlike manlik + manly manli mann mann + manna manna manner manner + mannerly mannerli manners manner + manningtree manningtre mannish mannish + manor manor manors manor + mans man mansion mansion + mansionry mansionri mansions mansion + manslaughter manslaught mantle mantl + mantled mantl mantles mantl + mantua mantua mantuan mantuan + manual manual manure manur + manured manur manus manu + many mani map map + mapp mapp maps map + mar mar marble marbl + marbled marbl marcade marcad + marcellus marcellu march march + marches march marcheth marcheth + marching march marchioness marchio + marchpane marchpan marcians marcian + marcius marciu marcus marcu + mardian mardian mare mare + mares mare marg marg + margarelon margarelon margaret margaret + marge marg margent margent + margery margeri maria maria + marian marian mariana mariana + maries mari marigold marigold + mariner marin mariners marin + maritime maritim marjoram marjoram + mark mark marked mark + market market marketable market + marketplace marketplac markets market + marking mark markman markman + marks mark marl marl + marle marl marmoset marmoset + marquess marquess marquis marqui + marr marr marriage marriag + marriages marriag married marri + marries marri marring mar + marrow marrow marrowless marrowless + marrows marrow marry marri + marrying marri mars mar + marseilles marseil marsh marsh + marshal marshal marshalsea marshalsea + marshalship marshalship mart mart + marted mart martem martem + martext martext martial martial + martin martin martino martino + martius martiu martlemas martlema + martlet martlet marts mart + martyr martyr martyrs martyr + marullus marullu marv marv + marvel marvel marvell marvel + marvellous marvel marvellously marvel + marvels marvel mary mari + mas ma masculine masculin + masham masham mask mask + masked mask masker masker + maskers masker masking mask + masks mask mason mason + masonry masonri masons mason + masque masqu masquers masquer + masques masqu masquing masqu + mass mass massacre massacr + massacres massacr masses mass + massy massi mast mast + mastcr mastcr master master + masterdom masterdom masterest masterest + masterless masterless masterly masterli + masterpiece masterpiec masters master + mastership mastership mastic mastic + mastiff mastiff mastiffs mastiff + masts mast match match + matches match matcheth matcheth + matching match matchless matchless + mate mate mated mate + mater mater material materi + mates mate mathematics mathemat + matin matin matron matron + matrons matron matter matter + matters matter matthew matthew + mattock mattock mattress mattress + mature matur maturity matur + maud maud maudlin maudlin + maugre maugr maul maul + maund maund mauri mauri + mauritania mauritania mauvais mauvai + maw maw maws maw + maxim maxim may mai + mayday maydai mayest mayest + mayor mayor maypole maypol + mayst mayst maz maz + maze maze mazed maze + mazes maze mazzard mazzard + me me meacock meacock + mead mead meadow meadow + meadows meadow meads mead + meagre meagr meal meal + meals meal mealy meali + mean mean meanders meander + meaner meaner meanest meanest + meaneth meaneth meaning mean + meanings mean meanly meanli + means mean meant meant + meantime meantim meanwhile meanwhil + measles measl measur measur + measurable measur measure measur + measured measur measureless measureless + measures measur measuring measur + meat meat meats meat + mechanic mechan mechanical mechan + mechanicals mechan mechanics mechan + mechante mechant med med + medal medal meddle meddl + meddler meddler meddling meddl + mede mede medea medea + media media mediation mediat + mediators mediat medice medic + medicinal medicin medicine medicin + medicines medicin meditate medit + meditates medit meditating medit + meditation medit meditations medit + mediterranean mediterranean mediterraneum mediterraneum + medlar medlar medlars medlar + meed meed meeds meed + meek meek meekly meekli + meekness meek meet meet + meeter meeter meetest meetest + meeting meet meetings meet + meetly meetli meetness meet + meets meet meg meg + mehercle mehercl meilleur meilleur + meiny meini meisen meisen + melancholies melancholi melancholy melancholi + melford melford mell mell + mellifluous melliflu mellow mellow + mellowing mellow melodious melodi + melody melodi melt melt + melted melt melteth melteth + melting melt melts melt + melun melun member member + members member memento memento + memorable memor memorandums memorandum + memorial memori memorials memori + memories memori memoriz memoriz + memorize memor memory memori + memphis memphi men men + menac menac menace menac + menaces menac menaphon menaphon + menas mena mend mend + mended mend mender mender + mending mend mends mend + menecrates menecr menelaus menelau + menenius meneniu mental mental + menteith menteith mention mention + mentis menti menton menton + mephostophilus mephostophilu mer mer + mercatante mercatant mercatio mercatio + mercenaries mercenari mercenary mercenari + mercer mercer merchandise merchandis + merchandized merchand merchant merchant + merchants merchant mercies merci + merciful merci mercifully mercifulli + merciless merciless mercurial mercuri + mercuries mercuri mercury mercuri + mercutio mercutio mercy merci + mere mere mered mere + merely mere merest merest + meridian meridian merit merit + merited merit meritorious meritori + merits merit merlin merlin + mermaid mermaid mermaids mermaid + merops merop merrier merrier + merriest merriest merrily merrili + merriman merriman merriment merriment + merriments merriment merriness merri + merry merri mervailous mervail + mes me mesh mesh + meshes mesh mesopotamia mesopotamia + mess mess message messag + messages messag messala messala + messaline messalin messenger messeng + messengers messeng messes mess + messina messina met met + metal metal metals metal + metamorphis metamorphi metamorphoses metamorphos + metaphor metaphor metaphysical metaphys + metaphysics metaphys mete mete + metellus metellu meteor meteor + meteors meteor meteyard meteyard + metheglin metheglin metheglins metheglin + methink methink methinks methink + method method methods method + methought methought methoughts methought + metre metr metres metr + metropolis metropoli mette mett + mettle mettl mettled mettl + meus meu mew mew + mewed mew mewling mewl + mexico mexico mi mi + mice mice michael michael + michaelmas michaelma micher micher + miching mich mickle mickl + microcosm microcosm mid mid + midas mida middest middest + middle middl middleham middleham + midnight midnight midriff midriff + midst midst midsummer midsumm + midway midwai midwife midwif + midwives midwiv mienne mienn + might might mightful might + mightier mightier mightiest mightiest + mightily mightili mightiness mighti + mightst mightst mighty mighti + milan milan milch milch + mild mild milder milder + mildest mildest mildew mildew + mildews mildew mildly mildli + mildness mild mile mile + miles mile milford milford + militarist militarist military militari + milk milk milking milk + milkmaid milkmaid milks milk + milksops milksop milky milki + mill mill mille mill + miller miller milliner millin + million million millioned million + millions million mills mill + millstones millston milo milo + mimic mimic minc minc + mince minc minces minc + mincing minc mind mind + minded mind minding mind + mindless mindless minds mind + mine mine mineral miner + minerals miner minerva minerva + mines mine mingle mingl + mingled mingl mingling mingl + minikin minikin minim minim + minime minim minimo minimo + minimus minimu mining mine + minion minion minions minion + minist minist minister minist + ministers minist ministration ministr + minnow minnow minnows minnow + minola minola minority minor + minos mino minotaurs minotaur + minstrel minstrel minstrels minstrel + minstrelsy minstrelsi mint mint + mints mint minute minut + minutely minut minutes minut + minx minx mio mio + mir mir mirable mirabl + miracle miracl miracles miracl + miraculous miracul miranda miranda + mire mire mirror mirror + mirrors mirror mirth mirth + mirthful mirth miry miri + mis mi misadventur misadventur + misadventure misadventur misanthropos misanthropo + misapplied misappli misbecame misbecam + misbecom misbecom misbecome misbecom + misbegot misbegot misbegotten misbegotten + misbeliever misbeliev misbelieving misbeliev + misbhav misbhav miscall miscal + miscalled miscal miscarried miscarri + miscarries miscarri miscarry miscarri + miscarrying miscarri mischance mischanc + mischances mischanc mischief mischief + mischiefs mischief mischievous mischiev + misconceived misconceiv misconst misconst + misconster misconst misconstruction misconstruct + misconstrued misconstru misconstrues misconstru + miscreant miscreant miscreate miscreat + misdeed misde misdeeds misde + misdemean misdemean misdemeanours misdemeanour + misdoubt misdoubt misdoubteth misdoubteth + misdoubts misdoubt misenum misenum + miser miser miserable miser + miserably miser misericorde misericord + miseries miseri misers miser + misery miseri misfortune misfortun + misfortunes misfortun misgive misgiv + misgives misgiv misgiving misgiv + misgoverned misgovern misgovernment misgovern + misgraffed misgraf misguide misguid + mishap mishap mishaps mishap + misheard misheard misinterpret misinterpret + mislead mislead misleader mislead + misleaders mislead misleading mislead + misled misl mislike mislik + misord misord misplac misplac + misplaced misplac misplaces misplac + mispris mispri misprised mispris + misprision mispris misprizing mispriz + misproud misproud misquote misquot + misreport misreport miss miss + missed miss misses miss + misshap misshap misshapen misshapen + missheathed missheath missing miss + missingly missingli missions mission + missive missiv missives missiv + misspoke misspok mist mist + mista mista mistak mistak + mistake mistak mistaken mistaken + mistakes mistak mistaketh mistaketh + mistaking mistak mistakings mistak + mistemp mistemp mistempered mistemp + misterm misterm mistful mist + misthink misthink misthought misthought + mistletoe mistleto mistook mistook + mistreadings mistread mistress mistress + mistresses mistress mistresss mistresss + mistriship mistriship mistrust mistrust + mistrusted mistrust mistrustful mistrust + mistrusting mistrust mists mist + misty misti misus misu + misuse misus misused misus + misuses misus mites mite + mithridates mithrid mitigate mitig + mitigation mitig mix mix + mixed mix mixture mixtur + mixtures mixtur mm mm + mnd mnd moan moan + moans moan moat moat + moated moat mobled mobl + mock mock mockable mockabl + mocker mocker mockeries mockeri + mockers mocker mockery mockeri + mocking mock mocks mock + mockvater mockvat mockwater mockwat + model model modena modena + moderate moder moderately moder + moderation moder modern modern + modest modest modesties modesti + modestly modestli modesty modesti + modicums modicum modo modo + module modul moe moe + moi moi moiety moieti + moist moist moisten moisten + moisture moistur moldwarp moldwarp + mole mole molehill molehil + moles mole molest molest + molestation molest mollification mollif + mollis molli molten molten + molto molto mome mome + moment moment momentary momentari + moming mome mon mon + monachum monachum monarch monarch + monarchies monarchi monarchize monarch + monarcho monarcho monarchs monarch + monarchy monarchi monast monast + monastery monasteri monastic monast + monday mondai monde mond + money monei moneys monei + mong mong monger monger + mongers monger monging mong + mongrel mongrel mongrels mongrel + mongst mongst monk monk + monkey monkei monkeys monkei + monks monk monmouth monmouth + monopoly monopoli mons mon + monsieur monsieur monsieurs monsieur + monster monster monsters monster + monstrous monstrou monstrously monstrous + monstrousness monstrous monstruosity monstruos + montacute montacut montage montag + montague montagu montagues montagu + montano montano montant montant + montez montez montferrat montferrat + montgomery montgomeri month month + monthly monthli months month + montjoy montjoi monument monument + monumental monument monuments monument + mood mood moods mood + moody moodi moon moon + moonbeams moonbeam moonish moonish + moonlight moonlight moons moon + moonshine moonshin moonshines moonshin + moor moor moorfields moorfield + moors moor moorship moorship + mop mop mope mope + moping mope mopping mop + mopsa mopsa moral moral + moraler moral morality moral + moralize moral mordake mordak + more more moreover moreov + mores more morgan morgan + mori mori morisco morisco + morn morn morning morn + mornings morn morocco morocco + morris morri morrow morrow + morrows morrow morsel morsel + morsels morsel mort mort + mortal mortal mortality mortal + mortally mortal mortals mortal + mortar mortar mortgaged mortgag + mortified mortifi mortifying mortifi + mortimer mortim mortimers mortim + mortis morti mortise mortis + morton morton mose mose + moss moss mossgrown mossgrown + most most mote mote + moth moth mother mother + mothers mother moths moth + motion motion motionless motionless + motions motion motive motiv + motives motiv motley motlei + mots mot mought mought + mould mould moulded mould + mouldeth mouldeth moulds mould + mouldy mouldi moult moult + moulten moulten mounch mounch + mounseur mounseur mounsieur mounsieur + mount mount mountain mountain + mountaineer mountain mountaineers mountain + mountainous mountain mountains mountain + mountant mountant mountanto mountanto + mountebank mountebank mountebanks mountebank + mounted mount mounteth mounteth + mounting mount mounts mount + mourn mourn mourned mourn + mourner mourner mourners mourner + mournful mourn mournfully mournfulli + mourning mourn mourningly mourningli + mournings mourn mourns mourn + mous mou mouse mous + mousetrap mousetrap mousing mous + mouth mouth mouthed mouth + mouths mouth mov mov + movables movabl move move + moveable moveabl moveables moveabl + moved move mover mover + movers mover moves move + moveth moveth moving move + movingly movingli movousus movousu + mow mow mowbray mowbrai + mower mower mowing mow + mows mow moy moi + moys moi moyses moys + mrs mr much much + muck muck mud mud + mudded mud muddied muddi + muddy muddi muffins muffin + muffl muffl muffle muffl + muffled muffl muffler muffler + muffling muffl mugger mugger + mugs mug mulberries mulberri + mulberry mulberri mule mule + mules mule muleteers mulet + mulier mulier mulieres mulier + muliteus muliteu mull mull + mulmutius mulmutiu multiplied multipli + multiply multipli multiplying multipli + multipotent multipot multitude multitud + multitudes multitud multitudinous multitudin + mum mum mumble mumbl + mumbling mumbl mummers mummer + mummy mummi mun mun + munch munch muniments muniment + munition munit murd murd + murder murder murdered murder + murderer murder murderers murder + murdering murder murderous murder + murders murder mure mure + murk murk murkiest murkiest + murky murki murmur murmur + murmurers murmur murmuring murmur + murrain murrain murray murrai + murrion murrion murther murther + murtherer murther murtherers murther + murthering murther murtherous murther + murthers murther mus mu + muscadel muscadel muscovites muscovit + muscovits muscovit muscovy muscovi + muse muse muses muse + mush mush mushrooms mushroom + music music musical music + musician musician musicians musician + musics music musing muse + musings muse musk musk + musket musket muskets musket + muskos musko muss muss + mussel mussel mussels mussel + must must mustachio mustachio + mustard mustard mustardseed mustardse + muster muster mustering muster + musters muster musty musti + mutability mutabl mutable mutabl + mutation mutat mutations mutat + mute mute mutes mute + mutest mutest mutine mutin + mutineer mutin mutineers mutin + mutines mutin mutinies mutini + mutinous mutin mutiny mutini + mutius mutiu mutter mutter + muttered mutter mutton mutton + muttons mutton mutual mutual + mutualities mutual mutually mutual + muzzl muzzl muzzle muzzl + muzzled muzzl mv mv + mww mww my my + mynheers mynheer myrmidon myrmidon + myrmidons myrmidon myrtle myrtl + myself myself myst myst + mysteries mysteri mystery mysteri + n n nag nag + nage nage nags nag + naiads naiad nail nail + nails nail nak nak + naked nake nakedness naked + nal nal nam nam + name name named name + nameless nameless namely name + names name namest namest + naming name nan nan + nance nanc nap nap + nape nape napes nape + napkin napkin napkins napkin + naples napl napless napless + napping nap naps nap + narbon narbon narcissus narcissu + narines narin narrow narrow + narrowly narrowli naso naso + nasty nasti nathaniel nathaniel + natifs natif nation nation + nations nation native nativ + nativity nativ natur natur + natural natur naturalize natur + naturally natur nature natur + natured natur natures natur + natus natu naught naught + naughtily naughtili naughty naughti + navarre navarr nave nave + navel navel navigation navig + navy navi nay nai + nayward nayward nayword nayword + nazarite nazarit ne ne + neaf neaf neamnoins neamnoin + neanmoins neanmoin neapolitan neapolitan + neapolitans neapolitan near near + nearer nearer nearest nearest + nearly nearli nearness near + neat neat neatly neatli + neb neb nebour nebour + nebuchadnezzar nebuchadnezzar nec nec + necessaries necessari necessarily necessarili + necessary necessari necessitied necess + necessities necess necessity necess + neck neck necklace necklac + necks neck nectar nectar + ned ned nedar nedar + need need needed need + needer needer needful need + needfull needful needing need + needle needl needles needl + needless needless needly needli + needs need needy needi + neer neer neeze neez + nefas nefa negation negat + negative neg negatives neg + neglect neglect neglected neglect + neglecting neglect neglectingly neglectingli + neglection neglect negligence neglig + negligent neglig negotiate negoti + negotiations negoti negro negro + neigh neigh neighbors neighbor + neighbour neighbour neighbourhood neighbourhood + neighbouring neighbour neighbourly neighbourli + neighbours neighbour neighing neigh + neighs neigh neither neither + nell nell nemean nemean + nemesis nemesi neoptolemus neoptolemu + nephew nephew nephews nephew + neptune neptun ner ner + nereides nereid nerissa nerissa + nero nero neroes nero + ners ner nerve nerv + nerves nerv nervii nervii + nervy nervi nessus nessu + nest nest nestor nestor + nests nest net net + nether nether netherlands netherland + nets net nettle nettl + nettled nettl nettles nettl + neuter neuter neutral neutral + nev nev never never + nevil nevil nevils nevil + new new newborn newborn + newer newer newest newest + newgate newgat newly newli + newness new news new + newsmongers newsmong newt newt + newts newt next next + nibbling nibbl nicanor nicanor + nice nice nicely nice + niceness nice nicer nicer + nicety niceti nicholas nichola + nick nick nickname nicknam + nicks nick niece niec + nieces niec niggard niggard + niggarding niggard niggardly niggardli + nigh nigh night night + nightcap nightcap nightcaps nightcap + nighted night nightgown nightgown + nightingale nightingal nightingales nightingal + nightly nightli nightmare nightmar + nights night nightwork nightwork + nihil nihil nile nile + nill nill nilus nilu + nimble nimbl nimbleness nimbl + nimbler nimbler nimbly nimbl + nine nine nineteen nineteen + ning ning ningly ningli + ninny ninni ninth ninth + ninus ninu niobe niob + niobes niob nip nip + nipp nipp nipping nip + nipple nippl nips nip + nit nit nly nly + nnight nnight nnights nnight + no no noah noah + nob nob nobility nobil + nobis nobi noble nobl + nobleman nobleman noblemen noblemen + nobleness nobl nobler nobler + nobles nobl noblesse nobless + noblest noblest nobly nobli + nobody nobodi noces noce + nod nod nodded nod + nodding nod noddle noddl + noddles noddl noddy noddi + nods nod noes noe + nointed noint nois noi + noise nois noiseless noiseless + noisemaker noisemak noises nois + noisome noisom nole nole + nominate nomin nominated nomin + nomination nomin nominativo nominativo + non non nonage nonag + nonce nonc none none + nonino nonino nonny nonni + nonpareil nonpareil nonsuits nonsuit + nony noni nook nook + nooks nook noon noon + noonday noondai noontide noontid + nor nor norbery norberi + norfolk norfolk norman norman + normandy normandi normans norman + north north northampton northampton + northamptonshire northamptonshir northerly northerli + northern northern northgate northgat + northumberland northumberland northumberlands northumberland + northward northward norway norwai + norways norwai norwegian norwegian + norweyan norweyan nos no + nose nose nosegays nosegai + noseless noseless noses nose + noster noster nostra nostra + nostril nostril nostrils nostril + not not notable notabl + notably notabl notary notari + notch notch note note + notebook notebook noted note + notedly notedli notes note + notest notest noteworthy noteworthi + nothing noth nothings noth + notice notic notify notifi + noting note notion notion + notorious notori notoriously notori + notre notr notwithstanding notwithstand + nought nought noun noun + nouns noun nourish nourish + nourished nourish nourisher nourish + nourishes nourish nourisheth nourisheth + nourishing nourish nourishment nourish + nous nou novel novel + novelties novelti novelty novelti + noverbs noverb novi novi + novice novic novices novic + novum novum now now + nowhere nowher noyance noyanc + ns ns nt nt + nubibus nubibu numa numa + numb numb number number + numbered number numbering number + numberless numberless numbers number + numbness numb nun nun + nuncio nuncio nuncle nuncl + nunnery nunneri nuns nun + nuntius nuntiu nuptial nuptial + nurs nur nurse nurs + nursed nurs nurser nurser + nursery nurseri nurses nurs + nurseth nurseth nursh nursh + nursing nurs nurtur nurtur + nurture nurtur nut nut + nuthook nuthook nutmeg nutmeg + nutmegs nutmeg nutriment nutriment + nuts nut nutshell nutshel + ny ny nym nym + nymph nymph nymphs nymph + o o oak oak + oaken oaken oaks oak + oared oar oars oar + oatcake oatcak oaten oaten + oath oath oathable oathabl + oaths oath oats oat + ob ob obduracy obduraci + obdurate obdur obedience obedi + obedient obedi obeisance obeis + oberon oberon obey obei + obeyed obei obeying obei + obeys obei obidicut obidicut + object object objected object + objections object objects object + oblation oblat oblations oblat + obligation oblig obligations oblig + obliged oblig oblique obliqu + oblivion oblivion oblivious oblivi + obloquy obloqui obscene obscen + obscenely obscen obscur obscur + obscure obscur obscured obscur + obscurely obscur obscures obscur + obscuring obscur obscurity obscur + obsequies obsequi obsequious obsequi + obsequiously obsequi observ observ + observance observ observances observ + observancy observ observant observ + observants observ observation observ + observe observ observed observ + observer observ observers observ + observing observ observingly observingli + obsque obsqu obstacle obstacl + obstacles obstacl obstinacy obstinaci + obstinate obstin obstinately obstin + obstruct obstruct obstruction obstruct + obstructions obstruct obtain obtain + obtained obtain obtaining obtain + occasion occas occasions occas + occident occid occidental occident + occulted occult occupat occupat + occupation occup occupations occup + occupied occupi occupies occupi + occupy occupi occurrence occurr + occurrences occurr occurrents occurr + ocean ocean oceans ocean + octavia octavia octavius octaviu + ocular ocular od od + odd odd oddest oddest + oddly oddli odds odd + ode od odes od + odious odiou odoriferous odorifer + odorous odor odour odour + odours odour ods od + oeillades oeillad oes oe + oeuvres oeuvr of of + ofephesus ofephesu off off + offal offal offence offenc + offenceful offenc offences offenc + offend offend offended offend + offendendo offendendo offender offend + offenders offend offendeth offendeth + offending offend offendress offendress + offends offend offense offens + offenseless offenseless offenses offens + offensive offens offer offer + offered offer offering offer + offerings offer offers offer + offert offert offic offic + office offic officed offic + officer offic officers offic + offices offic official offici + officious offici offspring offspr + oft oft often often + oftener often oftentimes oftentim + oh oh oil oil + oils oil oily oili + old old oldcastle oldcastl + olden olden older older + oldest oldest oldness old + olive oliv oliver oliv + olivers oliv olives oliv + olivia olivia olympian olympian + olympus olympu oman oman + omans oman omen omen + ominous omin omission omiss + omit omit omittance omitt + omitted omit omitting omit + omne omn omnes omn + omnipotent omnipot on on + once onc one on + ones on oneyers oney + ongles ongl onion onion + onions onion only onli + onset onset onward onward + onwards onward oo oo + ooze ooz oozes ooz + oozy oozi op op + opal opal ope op + open open opener open + opening open openly openli + openness open opens open + operant oper operate oper + operation oper operations oper + operative oper opes op + oph oph ophelia ophelia + opinion opinion opinions opinion + opportune opportun opportunities opportun + opportunity opportun oppos oppo + oppose oppos opposed oppos + opposeless opposeless opposer oppos + opposers oppos opposes oppos + opposing oppos opposite opposit + opposites opposit opposition opposit + oppositions opposit oppress oppress + oppressed oppress oppresses oppress + oppresseth oppresseth oppressing oppress + oppression oppress oppressor oppressor + opprest opprest opprobriously opprobri + oppugnancy oppugn opulency opul + opulent opul or or + oracle oracl oracles oracl + orange orang oration orat + orator orat orators orat + oratory oratori orb orb + orbed orb orbs orb + orchard orchard orchards orchard + ord ord ordain ordain + ordained ordain ordaining ordain + order order ordered order + ordering order orderless orderless + orderly orderli orders order + ordinance ordin ordinant ordin + ordinaries ordinari ordinary ordinari + ordnance ordnanc ords ord + ordure ordur ore or + organ organ organs organ + orgillous orgil orient orient + orifex orifex origin origin + original origin orisons orison + ork ork orlando orlando + orld orld orleans orlean + ornament ornament ornaments ornament + orodes orod orphan orphan + orphans orphan orpheus orpheu + orsino orsino ort ort + orthography orthographi orts ort + oscorbidulchos oscorbidulcho osier osier + osiers osier osprey osprei + osr osr osric osric + ossa ossa ost ost + ostent ostent ostentare ostentar + ostentation ostent ostents ostent + ostler ostler ostlers ostler + ostrich ostrich osw osw + oswald oswald othello othello + other other othergates otherg + others other otherwhere otherwher + otherwhiles otherwhil otherwise otherwis + otter otter ottoman ottoman + ottomites ottomit oublie oubli + ouches ouch ought ought + oui oui ounce ounc + ounces ounc ouphes ouph + our our ours our + ourself ourself ourselves ourselv + ousel ousel out out + outbids outbid outbrave outbrav + outbraves outbrav outbreak outbreak + outcast outcast outcries outcri + outcry outcri outdar outdar + outdare outdar outdares outdar + outdone outdon outfac outfac + outface outfac outfaced outfac + outfacing outfac outfly outfli + outfrown outfrown outgo outgo + outgoes outgo outgrown outgrown + outjest outjest outlaw outlaw + outlawry outlawri outlaws outlaw + outliv outliv outlive outliv + outlives outliv outliving outliv + outlook outlook outlustres outlustr + outpriz outpriz outrage outrag + outrageous outrag outrages outrag + outran outran outright outright + outroar outroar outrun outrun + outrunning outrun outruns outrun + outscold outscold outscorn outscorn + outsell outsel outsells outsel + outside outsid outsides outsid + outspeaks outspeak outsport outsport + outstare outstar outstay outstai + outstood outstood outstretch outstretch + outstretched outstretch outstrike outstrik + outstrip outstrip outstripped outstrip + outswear outswear outvenoms outvenom + outward outward outwardly outwardli + outwards outward outwear outwear + outweighs outweigh outwent outwent + outworn outworn outworths outworth + oven oven over over + overawe overaw overbear overbear + overblown overblown overboard overboard + overbold overbold overborne overborn + overbulk overbulk overbuys overbui + overcame overcam overcast overcast + overcharg overcharg overcharged overcharg + overcome overcom overcomes overcom + overdone overdon overearnest overearnest + overfar overfar overflow overflow + overflown overflown overglance overgl + overgo overgo overgone overgon + overgorg overgorg overgrown overgrown + overhead overhead overhear overhear + overheard overheard overhold overhold + overjoyed overjoi overkind overkind + overland overland overleather overleath + overlive overl overlook overlook + overlooking overlook overlooks overlook + overmaster overmast overmounting overmount + overmuch overmuch overpass overpass + overpeer overp overpeering overp + overplus overplu overrul overrul + overrun overrun overscutch overscutch + overset overset overshades overshad + overshine overshin overshines overshin + overshot overshot oversights oversight + overspread overspread overstain overstain + overswear overswear overt overt + overta overta overtake overtak + overtaketh overtaketh overthrow overthrow + overthrown overthrown overthrows overthrow + overtook overtook overtopp overtopp + overture overtur overturn overturn + overwatch overwatch overween overween + overweening overween overweigh overweigh + overwhelm overwhelm overwhelming overwhelm + overworn overworn ovid ovid + ovidius ovidiu ow ow + owe ow owed ow + owedst owedst owen owen + owes ow owest owest + oweth oweth owing ow + owl owl owls owl + own own owner owner + owners owner owning own + owns own owy owi + ox ox oxen oxen + oxford oxford oxfordshire oxfordshir + oxlips oxlip oyes oy + oyster oyster p p + pabble pabbl pabylon pabylon + pac pac pace pace + paced pace paces pace + pacified pacifi pacify pacifi + pacing pace pack pack + packet packet packets packet + packhorses packhors packing pack + packings pack packs pack + packthread packthread pacorus pacoru + paction paction pad pad + paddle paddl paddling paddl + paddock paddock padua padua + pagan pagan pagans pagan + page page pageant pageant + pageants pageant pages page + pah pah paid paid + pail pail pailfuls pail + pails pail pain pain + pained pain painful pain + painfully painfulli pains pain + paint paint painted paint + painter painter painting paint + paintings paint paints paint + pair pair paired pair + pairs pair pajock pajock + pal pal palabras palabra + palace palac palaces palac + palamedes palamed palate palat + palates palat palatine palatin + palating palat pale pale + paled pale paleness pale + paler paler pales pale + palestine palestin palfrey palfrei + palfreys palfrei palisadoes palisado + pall pall pallabris pallabri + pallas palla pallets pallet + palm palm palmer palmer + palmers palmer palms palm + palmy palmi palpable palpabl + palsied palsi palsies palsi + palsy palsi palt palt + palter palter paltry paltri + paly pali pamp pamp + pamper pamper pamphlets pamphlet + pan pan pancackes pancack + pancake pancak pancakes pancak + pandar pandar pandars pandar + pandarus pandaru pander pander + panderly panderli panders pander + pandulph pandulph panel panel + pang pang panging pang + pangs pang pannier pannier + pannonians pannonian pansa pansa + pansies pansi pant pant + pantaloon pantaloon panted pant + pantheon pantheon panther panther + panthino panthino panting pant + pantingly pantingli pantler pantler + pantry pantri pants pant + pap pap papal papal + paper paper papers paper + paphlagonia paphlagonia paphos papho + papist papist paps pap + par par parable parabl + paracelsus paracelsu paradise paradis + paradox paradox paradoxes paradox + paragon paragon paragons paragon + parallel parallel parallels parallel + paramour paramour paramours paramour + parapets parapet paraquito paraquito + parasite parasit parasites parasit + parca parca parcel parcel + parcell parcel parcels parcel + parch parch parched parch + parching parch parchment parchment + pard pard pardon pardon + pardona pardona pardoned pardon + pardoner pardon pardoning pardon + pardonne pardonn pardonner pardonn + pardonnez pardonnez pardons pardon + pare pare pared pare + parel parel parent parent + parentage parentag parents parent + parfect parfect paring pare + parings pare paris pari + parish parish parishioners parishion + parisians parisian paritors paritor + park park parks park + parle parl parler parler + parles parl parley parlei + parlez parlez parliament parliament + parlors parlor parlour parlour + parlous parlou parmacity parmac + parolles parol parricide parricid + parricides parricid parrot parrot + parrots parrot parsley parslei + parson parson part part + partake partak partaken partaken + partaker partak partakers partak + parted part parthia parthia + parthian parthian parthians parthian + parti parti partial partial + partialize partial partially partial + participate particip participation particip + particle particl particular particular + particularities particular particularize particular + particularly particularli particulars particular + parties parti parting part + partisan partisan partisans partisan + partition partit partizan partizan + partlet partlet partly partli + partner partner partners partner + partridge partridg parts part + party parti pas pa + pash pash pashed pash + pashful pash pass pass + passable passabl passado passado + passage passag passages passag + passant passant passed pass + passenger passeng passengers passeng + passes pass passeth passeth + passing pass passio passio + passion passion passionate passion + passioning passion passions passion + passive passiv passport passport + passy passi past past + paste past pasterns pastern + pasties pasti pastime pastim + pastimes pastim pastoral pastor + pastorals pastor pastors pastor + pastry pastri pasture pastur + pastures pastur pasty pasti + pat pat patay patai + patch patch patchery patcheri + patches patch pate pate + pated pate patent patent + patents patent paternal patern + pates pate path path + pathetical pathet paths path + pathway pathwai pathways pathwai + patience patienc patient patient + patiently patient patients patient + patines patin patrician patrician + patricians patrician patrick patrick + patrimony patrimoni patroclus patroclu + patron patron patronage patronag + patroness patro patrons patron + patrum patrum patter patter + pattern pattern patterns pattern + pattle pattl pauca pauca + paucas pauca paul paul + paulina paulina paunch paunch + paunches paunch pause paus + pauser pauser pauses paus + pausingly pausingli pauvres pauvr + pav pav paved pave + pavement pavement pavilion pavilion + pavilions pavilion pavin pavin + paw paw pawn pawn + pawns pawn paws paw + pax pax pay pai + payest payest paying pai + payment payment payments payment + pays pai paysan paysan + paysans paysan pe pe + peace peac peaceable peaceabl + peaceably peaceabl peaceful peac + peacemakers peacemak peaces peac + peach peach peaches peach + peacock peacock peacocks peacock + peak peak peaking peak + peal peal peals peal + pear pear peard peard + pearl pearl pearls pearl + pears pear peas pea + peasant peasant peasantry peasantri + peasants peasant peascod peascod + pease peas peaseblossom peaseblossom + peat peat peaten peaten + peating peat pebble pebbl + pebbled pebbl pebbles pebbl + peck peck pecks peck + peculiar peculiar pecus pecu + pedant pedant pedantical pedant + pedascule pedascul pede pede + pedestal pedest pedigree pedigre + pedlar pedlar pedlars pedlar + pedro pedro peds ped + peel peel peep peep + peeped peep peeping peep + peeps peep peer peer + peereth peereth peering peer + peerless peerless peers peer + peesel peesel peevish peevish + peevishly peevishli peflur peflur + peg peg pegasus pegasu + pegs peg peise peis + peised peis peize peiz + pelf pelf pelican pelican + pelion pelion pell pell + pella pella pelleted pellet + peloponnesus peloponnesu pelt pelt + pelting pelt pembroke pembrok + pen pen penalties penalti + penalty penalti penance penanc + pence penc pencil pencil + pencill pencil pencils pencil + pendant pendant pendent pendent + pendragon pendragon pendulous pendul + penelope penelop penetrable penetr + penetrate penetr penetrative penetr + penitence penit penitent penit + penitential penitenti penitently penit + penitents penit penker penker + penknife penknif penn penn + penned pen penning pen + pennons pennon penny penni + pennyworth pennyworth pennyworths pennyworth + pens pen pense pens + pension pension pensioners pension + pensive pensiv pensived pensiv + pensively pensiv pent pent + pentecost pentecost penthesilea penthesilea + penthouse penthous penurious penuri + penury penuri peopl peopl + people peopl peopled peopl + peoples peopl pepin pepin + pepper pepper peppercorn peppercorn + peppered pepper per per + peradventure peradventur peradventures peradventur + perceiv perceiv perceive perceiv + perceived perceiv perceives perceiv + perceiveth perceiveth perch perch + perchance perchanc percies perci + percussion percuss percy perci + perdie perdi perdita perdita + perdition perdit perdonato perdonato + perdu perdu perdurable perdur + perdurably perdur perdy perdi + pere pere peregrinate peregrin + peremptorily peremptorili peremptory peremptori + perfect perfect perfected perfect + perfecter perfect perfectest perfectest + perfection perfect perfections perfect + perfectly perfectli perfectness perfect + perfidious perfidi perfidiously perfidi + perforce perforc perform perform + performance perform performances perform + performed perform performer perform + performers perform performing perform + performs perform perfum perfum + perfume perfum perfumed perfum + perfumer perfum perfumes perfum + perge perg perhaps perhap + periapts periapt perigort perigort + perigouna perigouna peril peril + perilous peril perils peril + period period periods period + perish perish perished perish + perishest perishest perisheth perisheth + perishing perish periwig periwig + perjur perjur perjure perjur + perjured perjur perjuries perjuri + perjury perjuri perk perk + perkes perk permafoy permafoi + permanent perman permission permiss + permissive permiss permit permit + permitted permit pernicious pernici + perniciously pernici peroration peror + perpend perpend perpendicular perpendicular + perpendicularly perpendicularli perpetual perpetu + perpetually perpetu perpetuity perpetu + perplex perplex perplexed perplex + perplexity perplex pers per + persecuted persecut persecutions persecut + persecutor persecutor perseus perseu + persever persev perseverance persever + persevers persev persia persia + persian persian persist persist + persisted persist persistency persist + persistive persist persists persist + person person personae persona + personage personag personages personag + personal person personally person + personate person personated person + personates person personating person + persons person perspective perspect + perspectively perspect perspectives perspect + perspicuous perspicu persuade persuad + persuaded persuad persuades persuad + persuading persuad persuasion persuas + persuasions persuas pert pert + pertain pertain pertaining pertain + pertains pertain pertaunt pertaunt + pertinent pertin pertly pertli + perturb perturb perturbation perturb + perturbations perturb perturbed perturb + perus peru perusal perus + peruse perus perused perus + perusing perus perverse pervers + perversely pervers perverseness pervers + pervert pervert perverted pervert + peseech peseech pest pest + pester pester pestiferous pestifer + pestilence pestil pestilent pestil + pet pet petar petar + peter peter petit petit + petition petit petitionary petitionari + petitioner petition petitioners petition + petitions petit peto peto + petrarch petrarch petruchio petruchio + petter petter petticoat petticoat + petticoats petticoat pettiness petti + pettish pettish pettitoes pettito + petty petti peu peu + pew pew pewter pewter + pewterer pewter phaethon phaethon + phaeton phaeton phantasime phantasim + phantasimes phantasim phantasma phantasma + pharamond pharamond pharaoh pharaoh + pharsalia pharsalia pheasant pheasant + pheazar pheazar phebe phebe + phebes phebe pheebus pheebu + pheeze pheez phibbus phibbu + philadelphos philadelpho philario philario + philarmonus philarmonu philemon philemon + philip philip philippan philippan + philippe philipp philippi philippi + phillida phillida philo philo + philomel philomel philomela philomela + philosopher philosoph philosophers philosoph + philosophical philosoph philosophy philosophi + philostrate philostr philotus philotu + phlegmatic phlegmat phoebe phoeb + phoebus phoebu phoenicia phoenicia + phoenicians phoenician phoenix phoenix + phorbus phorbu photinus photinu + phrase phrase phraseless phraseless + phrases phrase phrygia phrygia + phrygian phrygian phrynia phrynia + physic physic physical physic + physician physician physicians physician + physics physic pia pia + pibble pibbl pible pibl + picardy picardi pick pick + pickaxe pickax pickaxes pickax + pickbone pickbon picked pick + pickers picker picking pick + pickle pickl picklock picklock + pickpurse pickpurs picks pick + pickt pickt pickthanks pickthank + pictur pictur picture pictur + pictured pictur pictures pictur + pid pid pie pie + piec piec piece piec + pieces piec piecing piec + pied pi piedness pied + pier pier pierc pierc + pierce pierc pierced pierc + pierces pierc pierceth pierceth + piercing pierc piercy pierci + piers pier pies pi + piety pieti pig pig + pigeon pigeon pigeons pigeon + pight pight pigmy pigmi + pigrogromitus pigrogromitu pike pike + pikes pike pil pil + pilate pilat pilates pilat + pilchers pilcher pile pile + piles pile pilf pilf + pilfering pilfer pilgrim pilgrim + pilgrimage pilgrimag pilgrims pilgrim + pill pill pillage pillag + pillagers pillag pillar pillar + pillars pillar pillicock pillicock + pillory pillori pillow pillow + pillows pillow pills pill + pilot pilot pilots pilot + pimpernell pimpernel pin pin + pinch pinch pinched pinch + pinches pinch pinching pinch + pindarus pindaru pine pine + pined pine pines pine + pinfold pinfold pining pine + pinion pinion pink pink + pinn pinn pinnace pinnac + pins pin pinse pins + pint pint pintpot pintpot + pioned pion pioneers pioneer + pioner pioner pioners pioner + pious piou pip pip + pipe pipe piper piper + pipers piper pipes pipe + piping pipe pippin pippin + pippins pippin pirate pirat + pirates pirat pisa pisa + pisanio pisanio pish pish + pismires pismir piss piss + pissing piss pistol pistol + pistols pistol pit pit + pitch pitch pitched pitch + pitcher pitcher pitchers pitcher + pitchy pitchi piteous piteou + piteously piteous pitfall pitfal + pith pith pithless pithless + pithy pithi pitie piti + pitied piti pities piti + pitiful piti pitifully pitifulli + pitiless pitiless pits pit + pittance pittanc pittie pitti + pittikins pittikin pity piti + pitying piti pius piu + plac plac place place + placed place placentio placentio + places place placeth placeth + placid placid placing place + plack plack placket placket + plackets placket plagu plagu + plague plagu plagued plagu + plagues plagu plaguing plagu + plaguy plagui plain plain + plainer plainer plainest plainest + plaining plain plainings plain + plainly plainli plainness plain + plains plain plainsong plainsong + plaintful plaint plaintiff plaintiff + plaintiffs plaintiff plaints plaint + planched planch planet planet + planetary planetari planets planet + planks plank plant plant + plantage plantag plantagenet plantagenet + plantagenets plantagenet plantain plantain + plantation plantat planted plant + planteth planteth plants plant + plash plash plashy plashi + plast plast plaster plaster + plasterer plaster plat plat + plate plate plated plate + plates plate platform platform + platforms platform plats plat + platted plat plausible plausibl + plausive plausiv plautus plautu + play plai played plai + player player players player + playeth playeth playfellow playfellow + playfellows playfellow playhouse playhous + playing plai plays plai + plea plea pleach pleach + pleached pleach plead plead + pleaded plead pleader pleader + pleaders pleader pleading plead + pleads plead pleas plea + pleasance pleasanc pleasant pleasant + pleasantly pleasantli please pleas + pleased pleas pleaser pleaser + pleasers pleaser pleases pleas + pleasest pleasest pleaseth pleaseth + pleasing pleas pleasure pleasur + pleasures pleasur plebeians plebeian + plebeii plebeii plebs pleb + pledge pledg pledges pledg + pleines plein plenitude plenitud + plenteous plenteou plenteously plenteous + plenties plenti plentiful plenti + plentifully plentifulli plenty plenti + pless pless plessed pless + plessing pless pliant pliant + plied pli plies pli + plight plight plighted plight + plighter plighter plod plod + plodded plod plodders plodder + plodding plod plods plod + plood plood ploody ploodi + plot plot plots plot + plotted plot plotter plotter + plough plough ploughed plough + ploughman ploughman ploughmen ploughmen + plow plow plows plow + pluck pluck plucked pluck + plucker plucker plucking pluck + plucks pluck plue plue + plum plum plume plume + plumed plume plumes plume + plummet plummet plump plump + plumpy plumpi plums plum + plung plung plunge plung + plunged plung plural plural + plurisy plurisi plus plu + pluto pluto plutus plutu + ply ply po po + pocket pocket pocketing pocket + pockets pocket pocky pocki + pody podi poem poem + poesy poesi poet poet + poetical poetic poetry poetri + poets poet poictiers poictier + poinards poinard poins poin + point point pointblank pointblank + pointed point pointing point + points point pois poi + poise pois poising pois + poison poison poisoned poison + poisoner poison poisoning poison + poisonous poison poisons poison + poke poke poking poke + pol pol polack polack + polacks polack poland poland + pold pold pole pole + poleaxe poleax polecat polecat + polecats polecat polemon polemon + poles pole poli poli + policies polici policy polici + polish polish polished polish + politic polit politician politician + politicians politician politicly politicli + polixenes polixen poll poll + polluted pollut pollution pollut + polonius poloniu poltroons poltroon + polusion polus polydamus polydamu + polydore polydor polyxena polyxena + pomander pomand pomegranate pomegran + pomewater pomewat pomfret pomfret + pomgarnet pomgarnet pommel pommel + pomp pomp pompeius pompeiu + pompey pompei pompion pompion + pompous pompou pomps pomp + pond pond ponder ponder + ponderous ponder ponds pond + poniard poniard poniards poniard + pont pont pontic pontic + pontifical pontif ponton ponton + pooh pooh pool pool + poole pool poop poop + poor poor poorer poorer + poorest poorest poorly poorli + pop pop pope pope + popedom popedom popilius popiliu + popingay popingai popish popish + popp popp poppy poppi + pops pop popular popular + popularity popular populous popul + porch porch porches porch + pore pore poring pore + pork pork porn porn + porpentine porpentin porridge porridg + porringer porring port port + portable portabl portage portag + portal portal portance portanc + portcullis portculli portend portend + portends portend portent portent + portentous portent portents portent + porter porter porters porter + portia portia portion portion + portly portli portotartarossa portotartarossa + portrait portrait portraiture portraitur + ports port portugal portug + pose pose posied posi + posies posi position posit + positive posit positively posit + posse poss possess possess + possessed possess possesses possess + possesseth possesseth possessing possess + possession possess possessions possess + possessor possessor posset posset + possets posset possibilities possibl + possibility possibl possible possibl + possibly possibl possitable possit + post post poste post + posted post posterior posterior + posteriors posterior posterity poster + postern postern posterns postern + posters poster posthorse posthors + posthorses posthors posthumus posthumu + posting post postmaster postmast + posts post postscript postscript + posture postur postures postur + posy posi pot pot + potable potabl potations potat + potato potato potatoes potato + potch potch potency potenc + potent potent potentates potent + potential potenti potently potent + potents potent pothecary pothecari + pother pother potion potion + potions potion potpan potpan + pots pot potter potter + potting pot pottle pottl + pouch pouch poulter poulter + poultice poultic poultney poultnei + pouncet pouncet pound pound + pounds pound pour pour + pourest pourest pouring pour + pourquoi pourquoi pours pour + pout pout poverty poverti + pow pow powd powd + powder powder power power + powerful power powerfully powerfulli + powerless powerless powers power + pox pox poys poi + poysam poysam prabbles prabbl + practic practic practice practic + practiced practic practicer practic + practices practic practicing practic + practis practi practisants practis + practise practis practiser practis + practisers practis practises practis + practising practis praeclarissimus praeclarissimu + praemunire praemunir praetor praetor + praetors praetor pragging prag + prague pragu prain prain + prains prain prais prai + praise prais praised prais + praises prais praisest praisest + praiseworthy praiseworthi praising prais + prancing pranc prank prank + pranks prank prat prat + prate prate prated prate + prater prater prating prate + prattle prattl prattler prattler + prattling prattl prave prave + prawls prawl prawns prawn + pray prai prayer prayer + prayers prayer praying prai + prays prai pre pre + preach preach preached preach + preachers preacher preaches preach + preaching preach preachment preachment + pread pread preambulate preambul + precedence preced precedent preced + preceding preced precept precept + preceptial precepti precepts precept + precinct precinct precious preciou + preciously precious precipice precipic + precipitating precipit precipitation precipit + precise precis precisely precis + preciseness precis precisian precisian + precor precor precurse precurs + precursors precursor predeceased predeceas + predecessor predecessor predecessors predecessor + predestinate predestin predicament predica + predict predict prediction predict + predictions predict predominance predomin + predominant predomin predominate predomin + preeches preech preeminence preemin + preface prefac prefer prefer + preferment prefer preferments prefer + preferr preferr preferreth preferreth + preferring prefer prefers prefer + prefiguring prefigur prefix prefix + prefixed prefix preformed preform + pregnancy pregnanc pregnant pregnant + pregnantly pregnantli prejudicates prejud + prejudice prejudic prejudicial prejudici + prelate prelat premeditated premedit + premeditation premedit premised premis + premises premis prenez prenez + prenominate prenomin prentice prentic + prentices prentic preordinance preordin + prepar prepar preparation prepar + preparations prepar prepare prepar + prepared prepar preparedly preparedli + prepares prepar preparing prepar + prepost prepost preposterous preposter + preposterously preposter prerogatifes prerogatif + prerogative prerog prerogatived prerogativ + presage presag presagers presag + presages presag presageth presageth + presaging presag prescience prescienc + prescribe prescrib prescript prescript + prescription prescript prescriptions prescript + prescripts prescript presence presenc + presences presenc present present + presentation present presented present + presenter present presenters present + presenteth presenteth presenting present + presently present presentment present + presents present preserv preserv + preservation preserv preservative preserv + preserve preserv preserved preserv + preserver preserv preservers preserv + preserving preserv president presid + press press pressed press + presser presser presses press + pressing press pressure pressur + pressures pressur prest prest + prester prester presume presum + presumes presum presuming presum + presumption presumpt presumptuous presumptu + presuppos presuppo pret pret + pretence pretenc pretences pretenc + pretend pretend pretended pretend + pretending pretend pretense pretens + pretext pretext pretia pretia + prettier prettier prettiest prettiest + prettily prettili prettiness pretti + pretty pretti prevail prevail + prevailed prevail prevaileth prevaileth + prevailing prevail prevailment prevail + prevails prevail prevent prevent + prevented prevent prevention prevent + preventions prevent prevents prevent + prey prei preyful prey + preys prei priam priam + priami priami priamus priamu + pribbles pribbl price price + prick prick pricked prick + pricket pricket pricking prick + pricks prick pricksong pricksong + pride pride prides pride + pridge pridg prie prie + pried pri prief prief + pries pri priest priest + priesthood priesthood priests priest + prig prig primal primal + prime prime primer primer + primero primero primest primest + primitive primit primo primo + primogenity primogen primrose primros + primroses primros primy primi + prince princ princely princ + princes princ princess princess + principal princip principalities princip + principality princip principle principl + principles principl princox princox + prings pring print print + printed print printing print + printless printless prints print + prioress prioress priories priori + priority prioriti priory priori + priscian priscian prison prison + prisoner prison prisoners prison + prisonment prison prisonnier prisonni + prisons prison pristine pristin + prithe prith prithee prithe + privacy privaci private privat + privately privat privates privat + privilage privilag privileg privileg + privilege privileg privileged privileg + privileges privileg privilegio privilegio + privily privili privity priviti + privy privi priz priz + prize prize prized prize + prizer prizer prizes prize + prizest prizest prizing prize + pro pro probable probabl + probal probal probation probat + proceed proce proceeded proceed + proceeders proceed proceeding proceed + proceedings proceed proceeds proce + process process procession process + proclaim proclaim proclaimed proclaim + proclaimeth proclaimeth proclaims proclaim + proclamation proclam proclamations proclam + proconsul proconsul procrastinate procrastin + procreant procreant procreants procreant + procreation procreat procrus procru + proculeius proculeiu procur procur + procurator procur procure procur + procured procur procures procur + procuring procur prodigal prodig + prodigality prodig prodigally prodig + prodigals prodig prodigies prodigi + prodigious prodigi prodigiously prodigi + prodigy prodigi proditor proditor + produc produc produce produc + produced produc produces produc + producing produc proface profac + profan profan profanation profan + profane profan profaned profan + profanely profan profaneness profan + profaners profan profaning profan + profess profess professed profess + professes profess profession profess + professions profess professors professor + proffer proffer proffered proffer + profferer proffer proffers proffer + proficient profici profit profit + profitable profit profitably profit + profited profit profiting profit + profitless profitless profits profit + profound profound profoundest profoundest + profoundly profoundli progenitors progenitor + progeny progeni progne progn + prognosticate prognost prognostication prognost + progress progress progression progress + prohibit prohibit prohibition prohibit + project project projection project + projects project prolixious prolixi + prolixity prolix prologue prologu + prologues prologu prolong prolong + prolongs prolong promethean promethean + prometheus prometheu promis promi + promise promis promised promis + promises promis promiseth promiseth + promising promis promontory promontori + promotion promot promotions promot + prompt prompt prompted prompt + promptement promptement prompter prompter + prompting prompt prompts prompt + prompture promptur promulgate promulg + prone prone prononcer prononc + prononcez prononcez pronoun pronoun + pronounc pronounc pronounce pronounc + pronounced pronounc pronouncing pronounc + pronouns pronoun proof proof + proofs proof prop prop + propagate propag propagation propag + propend propend propension propens + proper proper properer proper + properly properli propertied properti + properties properti property properti + prophecies propheci prophecy propheci + prophesied prophesi prophesier prophesi + prophesy prophesi prophesying prophesi + prophet prophet prophetess prophetess + prophetic prophet prophetically prophet + prophets prophet propinquity propinqu + propontic propont proportion proport + proportionable proportion proportions proport + propos propo propose propos + proposed propos proposer propos + proposes propos proposing propos + proposition proposit propositions proposit + propounded propound propp propp + propre propr propriety proprieti + props prop propugnation propugn + prorogue prorogu prorogued prorogu + proscription proscript proscriptions proscript + prose prose prosecute prosecut + prosecution prosecut proselytes proselyt + proserpina proserpina prosp prosp + prospect prospect prosper prosper + prosperity prosper prospero prospero + prosperous prosper prosperously prosper + prospers prosper prostitute prostitut + prostrate prostrat protect protect + protected protect protection protect + protector protector protectors protector + protectorship protectorship protectress protectress + protects protect protest protest + protestation protest protestations protest + protested protest protester protest + protesting protest protests protest + proteus proteu protheus protheu + protract protract protractive protract + proud proud prouder prouder + proudest proudest proudlier proudlier + proudly proudli prouds proud + prov prov provand provand + prove prove proved prove + provender provend proverb proverb + proverbs proverb proves prove + proveth proveth provide provid + provided provid providence provid + provident provid providently provid + provider provid provides provid + province provinc provinces provinc + provincial provinci proving prove + provision provis proviso proviso + provocation provoc provok provok + provoke provok provoked provok + provoker provok provokes provok + provoketh provoketh provoking provok + provost provost prowess prowess + prudence prudenc prudent prudent + prun prun prune prune + prunes prune pruning prune + pry pry prying pry + psalm psalm psalmist psalmist + psalms psalm psalteries psalteri + ptolemies ptolemi ptolemy ptolemi + public public publican publican + publication public publicly publicli + publicola publicola publish publish + published publish publisher publish + publishing publish publius publiu + pucelle pucel puck puck + pudder pudder pudding pud + puddings pud puddle puddl + puddled puddl pudency pudenc + pueritia pueritia puff puff + puffing puf puffs puff + pugging pug puis pui + puissance puissanc puissant puissant + puke puke puking puke + pulcher pulcher puling pule + pull pull puller puller + pullet pullet pulling pull + pulls pull pulpit pulpit + pulpiter pulpit pulpits pulpit + pulse puls pulsidge pulsidg + pump pump pumpion pumpion + pumps pump pun pun + punched punch punish punish + punished punish punishes punish + punishment punish punishments punish + punk punk punto punto + puny puni pupil pupil + pupils pupil puppet puppet + puppets puppet puppies puppi + puppy puppi pur pur + purblind purblind purchas purcha + purchase purchas purchased purchas + purchases purchas purchaseth purchaseth + purchasing purchas pure pure + purely pure purer purer + purest purest purg purg + purgation purgat purgative purg + purgatory purgatori purge purg + purged purg purgers purger + purging purg purifies purifi + purifying purifi puritan puritan + purity puriti purlieus purlieu + purple purpl purpled purpl + purples purpl purport purport + purpos purpo purpose purpos + purposed purpos purposely purpos + purposes purpos purposeth purposeth + purposing purpos purr purr + purs pur purse purs + pursents pursent purses purs + pursu pursu pursue pursu + pursued pursu pursuers pursuer + pursues pursu pursuest pursuest + pursueth pursueth pursuing pursu + pursuit pursuit pursuivant pursuiv + pursuivants pursuiv pursy pursi + purus puru purveyor purveyor + push push pushes push + pusillanimity pusillanim put put + putrefy putrefi putrified putrifi + puts put putter putter + putting put puttock puttock + puzzel puzzel puzzle puzzl + puzzled puzzl puzzles puzzl + py py pygmalion pygmalion + pygmies pygmi pygmy pygmi + pyramid pyramid pyramides pyramid + pyramids pyramid pyramis pyrami + pyramises pyramis pyramus pyramu + pyrenean pyrenean pyrrhus pyrrhu + pythagoras pythagora qu qu + quadrangle quadrangl quae quae + quaff quaff quaffing quaf + quagmire quagmir quail quail + quailing quail quails quail + quaint quaint quaintly quaintli + quak quak quake quak + quakes quak qualification qualif + qualified qualifi qualifies qualifi + qualify qualifi qualifying qualifi + qualite qualit qualities qualiti + quality qualiti qualm qualm + qualmish qualmish quam quam + quand quand quando quando + quantities quantiti quantity quantiti + quare quar quarrel quarrel + quarrell quarrel quarreller quarrel + quarrelling quarrel quarrelous quarrel + quarrels quarrel quarrelsome quarrelsom + quarries quarri quarry quarri + quart quart quarter quarter + quartered quarter quartering quarter + quarters quarter quarts quart + quasi quasi quat quat + quatch quatch quay quai + que que quean quean + queas quea queasiness queasi + queasy queasi queen queen + queens queen quell quell + queller queller quench quench + quenched quench quenching quench + quenchless quenchless quern quern + quest quest questant questant + question question questionable question + questioned question questioning question + questionless questionless questions question + questrists questrist quests quest + queubus queubu qui qui + quick quick quicken quicken + quickens quicken quicker quicker + quicklier quicklier quickly quickli + quickness quick quicksand quicksand + quicksands quicksand quicksilverr quicksilverr + quid quid quiddities quidditi + quiddits quiddit quier quier + quiet quiet quieter quieter + quietly quietli quietness quiet + quietus quietu quill quill + quillets quillet quills quill + quilt quilt quinapalus quinapalu + quince quinc quinces quinc + quintain quintain quintessence quintess + quintus quintu quip quip + quips quip quire quir + quiring quir quirk quirk + quirks quirk quis qui + quit quit quite quit + quits quit quittance quittanc + quitted quit quitting quit + quiver quiver quivering quiver + quivers quiver quo quo + quod quod quoifs quoif + quoint quoint quoit quoit + quoits quoit quondam quondam + quoniam quoniam quote quot + quoted quot quotes quot + quoth quoth quotidian quotidian + r r rabbit rabbit + rabble rabbl rabblement rabblement + race race rack rack + rackers racker racket racket + rackets racket racking rack + racks rack radiance radianc + radiant radiant radish radish + rafe rafe raft raft + rag rag rage rage + rages rage rageth rageth + ragg ragg ragged rag + raggedness ragged raging rage + ragozine ragozin rags rag + rah rah rail rail + railed rail railer railer + railest railest raileth raileth + railing rail rails rail + raiment raiment rain rain + rainbow rainbow raineth raineth + raining rain rainold rainold + rains rain rainy raini + rais rai raise rais + raised rais raises rais + raising rais raisins raisin + rak rak rake rake + rakers raker rakes rake + ral ral rald rald + ralph ralph ram ram + rambures rambur ramm ramm + rampallian rampallian rampant rampant + ramping ramp rampir rampir + ramps ramp rams ram + ramsey ramsei ramston ramston + ran ran rance ranc + rancorous rancor rancors rancor + rancour rancour random random + rang rang range rang + ranged rang rangers ranger + ranges rang ranging rang + rank rank ranker ranker + rankest rankest ranking rank + rankle rankl rankly rankli + rankness rank ranks rank + ransack ransack ransacking ransack + ransom ransom ransomed ransom + ransoming ransom ransomless ransomless + ransoms ransom rant rant + ranting rant rap rap + rape rape rapes rape + rapier rapier rapiers rapier + rapine rapin raps rap + rapt rapt rapture raptur + raptures raptur rar rar + rare rare rarely rare + rareness rare rarer rarer + rarest rarest rarities rariti + rarity rariti rascal rascal + rascalliest rascalliest rascally rascal + rascals rascal rased rase + rash rash rasher rasher + rashly rashli rashness rash + rat rat ratcatcher ratcatch + ratcliff ratcliff rate rate + rated rate rately rate + rates rate rather rather + ratherest ratherest ratified ratifi + ratifiers ratifi ratify ratifi + rating rate rational ration + ratolorum ratolorum rats rat + ratsbane ratsban rattle rattl + rattles rattl rattling rattl + rature ratur raught raught + rav rav rave rave + ravel ravel raven raven + ravening raven ravenous raven + ravens raven ravenspurgh ravenspurgh + raves rave ravin ravin + raving rave ravish ravish + ravished ravish ravisher ravish + ravishing ravish ravishments ravish + raw raw rawer rawer + rawly rawli rawness raw + ray rai rayed rai + rays rai raz raz + raze raze razed raze + razes raze razeth razeth + razing raze razor razor + razorable razor razors razor + razure razur re re + reach reach reaches reach + reacheth reacheth reaching reach + read read reader reader + readiest readiest readily readili + readiness readi reading read + readins readin reads read + ready readi real real + really realli realm realm + realms realm reap reap + reapers reaper reaping reap + reaps reap rear rear + rears rear rearward rearward + reason reason reasonable reason + reasonably reason reasoned reason + reasoning reason reasonless reasonless + reasons reason reave reav + rebate rebat rebato rebato + rebeck rebeck rebel rebel + rebell rebel rebelling rebel + rebellion rebellion rebellious rebelli + rebels rebel rebound rebound + rebuk rebuk rebuke rebuk + rebukeable rebuk rebuked rebuk + rebukes rebuk rebus rebu + recall recal recant recant + recantation recant recanter recant + recanting recant receipt receipt + receipts receipt receiv receiv + receive receiv received receiv + receiver receiv receives receiv + receivest receivest receiveth receiveth + receiving receiv receptacle receptacl + rechate rechat reciprocal reciproc + reciprocally reciproc recite recit + recited recit reciterai reciterai + reck reck recking reck + reckless reckless reckon reckon + reckoned reckon reckoning reckon + reckonings reckon recks reck + reclaim reclaim reclaims reclaim + reclusive reclus recognizance recogniz + recognizances recogniz recoil recoil + recoiling recoil recollected recollect + recomforted recomfort recomforture recomfortur + recommend recommend recommended recommend + recommends recommend recompens recompen + recompense recompens reconcil reconcil + reconcile reconcil reconciled reconcil + reconcilement reconcil reconciler reconcil + reconciles reconcil reconciliation reconcili + record record recordation record + recorded record recorder record + recorders record records record + recount recount recounted recount + recounting recount recountments recount + recounts recount recourse recours + recov recov recover recov + recoverable recover recovered recov + recoveries recoveri recovers recov + recovery recoveri recreant recreant + recreants recreant recreate recreat + recreation recreat rectify rectifi + rector rector rectorship rectorship + recure recur recured recur + red red redbreast redbreast + redder redder reddest reddest + rede rede redeem redeem + redeemed redeem redeemer redeem + redeeming redeem redeems redeem + redeliver redeliv redemption redempt + redime redim redness red + redoubled redoubl redoubted redoubt + redound redound redress redress + redressed redress redresses redress + reduce reduc reechy reechi + reed reed reeds reed + reek reek reeking reek + reeks reek reeky reeki + reel reel reeleth reeleth + reeling reel reels reel + refell refel refer refer + reference refer referr referr + referred refer refigured refigur + refin refin refined refin + reflect reflect reflecting reflect + reflection reflect reflex reflex + reform reform reformation reform + reformed reform refractory refractori + refrain refrain refresh refresh + refreshing refresh reft reft + refts reft refuge refug + refus refu refusal refus + refuse refus refused refus + refusest refusest refusing refus + reg reg regal regal + regalia regalia regan regan + regard regard regardance regard + regarded regard regardfully regardfulli + regarding regard regards regard + regenerate regener regent regent + regentship regentship regia regia + regiment regiment regiments regiment + regina regina region region + regions region regist regist + register regist registers regist + regreet regreet regreets regreet + regress regress reguerdon reguerdon + regular regular rehears rehear + rehearsal rehears rehearse rehears + reign reign reigned reign + reignier reignier reigning reign + reigns reign rein rein + reinforc reinforc reinforce reinforc + reinforcement reinforc reins rein + reiterate reiter reject reject + rejected reject rejoic rejoic + rejoice rejoic rejoices rejoic + rejoiceth rejoiceth rejoicing rejoic + rejoicingly rejoicingli rejoindure rejoindur + rejourn rejourn rel rel + relapse relaps relate relat + relates relat relation relat + relations relat relative rel + releas relea release releas + released releas releasing releas + relent relent relenting relent + relents relent reliances relianc + relics relic relief relief + reliev reliev relieve reliev + relieved reliev relieves reliev + relieving reliev religion religion + religions religion religious religi + religiously religi relinquish relinquish + reliques reliqu reliquit reliquit + relish relish relume relum + rely reli relying reli + remain remain remainder remaind + remainders remaind remained remain + remaineth remaineth remaining remain + remains remain remark remark + remarkable remark remediate remedi + remedied remedi remedies remedi + remedy remedi rememb rememb + remember rememb remembered rememb + remembers rememb remembrance remembr + remembrancer remembranc remembrances remembr + remercimens remercimen remiss remiss + remission remiss remissness remiss + remit remit remnant remnant + remnants remnant remonstrance remonstr + remorse remors remorseful remors + remorseless remorseless remote remot + remotion remot remov remov + remove remov removed remov + removedness removed remover remov + removes remov removing remov + remunerate remuner remuneration remuner + rence renc rend rend + render render rendered render + renders render rendezvous rendezv + renegado renegado renege reneg + reneges reneg renew renew + renewed renew renewest renewest + renounce renounc renouncement renounc + renouncing renounc renowmed renowm + renown renown renowned renown + rent rent rents rent + repaid repaid repair repair + repaired repair repairing repair + repairs repair repass repass + repast repast repasture repastur + repay repai repaying repai + repays repai repeal repeal + repealing repeal repeals repeal + repeat repeat repeated repeat + repeating repeat repeats repeat + repel repel repent repent + repentance repent repentant repent + repented repent repenting repent + repents repent repetition repetit + repetitions repetit repin repin + repine repin repining repin + replant replant replenish replenish + replenished replenish replete replet + replication replic replied repli + replies repli repliest repliest + reply repli replying repli + report report reported report + reporter report reportest reportest + reporting report reportingly reportingli + reports report reposal repos + repose repos reposeth reposeth + reposing repos repossess repossess + reprehend reprehend reprehended reprehend + reprehending reprehend represent repres + representing repres reprieve repriev + reprieves repriev reprisal repris + reproach reproach reproaches reproach + reproachful reproach reproachfully reproachfulli + reprobate reprob reprobation reprob + reproof reproof reprov reprov + reprove reprov reproveable reprov + reproves reprov reproving reprov + repugn repugn repugnancy repugn + repugnant repugn repulse repuls + repulsed repuls repurchas repurcha + repured repur reputation reput + repute reput reputed reput + reputeless reputeless reputes reput + reputing reput request request + requested request requesting request + requests request requiem requiem + requir requir require requir + required requir requires requir + requireth requireth requiring requir + requisite requisit requisites requisit + requit requit requital requit + requite requit requited requit + requites requit rer rer + rere rere rers rer + rescu rescu rescue rescu + rescued rescu rescues rescu + rescuing rescu resemblance resembl + resemble resembl resembled resembl + resembles resembl resembleth resembleth + resembling resembl reserv reserv + reservation reserv reserve reserv + reserved reserv reserves reserv + reside resid residence resid + resident resid resides resid + residing resid residue residu + resign resign resignation resign + resist resist resistance resist + resisted resist resisting resist + resists resist resolute resolut + resolutely resolut resolutes resolut + resolution resolut resolv resolv + resolve resolv resolved resolv + resolvedly resolvedli resolves resolv + resolveth resolveth resort resort + resorted resort resounding resound + resounds resound respeaking respeak + respect respect respected respect + respecting respect respective respect + respectively respect respects respect + respice respic respite respit + respites respit responsive respons + respose respos ress ress + rest rest rested rest + resteth resteth restful rest + resting rest restitution restitut + restless restless restor restor + restoration restor restorative restor + restore restor restored restor + restores restor restoring restor + restrain restrain restrained restrain + restraining restrain restrains restrain + restraint restraint rests rest + resty resti resum resum + resume resum resumes resum + resurrections resurrect retail retail + retails retail retain retain + retainers retain retaining retain + retell retel retention retent + retentive retent retinue retinu + retir retir retire retir + retired retir retirement retir + retires retir retiring retir + retold retold retort retort + retorts retort retourne retourn + retract retract retreat retreat + retrograde retrograd rets ret + return return returned return + returnest returnest returneth returneth + returning return returns return + revania revania reveal reveal + reveals reveal revel revel + reveler revel revell revel + reveller revel revellers revel + revelling revel revelry revelri + revels revel reveng reveng + revenge reveng revenged reveng + revengeful reveng revengement reveng + revenger reveng revengers reveng + revenges reveng revenging reveng + revengingly revengingli revenue revenu + revenues revenu reverb reverb + reverberate reverber reverbs reverb + reverenc reverenc reverence rever + reverend reverend reverent rever + reverently rever revers rever + reverse revers reversion revers + reverted revert review review + reviewest reviewest revil revil + revile revil revisits revisit + reviv reviv revive reviv + revives reviv reviving reviv + revok revok revoke revok + revokement revok revolt revolt + revolted revolt revolting revolt + revolts revolt revolution revolut + revolutions revolut revolve revolv + revolving revolv reward reward + rewarded reward rewarder reward + rewarding reward rewards reward + reword reword reworded reword + rex rex rey rei + reynaldo reynaldo rford rford + rful rful rfull rfull + rhapsody rhapsodi rheims rheim + rhenish rhenish rhesus rhesu + rhetoric rhetor rheum rheum + rheumatic rheumat rheums rheum + rheumy rheumi rhinoceros rhinocero + rhodes rhode rhodope rhodop + rhubarb rhubarb rhym rhym + rhyme rhyme rhymers rhymer + rhymes rhyme rhyming rhyme + rialto rialto rib rib + ribald ribald riband riband + ribands riband ribaudred ribaudr + ribb ribb ribbed rib + ribbon ribbon ribbons ribbon + ribs rib rice rice + rich rich richard richard + richer richer riches rich + richest richest richly richli + richmond richmond richmonds richmond + rid rid riddance riddanc + ridden ridden riddle riddl + riddles riddl riddling riddl + ride ride rider rider + riders rider rides ride + ridest ridest rideth rideth + ridge ridg ridges ridg + ridiculous ridicul riding ride + rids rid rien rien + ries ri rifle rifl + rift rift rifted rift + rig rig rigg rigg + riggish riggish right right + righteous righteou righteously righteous + rightful right rightfully rightfulli + rightly rightli rights right + rigol rigol rigorous rigor + rigorously rigor rigour rigour + ril ril rim rim + rin rin rinaldo rinaldo + rind rind ring ring + ringing ring ringleader ringlead + ringlets ringlet rings ring + ringwood ringwood riot riot + rioter rioter rioting riot + riotous riotou riots riot + rip rip ripe ripe + ripely ripe ripen ripen + ripened ripen ripeness ripe + ripening ripen ripens ripen + riper riper ripest ripest + riping ripe ripp ripp + ripping rip rise rise + risen risen rises rise + riseth riseth rish rish + rising rise rite rite + rites rite rivage rivag + rival rival rivality rival + rivall rival rivals rival + rive rive rived rive + rivelled rivel river river + rivers river rivet rivet + riveted rivet rivets rivet + rivo rivo rj rj + rless rless road road + roads road roam roam + roaming roam roan roan + roar roar roared roar + roarers roarer roaring roar + roars roar roast roast + roasted roast rob rob + roba roba robas roba + robb robb robbed rob + robber robber robbers robber + robbery robberi robbing rob + robe robe robed robe + robert robert robes robe + robin robin robs rob + robustious robusti rochester rochest + rochford rochford rock rock + rocks rock rocky rocki + rod rod rode rode + roderigo roderigo rods rod + roe roe roes roe + roger roger rogero rogero + rogue rogu roguery rogueri + rogues rogu roguish roguish + roi roi roisting roist + roll roll rolled roll + rolling roll rolls roll + rom rom romage romag + roman roman romano romano + romanos romano romans roman + rome rome romeo romeo + romish romish rondure rondur + ronyon ronyon rood rood + roof roof roofs roof + rook rook rooks rook + rooky rooki room room + rooms room root root + rooted root rootedly rootedli + rooteth rooteth rooting root + roots root rope rope + ropery roperi ropes rope + roping rope ros ro + rosalind rosalind rosalinda rosalinda + rosalinde rosalind rosaline rosalin + roscius rosciu rose rose + rosed rose rosemary rosemari + rosencrantz rosencrantz roses rose + ross ross rosy rosi + rot rot rote rote + roted rote rother rother + rotherham rotherham rots rot + rotted rot rotten rotten + rottenness rotten rotting rot + rotundity rotund rouen rouen + rough rough rougher rougher + roughest roughest roughly roughli + roughness rough round round + rounded round roundel roundel + rounder rounder roundest roundest + rounding round roundly roundli + rounds round roundure roundur + rous rou rouse rous + roused rous rousillon rousillon + rously rousli roussi roussi + rout rout routed rout + routs rout rove rove + rover rover row row + rowel rowel rowland rowland + rowlands rowland roy roi + royal royal royalize royal + royally royal royalties royalti + royalty royalti roynish roynish + rs rs rt rt + rub rub rubb rubb + rubbing rub rubbish rubbish + rubies rubi rubious rubiou + rubs rub ruby rubi + rud rud rudand rudand + rudder rudder ruddiness ruddi + ruddock ruddock ruddy ruddi + rude rude rudely rude + rudeness rude ruder ruder + rudesby rudesbi rudest rudest + rudiments rudiment rue rue + rued ru ruff ruff + ruffian ruffian ruffians ruffian + ruffle ruffl ruffling ruffl + ruffs ruff rug rug + rugby rugbi rugemount rugemount + rugged rug ruin ruin + ruinate ruinat ruined ruin + ruining ruin ruinous ruinou + ruins ruin rul rul + rule rule ruled rule + ruler ruler rulers ruler + rules rule ruling rule + rumble rumbl ruminaies ruminai + ruminat ruminat ruminate rumin + ruminated rumin ruminates rumin + rumination rumin rumor rumor + rumour rumour rumourer rumour + rumours rumour rump rump + run run runagate runag + runagates runag runaway runawai + runaways runawai rung rung + runn runn runner runner + runners runner running run + runs run rupture ruptur + ruptures ruptur rural rural + rush rush rushes rush + rushing rush rushling rushl + rushy rushi russet russet + russia russia russian russian + russians russian rust rust + rusted rust rustic rustic + rustically rustic rustics rustic + rustle rustl rustling rustl + rusts rust rusty rusti + rut rut ruth ruth + ruthful ruth ruthless ruthless + rutland rutland ruttish ruttish + ry ry rye rye + rything ryth s s + sa sa saba saba + sabbath sabbath sable sabl + sables sabl sack sack + sackbuts sackbut sackcloth sackcloth + sacked sack sackerson sackerson + sacks sack sacrament sacrament + sacred sacr sacrific sacrif + sacrifice sacrific sacrificers sacrific + sacrifices sacrific sacrificial sacrifici + sacrificing sacrif sacrilegious sacrilegi + sacring sacr sad sad + sadder sadder saddest saddest + saddle saddl saddler saddler + saddles saddl sadly sadli + sadness sad saf saf + safe safe safeguard safeguard + safely safe safer safer + safest safest safeties safeti + safety safeti saffron saffron + sag sag sage sage + sagittary sagittari said said + saidst saidst sail sail + sailing sail sailmaker sailmak + sailor sailor sailors sailor + sails sail sain sain + saint saint sainted saint + saintlike saintlik saints saint + saith saith sake sake + sakes sake sala sala + salad salad salamander salamand + salary salari sale sale + salerio salerio salicam salicam + salique saliqu salisbury salisburi + sall sall sallet sallet + sallets sallet sallies salli + sallow sallow sally salli + salmon salmon salmons salmon + salt salt salter salter + saltiers saltier saltness salt + saltpetre saltpetr salutation salut + salutations salut salute salut + saluted salut salutes salut + saluteth saluteth salv salv + salvation salvat salve salv + salving salv same same + samingo samingo samp samp + sampire sampir sample sampl + sampler sampler sampson sampson + samson samson samsons samson + sancta sancta sanctified sanctifi + sanctifies sanctifi sanctify sanctifi + sanctimonies sanctimoni sanctimonious sanctimoni + sanctimony sanctimoni sanctities sanctiti + sanctity sanctiti sanctuarize sanctuar + sanctuary sanctuari sand sand + sandal sandal sandbag sandbag + sanded sand sands sand + sandy sandi sandys sandi + sang sang sanguine sanguin + sanguis sangui sanity saniti + sans san santrailles santrail + sap sap sapient sapient + sapit sapit sapless sapless + sapling sapl sapphire sapphir + sapphires sapphir saracens saracen + sarcenet sarcenet sard sard + sardians sardian sardinia sardinia + sardis sardi sarum sarum + sat sat satan satan + satchel satchel sate sate + sated sate satiate satiat + satiety satieti satin satin + satire satir satirical satir + satis sati satisfaction satisfact + satisfied satisfi satisfies satisfi + satisfy satisfi satisfying satisfi + saturday saturdai saturdays saturdai + saturn saturn saturnine saturnin + saturninus saturninu satyr satyr + satyrs satyr sauc sauc + sauce sauc sauced sauc + saucers saucer sauces sauc + saucily saucili sauciness sauci + saucy sauci sauf sauf + saunder saunder sav sav + savage savag savagely savag + savageness savag savagery savageri + savages savag save save + saved save saves save + saving save saviour saviour + savory savori savour savour + savouring savour savours savour + savoury savouri savoy savoi + saw saw sawed saw + sawest sawest sawn sawn + sawpit sawpit saws saw + sawyer sawyer saxons saxon + saxony saxoni saxton saxton + say sai sayest sayest + saying sai sayings sai + says sai sayst sayst + sblood sblood sc sc + scab scab scabbard scabbard + scabs scab scaffold scaffold + scaffoldage scaffoldag scal scal + scald scald scalded scald + scalding scald scale scale + scaled scale scales scale + scaling scale scall scall + scalp scalp scalps scalp + scaly scali scamble scambl + scambling scambl scamels scamel + scan scan scandal scandal + scandaliz scandaliz scandalous scandal + scandy scandi scann scann + scant scant scanted scant + scanter scanter scanting scant + scantling scantl scants scant + scap scap scape scape + scaped scape scapes scape + scapeth scapeth scar scar + scarce scarc scarcely scarc + scarcity scarciti scare scare + scarecrow scarecrow scarecrows scarecrow + scarf scarf scarfed scarf + scarfs scarf scaring scare + scarlet scarlet scarr scarr + scarre scarr scars scar + scarus scaru scath scath + scathe scath scathful scath + scatt scatt scatter scatter + scattered scatter scattering scatter + scatters scatter scelera scelera + scelerisque scelerisqu scene scene + scenes scene scent scent + scented scent scept scept + scepter scepter sceptre sceptr + sceptred sceptr sceptres sceptr + schedule schedul schedules schedul + scholar scholar scholarly scholarli + scholars scholar school school + schoolboy schoolboi schoolboys schoolboi + schoolfellows schoolfellow schooling school + schoolmaster schoolmast schoolmasters schoolmast + schools school sciatica sciatica + sciaticas sciatica science scienc + sciences scienc scimitar scimitar + scion scion scions scion + scissors scissor scoff scoff + scoffer scoffer scoffing scof + scoffs scoff scoggin scoggin + scold scold scolding scold + scolds scold sconce sconc + scone scone scope scope + scopes scope scorch scorch + scorched scorch score score + scored score scores score + scoring score scorn scorn + scorned scorn scornful scorn + scornfully scornfulli scorning scorn + scorns scorn scorpion scorpion + scorpions scorpion scot scot + scotch scotch scotches scotch + scotland scotland scots scot + scottish scottish scoundrels scoundrel + scour scour scoured scour + scourg scourg scourge scourg + scouring scour scout scout + scouts scout scowl scowl + scrap scrap scrape scrape + scraping scrape scraps scrap + scratch scratch scratches scratch + scratching scratch scream scream + screams scream screech screech + screeching screech screen screen + screens screen screw screw + screws screw scribbl scribbl + scribbled scribbl scribe scribe + scribes scribe scrimers scrimer + scrip scrip scrippage scrippag + scripture scriptur scriptures scriptur + scrivener scriven scroll scroll + scrolls scroll scroop scroop + scrowl scrowl scroyles scroyl + scrubbed scrub scruple scrupl + scruples scrupl scrupulous scrupul + scuffles scuffl scuffling scuffl + scullion scullion sculls scull + scum scum scurril scurril + scurrility scurril scurrilous scurril + scurvy scurvi scuse scuse + scut scut scutcheon scutcheon + scutcheons scutcheon scylla scylla + scythe scyth scythed scyth + scythia scythia scythian scythian + sdeath sdeath se se + sea sea seacoal seacoal + seafaring seafar seal seal + sealed seal sealing seal + seals seal seam seam + seamen seamen seamy seami + seaport seaport sear sear + searce searc search search + searchers searcher searches search + searcheth searcheth searching search + seared sear seas sea + seasick seasick seaside seasid + season season seasoned season + seasons season seat seat + seated seat seats seat + sebastian sebastian second second + secondarily secondarili secondary secondari + seconded second seconds second + secrecy secreci secret secret + secretaries secretari secretary secretari + secretly secretli secrets secret + sect sect sectary sectari + sects sect secundo secundo + secure secur securely secur + securing secur security secur + sedg sedg sedge sedg + sedges sedg sedgy sedgi + sedition sedit seditious sediti + seduc seduc seduce seduc + seduced seduc seducer seduc + seducing seduc see see + seed seed seeded seed + seedness seed seeds seed + seedsman seedsman seein seein + seeing see seek seek + seeking seek seeks seek + seel seel seeling seel + seely seeli seem seem + seemed seem seemers seemer + seemest seemest seemeth seemeth + seeming seem seemingly seemingli + seemly seemli seems seem + seen seen seer seer + sees see seese sees + seest seest seethe seeth + seethes seeth seething seeth + seeting seet segregation segreg + seigneur seigneur seigneurs seigneur + seiz seiz seize seiz + seized seiz seizes seiz + seizeth seizeth seizing seiz + seizure seizur seld seld + seldom seldom select select + seleucus seleucu self self + selfsame selfsam sell sell + seller seller selling sell + sells sell selves selv + semblable semblabl semblably semblabl + semblance semblanc semblances semblanc + semblative sembl semi semi + semicircle semicircl semiramis semirami + semper semper sempronius semproniu + senate senat senator senat + senators senat send send + sender sender sendeth sendeth + sending send sends send + seneca seneca senior senior + seniory seniori senis seni + sennet sennet senoys senoi + sense sens senseless senseless + senses sens sensible sensibl + sensibly sensibl sensual sensual + sensuality sensual sent sent + sentenc sentenc sentence sentenc + sentences sentenc sententious sententi + sentinel sentinel sentinels sentinel + separable separ separate separ + separated separ separates separ + separation separ septentrion septentrion + sepulchre sepulchr sepulchres sepulchr + sepulchring sepulchr sequel sequel + sequence sequenc sequent sequent + sequest sequest sequester sequest + sequestration sequestr sere sere + serenis sereni serge serg + sergeant sergeant serious seriou + seriously serious sermon sermon + sermons sermon serpent serpent + serpentine serpentin serpents serpent + serpigo serpigo serv serv + servant servant servanted servant + servants servant serve serv + served serv server server + serves serv serveth serveth + service servic serviceable servic + services servic servile servil + servility servil servilius serviliu + serving serv servingman servingman + servingmen servingmen serviteur serviteur + servitor servitor servitors servitor + servitude servitud sessa sessa + session session sessions session + sestos sesto set set + setebos setebo sets set + setter setter setting set + settle settl settled settl + settlest settlest settling settl + sev sev seven seven + sevenfold sevenfold sevennight sevennight + seventeen seventeen seventh seventh + seventy seventi sever sever + several sever severally sever + severals sever severe sever + severed sever severely sever + severest severest severing sever + severity sever severn severn + severs sever sew sew + seward seward sewer sewer + sewing sew sex sex + sexes sex sexton sexton + sextus sextu seymour seymour + seyton seyton sfoot sfoot + sh sh shackle shackl + shackles shackl shade shade + shades shade shadow shadow + shadowed shadow shadowing shadow + shadows shadow shadowy shadowi + shady shadi shafalus shafalu + shaft shaft shafts shaft + shag shag shak shak + shake shake shaked shake + shaken shaken shakes shake + shaking shake shales shale + shall shall shallenge shalleng + shallow shallow shallowest shallowest + shallowly shallowli shallows shallow + shalt shalt sham sham + shambles shambl shame shame + shamed shame shameful shame + shamefully shamefulli shameless shameless + shames shame shamest shamest + shaming shame shank shank + shanks shank shap shap + shape shape shaped shape + shapeless shapeless shapen shapen + shapes shape shaping shape + shar shar shard shard + sharded shard shards shard + share share shared share + sharers sharer shares share + sharing share shark shark + sharp sharp sharpen sharpen + sharpened sharpen sharpens sharpen + sharper sharper sharpest sharpest + sharply sharpli sharpness sharp + sharps sharp shatter shatter + shav shav shave shave + shaven shaven shaw shaw + she she sheaf sheaf + sheal sheal shear shear + shearers shearer shearing shear + shearman shearman shears shear + sheath sheath sheathe sheath + sheathed sheath sheathes sheath + sheathing sheath sheaved sheav + sheaves sheav shed shed + shedding shed sheds shed + sheen sheen sheep sheep + sheepcote sheepcot sheepcotes sheepcot + sheeps sheep sheepskins sheepskin + sheer sheer sheet sheet + sheeted sheet sheets sheet + sheffield sheffield shelf shelf + shell shell shells shell + shelt shelt shelter shelter + shelters shelter shelves shelv + shelving shelv shelvy shelvi + shent shent shepherd shepherd + shepherdes shepherd shepherdess shepherdess + shepherdesses shepherdess shepherds shepherd + sher sher sheriff sheriff + sherris sherri shes she + sheweth sheweth shield shield + shielded shield shields shield + shift shift shifted shift + shifting shift shifts shift + shilling shill shillings shill + shin shin shine shine + shines shine shineth shineth + shining shine shins shin + shiny shini ship ship + shipboard shipboard shipman shipman + shipmaster shipmast shipmen shipmen + shipp shipp shipped ship + shipping ship ships ship + shipt shipt shipwreck shipwreck + shipwrecking shipwreck shipwright shipwright + shipwrights shipwright shire shire + shirley shirlei shirt shirt + shirts shirt shive shive + shiver shiver shivering shiver + shivers shiver shoal shoal + shoals shoal shock shock + shocks shock shod shod + shoe shoe shoeing shoe + shoemaker shoemak shoes shoe + shog shog shone shone + shook shook shoon shoon + shoot shoot shooter shooter + shootie shooti shooting shoot + shoots shoot shop shop + shops shop shore shore + shores shore shorn shorn + short short shortcake shortcak + shorten shorten shortened shorten + shortens shorten shorter shorter + shortly shortli shortness short + shot shot shotten shotten + shoughs shough should should + shoulder shoulder shouldering shoulder + shoulders shoulder shouldst shouldst + shout shout shouted shout + shouting shout shouts shout + shov shov shove shove + shovel shovel shovels shovel + show show showed show + shower shower showers shower + showest showest showing show + shown shown shows show + shreds shred shrew shrew + shrewd shrewd shrewdly shrewdli + shrewdness shrewd shrewish shrewish + shrewishly shrewishli shrewishness shrewish + shrews shrew shrewsbury shrewsburi + shriek shriek shrieking shriek + shrieks shriek shrieve shriev + shrift shrift shrill shrill + shriller shriller shrills shrill + shrilly shrilli shrimp shrimp + shrine shrine shrink shrink + shrinking shrink shrinks shrink + shriv shriv shrive shrive + shriver shriver shrives shrive + shriving shrive shroud shroud + shrouded shroud shrouding shroud + shrouds shroud shrove shrove + shrow shrow shrows shrow + shrub shrub shrubs shrub + shrug shrug shrugs shrug + shrunk shrunk shudd shudd + shudders shudder shuffl shuffl + shuffle shuffl shuffled shuffl + shuffling shuffl shun shun + shunless shunless shunn shunn + shunned shun shunning shun + shuns shun shut shut + shuts shut shuttle shuttl + shy shy shylock shylock + si si sibyl sibyl + sibylla sibylla sibyls sibyl + sicil sicil sicilia sicilia + sicilian sicilian sicilius siciliu + sicils sicil sicily sicili + sicinius siciniu sick sick + sicken sicken sickens sicken + sicker sicker sickle sickl + sicklemen sicklemen sicklied sickli + sickliness sickli sickly sickli + sickness sick sicles sicl + sicyon sicyon side side + sided side sides side + siege sieg sieges sieg + sienna sienna sies si + sieve siev sift sift + sifted sift sigeia sigeia + sigh sigh sighed sigh + sighing sigh sighs sigh + sight sight sighted sight + sightless sightless sightly sightli + sights sight sign sign + signal signal signet signet + signieur signieur significant signific + significants signific signified signifi + signifies signifi signify signifi + signifying signifi signior signior + signiories signiori signiors signior + signiory signiori signor signor + signories signori signs sign + signum signum silenc silenc + silence silenc silenced silenc + silencing silenc silent silent + silently silent silius siliu + silk silk silken silken + silkman silkman silks silk + silliest silliest silliness silli + silling sill silly silli + silva silva silver silver + silvered silver silverly silverli + silvia silvia silvius silviu + sima sima simile simil + similes simil simois simoi + simon simon simony simoni + simp simp simpcox simpcox + simple simpl simpleness simpl + simpler simpler simples simpl + simplicity simplic simply simpli + simular simular simulation simul + sin sin since sinc + sincere sincer sincerely sincer + sincerity sincer sinel sinel + sinew sinew sinewed sinew + sinews sinew sinewy sinewi + sinful sin sinfully sinfulli + sing sing singe sing + singeing sing singer singer + singes sing singeth singeth + singing sing single singl + singled singl singleness singl + singly singli sings sing + singular singular singulariter singularit + singularities singular singularity singular + singuled singul sinister sinist + sink sink sinking sink + sinks sink sinn sinn + sinner sinner sinners sinner + sinning sin sinon sinon + sins sin sip sip + sipping sip sir sir + sire sire siren siren + sirrah sirrah sirs sir + sist sist sister sister + sisterhood sisterhood sisterly sisterli + sisters sister sit sit + sith sith sithence sithenc + sits sit sitting sit + situate situat situation situat + situations situat siward siward + six six sixpence sixpenc + sixpences sixpenc sixpenny sixpenni + sixteen sixteen sixth sixth + sixty sixti siz siz + size size sizes size + sizzle sizzl skains skain + skamble skambl skein skein + skelter skelter skies ski + skilful skil skilfully skilfulli + skill skill skilless skilless + skillet skillet skillful skill + skills skill skim skim + skimble skimbl skin skin + skinker skinker skinny skinni + skins skin skip skip + skipp skipp skipper skipper + skipping skip skirmish skirmish + skirmishes skirmish skirr skirr + skirted skirt skirts skirt + skittish skittish skulking skulk + skull skull skulls skull + sky sky skyey skyei + skyish skyish slab slab + slack slack slackly slackli + slackness slack slain slain + slake slake sland sland + slander slander slandered slander + slanderer slander slanderers slander + slandering slander slanderous slander + slanders slander slash slash + slaught slaught slaughter slaughter + slaughtered slaughter slaughterer slaughter + slaughterman slaughterman slaughtermen slaughtermen + slaughterous slaughter slaughters slaughter + slave slave slaver slaver + slavery slaveri slaves slave + slavish slavish slay slai + slayeth slayeth slaying slai + slays slai sleave sleav + sledded sled sleek sleek + sleekly sleekli sleep sleep + sleeper sleeper sleepers sleeper + sleepest sleepest sleeping sleep + sleeps sleep sleepy sleepi + sleeve sleev sleeves sleev + sleid sleid sleided sleid + sleight sleight sleights sleight + slender slender slenderer slender + slenderly slenderli slept slept + slew slew slewest slewest + slice slice slid slid + slide slide slides slide + sliding slide slight slight + slighted slight slightest slightest + slightly slightli slightness slight + slights slight slily slili + slime slime slimy slimi + slings sling slink slink + slip slip slipp slipp + slipper slipper slippers slipper + slippery slipperi slips slip + slish slish slit slit + sliver sliver slobb slobb + slomber slomber slop slop + slope slope slops slop + sloth sloth slothful sloth + slough slough slovenly slovenli + slovenry slovenri slow slow + slower slower slowly slowli + slowness slow slubber slubber + slug slug sluggard sluggard + sluggardiz sluggardiz sluggish sluggish + sluic sluic slumb slumb + slumber slumber slumbers slumber + slumbery slumberi slunk slunk + slut slut sluts slut + sluttery slutteri sluttish sluttish + sluttishness sluttish sly sly + slys sly smack smack + smacking smack smacks smack + small small smaller smaller + smallest smallest smallness small + smalus smalu smart smart + smarting smart smartly smartli + smatch smatch smatter smatter + smear smear smell smell + smelling smell smells smell + smelt smelt smil smil + smile smile smiled smile + smiles smile smilest smilest + smilets smilet smiling smile + smilingly smilingli smirch smirch + smirched smirch smit smit + smite smite smites smite + smith smith smithfield smithfield + smock smock smocks smock + smok smok smoke smoke + smoked smoke smokes smoke + smoking smoke smoky smoki + smooth smooth smoothed smooth + smoothing smooth smoothly smoothli + smoothness smooth smooths smooth + smote smote smoth smoth + smother smother smothered smother + smothering smother smug smug + smulkin smulkin smutch smutch + snaffle snaffl snail snail + snails snail snake snake + snakes snake snaky snaki + snap snap snapp snapp + snapper snapper snar snar + snare snare snares snare + snarl snarl snarleth snarleth + snarling snarl snatch snatch + snatchers snatcher snatches snatch + snatching snatch sneak sneak + sneaking sneak sneap sneap + sneaping sneap sneck sneck + snip snip snipe snipe + snipt snipt snore snore + snores snore snoring snore + snorting snort snout snout + snow snow snowballs snowbal + snowed snow snowy snowi + snuff snuff snuffs snuff + snug snug so so + soak soak soaking soak + soaks soak soar soar + soaring soar soars soar + sob sob sobbing sob + sober sober soberly soberli + sobriety sobrieti sobs sob + sociable sociabl societies societi + society societi socks sock + socrates socrat sod sod + sodden sodden soe soe + soever soever soft soft + soften soften softens soften + softer softer softest softest + softly softli softness soft + soil soil soiled soil + soilure soilur soit soit + sojourn sojourn sol sol + sola sola solace solac + solanio solanio sold sold + soldat soldat solder solder + soldest soldest soldier soldier + soldiers soldier soldiership soldiership + sole sole solely sole + solem solem solemn solemn + solemness solem solemnities solemn + solemnity solemn solemniz solemniz + solemnize solemn solemnized solemn + solemnly solemnli soles sole + solicit solicit solicitation solicit + solicited solicit soliciting solicit + solicitings solicit solicitor solicitor + solicits solicit solid solid + solidares solidar solidity solid + solinus solinu solitary solitari + solomon solomon solon solon + solum solum solus solu + solyman solyman some some + somebody somebodi someone someon + somerset somerset somerville somervil + something someth sometime sometim + sometimes sometim somever somev + somewhat somewhat somewhere somewher + somewhither somewhith somme somm + son son sonance sonanc + song song songs song + sonnet sonnet sonneting sonnet + sonnets sonnet sons son + sont sont sonties sonti + soon soon sooner sooner + soonest soonest sooth sooth + soothe sooth soothers soother + soothing sooth soothsay soothsai + soothsayer soothsay sooty sooti + sop sop sophister sophist + sophisticated sophist sophy sophi + sops sop sorcerer sorcer + sorcerers sorcer sorceress sorceress + sorceries sorceri sorcery sorceri + sore sore sorel sorel + sorely sore sorer sorer + sores sore sorrier sorrier + sorriest sorriest sorrow sorrow + sorrowed sorrow sorrowest sorrowest + sorrowful sorrow sorrowing sorrow + sorrows sorrow sorry sorri + sort sort sortance sortanc + sorted sort sorting sort + sorts sort sossius sossiu + sot sot soto soto + sots sot sottish sottish + soud soud sought sought + soul soul sould sould + soulless soulless souls soul + sound sound sounded sound + sounder sounder soundest soundest + sounding sound soundless soundless + soundly soundli soundness sound + soundpost soundpost sounds sound + sour sour source sourc + sources sourc sourest sourest + sourly sourli sours sour + sous sou souse sous + south south southam southam + southampton southampton southerly southerli + southern southern southward southward + southwark southwark southwell southwel + souviendrai souviendrai sov sov + sovereign sovereign sovereignest sovereignest + sovereignly sovereignli sovereignty sovereignti + sovereignvours sovereignvour sow sow + sowing sow sowl sowl + sowter sowter space space + spaces space spacious spaciou + spade spade spades spade + spain spain spak spak + spake spake spakest spakest + span span spangle spangl + spangled spangl spaniard spaniard + spaniel spaniel spaniels spaniel + spanish spanish spann spann + spans span spar spar + spare spare spares spare + sparing spare sparingly sparingli + spark spark sparkle sparkl + sparkles sparkl sparkling sparkl + sparks spark sparrow sparrow + sparrows sparrow sparta sparta + spartan spartan spavin spavin + spavins spavin spawn spawn + speak speak speaker speaker + speakers speaker speakest speakest + speaketh speaketh speaking speak + speaks speak spear spear + speargrass speargrass spears spear + special special specialities special + specially special specialties specialti + specialty specialti specify specifi + speciously specious spectacle spectacl + spectacled spectacl spectacles spectacl + spectators spectat spectatorship spectatorship + speculation specul speculations specul + speculative specul sped sped + speech speech speeches speech + speechless speechless speed speed + speeded speed speedier speedier + speediest speediest speedily speedili + speediness speedi speeding speed + speeds speed speedy speedi + speens speen spell spell + spelling spell spells spell + spelt spelt spencer spencer + spend spend spendest spendest + spending spend spends spend + spendthrift spendthrift spent spent + sperato sperato sperm sperm + spero spero sperr sperr + spher spher sphere sphere + sphered sphere spheres sphere + spherical spheric sphery spheri + sphinx sphinx spice spice + spiced spice spicery spiceri + spices spice spider spider + spiders spider spied spi + spies spi spieth spieth + spightfully spightfulli spigot spigot + spill spill spilling spill + spills spill spilt spilt + spilth spilth spin spin + spinii spinii spinners spinner + spinster spinster spinsters spinster + spire spire spirit spirit + spirited spirit spiritless spiritless + spirits spirit spiritual spiritu + spiritualty spiritualti spirt spirt + spit spit spital spital + spite spite spited spite + spiteful spite spites spite + spits spit spitted spit + spitting spit splay splai + spleen spleen spleenful spleen + spleens spleen spleeny spleeni + splendour splendour splenitive splenit + splinter splinter splinters splinter + split split splits split + splitted split splitting split + spoil spoil spoils spoil + spok spok spoke spoke + spoken spoken spokes spoke + spokesman spokesman sponge spong + spongy spongi spoon spoon + spoons spoon sport sport + sportful sport sporting sport + sportive sportiv sports sport + spot spot spotless spotless + spots spot spotted spot + spousal spousal spouse spous + spout spout spouting spout + spouts spout sprag sprag + sprang sprang sprat sprat + sprawl sprawl spray sprai + sprays sprai spread spread + spreading spread spreads spread + sprighted spright sprightful spright + sprightly sprightli sprigs sprig + spring spring springe spring + springes spring springeth springeth + springhalt springhalt springing spring + springs spring springtime springtim + sprinkle sprinkl sprinkles sprinkl + sprite sprite sprited sprite + spritely sprite sprites sprite + spriting sprite sprout sprout + spruce spruce sprung sprung + spun spun spur spur + spurio spurio spurn spurn + spurns spurn spurr spurr + spurrer spurrer spurring spur + spurs spur spy spy + spying spy squabble squabbl + squadron squadron squadrons squadron + squand squand squar squar + square squar squarer squarer + squares squar squash squash + squeak squeak squeaking squeak + squeal squeal squealing squeal + squeezes squeez squeezing squeez + squele squel squier squier + squints squint squiny squini + squire squir squires squir + squirrel squirrel st st + stab stab stabb stabb + stabbed stab stabbing stab + stable stabl stableness stabl + stables stabl stablish stablish + stablishment stablish stabs stab + stacks stack staff staff + stafford stafford staffords stafford + staffordshire staffordshir stag stag + stage stage stages stage + stagger stagger staggering stagger + staggers stagger stags stag + staid staid staider staider + stain stain stained stain + staines stain staineth staineth + staining stain stainless stainless + stains stain stair stair + stairs stair stake stake + stakes stake stale stale + staled stale stalk stalk + stalking stalk stalks stalk + stall stall stalling stall + stalls stall stamford stamford + stammer stammer stamp stamp + stamped stamp stamps stamp + stanch stanch stanchless stanchless + stand stand standard standard + standards standard stander stander + standers stander standest standest + standeth standeth standing stand + stands stand staniel staniel + stanley stanlei stanze stanz + stanzo stanzo stanzos stanzo + staple stapl staples stapl + star star stare stare + stared stare stares stare + staring stare starings stare + stark stark starkly starkli + starlight starlight starling starl + starr starr starry starri + stars star start start + started start starting start + startingly startingli startle startl + startles startl starts start + starv starv starve starv + starved starv starvelackey starvelackei + starveling starvel starveth starveth + starving starv state state + statelier stateli stately state + states state statesman statesman + statesmen statesmen statilius statiliu + station station statist statist + statists statist statue statu + statues statu stature statur + statures statur statute statut + statutes statut stave stave + staves stave stay stai + stayed stai stayest stayest + staying stai stays stai + stead stead steaded stead + steadfast steadfast steadier steadier + steads stead steal steal + stealer stealer stealers stealer + stealing steal steals steal + stealth stealth stealthy stealthi + steed steed steeds steed + steel steel steeled steel + steely steeli steep steep + steeped steep steeple steepl + steeples steepl steeps steep + steepy steepi steer steer + steerage steerag steering steer + steers steer stelled stell + stem stem stemming stem + stench stench step step + stepdame stepdam stephano stephano + stephen stephen stepmothers stepmoth + stepp stepp stepping step + steps step sterile steril + sterility steril sterling sterl + stern stern sternage sternag + sterner sterner sternest sternest + sternness stern steterat steterat + stew stew steward steward + stewards steward stewardship stewardship + stewed stew stews stew + stick stick sticking stick + stickler stickler sticks stick + stiff stiff stiffen stiffen + stiffly stiffli stifle stifl + stifled stifl stifles stifl + stigmatic stigmat stigmatical stigmat + stile stile still still + stiller stiller stillest stillest + stillness still stilly stilli + sting sting stinging sting + stingless stingless stings sting + stink stink stinking stink + stinkingly stinkingli stinks stink + stint stint stinted stint + stints stint stir stir + stirr stirr stirred stir + stirrer stirrer stirrers stirrer + stirreth stirreth stirring stir + stirrup stirrup stirrups stirrup + stirs stir stitchery stitcheri + stitches stitch stithied stithi + stithy stithi stoccadoes stoccado + stoccata stoccata stock stock + stockfish stockfish stocking stock + stockings stock stockish stockish + stocks stock stog stog + stogs stog stoics stoic + stokesly stokesli stol stol + stole stole stolen stolen + stolest stolest stomach stomach + stomachers stomach stomaching stomach + stomachs stomach ston ston + stone stone stonecutter stonecutt + stones stone stonish stonish + stony stoni stood stood + stool stool stools stool + stoop stoop stooping stoop + stoops stoop stop stop + stope stope stopp stopp + stopped stop stopping stop + stops stop stor stor + store store storehouse storehous + storehouses storehous stores store + stories stori storm storm + stormed storm storming storm + storms storm stormy stormi + story stori stoup stoup + stoups stoup stout stout + stouter stouter stoutly stoutli + stoutness stout stover stover + stow stow stowage stowag + stowed stow strachy strachi + stragglers straggler straggling straggl + straight straight straightest straightest + straightway straightwai strain strain + strained strain straining strain + strains strain strait strait + straited strait straiter straiter + straitly straitli straitness strait + straits strait strand strand + strange strang strangely strang + strangeness strang stranger stranger + strangers stranger strangest strangest + strangle strangl strangled strangl + strangler strangler strangles strangl + strangling strangl strappado strappado + straps strap stratagem stratagem + stratagems stratagem stratford stratford + strato strato straw straw + strawberries strawberri strawberry strawberri + straws straw strawy strawi + stray strai straying strai + strays strai streak streak + streaks streak stream stream + streamers streamer streaming stream + streams stream streching strech + street street streets street + strength strength strengthen strengthen + strengthened strengthen strengthless strengthless + strengths strength stretch stretch + stretched stretch stretches stretch + stretching stretch strew strew + strewing strew strewings strew + strewments strewment stricken stricken + strict strict stricter stricter + strictest strictest strictly strictli + stricture strictur stride stride + strides stride striding stride + strife strife strifes strife + strik strik strike strike + strikers striker strikes strike + strikest strikest striking strike + string string stringless stringless + strings string strip strip + stripes stripe stripling stripl + striplings stripl stripp stripp + stripping strip striv striv + strive strive strives strive + striving strive strok strok + stroke stroke strokes stroke + strond strond stronds strond + strong strong stronger stronger + strongest strongest strongly strongli + strooke strook strossers strosser + strove strove strown strown + stroy stroi struck struck + strucken strucken struggle struggl + struggles struggl struggling struggl + strumpet strumpet strumpeted strumpet + strumpets strumpet strung strung + strut strut struts strut + strutted strut strutting strut + stubble stubbl stubborn stubborn + stubbornest stubbornest stubbornly stubbornli + stubbornness stubborn stuck stuck + studded stud student student + students student studied studi + studies studi studious studiou + studiously studious studs stud + study studi studying studi + stuff stuff stuffing stuf + stuffs stuff stumble stumbl + stumbled stumbl stumblest stumblest + stumbling stumbl stump stump + stumps stump stung stung + stupefy stupefi stupid stupid + stupified stupifi stuprum stuprum + sturdy sturdi sty sty + styga styga stygian stygian + styl styl style style + styx styx su su + sub sub subcontracted subcontract + subdu subdu subdue subdu + subdued subdu subduements subduement + subdues subdu subduing subdu + subject subject subjected subject + subjection subject subjects subject + submerg submerg submission submiss + submissive submiss submit submit + submits submit submitting submit + suborn suborn subornation suborn + suborned suborn subscrib subscrib + subscribe subscrib subscribed subscrib + subscribes subscrib subscription subscript + subsequent subsequ subsidies subsidi + subsidy subsidi subsist subsist + subsisting subsist substance substanc + substances substanc substantial substanti + substitute substitut substituted substitut + substitutes substitut substitution substitut + subtile subtil subtilly subtilli + subtle subtl subtleties subtleti + subtlety subtleti subtly subtli + subtractors subtractor suburbs suburb + subversion subvers subverts subvert + succedant succed succeed succe + succeeded succeed succeeders succeed + succeeding succeed succeeds succe + success success successantly successantli + successes success successful success + successfully successfulli succession success + successive success successively success + successor successor successors successor + succour succour succours succour + such such suck suck + sucker sucker suckers sucker + sucking suck suckle suckl + sucks suck sudden sudden + suddenly suddenli sue sue + sued su suerly suerli + sues sue sueth sueth + suff suff suffer suffer + sufferance suffer sufferances suffer + suffered suffer suffering suffer + suffers suffer suffic suffic + suffice suffic sufficed suffic + suffices suffic sufficeth sufficeth + sufficiency suffici sufficient suffici + sufficiently suffici sufficing suffic + sufficit sufficit suffigance suffig + suffocate suffoc suffocating suffoc + suffocation suffoc suffolk suffolk + suffrage suffrag suffrages suffrag + sug sug sugar sugar + sugarsop sugarsop suggest suggest + suggested suggest suggesting suggest + suggestion suggest suggestions suggest + suggests suggest suis sui + suit suit suitable suitabl + suited suit suiting suit + suitor suitor suitors suitor + suits suit suivez suivez + sullen sullen sullens sullen + sullied sulli sullies sulli + sully sulli sulph sulph + sulpherous sulpher sulphur sulphur + sulphurous sulphur sultan sultan + sultry sultri sum sum + sumless sumless summ summ + summa summa summary summari + summer summer summers summer + summit summit summon summon + summoners summon summons summon + sumpter sumpter sumptuous sumptuou + sumptuously sumptuous sums sum + sun sun sunbeams sunbeam + sunburning sunburn sunburnt sunburnt + sund sund sunday sundai + sundays sundai sunder sunder + sunders sunder sundry sundri + sung sung sunk sunk + sunken sunken sunny sunni + sunrising sunris suns sun + sunset sunset sunshine sunshin + sup sup super super + superficial superfici superficially superfici + superfluity superflu superfluous superflu + superfluously superflu superflux superflux + superior superior supernal supern + supernatural supernatur superpraise superprais + superscript superscript superscription superscript + superserviceable superservic superstition superstit + superstitious superstiti superstitiously superstiti + supersubtle supersubtl supervise supervis + supervisor supervisor supp supp + supper supper suppers supper + suppertime suppertim supping sup + supplant supplant supple suppl + suppler suppler suppliance supplianc + suppliant suppliant suppliants suppliant + supplicant supplic supplication supplic + supplications supplic supplie suppli + supplied suppli supplies suppli + suppliest suppliest supply suppli + supplyant supplyant supplying suppli + supplyment supplyment support support + supportable support supportance support + supported support supporter support + supporters support supporting support + supportor supportor suppos suppo + supposal suppos suppose suppos + supposed suppos supposes suppos + supposest supposest supposing suppos + supposition supposit suppress suppress + suppressed suppress suppresseth suppresseth + supremacy supremaci supreme suprem + sups sup sur sur + surance suranc surcease surceas + surd surd sure sure + surecard surecard surely sure + surer surer surest surest + sureties sureti surety sureti + surfeit surfeit surfeited surfeit + surfeiter surfeit surfeiting surfeit + surfeits surfeit surge surg + surgeon surgeon surgeons surgeon + surgere surger surgery surgeri + surges surg surly surli + surmis surmi surmise surmis + surmised surmis surmises surmis + surmount surmount surmounted surmount + surmounts surmount surnam surnam + surname surnam surnamed surnam + surpasseth surpasseth surpassing surpass + surplice surplic surplus surplu + surpris surpri surprise surpris + surprised surpris surrender surrend + surrey surrei surreys surrei + survey survei surveyest surveyest + surveying survei surveyor surveyor + surveyors surveyor surveys survei + survive surviv survives surviv + survivor survivor susan susan + suspect suspect suspected suspect + suspecting suspect suspects suspect + suspend suspend suspense suspens + suspicion suspicion suspicions suspicion + suspicious suspici suspiration suspir + suspire suspir sust sust + sustain sustain sustaining sustain + sutler sutler sutton sutton + suum suum swabber swabber + swaddling swaddl swag swag + swagg swagg swagger swagger + swaggerer swagger swaggerers swagger + swaggering swagger swain swain + swains swain swallow swallow + swallowed swallow swallowing swallow + swallows swallow swam swam + swan swan swans swan + sward sward sware sware + swarm swarm swarming swarm + swart swart swarth swarth + swarths swarth swarthy swarthi + swashers swasher swashing swash + swath swath swathing swath + swathling swathl sway swai + swaying swai sways swai + swear swear swearer swearer + swearers swearer swearest swearest + swearing swear swearings swear + swears swear sweat sweat + sweaten sweaten sweating sweat + sweats sweat sweaty sweati + sweep sweep sweepers sweeper + sweeps sweep sweet sweet + sweeten sweeten sweetens sweeten + sweeter sweeter sweetest sweetest + sweetheart sweetheart sweeting sweet + sweetly sweetli sweetmeats sweetmeat + sweetness sweet sweets sweet + swell swell swelling swell + swellings swell swells swell + swelter swelter sweno sweno + swept swept swerve swerv + swerver swerver swerving swerv + swift swift swifter swifter + swiftest swiftest swiftly swiftli + swiftness swift swill swill + swills swill swim swim + swimmer swimmer swimmers swimmer + swimming swim swims swim + swine swine swineherds swineherd + swing swing swinge swing + swinish swinish swinstead swinstead + switches switch swits swit + switzers switzer swol swol + swoll swoll swoln swoln + swoon swoon swooned swoon + swooning swoon swoons swoon + swoop swoop swoopstake swoopstak + swor swor sword sword + sworder sworder swords sword + swore swore sworn sworn + swounded swound swounds swound + swum swum swung swung + sy sy sycamore sycamor + sycorax sycorax sylla sylla + syllable syllabl syllables syllabl + syllogism syllog symbols symbol + sympathise sympathis sympathiz sympathiz + sympathize sympath sympathized sympath + sympathy sympathi synagogue synagogu + synod synod synods synod + syracuse syracus syracusian syracusian + syracusians syracusian syria syria + syrups syrup t t + ta ta taber taber + table tabl tabled tabl + tables tabl tablet tablet + tabor tabor taborer tabor + tabors tabor tabourines tabourin + taciturnity taciturn tack tack + tackle tackl tackled tackl + tackles tackl tackling tackl + tacklings tackl taddle taddl + tadpole tadpol taffeta taffeta + taffety taffeti tag tag + tagrag tagrag tah tah + tail tail tailor tailor + tailors tailor tails tail + taint taint tainted taint + tainting taint taints taint + tainture taintur tak tak + take take taken taken + taker taker takes take + takest takest taketh taketh + taking take tal tal + talbot talbot talbotites talbotit + talbots talbot tale tale + talent talent talents talent + taleporter taleport tales tale + talk talk talked talk + talker talker talkers talker + talkest talkest talking talk + talks talk tall tall + taller taller tallest tallest + tallies talli tallow tallow + tally talli talons talon + tam tam tambourines tambourin + tame tame tamed tame + tamely tame tameness tame + tamer tamer tames tame + taming tame tamora tamora + tamworth tamworth tan tan + tang tang tangle tangl + tangled tangl tank tank + tanlings tanl tann tann + tanned tan tanner tanner + tanquam tanquam tanta tanta + tantaene tantaen tap tap + tape tape taper taper + tapers taper tapestries tapestri + tapestry tapestri taphouse taphous + tapp tapp tapster tapster + tapsters tapster tar tar + tardied tardi tardily tardili + tardiness tardi tardy tardi + tarentum tarentum targe targ + targes targ target target + targets target tarpeian tarpeian + tarquin tarquin tarquins tarquin + tarr tarr tarre tarr + tarriance tarrianc tarried tarri + tarries tarri tarry tarri + tarrying tarri tart tart + tartar tartar tartars tartar + tartly tartli tartness tart + task task tasker tasker + tasking task tasks task + tassel tassel taste tast + tasted tast tastes tast + tasting tast tatt tatt + tatter tatter tattered tatter + tatters tatter tattle tattl + tattling tattl tattlings tattl + taught taught taunt taunt + taunted taunt taunting taunt + tauntingly tauntingli taunts taunt + taurus tauru tavern tavern + taverns tavern tavy tavi + tawdry tawdri tawny tawni + tax tax taxation taxat + taxations taxat taxes tax + taxing tax tc tc + te te teach teach + teacher teacher teachers teacher + teaches teach teachest teachest + teacheth teacheth teaching teach + team team tear tear + tearful tear tearing tear + tears tear tearsheet tearsheet + teat teat tedious tediou + tediously tedious tediousness tedious + teem teem teeming teem + teems teem teen teen + teeth teeth teipsum teipsum + telamon telamon telamonius telamoniu + tell tell teller teller + telling tell tells tell + tellus tellu temp temp + temper temper temperality temper + temperance temper temperate temper + temperately temper tempers temper + tempest tempest tempests tempest + tempestuous tempestu temple templ + temples templ temporal tempor + temporary temporari temporiz temporiz + temporize tempor temporizer tempor + temps temp tempt tempt + temptation temptat temptations temptat + tempted tempt tempter tempter + tempters tempter tempteth tempteth + tempting tempt tempts tempt + ten ten tenable tenabl + tenant tenant tenantius tenantiu + tenantless tenantless tenants tenant + tench tench tend tend + tendance tendanc tended tend + tender tender tendered tender + tenderly tenderli tenderness tender + tenders tender tending tend + tends tend tenedos tenedo + tenement tenement tenements tenement + tenfold tenfold tennis tenni + tenour tenour tenours tenour + tens ten tent tent + tented tent tenth tenth + tenths tenth tents tent + tenure tenur tenures tenur + tercel tercel tereus tereu + term term termagant termag + termed term terminations termin + termless termless terms term + terra terra terrace terrac + terram terram terras terra + terre terr terrene terren + terrestrial terrestri terrible terribl + terribly terribl territories territori + territory territori terror terror + terrors terror tertian tertian + tertio tertio test test + testament testament tested test + tester tester testern testern + testify testifi testimonied testimoni + testimonies testimoni testimony testimoni + testiness testi testril testril + testy testi tetchy tetchi + tether tether tetter tetter + tevil tevil tewksbury tewksburi + text text tgv tgv + th th thaes thae + thames thame than than + thane thane thanes thane + thank thank thanked thank + thankful thank thankfully thankfulli + thankfulness thank thanking thank + thankings thank thankless thankless + thanks thank thanksgiving thanksgiv + thasos thaso that that + thatch thatch thaw thaw + thawing thaw thaws thaw + the the theatre theatr + theban theban thebes thebe + thee thee theft theft + thefts theft thein thein + their their theirs their + theise theis them them + theme theme themes theme + themselves themselv then then + thence thenc thenceforth thenceforth + theoric theoric there there + thereabout thereabout thereabouts thereabout + thereafter thereaft thereat thereat + thereby therebi therefore therefor + therein therein thereof thereof + thereon thereon thereto thereto + thereunto thereunto thereupon thereupon + therewith therewith therewithal therewith + thersites thersit these these + theseus theseu thessalian thessalian + thessaly thessali thetis theti + thews thew they thei + thick thick thicken thicken + thickens thicken thicker thicker + thickest thickest thicket thicket + thickskin thickskin thief thief + thievery thieveri thieves thiev + thievish thievish thigh thigh + thighs thigh thimble thimbl + thimbles thimbl thin thin + thine thine thing thing + things thing think think + thinkest thinkest thinking think + thinkings think thinks think + thinkst thinkst thinly thinli + third third thirdly thirdli + thirds third thirst thirst + thirsting thirst thirsts thirst + thirsty thirsti thirteen thirteen + thirties thirti thirtieth thirtieth + thirty thirti this thi + thisby thisbi thisne thisn + thistle thistl thistles thistl + thither thither thitherward thitherward + thoas thoa thomas thoma + thorn thorn thorns thorn + thorny thorni thorough thorough + thoroughly thoroughli those those + thou thou though though + thought thought thoughtful thought + thoughts thought thousand thousand + thousands thousand thracian thracian + thraldom thraldom thrall thrall + thralled thrall thralls thrall + thrash thrash thrasonical thrason + thread thread threadbare threadbar + threaden threaden threading thread + threat threat threaten threaten + threatening threaten threatens threaten + threatest threatest threats threat + three three threefold threefold + threepence threepenc threepile threepil + threes three threescore threescor + thresher thresher threshold threshold + threw threw thrice thrice + thrift thrift thriftless thriftless + thrifts thrift thrifty thrifti + thrill thrill thrilling thrill + thrills thrill thrive thrive + thrived thrive thrivers thriver + thrives thrive thriving thrive + throat throat throats throat + throbbing throb throbs throb + throca throca throe throe + throes throe thromuldo thromuldo + thron thron throne throne + throned throne thrones throne + throng throng thronging throng + throngs throng throstle throstl + throttle throttl through through + throughfare throughfar throughfares throughfar + throughly throughli throughout throughout + throw throw thrower thrower + throwest throwest throwing throw + thrown thrown throws throw + thrum thrum thrumm thrumm + thrush thrush thrust thrust + thrusteth thrusteth thrusting thrust + thrusts thrust thumb thumb + thumbs thumb thump thump + thund thund thunder thunder + thunderbolt thunderbolt thunderbolts thunderbolt + thunderer thunder thunders thunder + thunderstone thunderston thunderstroke thunderstrok + thurio thurio thursday thursdai + thus thu thwack thwack + thwart thwart thwarted thwart + thwarting thwart thwartings thwart + thy thy thyme thyme + thymus thymu thyreus thyreu + thyself thyself ti ti + tib tib tiber tiber + tiberio tiberio tibey tibei + ticed tice tick tick + tickl tickl tickle tickl + tickled tickl tickles tickl + tickling tickl ticklish ticklish + tiddle tiddl tide tide + tides tide tidings tide + tidy tidi tie tie + tied ti ties ti + tiff tiff tiger tiger + tigers tiger tight tight + tightly tightli tike tike + til til tile tile + till till tillage tillag + tilly tilli tilt tilt + tilter tilter tilth tilth + tilting tilt tilts tilt + tiltyard tiltyard tim tim + timandra timandra timber timber + time time timeless timeless + timelier timeli timely time + times time timon timon + timor timor timorous timor + timorously timor tinct tinct + tincture tinctur tinctures tinctur + tinder tinder tingling tingl + tinker tinker tinkers tinker + tinsel tinsel tiny tini + tip tip tipp tipp + tippling tippl tips tip + tipsy tipsi tiptoe tipto + tir tir tire tire + tired tire tires tire + tirest tirest tiring tire + tirra tirra tirrits tirrit + tis ti tish tish + tisick tisick tissue tissu + titan titan titania titania + tithe tith tithed tith + tithing tith titinius titiniu + title titl titled titl + titleless titleless titles titl + tittle tittl tittles tittl + titular titular titus titu + tn tn to to + toad toad toads toad + toadstool toadstool toast toast + toasted toast toasting toast + toasts toast toaze toaz + toby tobi tock tock + tod tod today todai + todpole todpol tods tod + toe toe toes toe + tofore tofor toge toge + toged toge together togeth + toil toil toiled toil + toiling toil toils toil + token token tokens token + told told toledo toledo + tolerable toler toll toll + tolling toll tom tom + tomb tomb tombe tomb + tombed tomb tombless tombless + tomboys tomboi tombs tomb + tomorrow tomorrow tomyris tomyri + ton ton tongs tong + tongu tongu tongue tongu + tongued tongu tongueless tongueless + tongues tongu tonight tonight + too too took took + tool tool tools tool + tooth tooth toothache toothach + toothpick toothpick toothpicker toothpick + top top topas topa + topful top topgallant topgal + topless topless topmast topmast + topp topp topping top + topple toppl topples toppl + tops top topsail topsail + topsy topsi torch torch + torchbearer torchbear torchbearers torchbear + torcher torcher torches torch + torchlight torchlight tore tore + torment torment tormenta tormenta + tormente torment tormented torment + tormenting torment tormentors tormentor + torments torment torn torn + torrent torrent tortive tortiv + tortoise tortois tortur tortur + torture tortur tortured tortur + torturer tortur torturers tortur + tortures tortur torturest torturest + torturing tortur toryne toryn + toss toss tossed toss + tosseth tosseth tossing toss + tot tot total total + totally total tott tott + tottered totter totters totter + tou tou touch touch + touched touch touches touch + toucheth toucheth touching touch + touchstone touchston tough tough + tougher tougher toughness tough + touraine tourain tournaments tournament + tours tour tous tou + tout tout touze touz + tow tow toward toward + towardly towardli towards toward + tower tower towering tower + towers tower town town + towns town township township + townsman townsman townsmen townsmen + towton towton toy toi + toys toi trace trace + traces trace track track + tract tract tractable tractabl + trade trade traded trade + traders trader trades trade + tradesman tradesman tradesmen tradesmen + trading trade tradition tradit + traditional tradit traduc traduc + traduced traduc traducement traduc + traffic traffic traffickers traffick + traffics traffic tragedian tragedian + tragedians tragedian tragedies tragedi + tragedy tragedi tragic tragic + tragical tragic trail trail + train train trained train + training train trains train + trait trait traitor traitor + traitorly traitorli traitorous traitor + traitorously traitor traitors traitor + traitress traitress traject traject + trammel trammel trample trampl + trampled trampl trampling trampl + tranc tranc trance tranc + tranio tranio tranquil tranquil + tranquillity tranquil transcendence transcend + transcends transcend transferred transfer + transfigur transfigur transfix transfix + transform transform transformation transform + transformations transform transformed transform + transgress transgress transgresses transgress + transgressing transgress transgression transgress + translate translat translated translat + translates translat translation translat + transmigrates transmigr transmutation transmut + transparent transpar transport transport + transportance transport transported transport + transporting transport transports transport + transpose transpos transshape transshap + trap trap trapp trapp + trappings trap traps trap + trash trash travail travail + travails travail travel travel + traveler travel traveling travel + travell travel travelled travel + traveller travel travellers travel + travellest travellest travelling travel + travels travel travers traver + traverse travers tray trai + treacherous treacher treacherously treacher + treachers treacher treachery treacheri + tread tread treading tread + treads tread treason treason + treasonable treason treasonous treason + treasons treason treasure treasur + treasurer treasur treasures treasur + treasuries treasuri treasury treasuri + treat treat treaties treati + treatise treatis treats treat + treaty treati treble trebl + trebled trebl trebles trebl + trebonius treboniu tree tree + trees tree tremble trembl + trembled trembl trembles trembl + tremblest tremblest trembling trembl + tremblingly tremblingli tremor tremor + trempling trempl trench trench + trenchant trenchant trenched trench + trencher trencher trenchering trencher + trencherman trencherman trenchers trencher + trenches trench trenching trench + trent trent tres tre + trespass trespass trespasses trespass + tressel tressel tresses tress + treys trei trial trial + trials trial trib trib + tribe tribe tribes tribe + tribulation tribul tribunal tribun + tribune tribun tribunes tribun + tributaries tributari tributary tributari + tribute tribut tributes tribut + trice trice trick trick + tricking trick trickling trickl + tricks trick tricksy tricksi + trident trident tried tri + trier trier trifle trifl + trifled trifl trifler trifler + trifles trifl trifling trifl + trigon trigon trill trill + trim trim trimly trimli + trimm trimm trimmed trim + trimming trim trims trim + trinculo trinculo trinculos trinculo + trinkets trinket trip trip + tripartite tripartit tripe tripe + triple tripl triplex triplex + tripoli tripoli tripolis tripoli + tripp tripp tripping trip + trippingly trippingli trips trip + tristful trist triton triton + triumph triumph triumphant triumphant + triumphantly triumphantli triumpher triumpher + triumphers triumpher triumphing triumph + triumphs triumph triumvir triumvir + triumvirate triumvir triumvirs triumvir + triumviry triumviri trivial trivial + troat troat trod trod + trodden trodden troiant troiant + troien troien troilus troilu + troiluses troilus trojan trojan + trojans trojan troll troll + tromperies tromperi trompet trompet + troop troop trooping troop + troops troop trop trop + trophies trophi trophy trophi + tropically tropic trot trot + troth troth trothed troth + troths troth trots trot + trotting trot trouble troubl + troubled troubl troubler troubler + troubles troubl troublesome troublesom + troublest troublest troublous troublou + trough trough trout trout + trouts trout trovato trovato + trow trow trowel trowel + trowest trowest troy troi + troyan troyan troyans troyan + truant truant truce truce + truckle truckl trudge trudg + true true trueborn trueborn + truepenny truepenni truer truer + truest truest truie truie + trull trull trulls trull + truly truli trump trump + trumpery trumperi trumpet trumpet + trumpeter trumpet trumpeters trumpet + trumpets trumpet truncheon truncheon + truncheoners truncheon trundle trundl + trunk trunk trunks trunk + trust trust trusted trust + truster truster trusters truster + trusting trust trusts trust + trusty trusti truth truth + truths truth try try + ts ts tu tu + tuae tuae tub tub + tubal tubal tubs tub + tuck tuck tucket tucket + tuesday tuesdai tuft tuft + tufts tuft tug tug + tugg tugg tugging tug + tuition tuition tullus tullu + tully tulli tumble tumbl + tumbled tumbl tumbler tumbler + tumbling tumbl tumult tumult + tumultuous tumultu tun tun + tune tune tuneable tuneabl + tuned tune tuners tuner + tunes tune tunis tuni + tuns tun tupping tup + turban turban turbans turban + turbulence turbul turbulent turbul + turd turd turf turf + turfy turfi turk turk + turkey turkei turkeys turkei + turkish turkish turks turk + turlygod turlygod turmoil turmoil + turmoiled turmoil turn turn + turnbull turnbul turncoat turncoat + turncoats turncoat turned turn + turneth turneth turning turn + turnips turnip turns turn + turph turph turpitude turpitud + turquoise turquois turret turret + turrets turret turtle turtl + turtles turtl turvy turvi + tuscan tuscan tush tush + tut tut tutor tutor + tutored tutor tutors tutor + tutto tutto twain twain + twang twang twangling twangl + twas twa tway twai + tweaks tweak tween tween + twelfth twelfth twelve twelv + twelvemonth twelvemonth twentieth twentieth + twenty twenti twere twere + twice twice twig twig + twiggen twiggen twigs twig + twilight twilight twill twill + twilled twill twin twin + twine twine twink twink + twinkle twinkl twinkled twinkl + twinkling twinkl twinn twinn + twins twin twire twire + twist twist twisted twist + twit twit twits twit + twitting twit twixt twixt + two two twofold twofold + twopence twopenc twopences twopenc + twos two twould twould + tyb tyb tybalt tybalt + tybalts tybalt tyburn tyburn + tying ty tyke tyke + tymbria tymbria type type + types type typhon typhon + tyrannical tyrann tyrannically tyrann + tyrannize tyrann tyrannous tyrann + tyranny tyranni tyrant tyrant + tyrants tyrant tyrian tyrian + tyrrel tyrrel u u + ubique ubiqu udders udder + udge udg uds ud + uglier uglier ugliest ugliest + ugly ugli ulcer ulcer + ulcerous ulcer ulysses ulyss + um um umber umber + umbra umbra umbrage umbrag + umfrevile umfrevil umpire umpir + umpires umpir un un + unable unabl unaccommodated unaccommod + unaccompanied unaccompani unaccustom unaccustom + unaching unach unacquainted unacquaint + unactive unact unadvis unadvi + unadvised unadvis unadvisedly unadvisedli + unagreeable unagre unanel unanel + unanswer unansw unappeas unappea + unapproved unapprov unapt unapt + unaptness unapt unarm unarm + unarmed unarm unarms unarm + unassail unassail unassailable unassail + unattainted unattaint unattempted unattempt + unattended unattend unauspicious unauspici + unauthorized unauthor unavoided unavoid + unawares unawar unback unback + unbak unbak unbanded unband + unbar unbar unbarb unbarb + unbashful unbash unbated unbat + unbatter unbatt unbecoming unbecom + unbefitting unbefit unbegot unbegot + unbegotten unbegotten unbelieved unbeliev + unbend unbend unbent unbent + unbewail unbewail unbid unbid + unbidden unbidden unbind unbind + unbinds unbind unbitted unbit + unbless unbless unblest unblest + unbloodied unbloodi unblown unblown + unbodied unbodi unbolt unbolt + unbolted unbolt unbonneted unbonnet + unbookish unbookish unborn unborn + unbosom unbosom unbound unbound + unbounded unbound unbow unbow + unbowed unbow unbrac unbrac + unbraced unbrac unbraided unbraid + unbreathed unbreath unbred unbr + unbreech unbreech unbridled unbridl + unbroke unbrok unbruis unbrui + unbruised unbruis unbuckle unbuckl + unbuckles unbuckl unbuckling unbuckl + unbuild unbuild unburden unburden + unburdens unburden unburied unburi + unburnt unburnt unburthen unburthen + unbutton unbutton unbuttoning unbutton + uncapable uncap uncape uncap + uncase uncas uncasing uncas + uncaught uncaught uncertain uncertain + uncertainty uncertainti unchain unchain + unchanging unchang uncharge uncharg + uncharged uncharg uncharitably uncharit + unchary unchari unchaste unchast + uncheck uncheck unchilded unchild + uncivil uncivil unclaim unclaim + unclasp unclasp uncle uncl + unclean unclean uncleanliness uncleanli + uncleanly uncleanli uncleanness unclean + uncles uncl unclew unclew + unclog unclog uncoined uncoin + uncolted uncolt uncomeliness uncomeli + uncomfortable uncomfort uncompassionate uncompassion + uncomprehensive uncomprehens unconfinable unconfin + unconfirm unconfirm unconfirmed unconfirm + unconquer unconqu unconquered unconqu + unconsidered unconsid unconstant unconst + unconstrain unconstrain unconstrained unconstrain + uncontemn uncontemn uncontroll uncontrol + uncorrected uncorrect uncounted uncount + uncouple uncoupl uncourteous uncourt + uncouth uncouth uncover uncov + uncovered uncov uncropped uncrop + uncross uncross uncrown uncrown + unction unction unctuous unctuou + uncuckolded uncuckold uncurable uncur + uncurbable uncurb uncurbed uncurb + uncurls uncurl uncurrent uncurr + uncurse uncurs undaunted undaunt + undeaf undeaf undeck undeck + undeeded undeed under under + underbearing underbear underborne underborn + undercrest undercrest underfoot underfoot + undergo undergo undergoes undergo + undergoing undergo undergone undergon + underground underground underhand underhand + underlings underl undermine undermin + underminers undermin underneath underneath + underprizing underpr underprop underprop + understand understand understandeth understandeth + understanding understand understandings understand + understands understand understood understood + underta underta undertake undertak + undertakeing undertak undertaker undertak + undertakes undertak undertaking undertak + undertakings undertak undertook undertook + undervalu undervalu undervalued undervalu + underwent underw underwrit underwrit + underwrite underwrit undescried undescri + undeserved undeserv undeserver undeserv + undeservers undeserv undeserving undeserv + undetermin undetermin undid undid + undinted undint undiscernible undiscern + undiscover undiscov undishonoured undishonour + undispos undispo undistinguishable undistinguish + undistinguished undistinguish undividable undivid + undivided undivid undivulged undivulg + undo undo undoes undo + undoing undo undone undon + undoubted undoubt undoubtedly undoubtedli + undream undream undress undress + undressed undress undrown undrown + unduteous undut undutiful unduti + une un uneared unear + unearned unearn unearthly unearthli + uneasines uneasin uneasy uneasi + uneath uneath uneducated uneduc + uneffectual uneffectu unelected unelect + unequal unequ uneven uneven + unexamin unexamin unexecuted unexecut + unexpected unexpect unexperienc unexperienc + unexperient unexperi unexpressive unexpress + unfair unfair unfaithful unfaith + unfallible unfal unfam unfam + unfashionable unfashion unfasten unfasten + unfather unfath unfathered unfath + unfed unf unfeed unfe + unfeeling unfeel unfeigned unfeign + unfeignedly unfeignedli unfellowed unfellow + unfelt unfelt unfenced unfenc + unfilial unfili unfill unfil + unfinish unfinish unfirm unfirm + unfit unfit unfitness unfit + unfix unfix unfledg unfledg + unfold unfold unfolded unfold + unfoldeth unfoldeth unfolding unfold + unfolds unfold unfool unfool + unforc unforc unforced unforc + unforfeited unforfeit unfortified unfortifi + unfortunate unfortun unfought unfought + unfrequented unfrequ unfriended unfriend + unfurnish unfurnish ungain ungain + ungalled ungal ungart ungart + ungarter ungart ungenitur ungenitur + ungentle ungentl ungentleness ungentl + ungently ungent ungird ungird + ungodly ungodli ungor ungor + ungot ungot ungotten ungotten + ungovern ungovern ungracious ungraci + ungrateful ungrat ungravely ungrav + ungrown ungrown unguarded unguard + unguem unguem unguided unguid + unhack unhack unhair unhair + unhallow unhallow unhallowed unhallow + unhand unhand unhandled unhandl + unhandsome unhandsom unhang unhang + unhappied unhappi unhappily unhappili + unhappiness unhappi unhappy unhappi + unhardened unharden unharm unharm + unhatch unhatch unheard unheard + unhearts unheart unheedful unheed + unheedfully unheedfulli unheedy unheedi + unhelpful unhelp unhidden unhidden + unholy unholi unhop unhop + unhopefullest unhopefullest unhorse unhors + unhospitable unhospit unhous unhou + unhoused unhous unhurtful unhurt + unicorn unicorn unicorns unicorn + unimproved unimprov uninhabitable uninhabit + uninhabited uninhabit unintelligent unintellig + union union unions union + unite unit united unit + unity uniti universal univers + universe univers universities univers + university univers unjointed unjoint + unjust unjust unjustice unjustic + unjustly unjustli unkennel unkennel + unkept unkept unkind unkind + unkindest unkindest unkindly unkindli + unkindness unkind unking unk + unkinglike unkinglik unkiss unkiss + unknit unknit unknowing unknow + unknown unknown unlace unlac + unlaid unlaid unlawful unlaw + unlawfully unlawfulli unlearn unlearn + unlearned unlearn unless unless + unlesson unlesson unletter unlett + unlettered unlett unlick unlick + unlike unlik unlikely unlik + unlimited unlimit unlineal unlin + unlink unlink unload unload + unloaded unload unloading unload + unloads unload unlock unlock + unlocks unlock unlook unlook + unlooked unlook unloos unloo + unloose unloos unlov unlov + unloving unlov unluckily unluckili + unlucky unlucki unmade unmad + unmake unmak unmanly unmanli + unmann unmann unmanner unmann + unmannerd unmannerd unmannerly unmannerli + unmarried unmarri unmask unmask + unmasked unmask unmasking unmask + unmasks unmask unmast unmast + unmatch unmatch unmatchable unmatch + unmatched unmatch unmeasurable unmeasur + unmeet unmeet unmellowed unmellow + unmerciful unmerci unmeritable unmerit + unmeriting unmerit unminded unmind + unmindfull unmindful unmingled unmingl + unmitigable unmitig unmitigated unmitig + unmix unmix unmoan unmoan + unmov unmov unmoved unmov + unmoving unmov unmuffles unmuffl + unmuffling unmuffl unmusical unmus + unmuzzle unmuzzl unmuzzled unmuzzl + unnatural unnatur unnaturally unnatur + unnaturalness unnatur unnecessarily unnecessarili + unnecessary unnecessari unneighbourly unneighbourli + unnerved unnerv unnoble unnobl + unnoted unnot unnumb unnumb + unnumber unnumb unowed unow + unpack unpack unpaid unpaid + unparagon unparagon unparallel unparallel + unpartial unparti unpath unpath + unpaved unpav unpay unpai + unpeaceable unpeac unpeg unpeg + unpeople unpeopl unpeopled unpeopl + unperfect unperfect unperfectness unperfect + unpick unpick unpin unpin + unpink unpink unpitied unpiti + unpitifully unpitifulli unplagu unplagu + unplausive unplaus unpleas unplea + unpleasant unpleas unpleasing unpleas + unpolicied unpolici unpolish unpolish + unpolished unpolish unpolluted unpollut + unpossess unpossess unpossessing unpossess + unpossible unposs unpractis unpracti + unpregnant unpregn unpremeditated unpremedit + unprepar unprepar unprepared unprepar + unpress unpress unprevailing unprevail + unprevented unprev unpriz unpriz + unprizable unpriz unprofitable unprofit + unprofited unprofit unproper unprop + unproperly unproperli unproportion unproport + unprovide unprovid unprovided unprovid + unprovident unprovid unprovokes unprovok + unprun unprun unpruned unprun + unpublish unpublish unpurged unpurg + unpurpos unpurpo unqualitied unqual + unqueen unqueen unquestion unquest + unquestionable unquestion unquiet unquiet + unquietly unquietli unquietness unquiet + unraised unrais unrak unrak + unread unread unready unreadi + unreal unreal unreasonable unreason + unreasonably unreason unreclaimed unreclaim + unreconciled unreconcil unreconciliable unreconcili + unrecounted unrecount unrecuring unrecur + unregarded unregard unregist unregist + unrelenting unrel unremovable unremov + unremovably unremov unreprievable unrepriev + unresolv unresolv unrespected unrespect + unrespective unrespect unrest unrest + unrestor unrestor unrestrained unrestrain + unreveng unreveng unreverend unreverend + unreverent unrever unrevers unrev + unrewarded unreward unrighteous unright + unrightful unright unripe unrip + unripp unripp unrivall unrival + unroll unrol unroof unroof + unroosted unroost unroot unroot + unrough unrough unruly unruli + unsafe unsaf unsaluted unsalut + unsanctified unsanctifi unsatisfied unsatisfi + unsavoury unsavouri unsay unsai + unscalable unscal unscann unscann + unscarr unscarr unschool unschool + unscorch unscorch unscour unscour + unscratch unscratch unseal unseal + unseam unseam unsearch unsearch + unseason unseason unseasonable unseason + unseasonably unseason unseasoned unseason + unseconded unsecond unsecret unsecret + unseduc unseduc unseeing unse + unseeming unseem unseemly unseemli + unseen unseen unseminar unseminar + unseparable unsepar unserviceable unservic + unset unset unsettle unsettl + unsettled unsettl unsever unsev + unsex unsex unshak unshak + unshaked unshak unshaken unshaken + unshaped unshap unshapes unshap + unsheath unsheath unsheathe unsheath + unshorn unshorn unshout unshout + unshown unshown unshrinking unshrink + unshrubb unshrubb unshunn unshunn + unshunnable unshunn unsifted unsift + unsightly unsightli unsinew unsinew + unsisting unsist unskilful unskil + unskilfully unskilfulli unskillful unskil + unslipping unslip unsmirched unsmirch + unsoil unsoil unsolicited unsolicit + unsorted unsort unsought unsought + unsound unsound unsounded unsound + unspeak unspeak unspeakable unspeak + unspeaking unspeak unsphere unspher + unspoke unspok unspoken unspoken + unspotted unspot unsquar unsquar + unstable unstabl unstaid unstaid + unstain unstain unstained unstain + unstanched unstanch unstate unstat + unsteadfast unsteadfast unstooping unstoop + unstringed unstring unstuff unstuff + unsubstantial unsubstanti unsuitable unsuit + unsuiting unsuit unsullied unsulli + unsunn unsunn unsur unsur + unsure unsur unsuspected unsuspect + unsway unswai unswayable unsway + unswayed unswai unswear unswear + unswept unswept unsworn unsworn + untainted untaint untalk untalk + untangle untangl untangled untangl + untasted untast untaught untaught + untempering untemp untender untend + untent untent untented untent + unthankful unthank unthankfulness unthank + unthink unthink unthought unthought + unthread unthread unthrift unthrift + unthrifts unthrift unthrifty unthrifti + untie unti untied unti + until until untimber untimb + untimely untim untir untir + untirable untir untired untir + untitled untitl unto unto + untold untold untouch untouch + untoward untoward untowardly untowardli + untraded untrad untrain untrain + untrained untrain untread untread + untreasur untreasur untried untri + untrimmed untrim untrod untrod + untrodden untrodden untroubled untroubl + untrue untru untrussing untruss + untruth untruth untruths untruth + untucked untuck untun untun + untune untun untuneable untun + untutor untutor untutored untutor + untwine untwin unurg unurg + unus unu unused unus + unusual unusu unvalued unvalu + unvanquish unvanquish unvarnish unvarnish + unveil unveil unveiling unveil + unvenerable unvener unvex unvex + unviolated unviol unvirtuous unvirtu + unvisited unvisit unvulnerable unvulner + unwares unwar unwarily unwarili + unwash unwash unwatch unwatch + unwearied unweari unwed unw + unwedgeable unwedg unweeded unweed + unweighed unweigh unweighing unweigh + unwelcome unwelcom unwept unwept + unwhipp unwhipp unwholesome unwholesom + unwieldy unwieldi unwilling unwil + unwillingly unwillingli unwillingness unwilling + unwind unwind unwiped unwip + unwise unwis unwisely unwis + unwish unwish unwished unwish + unwitted unwit unwittingly unwittingli + unwonted unwont unwooed unwoo + unworthier unworthi unworthiest unworthiest + unworthily unworthili unworthiness unworthi + unworthy unworthi unwrung unwrung + unyok unyok unyoke unyok + up up upbraid upbraid + upbraided upbraid upbraidings upbraid + upbraids upbraid uphoarded uphoard + uphold uphold upholdeth upholdeth + upholding uphold upholds uphold + uplift uplift uplifted uplift + upmost upmost upon upon + upper upper uprear uprear + upreared uprear upright upright + uprighteously upright uprightness upright + uprise upris uprising upris + uproar uproar uproars uproar + uprous uprou upshoot upshoot + upshot upshot upside upsid + upspring upspr upstairs upstair + upstart upstart upturned upturn + upward upward upwards upward + urchin urchin urchinfield urchinfield + urchins urchin urg urg + urge urg urged urg + urgent urgent urges urg + urgest urgest urging urg + urinal urin urinals urin + urine urin urn urn + urns urn urs ur + ursa ursa ursley urslei + ursula ursula urswick urswick + us us usage usag + usance usanc usances usanc + use us used us + useful us useless useless + user user uses us + usest usest useth useth + usher usher ushered usher + ushering usher ushers usher + using us usual usual + usually usual usurer usur + usurers usur usuries usuri + usuring usur usurp usurp + usurpation usurp usurped usurp + usurper usurp usurpers usurp + usurping usurp usurpingly usurpingli + usurps usurp usury usuri + ut ut utensil utensil + utensils utensil utility util + utmost utmost utt utt + utter utter utterance utter + uttered utter uttereth uttereth + uttering utter utterly utterli + uttermost uttermost utters utter + uy uy v v + va va vacancy vacanc + vacant vacant vacation vacat + vade vade vagabond vagabond + vagabonds vagabond vagram vagram + vagrom vagrom vail vail + vailed vail vailing vail + vaillant vaillant vain vain + vainer vainer vainglory vainglori + vainly vainli vainness vain + vais vai valanc valanc + valance valanc vale vale + valence valenc valentine valentin + valentinus valentinu valentio valentio + valeria valeria valerius valeriu + vales vale valiant valiant + valiantly valiantli valiantness valiant + validity valid vallant vallant + valley vallei valleys vallei + vally valli valor valor + valorous valor valorously valor + valour valour valu valu + valuation valuat value valu + valued valu valueless valueless + values valu valuing valu + vane vane vanish vanish + vanished vanish vanishes vanish + vanishest vanishest vanishing vanish + vanities vaniti vanity vaniti + vanquish vanquish vanquished vanquish + vanquisher vanquish vanquishest vanquishest + vanquisheth vanquisheth vant vant + vantage vantag vantages vantag + vantbrace vantbrac vapians vapian + vapor vapor vaporous vapor + vapour vapour vapours vapour + vara vara variable variabl + variance varianc variation variat + variations variat varied vari + variest variest variety varieti + varld varld varlet varlet + varletry varletri varlets varlet + varletto varletto varnish varnish + varrius varriu varro varro + vary vari varying vari + vassal vassal vassalage vassalag + vassals vassal vast vast + vastidity vastid vasty vasti + vat vat vater vater + vaudemont vaudemont vaughan vaughan + vault vault vaultages vaultag + vaulted vault vaulting vault + vaults vault vaulty vaulti + vaumond vaumond vaunt vaunt + vaunted vaunt vaunter vaunter + vaunting vaunt vauntingly vauntingli + vaunts vaunt vauvado vauvado + vaux vaux vaward vaward + ve ve veal veal + vede vede vehemence vehem + vehemency vehem vehement vehement + vehor vehor veil veil + veiled veil veiling veil + vein vein veins vein + vell vell velure velur + velutus velutu velvet velvet + vendible vendibl venerable vener + venereal vener venetia venetia + venetian venetian venetians venetian + veneys venei venge veng + vengeance vengeanc vengeances vengeanc + vengeful veng veni veni + venial venial venice venic + venison venison venit venit + venom venom venomous venom + venomously venom vent vent + ventages ventag vented vent + ventidius ventidiu ventricle ventricl + vents vent ventur ventur + venture ventur ventured ventur + ventures ventur venturing ventur + venturous ventur venue venu + venus venu venuto venuto + ver ver verb verb + verba verba verbal verbal + verbatim verbatim verbosity verbos + verdict verdict verdun verdun + verdure verdur vere vere + verefore verefor verg verg + verge verg vergers verger + verges verg verier verier + veriest veriest verified verifi + verify verifi verily verili + veritable verit verite verit + verities veriti verity veriti + vermilion vermilion vermin vermin + vernon vernon verona verona + veronesa veronesa versal versal + verse vers verses vers + versing vers vert vert + very veri vesper vesper + vessel vessel vessels vessel + vestal vestal vestments vestment + vesture vestur vetch vetch + vetches vetch veux veux + vex vex vexation vexat + vexations vexat vexed vex + vexes vex vexest vexest + vexeth vexeth vexing vex + vi vi via via + vial vial vials vial + viand viand viands viand + vic vic vicar vicar + vice vice vicegerent viceger + vicentio vicentio viceroy viceroi + viceroys viceroi vices vice + vici vici vicious viciou + viciousness vicious vict vict + victims victim victor victor + victoress victoress victories victori + victorious victori victors victor + victory victori victual victual + victuall victual victuals victual + videlicet videlicet video video + vides vide videsne videsn + vidi vidi vie vie + vied vi vienna vienna + view view viewest viewest + vieweth vieweth viewing view + viewless viewless views view + vigil vigil vigilance vigil + vigilant vigil vigitant vigit + vigour vigour vii vii + viii viii vile vile + vilely vile vileness vile + viler viler vilest vilest + vill vill village villag + villager villag villagery villageri + villages villag villain villain + villainies villaini villainous villain + villainously villain villains villain + villainy villaini villanies villani + villanous villan villany villani + villiago villiago villian villian + villianda villianda villians villian + vinaigre vinaigr vincentio vincentio + vincere vincer vindicative vindic + vine vine vinegar vinegar + vines vine vineyard vineyard + vineyards vineyard vint vint + vintner vintner viol viol + viola viola violate violat + violated violat violates violat + violation violat violator violat + violence violenc violent violent + violenta violenta violenteth violenteth + violently violent violet violet + violets violet viper viper + viperous viper vipers viper + vir vir virgilia virgilia + virgin virgin virginal virgin + virginalling virginal virginity virgin + virginius virginiu virgins virgin + virgo virgo virtue virtu + virtues virtu virtuous virtuou + virtuously virtuous visag visag + visage visag visages visag + visard visard viscount viscount + visible visibl visibly visibl + vision vision visions vision + visit visit visitation visit + visitations visit visited visit + visiting visit visitings visit + visitor visitor visitors visitor + visits visit visor visor + vita vita vitae vita + vital vital vitement vitement + vitruvio vitruvio vitx vitx + viva viva vivant vivant + vive vive vixen vixen + viz viz vizaments vizament + vizard vizard vizarded vizard + vizards vizard vizor vizor + vlouting vlout vocation vocat + vocativo vocativo vocatur vocatur + voce voce voic voic + voice voic voices voic + void void voided void + voiding void voke voke + volable volabl volant volant + volivorco volivorco volley vollei + volquessen volquessen volsce volsc + volsces volsc volscian volscian + volscians volscian volt volt + voltemand voltemand volubility volubl + voluble volubl volume volum + volumes volum volumnia volumnia + volumnius volumniu voluntaries voluntari + voluntary voluntari voluptuously voluptu + voluptuousness voluptu vomissement vomiss + vomit vomit vomits vomit + vor vor vore vore + vortnight vortnight vot vot + votaries votari votarist votarist + votarists votarist votary votari + votre votr vouch vouch + voucher voucher vouchers voucher + vouches vouch vouching vouch + vouchsaf vouchsaf vouchsafe vouchsaf + vouchsafed vouchsaf vouchsafes vouchsaf + vouchsafing vouchsaf voudrais voudrai + vour vour vous vou + voutsafe voutsaf vow vow + vowed vow vowel vowel + vowels vowel vowing vow + vows vow vox vox + voyage voyag voyages voyag + vraiment vraiment vulcan vulcan + vulgar vulgar vulgarly vulgarli + vulgars vulgar vulgo vulgo + vulnerable vulner vulture vultur + vultures vultur vurther vurther + w w wad wad + waddled waddl wade wade + waded wade wafer wafer + waft waft waftage waftag + wafting waft wafts waft + wag wag wage wage + wager wager wagers wager + wages wage wagging wag + waggish waggish waggling waggl + waggon waggon waggoner waggon + wagon wagon wagoner wagon + wags wag wagtail wagtail + wail wail wailful wail + wailing wail wails wail + wain wain wainropes wainrop + wainscot wainscot waist waist + wait wait waited wait + waiter waiter waiteth waiteth + waiting wait waits wait + wak wak wake wake + waked wake wakefield wakefield + waken waken wakened waken + wakes wake wakest wakest + waking wake wales wale + walk walk walked walk + walking walk walks walk + wall wall walled wall + wallet wallet wallets wallet + wallon wallon walloon walloon + wallow wallow walls wall + walnut walnut walter walter + wan wan wand wand + wander wander wanderer wander + wanderers wander wandering wander + wanders wander wands wand + wane wane waned wane + wanes wane waning wane + wann wann want want + wanted want wanteth wanteth + wanting want wanton wanton + wantonly wantonli wantonness wanton + wantons wanton wants want + wappen wappen war war + warble warbl warbling warbl + ward ward warded ward + warden warden warder warder + warders warder wardrobe wardrob + wardrop wardrop wards ward + ware ware wares ware + warily warili warkworth warkworth + warlike warlik warm warm + warmed warm warmer warmer + warming warm warms warm + warmth warmth warn warn + warned warn warning warn + warnings warn warns warn + warp warp warped warp + warr warr warrant warrant + warranted warrant warranteth warranteth + warrantise warrantis warrantize warrant + warrants warrant warranty warranti + warren warren warrener warren + warring war warrior warrior + warriors warrior wars war + wart wart warwick warwick + warwickshire warwickshir wary wari + was wa wash wash + washed wash washer washer + washes wash washford washford + washing wash wasp wasp + waspish waspish wasps wasp + wassail wassail wassails wassail + wast wast waste wast + wasted wast wasteful wast + wasters waster wastes wast + wasting wast wat wat + watch watch watched watch + watchers watcher watches watch + watchful watch watching watch + watchings watch watchman watchman + watchmen watchmen watchword watchword + water water waterdrops waterdrop + watered water waterfly waterfli + waterford waterford watering water + waterish waterish waterpots waterpot + waterrugs waterrug waters water + waterton waterton watery wateri + wav wav wave wave + waved wave waver waver + waverer waver wavering waver + waves wave waving wave + waw waw wawl wawl + wax wax waxed wax + waxen waxen waxes wax + waxing wax way wai + waylaid waylaid waylay waylai + ways wai wayward wayward + waywarder wayward waywardness wayward + we we weak weak + weaken weaken weakens weaken + weaker weaker weakest weakest + weakling weakl weakly weakli + weakness weak weal weal + wealsmen wealsmen wealth wealth + wealthiest wealthiest wealthily wealthili + wealthy wealthi wealtlly wealtlli + wean wean weapon weapon + weapons weapon wear wear + wearer wearer wearers wearer + wearied weari wearies weari + weariest weariest wearily wearili + weariness weari wearing wear + wearisome wearisom wears wear + weary weari weasel weasel + weather weather weathercock weathercock + weathers weather weav weav + weave weav weaver weaver + weavers weaver weaves weav + weaving weav web web + wed wed wedded wed + wedding wed wedg wedg + wedged wedg wedges wedg + wedlock wedlock wednesday wednesdai + weed weed weeded weed + weeder weeder weeding weed + weeds weed weedy weedi + week week weeke week + weekly weekli weeks week + ween ween weening ween + weep weep weeper weeper + weeping weep weepingly weepingli + weepings weep weeps weep + weet weet weigh weigh + weighed weigh weighing weigh + weighs weigh weight weight + weightier weightier weightless weightless + weights weight weighty weighti + weird weird welcom welcom + welcome welcom welcomer welcom + welcomes welcom welcomest welcomest + welfare welfar welkin welkin + well well wells well + welsh welsh welshman welshman + welshmen welshmen welshwomen welshwomen + wench wench wenches wench + wenching wench wend wend + went went wept wept + weraday weradai were were + wert wert west west + western western westminster westminst + westmoreland westmoreland westward westward + wet wet wether wether + wetting wet wezand wezand + whale whale whales whale + wharf wharf wharfs wharf + what what whate whate + whatever whatev whatsoe whatso + whatsoever whatsoev whatsome whatsom + whe whe wheat wheat + wheaten wheaten wheel wheel + wheeling wheel wheels wheel + wheer wheer wheeson wheeson + wheezing wheez whelk whelk + whelks whelk whelm whelm + whelp whelp whelped whelp + whelps whelp when when + whenas whena whence whenc + whencesoever whencesoev whene whene + whenever whenev whensoever whensoev + where where whereabout whereabout + whereas wherea whereat whereat + whereby wherebi wherefore wherefor + wherein wherein whereinto whereinto + whereof whereof whereon whereon + whereout whereout whereso whereso + wheresoe whereso wheresoever wheresoev + wheresome wheresom whereto whereto + whereuntil whereuntil whereunto whereunto + whereupon whereupon wherever wherev + wherewith wherewith wherewithal wherewith + whet whet whether whether + whetstone whetston whetted whet + whew whew whey whei + which which whiff whiff + whiffler whiffler while while + whiles while whilst whilst + whin whin whine whine + whined whine whinid whinid + whining whine whip whip + whipp whipp whippers whipper + whipping whip whips whip + whipster whipster whipstock whipstock + whipt whipt whirl whirl + whirled whirl whirligig whirligig + whirling whirl whirlpool whirlpool + whirls whirl whirlwind whirlwind + whirlwinds whirlwind whisp whisp + whisper whisper whispering whisper + whisperings whisper whispers whisper + whist whist whistle whistl + whistles whistl whistling whistl + whit whit white white + whitehall whitehal whitely white + whiteness white whiter whiter + whites white whitest whitest + whither whither whiting white + whitmore whitmor whitsters whitster + whitsun whitsun whittle whittl + whizzing whizz who who + whoa whoa whoe whoe + whoever whoever whole whole + wholesom wholesom wholesome wholesom + wholly wholli whom whom + whoobub whoobub whoop whoop + whooping whoop whor whor + whore whore whoremaster whoremast + whoremasterly whoremasterli whoremonger whoremong + whores whore whoreson whoreson + whoresons whoreson whoring whore + whorish whorish whose whose + whoso whoso whosoe whoso + whosoever whosoev why why + wi wi wick wick + wicked wick wickednes wickedn + wickedness wicked wicket wicket + wicky wicki wid wid + wide wide widens widen + wider wider widow widow + widowed widow widower widow + widowhood widowhood widows widow + wield wield wife wife + wight wight wights wight + wild wild wildcats wildcat + wilder wilder wilderness wilder + wildest wildest wildfire wildfir + wildly wildli wildness wild + wilds wild wiles wile + wilful wil wilfull wilful + wilfully wilfulli wilfulnes wilfuln + wilfulness wil will will + willed will willers willer + willeth willeth william william + williams william willing will + willingly willingli willingness willing + willoughby willoughbi willow willow + wills will wilt wilt + wiltshire wiltshir wimpled wimpl + win win wince winc + winch winch winchester winchest + wincot wincot wind wind + winded wind windgalls windgal + winding wind windlasses windlass + windmill windmil window window + windows window windpipe windpip + winds wind windsor windsor + windy windi wine wine + wing wing winged wing + wingfield wingfield wingham wingham + wings wing wink wink + winking wink winks wink + winner winner winners winner + winning win winnow winnow + winnowed winnow winnows winnow + wins win winter winter + winterly winterli winters winter + wip wip wipe wipe + wiped wipe wipes wipe + wiping wipe wire wire + wires wire wiry wiri + wisdom wisdom wisdoms wisdom + wise wise wiselier wiseli + wisely wise wiser wiser + wisest wisest wish wish + wished wish wisher wisher + wishers wisher wishes wish + wishest wishest wisheth wisheth + wishful wish wishing wish + wishtly wishtli wisp wisp + wist wist wit wit + witb witb witch witch + witchcraft witchcraft witches witch + witching witch with with + withal withal withdraw withdraw + withdrawing withdraw withdrawn withdrawn + withdrew withdrew wither wither + withered wither withering wither + withers wither withheld withheld + withhold withhold withholds withhold + within within withold withold + without without withstand withstand + withstanding withstand withstood withstood + witless witless witness wit + witnesses wit witnesseth witnesseth + witnessing wit wits wit + witted wit wittenberg wittenberg + wittiest wittiest wittily wittili + witting wit wittingly wittingli + wittol wittol wittolly wittolli + witty witti wiv wiv + wive wive wived wive + wives wive wiving wive + wizard wizard wizards wizard + wo wo woe woe + woeful woeful woefull woeful + woefullest woefullest woes woe + woful woful wolf wolf + wolfish wolfish wolsey wolsei + wolves wolv wolvish wolvish + woman woman womanhood womanhood + womanish womanish womankind womankind + womanly womanli womb womb + wombs womb womby wombi + women women won won + woncot woncot wond wond + wonder wonder wondered wonder + wonderful wonder wonderfully wonderfulli + wondering wonder wonders wonder + wondrous wondrou wondrously wondrous + wont wont wonted wont + woo woo wood wood + woodbine woodbin woodcock woodcock + woodcocks woodcock wooden wooden + woodland woodland woodman woodman + woodmonger woodmong woods wood + woodstock woodstock woodville woodvil + wooed woo wooer wooer + wooers wooer wooes wooe + woof woof wooing woo + wooingly wooingli wool wool + woollen woollen woolly woolli + woolsack woolsack woolsey woolsei + woolward woolward woos woo + wor wor worcester worcest + word word words word + wore wore worins worin + work work workers worker + working work workings work + workman workman workmanly workmanli + workmanship workmanship workmen workmen + works work worky worki + world world worldlings worldl + worldly worldli worlds world + worm worm worms worm + wormwood wormwood wormy wormi + worn worn worried worri + worries worri worry worri + worrying worri worse wors + worser worser worship worship + worshipful worship worshipfully worshipfulli + worshipp worshipp worshipper worshipp + worshippers worshipp worshippest worshippest + worships worship worst worst + worsted worst wort wort + worth worth worthied worthi + worthier worthier worthies worthi + worthiest worthiest worthily worthili + worthiness worthi worthless worthless + worths worth worthy worthi + worts wort wot wot + wots wot wotting wot + wouid wouid would would + wouldest wouldest wouldst wouldst + wound wound wounded wound + wounding wound woundings wound + woundless woundless wounds wound + wouns woun woven woven + wow wow wrack wrack + wrackful wrack wrangle wrangl + wrangler wrangler wranglers wrangler + wrangling wrangl wrap wrap + wrapp wrapp wraps wrap + wrapt wrapt wrath wrath + wrathful wrath wrathfully wrathfulli + wraths wrath wreak wreak + wreakful wreak wreaks wreak + wreath wreath wreathed wreath + wreathen wreathen wreaths wreath + wreck wreck wrecked wreck + wrecks wreck wren wren + wrench wrench wrenching wrench + wrens wren wrest wrest + wrested wrest wresting wrest + wrestle wrestl wrestled wrestl + wrestler wrestler wrestling wrestl + wretch wretch wretchcd wretchcd + wretched wretch wretchedness wretched + wretches wretch wring wring + wringer wringer wringing wring + wrings wring wrinkle wrinkl + wrinkled wrinkl wrinkles wrinkl + wrist wrist wrists wrist + writ writ write write + writer writer writers writer + writes write writhled writhl + writing write writings write + writs writ written written + wrong wrong wronged wrong + wronger wronger wrongful wrong + wrongfully wrongfulli wronging wrong + wrongly wrongli wrongs wrong + wronk wronk wrote wrote + wroth wroth wrought wrought + wrung wrung wry wry + wrying wry wt wt + wul wul wye wye + x x xanthippe xanthipp + xi xi xii xii + xiii xiii xiv xiv + xv xv y y + yard yard yards yard + yare yare yarely yare + yarn yarn yaughan yaughan + yaw yaw yawn yawn + yawning yawn ycleped yclepe + ycliped yclipe ye ye + yea yea yead yead + year year yearly yearli + yearn yearn yearns yearn + years year yeas yea + yeast yeast yedward yedward + yell yell yellow yellow + yellowed yellow yellowing yellow + yellowness yellow yellows yellow + yells yell yelping yelp + yeoman yeoman yeomen yeomen + yerk yerk yes ye + yesterday yesterdai yesterdays yesterdai + yesternight yesternight yesty yesti + yet yet yew yew + yicld yicld yield yield + yielded yield yielder yielder + yielders yielder yielding yield + yields yield yok yok + yoke yoke yoked yoke + yokefellow yokefellow yokes yoke + yoketh yoketh yon yon + yond yond yonder yonder + yongrey yongrei yore yore + yorick yorick york york + yorkists yorkist yorks york + yorkshire yorkshir you you + young young younger younger + youngest youngest youngling youngl + younglings youngl youngly youngli + younker younker your your + yours your yourself yourself + yourselves yourselv youth youth + youthful youth youths youth + youtli youtli zanies zani + zany zani zeal zeal + zealous zealou zeals zeal + zed zed zenelophon zenelophon + zenith zenith zephyrs zephyr + zir zir zo zo + zodiac zodiac zodiacs zodiac + zone zone zounds zound + zwagger zwagger +} + + +set i 0 +foreach {in out} $test_vocab { + do_test "1.$i.($in -> $out)" { + lindex [sqlite3_fts5_tokenize db porter $in] 0 + } $out + incr i +} + + +finish_test diff --git a/ext/fts5/test/fts5porter2.test b/ext/fts5/test/fts5porter2.test new file mode 100644 index 0000000000000000000000000000000000000000..556060baa30c6aed9685b9fcbe4be13c3c49df19 --- /dev/null +++ b/ext/fts5/test/fts5porter2.test @@ -0,0 +1,69 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the fts5 porter stemmer implementation. +# +# These are extra tests added to those in fts5porter.test in order to +# improve test coverage of the porter stemmer implementation. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5porter2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set test_vocab { + tion tion + ation ation + vation vation + avation avat + vion vion + ion ion + relational relat + relation relat + relate relat + zzz zzz + ii ii + iiing ii + xtional xtional + xenci xenci + xlogi xlogi + realization realiz + realize realiz + xization xizat + capitalism capit + talism talism + xiveness xive + xfulness xful + xousness xous + xical xical + xicate xicat + xicity xiciti + ies ie + eed e + eing e + s s +} + +set i 0 +foreach {in out} $test_vocab { + do_test "1.$i.($in -> $out)" { + lindex [sqlite3_fts5_tokenize db porter $in] 0 + } $out + incr i +} + + +finish_test diff --git a/ext/fts5/test/fts5prefix.test b/ext/fts5/test/fts5prefix.test new file mode 100644 index 0000000000000000000000000000000000000000..7a29628ea11e9c822f0bbaa84cecb81dfc393f2d --- /dev/null +++ b/ext/fts5/test/fts5prefix.test @@ -0,0 +1,343 @@ +# 2015 Jan 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests focused on prefix indexes. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5prefix + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xx USING fts5(x, prefix=1); + INSERT INTO xx VALUES('one two three'); + INSERT INTO xx VALUES('four five six'); + INSERT INTO xx VALUES('seven eight nine ten'); +} + +do_execsql_test 1.1 { + SELECT rowid FROM xx WHERE xx MATCH 't*' +} {1 3} + + +#------------------------------------------------------------------------- +# Check that prefix indexes really do index n-character prefixes, not +# n-byte prefixes. Use the ascii tokenizer so as not to be confused by +# diacritic removal. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = ascii, prefix = 2) +} + +do_test 2.1 { + foreach {rowid string} { + 1 "\xCA\xCB\xCC\xCD" + 2 "\u1234\u5678\u4321\u8765" + } { + execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $string) } + } +} {} + +do_execsql_test 2.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +foreach {tn q res} { + 1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1 + 2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2 +} { + do_execsql_test 2.3.$tn $q $res +} + +#------------------------------------------------------------------------- +# Check that prefix queries with: +# +# * a column filter, and +# * no prefix index. +# +# work Ok. +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, b, c); + INSERT INTO t3(t3, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO t3 VALUES('acb ccc bba', 'cca bba bca', 'bbc ccc bca'); -- 1 + INSERT INTO t3 VALUES('cbb cac cab', 'abb aac bba', 'aab ccc cac'); -- 2 + INSERT INTO t3 VALUES('aac bcb aac', 'acb bcb caa', 'aca bab bca'); -- 3 + INSERT INTO t3 VALUES('aab ccb ccc', 'aca cba cca', 'aca aac cbb'); -- 4 + INSERT INTO t3 VALUES('bac aab bab', 'ccb bac cba', 'acb aba abb'); -- 5 + INSERT INTO t3 VALUES('bab abc ccb', 'acb cba abb', 'cbb aaa cab'); -- 6 + INSERT INTO t3 VALUES('cbb bbc baa', 'aab aca baa', 'bcc cca aca'); -- 7 + INSERT INTO t3 VALUES('abc bba abb', 'cac abc cba', 'acc aac cac'); -- 8 + INSERT INTO t3 VALUES('bbc bbc cab', 'bcb ccb cba', 'bcc cac acb'); -- 9 + COMMIT; +} + +foreach {tn match res} { + 1 "a : c*" {1 2 4 6 7 9} + 2 "b : c*" {1 3 4 5 6 8 9} + 3 "c : c*" {1 2 4 6 7 8 9} + 4 "a : b*" {1 3 5 6 7 8 9} + 5 "b : b*" {1 2 3 5 7 9} + 6 "c : b*" {1 3 7 9} + 7 "a : a*" {1 3 4 5 6 8} + 8 "b : a*" {2 3 4 6 7 8} + 9 "c : a*" {2 3 4 5 6 7 8 9} +} { + do_execsql_test 3.1.$tn { + SELECT rowid FROM t3($match) + } $res +} + +do_test 3.2 { + expr srand(0) + execsql { DELETE FROM t3 } + for {set i 0} {$i < 1000} {incr i} { + set a [fts5_rnddoc 3] + set b [fts5_rnddoc 8] + set c [fts5_rnddoc 20] + execsql { INSERT INTO t3 VALUES($a, $b, $c) } + } + execsql { INSERT INTO t3(t3) VALUES('integrity-check') } +} {} + +proc gmatch {col pattern} { + expr {[lsearch -glob $col $pattern]>=0} +} +db func gmatch gmatch + +proc ghl {col pattern} { + foreach t $col { + if {[string match $pattern $t]} { + lappend res "*$t*" + } else { + lappend res $t + } + } + set res +} +db func ghl ghl + +set COLS(a) 0 +set COLS(b) 1 +set COLS(c) 2 + +for {set x 0} {$x<2} {incr x} { + foreach {tn pattern} { + 1 {xa*} + 2 {xb*} + 3 {xc*} + 4 {xd*} + 5 {xe*} + 6 {xf*} + 7 {xg*} + 8 {xh*} + 9 {xi*} + 10 {xj*} + } { + foreach col {a b c} { + + # Check that the list of returned rowids is correct. + # + set res [db eval "SELECT rowid FROM t3 WHERE gmatch($col, '$pattern')"] + set query "$col : $pattern" + do_execsql_test 3.3.$x.$tn.$col.rowid { + SELECT rowid FROM t3($query); + } $res + + # Check that the highlight() function works. + # + set res [db eval \ + "SELECT ghl($col, '$pattern') FROM t3 WHERE gmatch($col, '$pattern')" + ] + set idx $COLS($col) + do_execsql_test 3.3.$x.$tn.$col.highlight { + SELECT highlight(t3, $idx, '*', '*') FROM t3($query); + } $res + } + + foreach colset {{a b} {b c} {c a} {a c} {b a}} { + # Check that the list of returned rowids is correct. + # + foreach {col1 col2} $colset {} + set expr "gmatch($col1, '$pattern') OR gmatch($col2, '$pattern')" + set res [db eval "SELECT rowid FROM t3 WHERE $expr"] + set query "{$colset} : $pattern" + do_execsql_test 3.3.$x.$tn.{$colset}.rowid { + SELECT rowid FROM t3($query); + } $res + + set resq "SELECT ghl($col1, '$pattern'), ghl($col2, '$pattern')" + append resq " FROM t3 WHERE $expr" + set res [db eval $resq] + set idx1 $COLS($col1) + set idx2 $COLS($col2) + do_execsql_test 3.3.$x.$tn.{$colset}.highlight { + SELECT highlight(t3, $idx1, '*', '*'), highlight(t3, $idx2, '*', '*') + FROM t3($query) + } $res + } + } + execsql { INSERT INTO t3(t3) VALUES('optimize') } + execsql { INSERT INTO t3(t3) VALUES('integrity-check') } +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(c1, c2); + INSERT INTO t2 VALUES('xa xb', 'xb xa'); + + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 2 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 4 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 8 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 16 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 32 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 64 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 128 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 256 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 512 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 1024 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 2048 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 4096 + + SELECT count(*) FROM t2('x*'); +} {4096} + +do_execsql_test 4.1 { + UPDATE t2 SET c2 = 'ya yb'; + SELECT count(*) FROM t2('c1:x*'); + SELECT count(*) FROM t2('c2:x*'); +} {4096 0} + +do_execsql_test 4.2 { + UPDATE t2 SET c2 = 'xa'; + SELECT count(*) FROM t2('c1:x*'); + SELECT count(*) FROM t2('c2:x*'); +} {4096 4096} + +#------------------------------------------------------------------------- +# +reset_db +proc rnddoc {n} { + set map [list a b c d] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc "x[lindex $map [expr int(rand()*4)]]" + } + set doc +} +set cols [list] +for {set i 1} {$i<250} {incr i} { + lappend cols "c$i" + lappend vals "'[rnddoc 10]'" +} + +do_test 5.0 { + execsql "CREATE VIRTUAL TABLE t4 USING fts5([join $cols ,])" + execsql {INSERT INTO t4(t4, rank) VALUES('pgsz', 32)} + execsql "INSERT INTO t4 VALUES([join $vals ,])" + execsql "INSERT INTO t4 VALUES([join $vals ,])" + execsql "INSERT INTO t4 VALUES([join $vals ,])" + execsql "INSERT INTO t4 VALUES([join $vals ,])" +} {} + +proc gmatch {col pattern} { + expr {[lsearch -glob $col $pattern]>=0} +} +db func gmatch gmatch +foreach {tn col pattern} { + 1 c100 {xa*} + 2 c200 {xb*} +} { + set res [db eval "SELECT rowid FROM t4 WHERE gmatch($col, \$pattern)"] + set query "$col : $pattern" + do_execsql_test 5.$tn { SELECT rowid FROM t4($query) } $res +} + +reset_db +db func fts5_rnddoc fts5_rnddoc +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t5 USING fts5(x, y); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + } +} {} + +proc gmatch {col pattern} { + expr {[lsearch -glob $col $pattern]>=0} +} +db func gmatch gmatch +foreach {tn col pattern} { + 1 y {xa*} + 2 y {xb*} + 3 y {xc*} + 4 x {xa*} + 5 x {xb*} + 6 x {xc*} +} { + set res [db eval "SELECT rowid FROM t5 WHERE gmatch($col, \$pattern)"] + set query "$col : $pattern" + do_execsql_test 6.$tn { SELECT rowid FROM t5($query) } $res +} + +#------------------------------------------------------------------------- +# Check that the various ways of creating prefix indexes produce the +# same database on disk. +# +save_prng_state +foreach {tn create} { + 1 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1,2,3") } + 2 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2 3") } + 3 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix=1, prefix=2, prefix=3) } + 4 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2", prefix=3) } +} { + execsql { DROP TABLE IF EXISTS tt } + restore_prng_state + execsql $create + execsql { + INSERT INTO tt VALUES('cc b ggg ccc aa eee hh', 'aa g b hh a e'); + INSERT INTO tt VALUES('cc bb cc gg j g cc', 'ii jjj ggg jjj cc cc'); + INSERT INTO tt VALUES('h eee cc h iii', 'aaa iii dd iii dd'); + INSERT INTO tt VALUES('jjj hh eee c e b gg', 'j bbb jj ddd jj'); + INSERT INTO tt VALUES('ii hhh aaa ff c hhh iii', 'j cc hh bb e'); + INSERT INTO tt VALUES('e fff hhh i aaa', 'g b aa gg c aa dd'); + INSERT INTO tt VALUES('i aaa ccc gg hhh aa h', 'j bbb bbb d ff'); + INSERT INTO tt VALUES('g f gg ff ff jjj d', 'jjj d j fff fff ee j'); + INSERT INTO tt VALUES('a cc e ccc jjj c', 'ccc iii d bb a eee g'); + INSERT INTO tt VALUES('jj hh hh bb bbb gg', 'j c jjj bb iii f'); + INSERT INTO tt VALUES('a ggg g cc ccc aa', 'jjj j j aaa c'); + INSERT INTO tt VALUES('ddd j dd b i', 'aaa bbb iii ggg ff ccc ddd'); + INSERT INTO tt VALUES('jj ii hh c ii h gg', 'hhh bbb ddd bbb hh g ggg'); + INSERT INTO tt VALUES('aa hhh ccc h ggg ccc', 'iii d jj a ff ii'); + } + + #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM tt_data} {puts $r} + + if {$tn==1} { + set ::checksum [execsql {SELECT md5sum(id, block) FROM tt_data}] + } else { + do_execsql_test 7.$tn { + SELECT md5sum(id, block) FROM tt_data + } [list $::checksum] + } +} + +finish_test diff --git a/ext/fts5/test/fts5prefix2.test b/ext/fts5/test/fts5prefix2.test new file mode 100644 index 0000000000000000000000000000000000000000..0860b3cddddf1c32fc9b9081369188c539ef400d --- /dev/null +++ b/ext/fts5/test/fts5prefix2.test @@ -0,0 +1,87 @@ +# 2020 Dec 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests focused on prefix indexes. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5prefix2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach p {3 2 1} { + reset_db + do_execsql_test 1.$p.0 " + CREATE VIRTUAL TABLE t1 USING fts5(xyz, prefix=$p); + " + do_execsql_test 1.$p.1 { + INSERT INTO t1 VALUES + ('May you do good and not evil.'), + ('May you find forgiveness for yourself and forgive others.'), + ('May you share freely, never taking more than you give f.'); + } + + do_execsql_test 1.$p.2 { + SELECT highlight(t1, 0, '[', ']') FROM t1('f*'); + } { + {May you [find] [forgiveness] [for] yourself and [forgive] others.} + {May you share [freely], never taking more than you give [f].} + } +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(one, prefix=3); + INSERT INTO t2 VALUES('top'); + INSERT INTO t2 VALUES('to'); + INSERT INTO t2 VALUES('tommy'); +} + +do_execsql_test 2.1 { + SELECT * FROM t2('to*'); +} {top to tommy} + +#------------------------------------------------------------------------- + +foreach {tn newrowid} { + 1 122 + 2 123 + 3 124 +} { + reset_db + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t12 USING fts5(x); + INSERT INTO t12(rowid, x) VALUES(123, 'wwww'); + } + do_execsql_test 3.$tn.1 { + BEGIN; + DELETE FROM t12 WHERE rowid=123; + SELECT * FROM t12('wwww*'); + INSERT INTO t12(rowid, x) VALUES($newrowid, 'wwww'); + SELECT * FROM t12('wwww*'); + END; + } {wwww} + do_execsql_test 3.$tn.2 { + INSERT INTO t12(t12) VALUES('integrity-check'); + } + do_execsql_test 3.$tn.3 { + SELECT rowid FROM t12('wwww*'); + } $newrowid +} + +finish_test + + + +finish_test diff --git a/ext/fts5/test/fts5query.test b/ext/fts5/test/fts5query.test new file mode 100644 index 0000000000000000000000000000000000000000..4e8bab8cf7f1d0c4c3575595c65261bc2f8707b5 --- /dev/null +++ b/ext/fts5/test/fts5query.test @@ -0,0 +1,91 @@ +# 2015 October 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5query + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} { + reset_db + do_test 1.$tn.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + BEGIN; + } + foreach x [list aaa bbb ccc ddd eee fff ggg hhh iii jjj] { + set doc [string repeat "$x " 30] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql COMMIT + } {} + + do_execsql_test 1.$tn.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + set ret 1 + foreach x [list a b c d e f g h i j] { + do_execsql_test 1.$tn.3.$ret { + SELECT rowid FROM t1 WHERE t1 MATCH $x || '*'; + } $ret + incr ret + } +} + +for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} { + reset_db + do_test 2.$tn.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + BEGIN; + } + foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] { + set doc [string repeat "$x " 30] + execsql { INSERT INTO t1 VALUES($doc) } + } + execsql COMMIT + } {} + + do_execsql_test 2.$tn.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + set ret 1 + foreach x [list a c e g i k m o q s u] { + do_execsql_test 2.$tn.3.$ret { + SELECT rowid FROM t1 WHERE t1 MATCH $x || '*'; + } {} + incr ret + } +} + +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota'); + INSERT INTO x1(rowid, a) VALUES(1, 'tarago'); +} +do_execsql_test 3.1 { + SELECT rowid FROM x1('t*'); +} {-1000000000000 1} + + +finish_test diff --git a/ext/fts5/test/fts5rank.test b/ext/fts5/test/fts5rank.test new file mode 100644 index 0000000000000000000000000000000000000000..7a700cb97f391e2419b6911979a708cd4f08e10a --- /dev/null +++ b/ext/fts5/test/fts5rank.test @@ -0,0 +1,207 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing queries that use the "rank" column. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5rank + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +#------------------------------------------------------------------------- +# "ORDER BY rank" + highlight() + large poslists. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xyz USING fts5(z); +} +do_test 1.1 { + set doc [string trim [string repeat "x y " 500]] + execsql { INSERT INTO xyz VALUES($doc) } +} {} +do_execsql_test 1.2 { + SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank +} [list [string map {x [x]} $doc]] + +do_execsql_test 1.3 { + SELECT highlight(xyz, 0, '[', ']') FROM xyz + WHERE xyz MATCH 'x AND y' ORDER BY rank +} [list [string map {x [x] y [y]} $doc]] + +#------------------------------------------------------------------------- +# Check that the 'rank' option really is persistent. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE tt USING fts5(a); + INSERT INTO tt VALUES('a x x x x'); + INSERT INTO tt VALUES('x x a a a'); + INSERT INTO tt VALUES('x a a x x'); +} + +proc firstinst {cmd} { + foreach {p c o} [$cmd xInst 0] {} + return $o +} +sqlite3_fts5_create_function db firstinst firstinst + +do_execsql_test 2.1 { + SELECT rowid FROM tt('a') ORDER BY rank; +} {2 3 1} + +do_execsql_test 2.2 { + SELECT rowid FROM tt('a', 'firstinst()') ORDER BY rank; +} {1 3 2} + +do_execsql_test 2.3 { + INSERT INTO tt(tt, rank) VALUES('rank', 'firstinst()'); + SELECT rowid FROM tt('a') ORDER BY rank; +} {1 3 2} + +do_test 2.4 { + sqlite3 db2 test.db + catchsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2 +} {1 {no such function: firstinst}} + +do_test 2.5 { + db2 close + sqlite3 db2 test.db + sqlite3_fts5_create_function db2 firstinst firstinst + execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2 +} {1 3 2} + +do_test 2.6 { + execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2 +} {1 3 2} + +do_test 2.7 { + execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db +} {1 3 2} + +db2 close + +#-------------------------------------------------------------------------- +# At one point there was a problem with queries such as: +# +# ... MATCH 'x OR y' ORDER BY rank; +# +# if there were zero occurrences of token 'y' in the dataset. The +# following tests verify that that problem has been addressed. +# +foreach_detail_mode $::testprefix { + do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%); + INSERT INTO y1 VALUES('test xyz'); + INSERT INTO y1 VALUES('test test xyz test'); + INSERT INTO y1 VALUES('test test xyz'); + } + + do_execsql_test 3.1.1 { + SELECT rowid FROM y1('test OR tset'); + } {1 2 3} + + do_execsql_test 3.1.2 { + SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1) + } {2 3 1} + + do_execsql_test 3.1.3 { + SELECT rowid FROM y1('test OR tset') ORDER BY +rank + } {2 3 1} + + do_execsql_test 3.1.4 { + SELECT rowid FROM y1('test OR tset') ORDER BY rank + } {2 3 1} + + do_execsql_test 3.1.5 { + SELECT rowid FROM y1('test OR xyz') ORDER BY rank + } {3 2 1} + + + do_execsql_test 3.2.1 { + CREATE VIRTUAL TABLE z1 USING fts5(a, detail=%DETAIL%); + INSERT INTO z1 VALUES('wrinkle in time'); + SELECT * FROM z1 WHERE z1 MATCH 'wrinkle in time OR a wrinkle in time'; + } {{wrinkle in time}} +} + +do_execsql_test 4.1 { + DROP TABLE IF EXISTS VTest; + CREATE virtual TABLE VTest USING FTS5( + Title, AUthor, tokenize ='porter unicode61 remove_diacritics 1', + columnsize='1', detail=full + ); + INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith'); + + SELECT * FROM VTest WHERE + VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank; +} {{wrinkle in time} {Bill Smith}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ttt USING fts5(a); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ttt SELECT 'word ' || i FROM s; +} + +do_execsql_test 5.1 { + SELECT rowid FROM ttt('word') WHERE rowid BETWEEN 30 AND 40 ORDER BY rank; +} {30 31 32 33 34 35 36 37 38 39 40} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE "My.Table" USING fts5(Text); + + INSERT INTO "My.Table" VALUES ('hello this is a test'); + INSERT INTO "My.Table" VALUES ('of trying to order by'); + INSERT INTO "My.Table" VALUES ('rank on an fts5 table'); + INSERT INTO "My.Table" VALUES ('that have periods in'); + INSERT INTO "My.Table" VALUES ('the table names.'); + INSERT INTO "My.Table" VALUES ('table table table'); +} +do_execsql_test 6.1 { + SELECT * FROM "My.Table" WHERE Text MATCH 'table' ORDER BY rank; +} { + {table table table} {the table names.} {rank on an fts5 table} +} + + +#------------------------------------------------------------------------- +# forum post: https://sqlite.org/forum/forumpost/a2dd636330 +# +reset_db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t USING fts5 (a, b); + INSERT INTO t (a, b) VALUES ('data1', 'sentence1'), ('data2', 'sentence2'); + INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)'); +} + +sqlite3 db2 test.db +do_execsql_test -db db2 1.1 { + SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK; +} {data1 sentence1 1 data2 sentence2 1} + +do_execsql_test 1.2 { + INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)'); +} +do_execsql_test -db db2 1.3 { + SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK; +} {data1 sentence1 1 data2 sentence2 1} +db2 close + +finish_test diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test new file mode 100644 index 0000000000000000000000000000000000000000..d74b148fb1e29adad975c38b84381ed4fa37c93b --- /dev/null +++ b/ext/fts5/test/fts5rebuild.test @@ -0,0 +1,67 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5rebuild + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1(a, b) VALUES('one', 'o n e'); + INSERT INTO f1(a, b) VALUES('two', 't w o'); + INSERT INTO f1(a, b) VALUES('three', 't h r e e'); +} + +do_execsql_test 1.2 { + INSERT INTO f1(f1) VALUES('integrity-check'); +} {} + +do_execsql_test 1.3 { + INSERT INTO f1(f1) VALUES('rebuild'); +} {} + +do_execsql_test 1.4 { + INSERT INTO f1(f1) VALUES('integrity-check'); +} {} + +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 1.5 { + DELETE FROM f1_data; +} {} + +do_catchsql_test 1.6 { + INSERT INTO f1(f1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +do_execsql_test 1.7 { + INSERT INTO f1(f1) VALUES('rebuild'); + INSERT INTO f1(f1) VALUES('integrity-check'); +} {} + + +#------------------------------------------------------------------------- +# Check that 'rebuild' may not be used with a contentless table. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE nc USING fts5(doc, content=); +} + +do_catchsql_test 2.2 { + INSERT INTO nc(nc) VALUES('rebuild'); +} {1 {'rebuild' may not be used with a contentless fts5 table}} +finish_test diff --git a/ext/fts5/test/fts5restart.test b/ext/fts5/test/fts5restart.test new file mode 100644 index 0000000000000000000000000000000000000000..da58fe3aed43b5ccd9707b4b344490a590ae6676 --- /dev/null +++ b/ext/fts5/test/fts5restart.test @@ -0,0 +1,152 @@ +# 2015 April 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing the planner (xBestIndex function). +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5restart + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE f1 USING fts5(ff); +} + +#------------------------------------------------------------------------- +# Run the 'optimize' command. Check that it does not disturb ongoing +# full-text queries. +# +unset -nocomplain lRowid +do_test 1.1 { + for {set i 1} {$i < 1000} {incr i} { + execsql { INSERT INTO f1 VALUES('a b c d e') } + lappend lRowid $i + } +} {} + +do_execsql_test 1.2 { + SELECT rowid FROM f1 WHERE f1 MATCH 'c'; +} $lRowid + +do_test 1.3 { + set res [list] + db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } { + if {$rowid == 100} { + execsql { INSERT INTO f1(f1) VALUES('optimize') } + } + lappend res $rowid + } + set res +} $lRowid + +do_test 1.4.1 { + sqlite3 db2 test.db + set res [list] + db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } { + if {$rowid == 100} { + set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }] + } + lappend res $rowid + } + set res +} $lRowid + +do_test 1.4.2 { + db2 close + set cres +} {1 {database is locked}} + +#------------------------------------------------------------------------- +# Open a couple of cursors. Then close them in the same order. +# +do_test 2.1 { + set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X] + set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X] + + sqlite3_step $::s1 +} {SQLITE_ROW} +do_test 2.2 { + sqlite3_step $::s2 +} {SQLITE_ROW} + +do_test 2.1 { + sqlite3_finalize $::s1 + sqlite3_finalize $::s2 +} {SQLITE_OK} + +#------------------------------------------------------------------------- +# Copy data between two FTS5 tables. +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE f2 USING fts5(gg); + INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d'; +} +do_execsql_test 3.2 { + SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e' +} $lRowid + +#------------------------------------------------------------------------- +# Remove the row that an FTS5 cursor is currently pointing to. And +# various other similar things. Check that this does not disturb +# ongoing scans. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE n4 USING fts5(n); + INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5'); + INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4'); + INSERT INTO n4(rowid, n) VALUES(300, '2 3 4'); + INSERT INTO n4(rowid, n) VALUES(400, '2 3'); + INSERT INTO n4(rowid, n) VALUES(500, '3'); +} + +do_test 4.1 { + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } { + if {$rowid==300} { + execsql { DELETE FROM n4 WHERE rowid=300 } + } + lappend res $rowid + } + set res +} {100 200 300 400 500} + +do_test 4.2 { + execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') } + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} { + if {$rowid==300} { + execsql { DELETE FROM n4 WHERE rowid=300 } + } + lappend res $rowid + } + set res +} {500 400 300 200 100} + +do_test 4.3 { + execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') } + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} { + if {$rowid==300} { + execsql { DELETE FROM n4 } + } + lappend res $rowid + } + set res +} {500 400 300} + + + +finish_test diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test new file mode 100644 index 0000000000000000000000000000000000000000..e4e4f6b844e4443b336ddaae8475a20b0b8eff25 --- /dev/null +++ b/ext/fts5/test/fts5rowid.test @@ -0,0 +1,219 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the scalar fts5_rowid() and fts5_decode() functions. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5rowid + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_catchsql_test 1.1 { + SELECT fts5_rowid() +} {1 {should be: fts5_rowid(subject, ....)}} + +do_catchsql_test 1.2 { + SELECT fts5_rowid('segment') +} {1 {should be: fts5_rowid('segment', segid, pgno))}} + +do_execsql_test 1.3 { + SELECT fts5_rowid('segment', 1, 1) +} {137438953473} + +do_catchsql_test 1.4 { + SELECT fts5_rowid('nosucharg'); +} {1 {first arg to fts5_rowid() must be 'segment'}} + + +#------------------------------------------------------------------------- +# Tests of the fts5_decode() function. +# +reset_db +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); +} {} + +proc rnddoc {n} { + set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc [string map $map [format %.3d [expr int(rand()*100)]]] + } + set doc +} +db func rnddoc rnddoc + +do_execsql_test 2.2 { + WITH r(a, b) AS ( + SELECT rnddoc(6), rnddoc(6) UNION ALL + SELECT rnddoc(6), rnddoc(6) FROM r + ) + INSERT INTO x1 SELECT * FROM r LIMIT 10000; + DELETE FROM x1 WHERE (rowid%2); +} + +set res [db one {SELECT count(*) FROM x1_data}] +do_execsql_test 2.3 { + SELECT count(fts5_decode(rowid, block)) FROM x1_data; +} $res +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 2.4 { + UPDATE x1_data SET block = X''; + SELECT count(fts5_decode(rowid, block)) FROM x1_data; +} $res + +do_execsql_test 2.5 { + INSERT INTO x1(x1, rank) VALUES('pgsz', 1024); + INSERT INTO x1(x1) VALUES('rebuild'); +} + +set res [db one {SELECT count(*) FROM x1_data}] +do_execsql_test 2.6 { + SELECT count(fts5_decode(rowid, block)) FROM x1_data; +} $res + +# This is really a corruption test... +#do_execsql_test 2.7 { +# UPDATE x1_data SET block = X''; +# SELECT count(fts5_decode(rowid, block)) FROM x1_data; +#} $res + +do_execsql_test 2.8 { + SELECT fts5_decode(fts5_rowid('segment', 1000, 1), X'AB') +} {corrupt} + +#------------------------------------------------------------------------- +# Tests with very large tokens. +# +set strlist [list \ + "[string repeat x 400]" \ + "[string repeat x 300][string repeat w 100]" \ + "[string repeat x 300][string repeat y 100]" \ + "[string repeat x 300][string repeat z 600]" \ +] +do_test 3.0 { + execsql { + BEGIN; + CREATE VIRTUAL TABLE x2 USING fts5(a); + } + foreach str $strlist { execsql { INSERT INTO x2 VALUES($str) } } + execsql COMMIT +} {} + +for {set tn 0} {$tn<[llength $strlist]} {incr tn} { + set str [lindex $strlist $tn] + do_execsql_test 3.1.$tn { + SELECT rowid FROM x2 WHERE x2 MATCH $str + } [expr $tn+1] +} + +set res [db one {SELECT count(*) FROM x2_data}] +do_execsql_test 3.2 { + SELECT count(fts5_decode(rowid, block)) FROM x2_data; +} $res + +#------------------------------------------------------------------------- +# Leaf pages with no terms or rowids at all. +# +set strlist [list \ + "[string repeat {w } 400]" \ + "[string repeat {x } 400]" \ + "[string repeat {y } 400]" \ + "[string repeat {z } 400]" \ +] +do_test 4.0 { + execsql { + BEGIN; + CREATE VIRTUAL TABLE x3 USING fts5(a); + INSERT INTO x3(x3, rank) VALUES('pgsz', 32); + } + foreach str $strlist { execsql { INSERT INTO x3 VALUES($str) } } + execsql COMMIT +} {} + +for {set tn 0} {$tn<[llength $strlist]} {incr tn} { + set str [lindex $strlist $tn] + do_execsql_test 4.1.$tn { + SELECT rowid FROM x3 WHERE x3 MATCH $str + } [expr $tn+1] +} + +set res [db one {SELECT count(*) FROM x3_data}] +do_execsql_test 4.2 { + SELECT count(fts5_decode(rowid, block)) FROM x3_data; +} $res + +#------------------------------------------------------------------------- +# Position lists with large values. +# +set strlist [list \ + "[string repeat {w } 400]a" \ + "[string repeat {x } 400]a" \ + "[string repeat {y } 400]a" \ + "[string repeat {z } 400]a" \ +] +do_test 5.0 { + execsql { + BEGIN; + CREATE VIRTUAL TABLE x4 USING fts5(a); + INSERT INTO x4(x4, rank) VALUES('pgsz', 32); + } + foreach str $strlist { execsql { INSERT INTO x4 VALUES($str) } } + execsql COMMIT +} {} + +do_execsql_test 5.1 { + SELECT rowid FROM x4 WHERE x4 MATCH 'a' +} {1 2 3 4} + +set res [db one {SELECT count(*) FROM x4_data}] +do_execsql_test 5.2 { + SELECT count(fts5_decode(rowid, block)) FROM x4_data; +} $res + +#------------------------------------------------------------------------- +# + +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x5 USING fts5(x, detail=none); + INSERT INTO x5(x5, rank) VALUES('pgsz', 32); + INSERT INTO x5 VALUES('a b c d e f'); + INSERT INTO x5 VALUES('a b c d e f'); + INSERT INTO x5 VALUES('a b c d e f'); + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) INSERT INTO x5 SELECT 'a b c d e f' FROM s; + COMMIT; + SELECT count(fts5_decode_none(rowid, block)) FROM x5_data; +} {32} + +do_execsql_test 6.1 { + DELETE FROM x5 WHERE rowid <= 2; + SELECT count(fts5_decode_none(rowid, block)) FROM x5_data; +} {34} + +do_execsql_test 6.2 { + UPDATE x5 SET x='a b c d e f' WHERE rowid=3; + SELECT count(fts5_decode_none(rowid, block)) FROM x5_data; +} {36} + +#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r} + + + +finish_test diff --git a/ext/fts5/test/fts5savepoint.test b/ext/fts5/test/fts5savepoint.test new file mode 100644 index 0000000000000000000000000000000000000000..fdb0a25ba53366499ae4dabfb3cd3b125fb03833 --- /dev/null +++ b/ext/fts5/test/fts5savepoint.test @@ -0,0 +1,85 @@ +# 2019 Dec 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5savepoint + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(c); + BEGIN; + SAVEPOINT one; + INSERT INTO ft VALUES('a'); + SAVEPOINT two; + INSERT INTO ft VALUES('b'); + RELEASE two; + SAVEPOINT four; + INSERT INTO ft VALUES('c'); + RELEASE four; + SAVEPOINT three; + INSERT INTO ft VALUES('d'); + ROLLBACK TO three; + COMMIT; + SELECT * FROM ft +} {a b c} + +reset_db +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(c); + CREATE VIRTUAL TABLE ft2 USING fts5(c); + DROP TABLE ft2_idx; + BEGIN; + INSERT INTO ft2 VALUES('a'); + INSERT INTO ft1 VALUES('a'); + SAVEPOINT two; + INSERT INTO ft1 VALUES('b'); + COMMIT; +} {1 {SQL logic error}} + +reset_db +ifcapable fts3 { + do_execsql_test 3.0 { + CREATE VIRTUAL TABLE vt0 USING fts5(c0); + CREATE VIRTUAL TABLE vt1 USING fts4(c0); + INSERT INTO vt1(c0) VALUES(0); + } + + do_execsql_test 3.1 { + BEGIN; + UPDATE vt1 SET c0 = 0; + INSERT INTO vt1(c0) VALUES (0), (0); + UPDATE vt0 SET c0 = 0; + INSERT INTO vt1(c0) VALUES (0); + UPDATE vt1 SET c0 = 0; + INSERT INTO vt1(vt1) VALUES('automerge=1'); + UPDATE vt1 SET c0 = 0; + } + + do_catchsql_test 3.2 { + DROP TABLE vt1; + } {0 {}} + + do_execsql_test 3.3 { + SAVEPOINT x; + INSERT INTO vt0 VALUES('x'); + COMMIT; + INSERT INTO vt0(vt0) VALUES('integrity-check'); + } +} + +finish_test + diff --git a/ext/fts5/test/fts5secure.test b/ext/fts5/test/fts5secure.test new file mode 100644 index 0000000000000000000000000000000000000000..73149461629e276adb0510749026c7e0acc74a60 --- /dev/null +++ b/ext/fts5/test/fts5secure.test @@ -0,0 +1,348 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure + +proc dump {tname} { + execsql_pp "SELECT * FROM ${tname}_idx" + execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data" +} + + +do_execsql_test 0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + CREATE VIRTUAL TABLE v1 USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, ab) VALUES + (0,'abc'), (1,'abc'), (2,'abc'), (3,'abc'), (4,'def'); +} + +do_execsql_test 0.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 0.2 { + DELETE FROM t1 WHERE rowid=2; +} + +do_execsql_test 0.3 { + SELECT count(*) FROM t1_data +} 3 + +do_execsql_test 0.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 0.5 { + DELETE FROM t1 WHERE rowid=3; +} + +do_execsql_test 0.6 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 0.7 { + DELETE FROM t1 WHERE rowid=0; +} + +do_execsql_test 0.8 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#---------------------------------- + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(ab); + INSERT INTO t2(rowid, ab) VALUES (5, 'key'), (6, 'value'); + INSERT INTO t2(t2, rank) VALUES('secure-delete', 1); +} + +#execsql_pp { SELECT id, quote(block) FROM t1_data } +#execsql_pp { SELECT segid, quote(term), pgno FROM t1_idx } + +do_execsql_test 1.1 { + DELETE FROM t2 WHERE rowid = 5; +} + +do_execsql_test 1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 1.3 { + DELETE FROM t2 WHERE rowid = 6; +} + +do_execsql_test 1.4 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 1.5 { + SELECT * FROM t2('value'); + SELECT * FROM t2('v*'); +} + +do_execsql_test 1.6 { + SELECT * FROM t2('value') ORDER BY rowid DESC; + SELECT * FROM t2('v*') ORDER BY rowid DESC; +} +execsql_pp { + SELECT id, quote(block) FROM t2_data; +} + +#---------------------------------- + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(ab); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', 'instance'); + INSERT INTO ft(rowid, ab) VALUES + (1, 'one'), + (2, 'two'), + (3, 'three'), + (4, 'four'), + (5, 'one one'), + (6, 'one two'), + (7, 'one three'), + (8, 'one four'), + (9, 'two one'), + (10, 'two two'), + (11, 'two three'), + (12, 'two four'), + (13, 'three one'), + (14, 'three two'), + (15, 'three three'), + (16, 'three four'); +} + +do_execsql_test 2.1 { + SELECT count(*) FROM ft_data; +} {3} + +do_execsql_test 2.2 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.3 { + DELETE FROM ft WHERE rowid=9; +} + +do_execsql_test 2.4 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.5 { + DELETE FROM ft WHERE ab LIKE '%two%' +} + +do_execsql_test 2.6 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.7 { + SELECT count(*) FROM ft_data; +} {3} + +#---------------------------------- +reset_db + +set ::vocab { + one two three four five six seven eight nine ten + eleven twelve thirteen fourteen fifteen sixteen + seventeen eighteen nineteen twenty +} +proc rnddoc {} { + set nVocab [llength $::vocab] + set ret [list] + for {set ii 0} {$ii < 8} {incr ii} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * $nVocab)]] + } + set ret +} + +proc contains {list val} { + expr {[lsearch $list $val]>=0} +} + +foreach {tn pgsz} { + 2 64 + 1 1000 +} { + reset_db + db function rnddoc rnddoc + db function contains contains + + expr srand(1) + + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + WITH s(i) AS ( + VALUES(1) UNION SELECT i+1 FROM s WHERE i<20 + ) + INSERT INTO t1 SELECT rnddoc() FROM s; + } + + do_execsql_test 3.$tn.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + foreach {rowid} { + 6 16 3 4 9 14 13 7 20 15 19 10 11 2 5 18 17 1 12 8 + } { + + do_execsql_test 3.$tn.2.$rowid { + DELETE FROM t1 WHERE rowid=$rowid; + } + do_execsql_test 3.$tn.2.$rowid.ic { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + foreach v $::vocab { + do_execsql_test 3.$tn.2.$rowid.q.$v { + SELECT rowid FROM t1($v) + } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v)}] + + do_execsql_test 3.$tn.2.$rowid.q.$v.DESC { + SELECT rowid FROM t1($v) ORDER BY 1 DESC + } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v) ORDER BY 1 DESC}] + } + } +} + +do_execsql_test 3.3 { + INSERT INTO t1(x) VALUES('optimize'); + INSERT INTO t1(t1) VALUES('optimize'); + SELECT count(*) FROM t1_data; +} {3} + +#---------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +set L1 [string repeat abcdefghij 10] +set L2 [string repeat 1234567890 10] + +do_execsql_test 4.1 { + INSERT INTO t1 VALUES('aa' || $L1 || ' ' || $L2); +} +do_execsql_test 4.2 { + DELETE FROM t1 WHERE rowid=1 +} +do_execsql_test 4.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#---------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +set doc "aa [string repeat {abc } 60]" + +do_execsql_test 5.1 { + BEGIN; + INSERT INTO t1 VALUES($doc); + INSERT INTO t1 VALUES('aa abc'); + COMMIT; +} + +do_execsql_test 5.2 { + DELETE FROM t1 WHERE rowid = 1; +} + +do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2 +do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2 + +#------------------------------------------------------------------------- +# Tests for the bug fixed by https://sqlite.org/src/info/4b60a1c3 +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE fts USING fts5(content); + INSERT INTO fts(fts, rank) VALUES ('secure-delete', 1); + INSERT INTO fts(rowid, content) VALUES + (3407, 'profile profile profile profile profile profile profile profile pull pulling pulling really'); + DELETE FROM fts WHERE rowid IS 3407; + INSERT INTO fts(fts) VALUES ('integrity-check'); +} + +foreach {tn detail} { + 1 full + 2 column + 3 none +} { + do_execsql_test 6.1.$detail " + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=$detail); + " + + do_execsql_test 6.2.$detail { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + for {set ii 1} {$ii < 100} {incr ii} { + do_execsql_test 6.3.$detail.$ii.1 { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'word1'); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i0} { + set idx [expr int(abs(rand()) * [llength $in])] + lappend out [lindex $in $idx] + set in [lreplace $in $idx $idx] + } + set out + } + + #dump fff + + set iTest 1 + foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { + #if {$iTest==1} { dump fff } + #if {$iTest==1} { breakpoint } + do_execsql_test 3.$tn.1.$iTest.$ii { + DELETE FROM fff WHERE rowid=$ii; + } + #if {$iTest==1} { dump fff } + if {($iTest % 20)==0} { + do_execsql_test 3.$tn.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } + } + #if {$iTest==1} { break } + incr iTest + } +} + +#execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC } +#breakpoint +#execsql_pp { +# SELECT rowid FROM fff('post') ORDER BY rowid DESC +#} +# +#dump fff + + +finish_test + diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test new file mode 100644 index 0000000000000000000000000000000000000000..7588a34683fcd6c130fc5e6125ed6a4fbac2615f --- /dev/null +++ b/ext/fts5/test/fts5secure4.test @@ -0,0 +1,170 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set ::testprefix fts5secure4 + +#------------------------------------------------------------------------- +# Test using the 'delete' command to attempt to delete a token that +# is not present in the index in secure-delete mode. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, content=x1); + + CREATE TABLE x1(rowid INTEGER PRIMARY KEY, a, b); + INSERT INTO x1 VALUES + (1, 'hello world', 'today xyz'), + (2, 'not the day', 'crunch crumble and chomp'), + (3, 'one', 'two'); + INSERT INTO t1(t1) VALUES('rebuild'); +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.2 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 4, 'nosuchtoken', ''); +} + +do_execsql_test 1.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 1.4 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'crunch', ''); +} + +do_execsql_test 1.5 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 3, 'crunch', ''); +} + +do_execsql_test 1.6 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 1.7 { +CREATE VIRTUAL TABLE y1 USING fts5(xx, prefix='1,2'); +INSERT INTO y1(y1, rank) VALUES('pgsz', 64); +INSERT INTO y1(y1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 1.8 { + BEGIN; + INSERT INTO y1(rowid, xx) VALUES(1, 'abc def'); + INSERT INTO y1(rowid, xx) VALUES(2, 'reallyreallylongtoken'); + COMMIT; +} +do_execsql_test 1.9 { + DELETE FROM y1 WHERE rowid=1; + INSERT INTO y1(y1) VALUES('integrity-check'); +} + +do_execsql_test 1.10 { + CREATE VIRTUAL TABLE w1 USING fts5(ww, content=""); + INSERT INTO w1(rowid, ww) VALUES(123, ''); +} +do_catchsql_test 1.11 { + INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz'); +} {1 {database disk image is malformed}} +do_catchsql_test 1.12 { + DROP TABLE w1; + CREATE VIRTUAL TABLE w1 USING fts5(ww, content=""); + INSERT INTO w1(rowid, ww) VALUES(123, ''); + DELETE FROM w1_data WHERE id>10; + INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# Test using secure-delete with detail=none or detail=col. +# +foreach {tn d} {1 full 2 none 3 column} { + reset_db + do_execsql_test 2.$tn.1 " + CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20'); + INSERT INTO x1(x1, rank) VALUES('pgsz', 64); + INSERT INTO x1(x1, rank) VALUES('secure-delete', 1); + " + + do_execsql_test 2.$tn.2 { + BEGIN; + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + COMMIT; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.3 { + DELETE FROM x1 WHERE rowid IN (2, 4, 6); + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.4 { + DELETE FROM x1 WHERE rowid IN (1, 3, 5); + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.5 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO x1 + SELECT 'seems to be', 'used brew to', 'everything is working' FROM s + UNION ALL + SELECT 'used brew to', 'everything is working', 'seems to be' FROM s + UNION ALL + SELECT 'everything is working', 'seems to be', 'used brew to' FROM s + UNION ALL + SELECT 'abc', 'zzz', 'a b c d' + UNION ALL + SELECT 'z', 'z', 'z' FROM s + } + + do_test 2.$tn.6 { + for {set i 300} {$i > 200} {incr i -1} { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_test 2.$tn.7 { + for {set i 1} {$i < 100} {incr i} { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_test 2.$tn.8 { + foreach i [db eval {SELECT rowid FROM x1}] { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_execsql_test 2.$tn.9 { + SELECT * FROM x1 + } {} +} + + + +finish_test + diff --git a/ext/fts5/test/fts5secure5.test b/ext/fts5/test/fts5secure5.test new file mode 100644 index 0000000000000000000000000000000000000000..ca8570211cb62bc9730f59dfa150b04624c17abf --- /dev/null +++ b/ext/fts5/test/fts5secure5.test @@ -0,0 +1,129 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set ::testprefix fts5secure5 +return_if_no_fts5 + +proc dump {} { + execsql_pp { + SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft1(rowid, a) VALUES(1, 'abcd'); + INSERT INTO ft1(rowid, a) VALUES(2, 'abcd'); + INSERT INTO ft1(rowid, a) VALUES(3, 'abcd'); + COMMIT; +} +do_execsql_test 1.2 { + DELETE FROM ft1 WHERE rowid=1; +} +do_execsql_test 1.3 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 1.4 { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 1.5 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 1.6 { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 1.7 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one'); + INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three'); + COMMIT; +} + +do_execsql_test 2.2a { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 2.2b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 2.3a { + DELETE FROM ft1 WHERE rowid=2; +} +do_execsql_test 2.3b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 2.4a { + DELETE FROM ft1 WHERE rowid=4; +} +do_execsql_test 2.4b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); +} +do_execsql_test 3.1 { + BEGIN; + INSERT INTO ft1(a) VALUES('c'); + COMMIT; +} +do_execsql_test 3.2 { + DELETE FROM ft1 WHERE rowid IN (1); + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 + ) + INSERT INTO ft1 SELECT 'abcdefg' FROM s; +} + +do_test 4.1 { + for {set i 500} {$i > 0} {incr i -1} { + execsql { DELETE FROM ft1 WHERE rowid=$i } + execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') } + } +} {} + +finish_test + diff --git a/ext/fts5/test/fts5secure6.test b/ext/fts5/test/fts5secure6.test new file mode 100644 index 0000000000000000000000000000000000000000..e2f4ceabc8d5bf0778745d7b1015f56e8a98dc3e --- /dev/null +++ b/ext/fts5/test/fts5secure6.test @@ -0,0 +1,141 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure6 + +db progress 1 progress_handler +set ::PHC 0 +proc progress_handler {args} { + incr ::PHC + # if {($::PHC % 100000)==0} breakpoint + return 0 +} + +proc setup {} { + db eval { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1 SELECT 'a b c d e f g h i j k' FROM s; + } +} + +foreach {tn sd} { + 1 0 + 2 1 +} { + setup + do_execsql_test 1.$tn.0 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) + } + set PHC 0 + do_execsql_test 1.$tn.1 { DELETE FROM t1; } + set phc($tn) $PHC +} + +do_test 1.3 { + expr $phc(1)*5 < $phc(2) +} {1} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) +} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(-100000, 'abc def ghi'); + INSERT INTO t1(rowid, x) VALUES(-99999, 'abc def ghi'); + INSERT INTO t1(rowid, x) VALUES(9223372036854775800, 'abc def ghi'); + COMMIT; +} + +do_execsql_test 2.2 { + SELECT rowid FROM t1('def') +} {-100000 -99999 9223372036854775800} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) +} + +do_execsql_test 3.1 { + BEGIN; + INSERT INTO t1(rowid, x) + VALUES(51869, 'when whenever where weress what turn'), + (51871, 'to were'); + COMMIT; +} + +do_execsql_test 3.2 { + DELETE FROM t1 WHERE rowid=51871; + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(rowid, x) VALUES(10, 'one two'); +} +do_execsql_test 4.1 { + UPDATE t1 SET x = 'one three' WHERE rowid=10; + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 4.2 { + DELETE FROM t1 WHERE rowid=10; +} +do_execsql_test 4.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(content); + + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); + INSERT INTO t1 VALUES('active'),('boomer'),('atom'),('atomic'), + ('alpha channel backup abandon test aback boomer atom alpha active'); + DELETE FROM t1 WHERE t1 MATCH 'abandon'; +} + +do_execsql_test 5.1 { + INSERT INTO t1(t1) VALUES('rebuild'); +} + +do_execsql_test 5.2 { + DELETE FROM t1 WHERE rowid NOTNULL<5; +} + +db close +sqlite3 db test.db + +do_execsql_test 5.3 { + PRAGMA integrity_check; +} {ok} + + +finish_test + diff --git a/ext/fts5/test/fts5secure7.test b/ext/fts5/test/fts5secure7.test new file mode 100644 index 0000000000000000000000000000000000000000..16a044f5389005a4c10f8513cc4340469abdcc6e --- /dev/null +++ b/ext/fts5/test/fts5secure7.test @@ -0,0 +1,116 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# TESTRUNNER: slow +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure7 + + +set NVOCAB 500 +set NDOC [expr 1000] + +set NREP 100 +set nDeletePerRep [expr 5] + +set VOCAB [list] + +proc select_one {list} { + set n [llength $list] + lindex $list [expr {abs(int(rand()*$n))}] +} + +proc init_vocab {} { + set L [split "abcdefghijklmnopqrstuvwxyz" {}] + set nL [llength $L] + for {set i 0} {$i < $::NVOCAB} {incr i} { + set n [expr {6 + int(rand()*8)}] + set word "" + for {set j 0} {$j < $n} {incr j} { + append word [select_one $L] + } + lappend ::VOCAB $word + } +} + +proc get_word {} { + select_one $::VOCAB +} + +proc get_document {nWord} { + set ret [list] + for {set i 0} {$i < $nWord} {incr i} { + lappend ret [get_word] + } + return $ret +} + +init_vocab + +db func document [list get_document 12] + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(body); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 1.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC + ) + INSERT INTO t1 SELECT document() FROM s; +} + +for {set iRep 0} {$iRep < $NREP} {incr iRep} { + set lRowid [db eval {SELECT rowid FROM t1}] + for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} { + set idx [select_one $lRowid] + db eval { + DELETE FROM t1 WHERE rowid=$idx + } + } + db eval { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep + ) + INSERT INTO t1 SELECT document() FROM s; + } + do_execsql_test 1.2.$iRep { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} + +reset_db +db func document [list get_document 12] +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(body); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + INSERT INTO t1(t1, rank) VALUES('pgsz', 128); +} +do_execsql_test 2.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC + ) + INSERT INTO t1 SELECT document() FROM s; +} +for {set ii 0} {$ii < $NDOC} {incr ii} { + set lRowid [db eval {SELECT rowid FROM t1}] + set idx [select_one $lRowid] + db eval { DELETE FROM t1 WHERE rowid=$idx } + do_execsql_test 2.2.$ii { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} + +finish_test + + diff --git a/ext/fts5/test/fts5secure8.test b/ext/fts5/test/fts5secure8.test new file mode 100644 index 0000000000000000000000000000000000000000..8b65b7c59f478e50b1f87fbf496ba13ae4e2fd53 --- /dev/null +++ b/ext/fts5/test/fts5secure8.test @@ -0,0 +1,71 @@ +# 2023 Nov 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure8 + +proc sql_repeat {txt n} { + string repeat $txt $n +} +db func repeat sql_repeat + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + + INSERT INTO ft(rowid, x) VALUES(100, 'hello world'); + INSERT INTO ft(rowid, x) VALUES(200, 'one day'); + + BEGIN; + INSERT INTO ft(rowid, x) VALUES(45, 'one two three'); + UPDATE ft SET x = repeat('hello world ', 500) WHERE rowid=100; + COMMIT +} + +do_execsql_test 1.1 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); + DELETE FROM ft WHERE rowid=100; +} + +do_execsql_test 1.2 { + PRAGMA integrity_check; +} {ok} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5 ( + name, + content='' + ); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + INSERT INTO xyz (rowid, name) VALUES(1, 'A'); + INSERT INTO xyz (rowid, name) VALUES(2, 'A'); + INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A'); +} + +do_execsql_test 2.1 { + pragma quick_check; +} {ok} + +do_catchsql_test 2.2 { + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 'hello world'); +} {1 {SQL logic error}} + + + + + +finish_test + + diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test new file mode 100644 index 0000000000000000000000000000000000000000..2959ab65ce89bb25f3d577196ed2dd89ace7df1e --- /dev/null +++ b/ext/fts5/test/fts5securefault.test @@ -0,0 +1,225 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5securefault + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +return_if_no_fts5 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(rowid, ab) VALUES + (0, 'abc'), (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'def'); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid=2 } +} -test { + faultsim_test_result {0 {}} +} +do_faultsim_test 1.2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid IN(0, 1, 2, 3, 4) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set big [string repeat abcdefghij 5] +set big2 [string repeat klmnopqrst 5] +set doc "$big $big2" + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<4 + ) + INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s; +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 3 } + execsql { DELETE FROM t1 WHERE rowid = 4 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set big [string repeat abcdefghij 5] +set big2 [string repeat klmnopqrst 5] +set doc "$big $big2" + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<25 + ) + INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s; + + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + DELETE FROM t1 WHERE rowid BETWEEN 3 AND 23; +} +faultsim_save_and_close + +do_faultsim_test 3.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 24 } + execsql { DELETE FROM t1 WHERE rowid = 25 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set doc [string repeat "tok " 400] + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1(rowid, ab) VALUES(1, $doc), (2, $doc), (3, $doc); +} +faultsim_save_and_close + +do_faultsim_test 4.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 2 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db + +set doc1 [string repeat "abc " 10] +set doc2 [string repeat "def " 10] + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + BEGIN; + } + for {set i 0} {$i < 50} {incr i} { + execsql { + INSERT INTO t1(rowid, ab) VALUES($i, 'abcdefg'); + } + } + execsql { + INSERT INTO t1(rowid, ab) VALUES(105, 'def'); + COMMIT; + } +} {} +faultsim_save_and_close + +do_faultsim_test 5.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 105 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + BEGIN; + INSERT INTO t1(rowid, ab) VALUES(1, 'abcdefg'); + INSERT INTO t1(rowid, ab) VALUES(2, 'abcdefg'); + INSERT INTO t1(rowid, ab) VALUES(3, 'abcdefg'); + COMMIT; + } +} {} +faultsim_save_and_close + +do_faultsim_test 6.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { + UPDATE t1 SET ab='abcdefg' WHERE rowid=2; + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +do_test 7.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} {} +faultsim_save_and_close + +do_faultsim_test 7.1 -faults oom* -prep { + faultsim_restore_and_reopen + set big1 "[string repeat x 50] [string repeat y 50] [string repeat z 50]" + execsql { + BEGIN; + INSERT INTO t1 VALUES($big1); + } +} -body { + execsql { COMMIT } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test new file mode 100644 index 0000000000000000000000000000000000000000..ad59bf0d9e660c4e13663ef35a47010eda21eae7 --- /dev/null +++ b/ext/fts5/test/fts5simple.test @@ -0,0 +1,512 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if 1 { + +#------------------------------------------------------------------------- +# +set doc "x x [string repeat {y } 50]z z" +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO t1 VALUES($doc); + COMMIT; +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('d e f'); + INSERT INTO t1(t1) VALUES('optimize'); +} + +do_execsql_test 2.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, prefix='1,2'); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO t1 VALUES('one'); + SELECT * FROM t1 WHERE t1 MATCH 'o*'; +} {one} + +do_execsql_test 3.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t11 USING fts5(content); + INSERT INTO t11(t11, rank) VALUES('pgsz', 32); + INSERT INTO t11 VALUES('another'); + INSERT INTO t11 VALUES('string'); + INSERT INTO t11 VALUES('of'); + INSERT INTO t11 VALUES('text'); +} +do_test 4.2 { + execsql { INSERT INTO t11(t11) VALUES('optimize') } +} {} +do_execsql_test 4.3 { + INSERT INTO t11(t11) VALUES('integrity-check'); +} {} + +#db eval { SELECT fts5_decode(rowid, block) as x FROM t11_data } { puts $x } + +#------------------------------------------------------------------------- +reset_db +set doc [string repeat "x y " 5] +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE yy USING fts5(content); + INSERT INTO yy(yy, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + COMMIT; +} + +do_execsql_test 5.2 { + SELECT rowid FROM yy WHERE yy MATCH 'y' ORDER BY rowid ASC +} {1 2 3 4 5 6 7 8} + +do_execsql_test 5.3 { + SELECT rowid FROM yy WHERE yy MATCH 'y' ORDER BY rowid DESC +} {8 7 6 5 4 3 2 1} + +#db eval { SELECT fts5_decode(rowid, block) as x FROM yy_data } { puts $x } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE tt USING fts5(content); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + INSERT INTO tt VALUES('aa'); +} + +do_execsql_test 5.2 { + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1} + +do_execsql_test 5.3 { + DELETE FROM tt; + BEGIN; + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + COMMIT; +} {} + +do_execsql_test 5.4 { + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1 2} + +do_execsql_test 5.5 { + DELETE FROM tt; + BEGIN; + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + COMMIT; + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1 2 3 4 5 6 7 8} + +do_execsql_test 5.6 { + INSERT INTO tt(tt) VALUES('integrity-check'); +} + +reset_db +do_execsql_test 5.7 { + CREATE VIRTUAL TABLE tt USING fts5(content); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + INSERT INTO tt VALUES('aa ab ac ad ae af'); +} + +do_execsql_test 5.8 { + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1} + +#------------------------------------------------------------------------- + +reset_db +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE xyz USING fts5(x, y, z); + INSERT INTO xyz VALUES('x', 'y', 'z'); +} + +do_catchsql_test 6.2 { + SELECT * FROM xyz WHERE xyz MATCH '' +} {1 {fts5: syntax error near ""}} +do_catchsql_test 6.3 { + SELECT * FROM xyz WHERE xyz MATCH NULL +} {1 {fts5: syntax error near ""}} + +#------------------------------------------------------------------------- + +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b c'); + INSERT INTO ft2(rowid, content) VALUES(2, 'a b d'); +} + +do_catchsql_test 7.2 { + BEGIN; + UPDATE ft2 SET rowid=2 WHERE rowid=1; +} {1 {constraint failed}} + +do_execsql_test 7.3 { + COMMIT; + INSERT INTO ft2(ft2) VALUES('integrity-check'); +} {} + +do_execsql_test 7.4 { + SELECT * FROM ft2; +} {{a b c} {a b d}} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b'); +} + +do_execsql_test 8.2 { + BEGIN; + INSERT INTO ft2(rowid, content) VALUES(4, 'a x'); +} + +do_execsql_test 8.3 { + INSERT INTO ft2(ft2) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Check that the "table function" syntax works. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b'); + INSERT INTO ft2(rowid, content) VALUES(2, 'a b c d'); + INSERT INTO ft2(rowid, content) VALUES(3, 'c d e f'); +} + +do_execsql_test 9.2 { + SELECT rowid FROM ft2('a'); +} {1 2} + +do_execsql_test 9.3 { + SELECT rowid FROM ft2('b AND c'); +} {2} + +#------------------------------------------------------------------------- +# +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, b, c); + INSERT INTO t3 VALUES('bac aab bab', 'c bac c', 'acb aba abb'); -- 1 + INSERT INTO t3 VALUES('bab abc c', 'acb c abb', 'c aaa c'); -- 2 +} + +do_execsql_test 10.1 { + SELECT rowid FROM t3('c: c*'); +} {2} + +do_execsql_test 10.2 { + SELECT rowid FROM t3('b: acb'); +} {2} + +#------------------------------------------------------------------------- +# Test that character 0x1A is allowed in fts5 barewords. +# +do_test 11.0 { + execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")" + execsql " + INSERT INTO t4 VALUES('a b c \x1A'); + INSERT INTO t4 VALUES('a b c d\x1A'); + INSERT INTO t4 VALUES('a b c \x1Ag'); + INSERT INTO t4 VALUES('a b c d'); + " +} {} + +do_test 11.1 { + execsql "SELECT rowid FROM t4('\x1A')" +} {1} +do_test 11.2 { + execsql "SELECT rowid FROM t4('\x1A*')" +} {1 3} +do_test 11.3 { + execsql "SELECT rowid FROM t4('d\x1A')" +} {2} + +do_test 11.4 { + catchsql "SELECT rowid FROM t4('d\x1B')" +} {/fts5: syntax error/} +do_test 11.5 { + catchsql "SELECT rowid FROM t4('d\x19')" +} {/fts5: syntax error/} + +#------------------------------------------------------------------------- +# +do_test 12.1 { + execsql { + CREATE VIRTUAL TABLE xx USING fts5(x,y); + BEGIN; + INSERT INTO xx VALUES('1 2 3', 'a b c'); + } +} {} + +do_execsql_test 12.2 { + SELECT rowid FROM xx('x:a'); + COMMIT; +} {} + +#------------------------------------------------------------------------- +# Try an UPDATE OR REPLACE query. +# +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE xy USING fts5(x); + INSERT INTO xy(rowid, x) VALUES(1, '1 2 3'); + INSERT INTO xy(rowid, x) VALUES(2, '2 3 4'); + INSERT INTO xy(rowid, x) VALUES(3, '3 4 5'); +} + +do_execsql_test 13.2 { + UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2; + SELECT rowid, x FROM xy; +} { + 1 {1 2 3} + 3 {2 3 4} +} + +do_execsql_test 13.3 { + INSERT INTO xy(xy) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE ttt USING fts5(x); + BEGIN; + INSERT INTO ttt(rowid, x) VALUES(1, 'a b c'); + INSERT INTO ttt(rowid, x) VALUES(2, 'a b c'); + INSERT INTO ttt(rowid, x) VALUES(3, 'a b c'); + COMMIT; +} +do_test 14.2 { + fts5_level_segs ttt +} {1} + +#------------------------------------------------------------------------- +db func rnddoc fts5_rnddoc +do_execsql_test 14.3 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + + WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 ) + INSERT INTO x1 SELECT rnddoc(5) FROM ii; +} + +do_execsql_test 14.4 { + SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' +} {0 {} 2} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 15.0 { + CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1); + INSERT INTO x2 VALUES('ab'); +} + +do_execsql_test 15.1 { + INSERT INTO x2(x2) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +foreach_detail_mode $testprefix { + reset_db + fts5_aux_test_functions db + do_execsql_test 16.0 { + CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); + INSERT INTO x3 VALUES('a b c d e f'); + } + do_execsql_test 16.1 { + SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); + } {2.0.2} + + do_execsql_test 16.1 { + SELECT fts5_test_poslist(x3) FROM x3('a OR c'); + } {{0.0.0 1.0.2}} +} + +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 17.0 { + CREATE VIRTUAL TABLE x3 USING fts5(x); + INSERT INTO x3 VALUES('a b c'); +} + +do_execsql_test 17.1 { + SELECT rowid FROM x3('b AND d'); +} + +#------------------------------------------------------------------------- +do_execsql_test 18.1 { + CREATE VIRTUAL TABLE x4 USING fts5(x); + SELECT rowid FROM x4('""'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 19.1 { + CREATE VIRTUAL TABLE x1 USING fts5(a,b,c); +} + +do_catchsql_test 19.2 { + SELECT * FROM x1 WHERE x1 MATCH 'c0 AND (c1 AND (c2 AND (c3 AND (c4 AND (c5 AND (c6 AND (c7 AND (c8 AND (c9 AND (c10 AND (c11 AND (c12 AND (c13 AND (c14 AND (c15 AND (c16 AND (c17 AND (c18 AND (c19 AND (c20 AND (c21 AND (c22 AND (c23 AND (c24 AND (c25 AND (c26 AND (c27 AND (c28 AND (c29 AND (c30 AND (c31 AND (c32 AND (c33 AND (c34 AND (c35 AND (c36 AND (c37 AND (c38 AND (c39 AND (c40 AND (c41 AND (c42 AND (c43 AND (c44 AND (c45 AND (c46 AND (c47 AND (c48 AND (c49 AND (c50 AND (c51 AND (c52 AND (c53 AND (c54 AND (c55 AND (c56 AND (c57 AND (c58 AND (c59 AND (c60 AND (c61 AND (c62 AND (c63 AND (c64 AND (c65 AND (c66 AND (c67 AND (c68 AND (c69 AND (c70 AND (c71 AND (c72 AND (c73 AND (c74 AND (c75 AND (c76 AND (c77 AND (c78 AND (c79 AND (c80 AND (c81 AND (c82 AND (c83 AND (c84 AND (c85 AND (c86 AND (c87 AND (c88 AND (c89 AND (c90 AND (c91 AND (c92 AND (c93 AND (c94 AND (c95 AND (c96 AND (c97 AND (c98 AND (c99 AND (c100 AND (c101 AND (c102 AND (c103 AND (c104 AND (c105 AND (c106 AND (c107 AND (c108 AND (c109 AND (c110 AND (c111 AND (c112 AND (c113 AND (c114 AND (c115 AND (c116 AND (c117 AND (c118 AND (c119 AND (c120 AND (c121 AND (c122 AND (c123 AND (c124 AND (c125 AND (c126 AND (c127 AND (c128 AND (c129 AND (c130 AND (c131 AND (c132 AND (c133 AND (c134 AND (c135 AND (c136 AND (c137 AND (c138 AND (c139 AND (c140 AND (c141 AND (c142 AND (c143 AND (c144 AND (c145 AND (c146 AND (c147 AND (c148 AND (c149 AND (c150 AND (c151 AND (c152 AND (c153 AND (c154 AND (c155 AND (c156 AND (c157 AND (c158 AND (c159 AND (c160 AND (c161 AND (c162 AND (c163 AND (c164 AND (c165 AND (c166 AND (c167 AND (c168 AND (c169 AND (c170 AND (c171 AND (c172 AND (c173 AND (c174 AND (c175 AND (c176 AND (c177 AND (c178 AND (c179 AND (c180 AND (c181 AND (c182 AND (c183 AND (c184 AND (c185 AND (c186 AND (c187 AND (c188 AND (c189 AND (c190 AND (c191 AND (c192 AND (c193 AND (c194 AND (c195 AND (c196 AND (c197 AND (c198 AND (c199 AND c200)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))'; +} {1 {fts5: parser stack overflow}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree'); +} +do_test 20.1 { + for {set i 1} {$i <= 200} {incr i} { + execsql { INSERT INTO x1(rowid, x) VALUES($i, 'one two three'); } + } + execsql { INSERT INTO x1(x1) VALUES('optimize'); } + execsql { DELETE FROM x1 WHERE rowid = 4; } +} {} +do_execsql_test 20.2 { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x1(x1) VALUES('integrity-check'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree'); +} +do_test 20.1 { + for {set i 1} {$i <= 200} {incr i} { + execsql { INSERT INTO x1(rowid, x) VALUES($i, 'one two three'); } + } + execsql { INSERT INTO x1(x1) VALUES('optimize'); } + execsql { DELETE FROM x1 WHERE rowid = 4; } +} {} +do_execsql_test 20.2 { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x1(x1) VALUES('integrity-check'); +} {} + +#------------------------------------------------------------------------- +reset_db +set doc "a b [string repeat x 100000]" +do_execsql_test 21.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(rowid, x) VALUES(11111, $doc); + INSERT INTO x1(rowid, x) VALUES(11112, $doc); +} +do_execsql_test 21.1 { + INSERT INTO x1(x1) VALUES('integrity-check'); +} +do_execsql_test 21.2 { + SELECT rowid FROM x1($doc); +} {11111 11112} +do_execsql_test 21.3 { + DELETE FROM x1 WHERE rowid=11111; + INSERT INTO x1(x1) VALUES('integrity-check'); + SELECT rowid FROM x1($doc); +} {11112} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 22.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x) VALUES('a b c'); + INSERT INTO x1(x) VALUES('x y z'); + INSERT INTO x1(x) VALUES('c b a'); + INSERT INTO x1(x) VALUES('z y x'); +} + +do_catchsql_test 22.1 {SELECT * FROM x1('')} {1 {fts5: syntax error near ""}} +do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.1 { + BEGIN; + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); +} + +do_execsql_test 23.2 { + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.3 { + INSERT INTO x1(x1) VALUES('flush'); + SELECT count(*) FROM x1_data; +} {3} + +do_execsql_test 23.4 { + ROLLBACK; + SELECT count(*) FROM x1_data; +} {2} + + +finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test new file mode 100644 index 0000000000000000000000000000000000000000..01c590c9f73de9db2212767c5a7f92758a6740f1 --- /dev/null +++ b/ext/fts5/test/fts5simple2.test @@ -0,0 +1,374 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c'); +} +do_execsql_test 1.1 { + SELECT rowid FROM t1('c a b') +} {1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('b c d'); + INSERT INTO t1 VALUES('b c d'); + COMMIT; +} +do_execsql_test 2.1 { + SELECT rowid FROM t1('b c d') +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('b c d'); + INSERT INTO t1 VALUES('b c d'); +} +do_execsql_test 3.1 { + SELECT rowid FROM t1('b c d'); COMMIT; +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a3 b3 c3'); + COMMIT; +} +do_execsql_test 4.1 { + SELECT rowid FROM t1('b*'); +} {1 2 3} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); + COMMIT; +} +do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a1 b1 c1'); + COMMIT; +} + +do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1'); + INSERT INTO t1 VALUES('a1 b2'); + COMMIT; +} +do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1} +do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); +} +do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3} +do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3} +do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2} + +do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1} +do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2} + +do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') } + +do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3} +do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3} +do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2} + +do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1} +do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1} +do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 9.0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); +} +do_execsql_test 9.0.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +reset_db +do_execsql_test 9.1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1', 'x y z'); + INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3'); + INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z'); +} +do_execsql_test 9.2.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('b1'); + INSERT INTO t1 VALUES('b1'); + DELETE FROM t1 WHERE rowid=1; +} + +do_execsql_test 10.1 { + SELECT rowid FROM t1('b1'); +} {2} + +do_execsql_test 10.2 { + SELECT rowid FROM t1('b1') ORDER BY rowid DESC; +} {2} + +do_execsql_test 10.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH d(x,y) AS ( + SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d + ) + INSERT INTO t1 SELECT * FROM d LIMIT 23; +} + +#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" } +do_execsql_test 11.2 { + SELECT rowid FROM t1; +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.3 { + SELECT rowid FROM t1('xyz'); +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, detail=none); + INSERT INTO yy VALUES('in if'); + INSERT INTO yy VALUES('if'); +} {} + +do_execsql_test 12.1 { + SELECT rowid FROM yy('i*'); +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none); +} {} +foreach {rowid a} { + 0 {f} + 1 {u} + 2 {k} + 3 {a} + 4 {a} + 5 {u} + 6 {u} + 7 {u} + 8 {f} + 9 {f} + 10 {a} + 11 {p} + 12 {f} + 13 {u} + 14 {a} + 15 {a} +} { + do_execsql_test 13.1.$rowid { + INSERT INTO t1(rowid, a) VALUES($rowid, $a); + } +} + +#------------------------------------------------------------------------- +# +reset_db +fts5_aux_test_functions db +do_execsql_test 14.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c d'); +} {} + +do_execsql_test 14.1 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {0.0.1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(1, 'sqlite'); + INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); + COMMIT; +} {} + +do_test 15.1 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +do_test 15.2 { + execsql { DELETE FROM t1 } +} {} + +do_execsql_test 15.3.1 { + SELECT rowid FROM t1('sqlite'); +} {} + +do_execsql_test 15.3.2 { + SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC; +} {} + +do_test 15.4 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 16.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t2(rowid, x) VALUES(1, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(456, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(1000, 'a b c'); + COMMIT; + UPDATE t2 SET x=x; +} + +do_execsql_test 16.1 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} {} + +do_execsql_test 16.2 { + SELECT rowid FROM t2('b') ORDER BY rowid DESC +} {1000 456 1} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 16.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t2(rowid, x) VALUES(1, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(456, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(1000, 'a b c'); + COMMIT; + UPDATE t2 SET x=x; + DELETE FROM t2; +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 17.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, y); + BEGIN; + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + COMMIT; +} +do_execsql_test 17.1 { + SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 +} +do_execsql_test 17.2 { + BEGIN; + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 ; +} +do_execsql_test 17.3 { + COMMIT +} + +reset_db +do_execsql_test 17.4 { + CREATE VIRTUAL TABLE t2 USING fts5(x, y); + BEGIN; + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + SELECT * FROM t2('y:a*') WHERE rowid>66; +} +do_execsql_test 17.5 { SELECT * FROM t2('x:b* OR y:a*') } +do_execsql_test 17.5 { COMMIT ; SELECT * FROM t2('x:b* OR y:a*') } +do_execsql_test 17.6 { + SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55 +} + +#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r} + +finish_test diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test new file mode 100644 index 0000000000000000000000000000000000000000..680448081dffa99a0ebcbca9aa2d3da7031836f0 --- /dev/null +++ b/ext/fts5/test/fts5simple3.test @@ -0,0 +1,118 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('x', 'x', 'x'); +} + +do_execsql_test 1.1 { + SELECT rowid, fts5_test_collist(t1) FROM t1('a:a'); +} {1 0.0} + +do_execsql_test 1.2 { + SELECT rowid, fts5_test_collist(t1) FROM t1('b:x'); +} {2 0.1} + +do_execsql_test 1.3 { + SELECT rowid, fts5_test_collist(t1) FROM t1('b:a'); +} {} + +#------------------------------------------------------------------------- +# Create detail=col and detail=full tables with 998 columns. +# +foreach_detail_mode $testprefix { + if {[detail_is_none]} continue + + do_test 2.1 { + execsql { DROP TABLE IF EXISTS t2 } + set cols [list] + set vals [list] + for {set i 1} {$i <= 998} {incr i} { + lappend cols "c$i" + lappend vals "'val$i'" + } + execsql "CREATE VIRTUAL TABLE t2 USING fts5(detail=%DETAIL%,[join $cols ,])" + } {} + + do_test 2.2 { + execsql "INSERT INTO t2 VALUES([join $vals ,])" + } {} + + foreach {tn q res} { + 1 { c1:val1 } 1 + 2 { c300:val300 } 1 + 3 { c300:val1 } {} + 4 { c1:val300 } {} + } { + do_execsql_test 2.3.$tn { + SELECT rowid FROM t2($q) + } $res + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x3 USING fts5(one); + INSERT INTO x3 VALUES('a b c'); + INSERT INTO x3 VALUES('c b a'); + INSERT INTO x3 VALUES('o t t'); + SELECT * FROM x3('x OR y OR z'); +} + +#------------------------------------------------------------------------- +# Test that a crash occuring when the second or subsequent tokens in a +# phrase matched zero rows has been fixed. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1 VALUES('ab'); + INSERT INTO t1 VALUES('cd'); + INSERT INTO t1 VALUES('ab cd'); + INSERT INTO t1 VALUES('ab cdXXX'); + INSERT INTO t1 VALUES('abXXX cd'); +} +do_execsql_test 4.1 { + SELECT * FROM t1('"ab cd" OR "ab cd" *'); +} {{ab cd} {ab cdXXX}} +do_execsql_test 4.2 { + SELECT * FROM t1('"xy zz" OR "ab cd" *'); +} {{ab cd} {ab cdXXX}} +do_execsql_test 4.3 { + SELECT * FROM t1('"xy zz" OR "xy zz" *'); +} +do_execsql_test 4.4 { + SELECT * FROM t1('"ab cd" OR "xy zz" *'); +} {{ab cd}} +do_execsql_test 4.5 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('ab'); + INSERT INTO t2 VALUES('cd'); + INSERT INTO t2 VALUES('ef'); +} +do_execsql_test 4.6 { + SELECT * FROM t2('ab + xyz'); +} + + +finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test new file mode 100644 index 0000000000000000000000000000000000000000..55e2f186a908e012429a26d66f2ae58e0b77dafe --- /dev/null +++ b/ext/fts5/test/fts5synonym.test @@ -0,0 +1,423 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc tcl_create {args} { return "tcl_tokenize" } + +foreach_detail_mode $testprefix { + +#------------------------------------------------------------------------- +# Warm body test for the code in fts5_tcl.c. +# +fts5_tclnum_register db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%); + INSERT INTO ft VALUES('abc def ghi'); + INSERT INTO ft VALUES('jkl mno pqr'); + SELECT rowid, x FROM ft WHERE ft MATCH 'def'; + SELECT x, rowid FROM ft WHERE ft MATCH 'pqr'; +} {1 {abc def ghi} {jkl mno pqr} 2} + +#------------------------------------------------------------------------- +# Test a tokenizer that supports synonyms by adding extra entries to the +# FTS index. +# +reset_db +fts5_tclnum_register db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize = "tclnum document", detail=%DETAIL% + ); + INSERT INTO ft VALUES('one two three'); + INSERT INTO ft VALUES('four five six'); + INSERT INTO ft VALUES('eight nine ten'); +} {} + +foreach {tn expr res} { + 1 "3" 1 + 2 "eight OR 8 OR 5" {2 3} + 3 "10" {} + 4 "1*" {1} + 5 "1 + 2" {1} +} { + if {![fts5_expr_ok $expr ft]} continue + do_execsql_test 2.1.$tn { + SELECT rowid FROM ft WHERE ft MATCH $expr + } $res +} + +#------------------------------------------------------------------------- +# Test some broken tokenizers: +# +# 3.1.*: A tokenizer that declares the very first token to be colocated. +# +# 3.2.*: A tokenizer that reports two identical tokens at the same position. +# This is allowed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + set bColo 1 + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + if {$bColo} { + sqlite3_fts5_token -colo $w $iStart $iEnd + set bColo 0 + } { + sqlite3_fts5_token $w $iStart $iEnd + } + } +} +do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 1 three 1 1 two 1 1 +} + +do_execsql_test 3.1.1 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} {} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 3.1.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three' +} {1} + +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + sqlite3_fts5_token -colo $w $iStart $iEnd + } +} +do_execsql_test 3.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 4 three 1 2 two 1 2 +} +do_execsql_test 3.2.1 { + SELECT rowid FROM ft WHERE ft MATCH 'one'; +} {1} +do_execsql_test 3.2.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three'; +} {1} +do_execsql_test 3.2.3 { + SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; +} {1} +do_execsql_test 3.2.4 { + SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; +} {1} +do_execsql_test 3.2.5 { + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {} + +#------------------------------------------------------------------------- +# Check that expressions with synonyms can be parsed and executed. +# +reset_db +fts5_tclnum_register db + +foreach {tn expr res} { + 1 {abc} {"abc"} + 2 {one} {"one"|"i"|"1"} + 3 {3} {"3"|"iii"|"three"} + 4 {3*} {"3" *} +} { + do_execsql_test 4.1.$tn { + SELECT fts5_expr($expr, 'tokenize=tclnum') + } [list $res] +} + +do_execsql_test 4.2.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%); + INSERT INTO xx VALUES('one two'); + INSERT INTO xx VALUES('three four'); +} + +do_execsql_test 4.2.2 { + SELECT rowid FROM xx WHERE xx MATCH '2' +} {1} + +do_execsql_test 4.2.3 { + SELECT rowid FROM xx WHERE xx MATCH '3' +} {2} + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%) + } + foreach {rowid a b} { + 1 {four v 4 i three} {1 3 five five 4 one} + 2 {5 1 3 4 i} {2 2 v two 4} + 3 {5 i 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one 5 1 iii 4 3} + 5 {three i v i four 4 1} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv} + 7 {ii ii two three 2 5} {iii i ii iii iii one one} + 8 {2 ii i two 3 three 2} {two iv v iii 3 five} + 9 {i 2 iv 3 five four v} {iii 4 three i three ii 1} + } { + execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + + +foreach {tn q res} { + 1 {one} { + 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} + 2 {5 [1] 3 4 [i]} {2 2 v two 4} + 3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]} + 4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3} + 5 {three [i] v [i] four 4 [1]} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv} + 7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]} + 8 {2 ii [i] two 3 three 2} {two iv v iii 3 five} + 9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]} + } + 2 {five four} { + 1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one} + 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} + 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} + 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} + 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} + 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} + 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} + } + 3 {one OR two OR iii OR 4 OR v} { + 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]} + 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]} + 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]} + 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]} + 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]} + 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]} + 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]} + 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} + 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} + } + + 4 {5 + 1} { + 2 {[5 1] 3 4 i} {2 2 v two 4} + 3 {[5 i] 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one [5 1] iii 4 3} + 5 {three i [v i] four 4 1} {ii five five five iii} + } + + 5 {one + two + three} { + 7 {ii ii two three 2 5} {iii [i ii iii] iii one one} + 8 {2 ii [i two 3] three 2} {two iv v iii 3 five} + } + + 6 {"v v"} { + 1 {four v 4 i three} {1 3 [five five] 4 one} + 5 {three i v i four 4 1} {ii [five five five] iii} + } +} { + if {![fts5_expr_ok $q t1]} continue + do_execsql_test 5.1.$tn { + SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') + FROM t1 WHERE t1 MATCH $q + } $res +} + +# Test that the xQueryPhrase() API works with synonyms. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit +sqlite3_fts5_register_matchinfo db + +foreach {tn q res} { + 1 {one} { + 1 {1 11 7 2 12 6} 2 {2 11 7 0 12 6} + 3 {2 11 7 1 12 6} 4 {1 11 7 2 12 6} + 5 {3 11 7 0 12 6} 6 {0 11 7 2 12 6} + 7 {0 11 7 3 12 6} 8 {1 11 7 0 12 6} + 9 {1 11 7 2 12 6} + } +} { + do_execsql_test 5.2.$tn { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q + } $res +} + +#------------------------------------------------------------------------- +# Test terms with more than 4 synonyms. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_execsql_test 6.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%); + INSERT INTO t1 VALUES('yy xx qq'); + INSERT INTO t1 VALUES('yy xx xx'); +} +if {[fts5_expr_ok "NEAR(y q)" t1]} { + do_execsql_test 6.0.2 { + SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)'; + } {{yy xx qq}} +} + +do_test 6.0.3 { + execsql { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%) + } + foreach {rowid a b} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii} + 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq} + 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu} + 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss} + 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } { + execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {a} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]} + 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq} + } + + 2 {a AND q} { + 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]} + } + + 3 {o OR (q AND a)} { + 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} + 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } + + 4 {NEAR(q y, 20)} { + 1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} + } +} { + if {![fts5_expr_ok $q t2]} continue + + do_execsql_test 6.1.$tn.asc { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q + } $res + + set res2 [list] + foreach {rowid a b} $res { + set res2 [concat [list $rowid $a $b] $res2] + } + + do_execsql_test 6.1.$tn.desc { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q ORDER BY rowid DESC + } $res2 +} + +do_execsql_test 6.2.1 { + INSERT INTO t2(rowid, a, b) VALUES(13, + 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy' + ); + SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')') + FROM t2 WHERE t2 MATCH 'x OR y' +} { + 1 { vvvvv qq oo vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq} + 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii} + 13 { } + {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} +} + +#------------------------------------------------------------------------- +# Test that the xColumnSize() API is not confused by colocated tokens. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_aux_test_functions db +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {[string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_execsql_test 7.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%); + INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} + +do_execsql_test 7.0.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%); + INSERT INTO t2 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} + +do_execsql_test 7.1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +} ;# foreach_detail_mode + +finish_test diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test new file mode 100644 index 0000000000000000000000000000000000000000..ec8b750c579139618809d9d7c61992aaf2959636 --- /dev/null +++ b/ext/fts5/test/fts5synonym2.test @@ -0,0 +1,166 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach tok {query document} { +foreach_detail_mode $testprefix { + +fts5_tclnum_register db +fts5_aux_test_functions db + +proc fts5_test_bothlist {cmd} { + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + set bFirst 1 + $cmd xPhraseColumnForeach $i c { + lappend CL $i.$c + if {$bFirst} { $cmd xPhraseForeach $i c o { lappend PL $i.$c.$o } } + set bFirst 0 + } + } + + list [sort_poslist $PL] $CL +} +sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist + +proc fts5_rowid {cmd} { expr [$cmd xRowid] } +sqlite3_fts5_create_function db fts5_rowid fts5_rowid + +do_execsql_test 1.$tok.0.1 " + CREATE VIRTUAL TABLE ss USING fts5(a, b, + tokenize='tclnum $tok', detail=%DETAIL%); + INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()'); +" + +do_execsql_test 1.$tok.0.2 { + INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); + INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); + INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); + INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7'); + INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii'); + INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4'); + INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4'); + INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii'); + INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five'); + INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine'); + INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven'); + INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three'); + INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9'); + INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight'); + INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven'); + INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four'); + INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2'); + INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six'); + INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi'); + INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi'); + INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii'); + INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii'); + INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4'); + INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii'); + INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4'); + INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii'); + INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four'); + INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five'); + INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one'); + INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8'); + INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i'); + INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9'); + INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five'); + INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6'); + INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4'); + INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv'); + INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine'); + INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five'); + INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi'); + INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii'); + INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two'); + INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one'); + INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii'); + INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4'); + INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii'); + INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii'); + INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4'); + INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii'); + INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one'); +} + +foreach {tn expr} { + 2.1 "one OR two OR three OR four" + + 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" + 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" + 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" + + 2.1 "one OR two OR three OR four" + 2.2 "(one AND two) OR (three AND four)" + 2.3 "(one AND two) OR (three AND four) NOT five" + 2.4 "(one AND two) NOT 6" + + 3.1 "b:one AND a:two" + 3.2 "b:one OR a:two" + 3.3 "a:one OR b:1 OR {a b} : i" + + 4.1 "NEAR(one two, 2)" + 4.2 "NEAR(one two three, 2)" + 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" + + 5.1 "one + two" + 5.2 "1 + two" +} { + if {[fts5_expr_ok $expr ss]==0} { + do_test 1.$tok.$tn.OMITTED { list } [list] + continue + } + + set res [fts5_query_data $expr ss ASC ::tclnum_syn] + do_execsql_test 1.$tok.$tn.[llength $res].asc.1 { + SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr) + } $res + + do_execsql_test 1.$tok.$tn.[llength $res].asc.2 { + SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + } $res + + do_execsql_test 1.$tok.$tn.[llength $res].asc.2 { + SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr) + ORDER BY rank ASC + } $res + + set res2 [list] + foreach {a b c} $res { lappend res2 $a $c $b } + do_execsql_test 1.$tok.$tn.[llength $res].asc.3 { + SELECT rowid, fts5_test_collist(ss), fts5_test_poslist2(ss) FROM ss($expr) + } $res2 + + set res3 [list] + foreach {a b c} $res { lappend res3 $a [list $b $c] } + do_execsql_test 1.$tok.$tn.[llength $res].asc.3 { + SELECT rowid, fts5_test_bothlist(ss) FROM ss($expr) + } $res3 + + +} + +} +} + +finish_test diff --git a/ext/fts5/test/fts5tok1.test b/ext/fts5/test/fts5tok1.test new file mode 100644 index 0000000000000000000000000000000000000000..c605ce36172399d2b43f1081642d07b3ec8a342f --- /dev/null +++ b/ext/fts5/test/fts5tok1.test @@ -0,0 +1,150 @@ +# 2016 Jan 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5tok1 + + +sqlite3_fts5_register_fts5tokenize db + +#------------------------------------------------------------------------- +# Simple test cases. Using the default (ascii) tokenizer. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii); + CREATE VIRTUAL TABLE t2 USING fts5tokenize(); + CREATE VIRTUAL TABLE t3 USING fts5tokenize( + ascii, 'separators', 'xyz', tokenchars, '''' + ); +} + +foreach {tn tbl} {1 t1 2 t2 3 t3} { + do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" { + {one two three} one 0 3 0 + {one two three} two 4 7 1 + {one two three} three 8 13 2 + } + + do_execsql_test 1.$tn.2 " + SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe' + " { + one two three + } +} + +do_execsql_test 1.4 { + SELECT token FROM t3 WHERE input = '1x2x3x' +} {1 2 3} + +do_execsql_test 1.5 { + SELECT token FROM t1 WHERE input = '1x2x3x' +} {1x2x3x} + +do_execsql_test 1.6 { + SELECT token FROM t3 WHERE input = '1''2x3x' +} {1'2 3} + +do_execsql_test 1.7 { + SELECT token FROM t3 WHERE input = '' +} {} + +do_execsql_test 1.8 { + SELECT token FROM t3 WHERE input = NULL +} {} + +do_execsql_test 1.9 { + SELECT input, * FROM t3 WHERE input = 123 +} {123 123 0 3 0} + +do_execsql_test 1.10 { + SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.11 { + SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.12 { + SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c'; +} { + {a b c} a 0 1 0 + {a b c} b 2 3 1 + {a b c} c 4 5 2 +} + +do_execsql_test 1.13.1 { + CREATE TABLE c1(x); + INSERT INTO c1(x) VALUES('a b c'); + INSERT INTO c1(x) VALUES('d e f'); +} +do_execsql_test 1.13.2 { + SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid; +} { + {a b c} {a b c} a 0 1 0 + {d e f} {d e f} e 2 3 1 +} + + +#------------------------------------------------------------------------- +# Error cases. +# +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer); +} {1 {vtable constructor failed: tX}} + +do_catchsql_test 2.1 { + CREATE VIRTUAL TABLE t4 USING fts5tokenize; + SELECT * FROM t4; +} {1 {SQL logic error}} + +#------------------------------------------------------------------------- +# Embedded 0x00 characters. +# +reset_db +do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(z); + CREATE VIRTUAL TABLE tt USING fts5vocab(t1, 'instance'); + INSERT INTO t1 VALUES('abc' || char(0) || 'def'); + SELECT * FROM tt; +} { abc 1 z 0 def 1 z 1 } +do_execsql_test 3.1.1 { + SELECT hex(z) FROM t1; +} {61626300646566} +do_execsql_test 3.1.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +do_execsql_test 3.2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(z, + tokenize="unicode61 categories 'L* N* Co Cc'" + ); + CREATE VIRTUAL TABLE tu USING fts5vocab(t2, 'instance'); + + INSERT INTO t2 VALUES('abc' || char(0) || 'def'); + SELECT * FROM tu; +} { abc 1 z 0 def 1 z 1 } + +do_execsql_test 3.2.1 { + SELECT hex(z) FROM t1; +} {61626300646566} + +do_execsql_test 3.2.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + + +finish_test diff --git a/ext/fts5/test/fts5tok2.test b/ext/fts5/test/fts5tok2.test new file mode 100644 index 0000000000000000000000000000000000000000..77c7e1ec8e647014bc5196d765043270f8102b16 --- /dev/null +++ b/ext/fts5/test/fts5tok2.test @@ -0,0 +1,47 @@ +# 2016 Jan 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5||!fts3 { finish_test ; return } +set ::testprefix fts5tok2 + +sqlite3_fts5_register_fts5tokenize db + +#------------------------------------------------------------------------- +# Simple test cases. Using the default (ascii) tokenizer. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61); + CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61); +} + +do_test 1.1 { + array unset -nocomplain A + + for {set i 1} {$i < 65536} {incr i} { + set input [format "abc%cxyz" $i] + set expect [execsql { + SELECT input, token, start, end FROM t3 WHERE input=$input + }] + + incr A([llength $expect]) + + set res [execsql { + SELECT input, token, start, end FROM t5($input) + }] + if {$res != $expect} {error "failed at i=$i"} + } +} {} + +do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {} + +finish_test diff --git a/ext/fts5/test/fts5tokendata.test b/ext/fts5/test/fts5tokendata.test new file mode 100644 index 0000000000000000000000000000000000000000..7f75f4fa8ed6ebc9825e8d402c71b14f086c3f56 --- /dev/null +++ b/ext/fts5/test/fts5tokendata.test @@ -0,0 +1,105 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokendata + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + + sqlite3_fts5_register_origintext db + fts5_aux_test_functions db + proc b {x} { string map [list "\0" "."] $x } + db func b b + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, tokendata=1, + tokenize="origintext unicode61", detail=%DETAIL% + ); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); + } + + do_execsql_test 1.1 { + INSERT INTO ft(rowid, a, b) VALUES + (1, 'Pedagog Pedal Pedant', 'Peculier Day Today'), + (2, 'Pedant pedantic pecked', 'Peck Penalize Pen'); + + INSERT INTO ft(rowid, a, b) VALUES + (3, 'Penalty Pence Penciled', 'One Two Three'), + (4, 'Pedant Pedal Pedant', 'Peculier Day Today'); + } + + do_execsql_test 1.2 { + SELECT DISTINCT b(term) FROM vocab + } { + day.Day one.One peck.Peck pecked peculier.Peculier pedagog.Pedagog + pedal.Pedal pedant.Pedant pedantic pen.Pen penalize.Penalize + penalty.Penalty pence.Pence penciled.Penciled three.Three + today.Today two.Two + } + + do_execsql_test 1.3.1 { + SELECT rowid FROM ft('pe*') + } { + 1 2 3 4 + } + + do_execsql_test 1.3.2 { + SELECT rowid FROM ft('pe*') ORDER BY rowid DESC + } { + 4 3 2 1 + } + + if {"%DETAIL%"!="none"} { + do_execsql_test 1.3.3 { + SELECT rowid FROM ft WHERE a MATCH 'pe*' ORDER BY rowid DESC + } { + 4 3 2 1 + } + } + + do_execsql_test 1.4 { + SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pedant') + } { + 1 pedant.Pedant + 2 pedant.Pedant + 4 pedant.Pedant + } + + do_execsql_test 1.5 { + SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pe*') + } { + 1 pedagog.Pedagog + 2 pedant.Pedant + 3 penalty.Penalty + 4 pedant.Pedant + } + + do_execsql_test 1.6 { + SELECT rowid, fts5_test_poslist(ft) FROM ft('pe*') + } { + 1 {0.0.0 0.0.1 0.0.2 0.1.0} + 2 {0.0.0 0.0.1 0.0.2 0.1.0 0.1.1 0.1.2} + 3 {0.0.0 0.0.1 0.0.2} + 4 {0.0.0 0.0.1 0.0.2 0.1.0} + } +} + +finish_test + diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test new file mode 100644 index 0000000000000000000000000000000000000000..a828e3a22b7b5bf054597753329015b258a740b0 --- /dev/null +++ b/ext/fts5/test/fts5tokenizer.test @@ -0,0 +1,374 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter); + DROP TABLE ft1; +} +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize='porter'); + DROP TABLE ft1; +} +do_execsql_test 1.2 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = porter); + DROP TABLE ft1; +} +do_execsql_test 1.3 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter'); + DROP TABLE ft1; +} +do_execsql_test 1.4 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii'); + DROP TABLE ft1; +} + +do_catchsql_test 1.5 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch'); +} {1 {no such tokenizer: nosuch}} + +do_catchsql_test 1.6 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch'); +} {1 {error in tokenizer constructor}} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter); + INSERT INTO ft1 VALUES('embedded databases'); +} +do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1 +do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1 +do_execsql_test 2.3 { + SELECT rowid FROM ft1 WHERE ft1 MATCH 'database embedding' +} 1 + +proc tcl_create {args} { + set ::targs $args + error "failed" +} +sqlite3_fts5_create_tokenizer db tcl tcl_create + +foreach {tn directive expected} { + 1 {tokenize='tcl a b c'} {a b c} + 2 {tokenize='tcl ''d'' ''e'' ''f'''} {d e f} + 3 {tokenize="tcl 'g' 'h' 'i'"} {g h i} + 4 {tokenize = tcl} {} +} { + do_catchsql_test 3.$tn.1 " + CREATE VIRTUAL TABLE ft2 USING fts5(x, $directive) + " {1 {error in tokenizer constructor}} + do_test 3.$tn.2 { set ::targs } $expected +} + +do_catchsql_test 4.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize = tcl abc); +} {1 {parse error in "tokenize = tcl abc"}} +do_catchsql_test 4.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(x y) +} {1 {unrecognized column option: y}} + +#------------------------------------------------------------------------- +# Test the "separators" and "tokenchars" options a bit. +# +foreach {tn tokenizer} {1 ascii 2 unicode61} { + reset_db + set T "$tokenizer tokenchars ',.:' separators 'xyz'" + execsql "CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = \"$T\")" + do_execsql_test 5.$tn.1 { + INSERT INTO t1 VALUES('abcxdefyghizjkl.mno,pqr:stu/vwx+yz'); + } + foreach {tn2 token res} { + 1 abc 1 2 def 1 3 ghi 1 4 jkl {} + 5 mno {} 6 pqr {} 7 stu {} 8 jkl.mno,pqr:stu 1 + 9 vw 1 + } { + do_execsql_test 5.$tn.2.$tn2 " + SELECT rowid FROM t1 WHERE t1 MATCH '\"$token\"' + " $res + } +} + +#------------------------------------------------------------------------- +# Miscellaneous tests for the ascii tokenizer. +# +# 5.1.*: Test that the ascii tokenizer ignores non-ASCII characters in the +# 'separators' option. But unicode61 does not. +# +# 5.2.*: An option without an argument is an error. +# + +do_test 5.1.1 { + execsql " + CREATE VIRTUAL TABLE a1 USING fts5(x, tokenize=`ascii separators '\u1234'`); + INSERT INTO a1 VALUES('abc\u1234def'); + " + execsql { SELECT rowid FROM a1 WHERE a1 MATCH 'def' } +} {} + +do_test 5.1.2 { + execsql " + CREATE VIRTUAL TABLE a2 USING fts5( + x, tokenize=`unicode61 separators '\u1234'`); + INSERT INTO a2 VALUES('abc\u1234def'); + " + execsql { SELECT rowid FROM a2 WHERE a2 MATCH 'def' } +} {1} + +do_catchsql_test 5.2 { + CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'ascii tokenchars'); +} {1 {error in tokenizer constructor}} +do_catchsql_test 5.3 { + CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'ascii opt arg'); +} {1 {error in tokenizer constructor}} + +#------------------------------------------------------------------------- +# Test that the ASCII and unicode61 tokenizers both handle SQLITE_DONE +# correctly. +# + +proc test_token_cb {varname token iStart iEnd} { + upvar $varname var + lappend var $token + if {[llength $var]==3} { return "SQLITE_DONE" } + return "SQLITE_OK" +} + +proc tokenize {cmd} { + set res [list] + $cmd xTokenize [$cmd xColumnText 0] [list test_token_cb res] + set res +} +sqlite3_fts5_create_function db tokenize tokenize + +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, tokenize=ascii); + INSERT INTO x1 VALUES('q w e r t y'); + INSERT INTO x1 VALUES('y t r e w q'); + SELECT tokenize(x1) FROM x1 WHERE x1 MATCH 'e AND r'; +} { + {q w e} {y t r} +} + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE x2 USING fts5(a, tokenize=unicode61); + INSERT INTO x2 VALUES('q w e r t y'); + INSERT INTO x2 VALUES('y t r e w q'); + SELECT tokenize(x2) FROM x2 WHERE x2 MATCH 'e AND r'; +} { + {q w e} {y t r} +} + + +#------------------------------------------------------------------------- +# Miscellaneous tests for the unicode tokenizer. +# +do_catchsql_test 6.1 { + CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars'); +} {1 {error in tokenizer constructor}} +do_catchsql_test 6.2 { + CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b'); +} {1 {error in tokenizer constructor}} +do_catchsql_test 6.3 { + CREATE VIRTUAL TABLE a3 USING fts5( + x, y, tokenize = 'unicode61 remove_diacritics 3' + ); +} {1 {error in tokenizer constructor}} +do_catchsql_test 6.4 { + CREATE VIRTUAL TABLE a3 USING fts5( + x, y, tokenize = 'unicode61 remove_diacritics 10' + ); +} {1 {error in tokenizer constructor}} + +#------------------------------------------------------------------------- +# Porter tokenizer with very large tokens. +# +set a [string repeat a 100] +set b [string repeat b 500] +set c [string repeat c 1000] +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE e5 USING fts5(x, tokenize=porter); + INSERT INTO e5 VALUES($a || ' ' || $b); + INSERT INTO e5 VALUES($b || ' ' || $c); + INSERT INTO e5 VALUES($c || ' ' || $a); +} + +do_execsql_test 7.1 {SELECT rowid FROM e5 WHERE e5 MATCH $a} { 1 3 } +do_execsql_test 7.2 {SELECT rowid FROM e5 WHERE e5 MATCH $b} { 1 2 } +do_execsql_test 7.3 {SELECT rowid FROM e5 WHERE e5 MATCH $c} { 2 3 } + +#------------------------------------------------------------------------- +# Test the 'separators' option with the unicode61 tokenizer. +# +do_execsql_test 8.1 { + BEGIN; + CREATE VIRTUAL TABLE e6 USING fts5(x, + tokenize="unicode61 separators ABCDEFGHIJKLMNOPQRSTUVWXYZ" + ); + INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog'); + CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row'); + SELECT term FROM e7; + ROLLBACK; +} { + brown dog fox jumped lazy over quick the +} + +do_execsql_test 8.2 [subst { + BEGIN; + CREATE VIRTUAL TABLE e6 USING fts5(x, + tokenize="unicode61 separators '\u0E01\u0E02\u0E03\u0E04\u0E05\u0E06\u0E07'" + ); + INSERT INTO e6 VALUES('the\u0E01quick\u0E01brown\u0E01fox\u0E01' + || 'jumped\u0E01over\u0E01the\u0E01lazy\u0E01dog' + ); + INSERT INTO e6 VALUES('\u0E08\u0E07\u0E09'); + CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row'); + SELECT term FROM e7; + ROLLBACK; +}] [subst { + brown dog fox jumped lazy over quick the \u0E08 \u0E09 +}] + +# Test that the porter tokenizer correctly passes arguments through to +# its parent tokenizer. +do_execsql_test 8.3 { + BEGIN; + CREATE VIRTUAL TABLE e6 USING fts5(x, + tokenize="porter unicode61 separators ABCDEFGHIJKLMNOPQRSTUVWXYZ" + ); + INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog'); + CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row'); + SELECT term FROM e7; + ROLLBACK; +} { + brown dog fox jump lazi over quick the +} + +#------------------------------------------------------------------------- +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer +# implementation. +# +reset_db +proc tcl_create {args} { return "tcl_tokenize" } +sqlite3_fts5_create_tokenizer db tcl tcl_create +set ::flags [list] +proc tcl_tokenize {tflags text} { + lappend ::flags $tflags + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 9.1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('xyz'); +} {} +do_test 9.1.2 { set ::flags } {document document} + +set ::flags [list] +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc} +do_test 9.2.2 { set ::flags } {query} + +set ::flags [list] +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc} +do_test 9.3.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {} +do_test 9.4.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} +do_test 9.5.2 { set ::flags } {query} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61); + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="unicode61 error");' + WHERE name = 'x1'; +} + +db close +sqlite3 db test.db + +do_catchsql_test 10.2 { + SELECT * FROM x1('abc'); +} {1 {error in tokenizer constructor}} + +do_catchsql_test 10.3 { + INSERT INTO x1 VALUES('abc'); +} {1 {error in tokenizer constructor}} + +do_execsql_test 10.4 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="nosuch error");' + WHERE name = 'x1'; +} + +db close +sqlite3 db test.db + +do_catchsql_test 10.5 { + SELECT * FROM x1('abc'); +} {1 {no such tokenizer: nosuch}} +do_catchsql_test 10.6 { + INSERT INTO x1 VALUES('abc'); +} {1 {no such tokenizer: nosuch}} + +do_execsql_test 10.7 { + DROP TABLE x1; + SELECT * FROM sqlite_schema; +} + +reset_db +do_execsql_test 10.8 { + CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61); + INSERT INTO x1 VALUES('a b c'), ('d e f'), ('a b c'); + CREATE VIRTUAL TABLE x1v USING fts5vocab(x1, row); + + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=simplify);' + WHERE name = 'x1'; +} + +do_execsql_test 10.9 { + SELECT * FROM x1v +} { + a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1 +} + +db close +sqlite3 db test.db + +do_execsql_test 10.10 { + SELECT * FROM x1v +} { + a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1 +} + +finish_test diff --git a/ext/fts5/test/fts5tokenizer2.test b/ext/fts5/test/fts5tokenizer2.test new file mode 100644 index 0000000000000000000000000000000000000000..4fe31d22c4e77d832022b99da008fe8c09ec7377 --- /dev/null +++ b/ext/fts5/test/fts5tokenizer2.test @@ -0,0 +1,109 @@ +# 2023 Nov 03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer +proc get_tst_tokenizer {args} { + return "tst_tokenizer" +} +proc tst_tokenizer {flags txt} { + set token "" + set lTok [list] + + foreach c [split $txt {}] { + if {$token==""} { + append token $c + } else { + set t1 [string is upper $token] + set t2 [string is upper $c] + + if {$t1!=$t2} { + lappend lTok $token + set token "" + } + append token $c + } + } + if {$token!=""} { lappend lTok $token } + + set iOff 0 + foreach t $lTok { + set n [string length $t] + sqlite3_fts5_token $t $iOff [expr $iOff+$n] + incr iOff $n + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('AAdontBBmess'); +} + +do_execsql_test 1.2 { + SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB'); +} {AAdont>BB', '<') FROM t1('BB'); +} {AAdont>BB', '<') FROM t1('AA'); +} {>AA', '<') FROM t1('dont'); +} {AA>dont', '<') FROM t1('mess'); +} {AAdontBB>mess<} + +do_execsql_test 1.7 { + SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess'); +} {AAdont>BBmess<} + +# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd +# Error handling of tokenize= arguments. +# +foreach {n tkz} { + 1 {ascii none} + 2 {unicode61 none} + 3 {porter none} + 4 {trigram none} + 5 {ascii none 0} + 6 {unicode61 none 0} + 7 {porter none 0} + 8 {trigram none 0} +} { + db eval {DROP TABLE IF EXISTS t2;} + do_catchsql_test 2.$n " + DROP TABLE IF EXISTS t2; + CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz'); + " {1 {error in tokenizer constructor}} +} + + +finish_test diff --git a/ext/fts5/test/fts5tokenizer3.test b/ext/fts5/test/fts5tokenizer3.test new file mode 100644 index 0000000000000000000000000000000000000000..5cdab743c2b218df2fd4a0e48c620d3f9d97e8f1 --- /dev/null +++ b/ext/fts5/test/fts5tokenizer3.test @@ -0,0 +1,77 @@ +# 2024 Aug 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +proc get_sod {args} { return "split_on_dot" } +proc get_lowercase {args} { return "lowercase" } + +proc lowercase {flags txt} { + set n [string length $txt] + sqlite3_fts5_token [string tolower $txt] 0 $n + return 0 +} + +proc split_on_dot {flags txt} { + set iOff 0 + foreach t [split $txt "."] { + set n [string length $txt] + sqlite3_fts5_token $t $iOff [expr $iOff+$n] + incr iOff [expr {$n+1}] + } + return "" +} + +foreach {tn script} { + 1 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 2 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 3 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } + 4 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } +} { + reset_db + eval $script + + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot); + CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance); + INSERT INTO t1 VALUES('ABC.Def.ghi'); + } + + do_execsql_test 1.$tn.1 { + SELECT term FROM t1vocab ORDER BY 1 + } {abc def ghi} +} + + +finish_test diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test new file mode 100644 index 0000000000000000000000000000000000000000..377e3f7813308ff40943356695d1f250607bc44a --- /dev/null +++ b/ext/fts5/test/fts5trigram.test @@ -0,0 +1,366 @@ +# 2020 September 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Tests for the fts5 "trigram" tokenizer. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5trigram + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram); + INSERT INTO t1 VALUES('abcdefghijklm'); + INSERT INTO t1 VALUES('กรุงเทพมหานคร'); +} + +foreach {tn s res} { + 1 abc "(abc)defghijklm" + 2 defgh "abc(defgh)ijklm" + 3 abcdefghijklm "(abcdefghijklm)" + 4 กรุ "(กรุ)งเทพมหานคร" + 5 งเทพมห "กรุ(งเทพมห)านคร" + 6 กรุงเทพมหานคร "(กรุงเทพมหานคร)" + 7 Abc "(abc)defghijklm" + 8 deFgh "abc(defgh)ijklm" + 9 aBcdefGhijKlm "(abcdefghijklm)" +} { + do_execsql_test 1.1.$tn { + SELECT highlight(t1, 0, '(', ')') FROM t1($s) + } $res +} + +do_execsql_test 1.2.0 { + SELECT fts5_expr('ABCD', 'tokenize=trigram') +} {{"abc" + "bcd"}} + +do_execsql_test 1.2.1 { + SELECT * FROM t1 WHERE y LIKE ? ESCAPE 'a' +} + +foreach {tn like res} { + 1 {%cDef%} 1 + 2 {cDef%} {} + 3 {%f%} 1 + 4 {%f_h%} 1 + 5 {%f_g%} {} + 6 {abc%klm} 1 + 7 {ABCDEFG%} 1 + 8 {%รุงเ%} 2 + 9 {%งเ%} 2 + 10 {%"งเ"%} {} +} { + do_execsql_test 1.3.$tn { + SELECT rowid FROM t1 WHERE y LIKE $like + } $res +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize="trigram case_sensitive 1"); + INSERT INTO t1 VALUES('abcdefghijklm'); + INSERT INTO t1 VALUES('กรุงเทพมหานคร'); +} +do_catchsql_test 2.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram case_sensitive'); +} {1 {error in tokenizer constructor}} + +foreach {tn s res} { + 1 abc "(abc)defghijklm" + 2 defgh "abc(defgh)ijklm" + 3 abcdefghijklm "(abcdefghijklm)" + 4 กรุ "(กรุ)งเทพมหานคร" + 5 งเทพมห "กรุ(งเทพมห)านคร" + 6 กรุงเทพมหานคร "(กรุงเทพมหานคร)" + 7 Abc "" + 8 deFgh "" + 9 aBcdefGhijKlm "" +} { + do_execsql_test 2.1.$tn { + SELECT highlight(t1, 0, '(', ')') FROM t1($s) + } $res +} +foreach {tn like res} { + 1 {%cDef%} 1 + 2 {cDef%} {} + 3 {%f%} 1 + 4 {%f_h%} 1 + 5 {%f_g%} {} + 6 {abc%klm} 1 + 7 {ABCDEFG%} 1 + 8 {%รุงเ%} 2 +} { + do_execsql_test 2.2.$tn { + SELECT rowid FROM t1 WHERE y LIKE $like + } $res +} +foreach {tn like res} { + 1 {*cdef*} 1 + 2 {cdef*} {} + 3 {*f*} 1 + 4 {*f?h*} 1 + 5 {*f?g*} {} + 6 {abc*klm} 1 + 7 {abcdefg*} 1 + 8 {*รุงเ*} 2 + 9 {abc[d]efg*} 1 + 10 {abc[]d]efg*} 1 + 11 {abc[^]d]efg*} {} + 12 {abc[^]XYZ]efg*} 1 +} { + do_execsql_test 2.3.$tn { + SELECT rowid FROM t1 WHERE y GLOB $like + } $res +} + +do_execsql_test 2.3.null.1 { + SELECT rowid FROM t1 WHERE y LIKE NULL +} + +#------------------------------------------------------------------------- +reset_db +do_catchsql_test 3.1 { + CREATE VIRTUAL TABLE ttt USING fts5(c, tokenize="trigram case_sensitive 2"); +} {1 {error in tokenizer constructor}} +do_catchsql_test 3.2 { + CREATE VIRTUAL TABLE ttt USING fts5(c, tokenize="trigram case_sensitive 11"); +} {1 {error in tokenizer constructor}} +do_catchsql_test 3.3 { + CREATE VIRTUAL TABLE ttt USING fts5(c, "tokenize=trigram case_sensitive 1"); +} {0 {}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t0 USING fts5(b, tokenize = "trigram"); +} +do_execsql_test 4.1 { + INSERT INTO t0 VALUES (x'000b01'); +} +do_execsql_test 4.2 { + INSERT INTO t0(t0) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +foreach_detail_mode $::testprefix { + foreach {ci} {0 1} { + reset_db + do_execsql_test 5.cs=$ci.0.1 " + CREATE VIRTUAL TABLE t1 USING fts5( + y, tokenize=\"trigram case_sensitive $ci\", detail=%DETAIL% + ); + " + do_execsql_test 5.cs=$ci.0.2 { + INSERT INTO t1 VALUES('abcdefghijklm'); + INSERT INTO t1 VALUES('กรุงเทพมหานคร'); + } + + foreach {tn like res} { + 1 {%cDef%} 1 + 2 {cDef%} {} + 3 {%f%} 1 + 4 {%f_h%} 1 + 5 {%f_g%} {} + 6 {abc%klm} 1 + 7 {ABCDEFG%} 1 + 8 {%รุงเ%} 2 + } { + do_execsql_test 5.cs=$ci.1.$tn { + SELECT rowid FROM t1 WHERE y LIKE $like + } $res + } + } +} + +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ci0 USING fts5(x, tokenize="trigram"); + CREATE VIRTUAL TABLE ci1 USING fts5(x, tokenize="trigram case_sensitive 1"); +} + +# LIKE and GLOB both work with case-insensitive tokenizers. Only GLOB works +# with case-sensitive. +do_eqp_test 6.1 { + SELECT * FROM ci0 WHERE x LIKE ? +} {VIRTUAL TABLE INDEX 0:L0} +do_eqp_test 6.2 { + SELECT * FROM ci0 WHERE x GLOB ? +} {VIRTUAL TABLE INDEX 0:G0} +do_eqp_test 6.3 { + SELECT * FROM ci1 WHERE x LIKE ? +} {{SCAN ci1 VIRTUAL TABLE INDEX 0:}} +do_eqp_test 6.4 { + SELECT * FROM ci1 WHERE x GLOB ? +} {VIRTUAL TABLE INDEX 0:G0} +do_eqp_test 6.5 { + SELECT * FROM ci1 WHERE x < ? +} {{SCAN ci1 VIRTUAL TABLE INDEX 0:}} +do_eqp_test 6.6 { + SELECT * FROM ci0 WHERE x < ? +} {{SCAN ci0 VIRTUAL TABLE INDEX 0:}} + +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE f USING FTS5(filename, tokenize="trigram"); + INSERT INTO f (rowid, filename) VALUES + (10, "giraffe.png"), + (20, "жираф.png"), + (30, "cat.png"), + (40, "кот.png"), + (50, "misic-🎵-.mp3"); +} +do_execsql_test 7.1 { + SELECT rowid FROM f WHERE +filename GLOB '*ир*'; +} {20} +do_execsql_test 7.2 { + SELECT rowid FROM f WHERE filename GLOB '*ир*'; +} {20} + + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram); + INSERT INTO t1 VALUES('abcdefghijklm'); +} + +foreach {tn match res} { + 1 "abc ghi" "(abc)def(ghi)jklm" + 2 "def ghi" "abc(defghi)jklm" + 3 "efg ghi" "abcd(efghi)jklm" + 4 "efghi" "abcd(efghi)jklm" + 5 "abcd jklm" "(abcd)efghi(jklm)" + 6 "ijkl jklm" "abcdefgh(ijklm)" + 7 "ijk ijkl hijk" "abcdefg(hijkl)m" + +} { + do_execsql_test 8.1.$tn { + SELECT highlight(t1, 0, '(', ')') FROM t1($match) + } $res +} + +do_execsql_test 8.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(a, tokenize="trigram"); + INSERT INTO ft2 VALUES('abc x cde'); + INSERT INTO ft2 VALUES('abc cde'); + INSERT INTO ft2 VALUES('abcde'); +} + +do_execsql_test 8.3 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'abc AND cde'; +} { + {[abc] x [cde]} + {[abc] [cde]} + {[abcde]} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + tokenize=trigram + ); + + INSERT INTO t1(rowid, a12) VALUES(111, 'thats a tricky case though'); + INSERT INTO t1(rowid, a12) VALUES(222, 'the query planner cannot do'); +} + +do_execsql_test 9.1 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' +} {111} + +do_execsql_test 9.2 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' AND a12 LIKE '%case%' +} {111} + +do_execsql_test 9.3 { + SELECT rowid FROM t1 WHERE a12 LIKE NULL +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=trigram); +} + +do_test 10.1 { + foreach {val} { + "abc \UFFjkl\UFF" + "abc \UFFFjkl\UFFF" + "abc \UFFFFjkl\UFFFF" + "abc \UFFFFFjkl\UFFFFF" + "\UFFjkl\UFF abc" + "\UFFFjkl\UFFF abc" + "\UFFFFjkl\UFFFF abc" + "\UFFFFFjkl\UFFFFF abc" + "\U10001jkl\U10001 abc" + } { + execsql { INSERT INTO t1 VALUES( $val ) } + } +} {} + +do_test 10.2 { + foreach {val} { + X'E18000626320646566' + X'61EDA0806320646566' + X'61EDA0806320646566' + X'61EFBFBE6320646566' + X'76686920E18000626320646566' + X'7668692061EDA0806320646566' + X'7668692061EDA0806320646566' + X'7668692061EFBFBE6320646566' + } { + execsql " INSERT INTO t1 VALUES( $val ) " + } +} {} + +do_test 10.3 { + set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] + set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] + set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + execsql { + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + + INSERT INTO t1 VALUES('abcd' || $a); + INSERT INTO t1 VALUES('abcd' || $b); + INSERT INTO t1 VALUES('abcd' || $c); + INSERT INTO t1 VALUES('abcd' || $d); + } +} {} + +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE t4 USING fts5(y, tokenize=trigram); +} +sqlite3_fts5_register_str db +do_execsql_test 11.1 { + INSERT INTO t4 VALUES( str('') ); +} + +do_test 12.0 { + sqlite3_fts5_tokenize db trigram "abcd" +} {abc 0 3 bcd 1 4} + +do_test 12.1 { + sqlite3_fts5_tokenize db trigram "a" +} {} + +do_test 12.2 { + sqlite3_fts5_tokenize db trigram "" +} {} + +finish_test + diff --git a/ext/fts5/test/fts5trigram2.test b/ext/fts5/test/fts5trigram2.test new file mode 100644 index 0000000000000000000000000000000000000000..c81684a22b9974e88d8bfa47b2d5bc4adb8b6a7f --- /dev/null +++ b/ext/fts5/test/fts5trigram2.test @@ -0,0 +1,137 @@ +# 2023 October 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Tests for the fts5 "trigram" tokenizer. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5trigram2 + +do_execsql_test 1.0 " + CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1'); + INSERT INTO t1 VALUES('abc\u0303defghijklm'); + INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm'); +" +do_catchsql_test 1.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram remove_diacritics'); +} {1 {error in tokenizer constructor}} + +do_execsql_test 1.1 { + SELECT highlight(t1, 0, '(', ')') FROM t1('abc'); +} [list \ + "(abc\u0303)defghijklm" \ + "(a\u0303b\u0303c\u0303)defghijklm" \ +] + +do_execsql_test 1.2 { + SELECT highlight(t1, 0, '(', ')') FROM t1('bcde'); +} [list \ + "a(bc\u0303de)fghijklm" \ + "a\u0303(b\u0303c\u0303de)fghijklm" \ +] + +do_execsql_test 1.3 { + SELECT highlight(t1, 0, '(', ')') FROM t1('cdef'); +} [list \ + "ab(c\u0303def)ghijklm" \ + "a\u0303b\u0303(c\u0303def)ghijklm" \ +] + +do_execsql_test 1.4 { + SELECT highlight(t1, 0, '(', ')') FROM t1('def'); +} [list \ + "abc\u0303(def)ghijklm" \ + "a\u0303b\u0303c\u0303(def)ghijklm" \ +] + + +#------------------------------------------------------------------------- +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5( + z, tokenize='trigram case_sensitive 1 remove_diacritics 1' + ); +} {1 {error in tokenizer constructor}} + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t2 USING fts5( + z, tokenize='trigram case_sensitive 0 remove_diacritics 1' + ); +} +do_execsql_test 2.2 " + INSERT INTO t2 VALUES('\u00E3bcdef'); + INSERT INTO t2 VALUES('b\u00E3cdef'); + INSERT INTO t2 VALUES('bc\u00E3def'); + INSERT INTO t2 VALUES('bcd\u00E3ef'); +" + +do_execsql_test 2.3 { + SELECT highlight(t2, 0, '(', ')') FROM t2('abc'); +} "(\u00E3bc)def" +do_execsql_test 2.4 { + SELECT highlight(t2, 0, '(', ')') FROM t2('bac'); +} "(b\u00E3c)def" +do_execsql_test 2.5 { + SELECT highlight(t2, 0, '(', ')') FROM t2('bca'); +} "(bc\u00E3)def" +do_execsql_test 2.6 " + SELECT highlight(t2, 0, '(', ')') FROM t2('\u00E3bc'); +" "(\u00E3bc)def" + +#------------------------------------------------------------------------- +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5( + z, tokenize='trigram remove_diacritics 1' + ); +} {} +do_execsql_test 3.1 " + INSERT INTO t3 VALUES ('\u0303abc\u0303'); +" +do_execsql_test 3.2 { + SELECT highlight(t3, 0, '(', ')') FROM t3('abc'); +} "\u0303(abc\u0303)" + +#------------------------------------------------------------------------- +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram); +} {} + +do_execsql_test 4.1 { + INSERT INTO t4 VALUES('ABCD'); + INSERT INTO t4 VALUES('DEFG'); +} {} + +db close +sqlite3 db test.db + +do_eqp_test 4.1 { + SELECT rowid FROM t4 WHERE z LIKE '%abc%' +} {VIRTUAL TABLE INDEX 0:L0} + +do_execsql_test 4.2 { + SELECT rowid FROM t4 WHERE z LIKE '%abc%' +} {1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts5( + c1, tokenize='trigram', detail='none' + ); + INSERT INTO t5(rowid, c1) VALUES(1, 'abc_____xyx_yxz'); + INSERT INTO t5(rowid, c1) VALUES(2, 'abc_____xyxz'); + INSERT INTO t5(rowid, c1) VALUES(3, 'ac_____xyxz'); +} {} +do_execsql_test 5.1 { + SELECT rowid FROM t5 WHERE c1 LIKE 'abc%xyxz' +} {2} + +finish_test diff --git a/ext/fts5/test/fts5ubsan.test b/ext/fts5/test/fts5ubsan.test new file mode 100644 index 0000000000000000000000000000000000000000..76382a1e154e34963b9d67b320151dae5c3a9ffc --- /dev/null +++ b/ext/fts5/test/fts5ubsan.test @@ -0,0 +1,60 @@ +# 2022 August 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on edge cases that cause ubsan errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ubsan + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +set BIG 9000000000000000000 +set SMALL -9000000000000000000 + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO x1 (rowid, x) VALUES($BIG, 'aaa aba acc'); + INSERT INTO x1 (rowid, x) VALUES($SMALL, 'aaa abc acb'); + COMMIT; +} + +do_execsql_test 1.2 { + SELECT rowid, x FROM x1('ab*'); +} [list $SMALL {aaa abc acb} $BIG {aaa aba acc}] + +do_execsql_test 1.3 { + SELECT rowid, x FROM x1('ac*'); +} [list $SMALL {aaa abc acb} $BIG {aaa aba acc}] + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +do_execsql_test 2.1 { + INSERT INTO x1 (rowid, x) VALUES($BIG, 'aaa aba acc'); + INSERT INTO x1 (rowid, x) VALUES($SMALL, 'aaa abc acb'); +} + +do_execsql_test 2.2 { + INSERT INTO x1 (x1) VALUES('optimize'); +} + +finish_test diff --git a/ext/fts5/test/fts5umlaut.test b/ext/fts5/test/fts5umlaut.test new file mode 100644 index 0000000000000000000000000000000000000000..fa948107a368e5f356edb99b64294e38f1fe3b37 --- /dev/null +++ b/ext/fts5/test/fts5umlaut.test @@ -0,0 +1,65 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5umlaut + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t2 USING fts5( + x, + tokenize="unicode61 remove_diacritics 2" + ); +} + +foreach {tn q res1 res2} { + 1 "Hà Nội" 0 1 + 2 "Hà Noi" 1 1 + 3 "Ha Noi" 1 1 + 4 "Ha N\u1ed9i" 0 1 + 5 "Ha N\u006fi" 1 1 + 6 "Ha N\u006f\u0302i" 1 1 + 7 "Ha N\u006f\u0323\u0302i" 1 1 +} { + do_execsql_test 1.$tn.1 { + DELETE FROM t1; + INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi'); + SELECT count(*) FROM t1($q) + } $res1 + do_execsql_test 1.$tn.2 { + DELETE FROM t1; + INSERT INTO t1(rowid, x) VALUES (1, $q); + SELECT count(*) FROM t1('Ha Noi') + } $res1 + + do_execsql_test 1.$tn.2 { + DELETE FROM t2; + INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi'); + SELECT count(*) FROM t2($q) + } $res2 + do_execsql_test 1.$tn.2 { + DELETE FROM t2; + INSERT INTO t2(rowid, x) VALUES (1, $q); + SELECT count(*) FROM t2('Ha Noi') + } $res2 +} + +finish_test + diff --git a/ext/fts5/test/fts5unicode.test b/ext/fts5/test/fts5unicode.test new file mode 100644 index 0000000000000000000000000000000000000000..f10e0d02d843616e7d704e82d69b34bfb90fd50e --- /dev/null +++ b/ext/fts5/test/fts5unicode.test @@ -0,0 +1,86 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the fts5 tokenizers +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5unicode + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc tokenize_test {tn tokenizer input output} { + uplevel [list do_test $tn [subst -nocommands { + set ret {} + foreach {z s e} [sqlite3_fts5_tokenize db {$tokenizer} {$input}] { + lappend ret [set z] + } + set ret + }] [list {*}$output]] +} + +foreach {tn t} {1 ascii 2 unicode61} { + tokenize_test 1.$tn.0 $t {A B C D} {a b c d} + tokenize_test 1.$tn.1 $t {May you share freely,} {may you share freely} + tokenize_test 1.$tn.2 $t {..May...you.shAre.freely} {may you share freely} + tokenize_test 1.$tn.3 $t {} {} +} + +#------------------------------------------------------------------------- +# Check that "unicode61" really is the default tokenizer. +# +do_execsql_test 2.0 " + CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61); + CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii); + INSERT INTO t1 VALUES('\xC0\xC8\xCC'); + INSERT INTO t2 VALUES('\xC0\xC8\xCC'); + INSERT INTO t3 VALUES('\xC0\xC8\xCC'); +" +do_execsql_test 2.1 " + SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC'; + SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC'; + SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC'; +" {t1 t2} + +#------------------------------------------------------------------------- +# Check that codepoints that require 4 bytes to store in utf-8 (those that +# require 17 or more bits to store). +# + +unset -nocomplain A B C D +set A [db one {SELECT char(0x1F75E)}] ;# Type So +set B [db one {SELECT char(0x1F5FD)}] ;# Type So +set C [db one {SELECT char(0x2F802)}] ;# Type Lo +set D [db one {SELECT char(0x2F808)}] ;# Type Lo + +do_execsql_test 3.0 " + CREATE VIRTUAL TABLE xyz USING fts5(x, + tokenize = \"unicode61 separators '$C' tokenchars '$A'\" + ); + CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row); + + INSERT INTO xyz VALUES('$A$B$C$D'); +" + +do_execsql_test 3.1 { + SELECT * FROM xyz_v; +} [list $A 1 1 $D 1 1] + + + + + +finish_test diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test new file mode 100644 index 0000000000000000000000000000000000000000..7a49a1d83fe0b16c01ba1f2687fffe79893dcb55 --- /dev/null +++ b/ext/fts5/test/fts5unicode2.test @@ -0,0 +1,493 @@ +# 2012 May 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# The tests in this file focus on testing the "unicode" FTS tokenizer. +# +# This is a modified copy of FTS4 test file "fts4_unicode.test". +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5unicode2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc do_unicode_token_test {tn input res} { + uplevel [list do_test $tn [list \ + sqlite3_fts5_tokenize -subst db "unicode61 remove_diacritics 0" $input + ] [list {*}$res]] +} + +proc do_unicode_token_test2 {tn input res} { + uplevel [list do_test $tn [list \ + sqlite3_fts5_tokenize -subst db "unicode61" $input + ] [list {*}$res]] +} + +proc do_unicode_token_test3 {tn args} { + set tokenizer [concat unicode61 {*}[lrange $args 0 end-2]] + set input [lindex $args end-1] + set res [lindex $args end] + uplevel [list do_test $tn [list \ + sqlite3_fts5_tokenize -subst db $tokenizer $input + ] [list {*}$res]] +} + +do_unicode_token_test 1.0 {a B c D} {a a b B c c d D} + +do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \ + "\uE4 \uC4 \uF6 \uD6 \uFC \uDC" + +do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \ + "x\uE4x x\uC4x x\uF6x x\uD6x x\uFCx x\uDCx" + +# 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s. +do_unicode_token_test 1.3 "\uDF" "\uDF \uDF" +do_unicode_token_test 1.4 "\u1E9E" "\uDF \u1E9E" + +do_unicode_token_test 1.5 "The quick brown fox" { + the The quick quick brown brown fox fox +} +do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" { + the The quick quick brown brown fox fox +} + +do_unicode_token_test2 1.7 {a B c D} {a a b B c c d D} +do_unicode_token_test2 1.8 "\uC4 \uD6 \uDC" "a \uC4 o \uD6 u \uDC" + +do_unicode_token_test2 1.9 "x\uC4x x\uD6x x\uDCx" \ + "xax x\uC4x xox x\uD6x xux x\uDCx" + +# Check that diacritics are removed if remove_diacritics=1 is specified. +# And that they do not break tokens. +do_unicode_token_test2 1.10 "xx\u0301xx" "xxxx xx\u301xx" + +# Title-case mappings work +do_unicode_token_test 1.11 "\u01c5" "\u01c6 \u01c5" + +do_unicode_token_test 1.12 "\u00C1abc\u00C2 \u00D1def\u00C3" \ + "\u00E1abc\u00E2 \u00C1abc\u00C2 \u00F1def\u00E3 \u00D1def\u00C3" + +do_unicode_token_test 1.13 "\u00A2abc\u00A3 \u00A4def\u00A5" \ + "abc abc def def" + +#------------------------------------------------------------------------- +# +set docs [list { + Enhance the INSERT syntax to allow multiple rows to be inserted via the + VALUES clause. +} { + Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause. +} { + Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp(). +} { + Added the sqlite3_db_readonly() interface. +} { + Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the + ability to add new PRAGMA statements or to override built-in PRAGMAs. +} { + Queries of the form: "SELECT max(x), y FROM table" returns the value of y on + the same row that contains the maximum x value. +} { + Added support for the FTS4 languageid option. +} { + Documented support for the FTS4 content option. This feature has actually + been in the code since version 3.7.9 but is only now considered to be + officially supported. +} { + Pending statements no longer block ROLLBACK. Instead, the pending statement + will return SQLITE_ABORT upon next access after the ROLLBACK. +} { + Improvements to the handling of CSV inputs in the command-line shell +} { + Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be + incorrectly converted into an INNER JOIN if the WHERE clause indexable terms + connected by OR. +}] + +unset -nocomplain map +set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS +set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS +set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS +set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS +set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS +set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS +set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS +set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS +set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS +foreach k [array names map] { + lappend mappings [string toupper $k] [lindex $map($k) 0] + lappend mappings $k [lindex $map($k) 1] +} +proc mapdoc {doc} { + set doc [regsub -all {[[:space:]]+} $doc " "] + string map $::mappings [string trim $doc] +} + +do_test 2.0 { + execsql { CREATE VIRTUAL TABLE t2 USING fts5(tokenize=unicode61, x); } + foreach doc $docs { + set d [mapdoc $doc] + execsql { INSERT INTO t2 VALUES($d) } + } +} {} + +do_test 2.1 { + set q [mapdoc "row"] + execsql { SELECT * FROM t2 WHERE t2 MATCH $q } +} [list [mapdoc { + Queries of the form: "SELECT max(x), y FROM table" returns the value of y on + the same row that contains the maximum x value. +}]] + +foreach {tn query snippet} { + 2 "row" { + ...returns the value of y on the same [row] that contains + the maximum x value. + } + 3 "ROW" { + ...returns the value of y on the same [row] that contains + the maximum x value. + } + 4 "rollback" { + Pending statements no longer block [ROLLBACK]. Instead, the pending + statement will return SQLITE_ABORT upon... + } + 5 "rOllback" { + Pending statements no longer block [ROLLBACK]. Instead, the pending + statement will return SQLITE_ABORT upon... + } + 6 "lang*" { + Added support for the FTS4 [languageid] option. + } +} { + do_test 2.$tn { + set q [mapdoc $query] + execsql { + SELECT snippet(t2, -1, '[', ']', '...', 15) FROM t2 WHERE t2 MATCH $q + } + } [list [mapdoc $snippet]] +} + +#------------------------------------------------------------------------- +# Make sure the unicode61 tokenizer does not crash if it is passed a +# NULL pointer. +reset_db +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x, y); + INSERT INTO t1 VALUES(NULL, 'a b c'); +} + +do_execsql_test 3.2 { + SELECT snippet(t1, -1, '[', ']', '...', 15) FROM t1 WHERE t1 MATCH 'b' +} {{a [b] c}} + +do_execsql_test 3.3 { + BEGIN; + DELETE FROM t1; + INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b'); + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 VALUES('a b c', NULL); + INSERT INTO t1 VALUES('a x c', NULL); + COMMIT; +} + +do_execsql_test 3.4 { + SELECT * FROM t1 WHERE t1 MATCH 'a b'; +} {{a b c} {}} + +#------------------------------------------------------------------------- +# +reset_db + +do_test 4.1 { + set a "abc\uFFFEdef" + set b "abc\uD800def" + set c "\uFFFEdef" + set d "\uD800def" + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x); + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + } + + execsql "CREATE VIRTUAL TABLE t8 USING fts5( + a, b, tokenize=\"unicode61 separators '\uFFFE\uD800\u00BF'\" + )" +} {} + +do_test 4.2 { + set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] + set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] + set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + execsql { + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + } +} {} + +do_test 4.3 { + set a [binary format c* {0xF7 0xBF 0xBF 0xBF}] + set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}] + set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}] + set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}] + execsql { + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + } +} {} + +do_test 4.4 { + sqlite3_exec_hex db { + CREATE VIRTUAL TABLE t9 USING fts5(a, b, + tokenize="unicode61 separators '%C09004'" + ); + INSERT INTO t9(a) VALUES('abc%88def %89ghi%90'); + } +} {0 {}} + + +#------------------------------------------------------------------------- + +do_unicode_token_test3 5.1 {tokenchars {}} { + sqlite3_reset sqlite3_column_int +} { + sqlite3 sqlite3 + reset reset + sqlite3 sqlite3 + column column + int int +} + +do_unicode_token_test3 5.2 {tokenchars _} { + sqlite3_reset sqlite3_column_int +} { + sqlite3_reset sqlite3_reset + sqlite3_column_int sqlite3_column_int +} + +do_unicode_token_test3 5.3 {separators xyz} { + Laotianxhorseyrunszfast +} { + laotian Laotian + horse horse + runs runs + fast fast +} + +do_unicode_token_test3 5.4 {tokenchars xyz} { + Laotianxhorseyrunszfast +} { + laotianxhorseyrunszfast Laotianxhorseyrunszfast +} + +do_unicode_token_test3 5.5 {tokenchars _} {separators zyx} { + sqlite3_resetxsqlite3_column_intyhonda_phantom +} { + sqlite3_reset sqlite3_reset + sqlite3_column_int sqlite3_column_int + honda_phantom honda_phantom +} + +do_unicode_token_test3 5.6 "separators \u05D1" "abc\u05D1def" { + abc abc def def +} + +do_unicode_token_test3 5.7 \ + "tokenchars \u2444\u2445" \ + "separators \u05D0\u05D1\u05D2" \ + "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \ + [list \ + \u2444fre\u2445sh \u2444fre\u2445sh \ + water water \ + fish fish \ + \u2445timer \u2445timer \ + ] + +# Check that it is not possible to add a standalone diacritic codepoint +# to either separators or tokenchars. +do_unicode_token_test3 5.8 "separators \u0301" \ + "hello\u0301world \u0301helloworld" \ + "helloworld hello\u0301world helloworld helloworld" + +do_unicode_token_test3 5.9 "tokenchars \u0301" \ + "hello\u0301world \u0301helloworld" \ + "helloworld hello\u0301world helloworld helloworld" + +do_unicode_token_test3 5.10 "separators \u0301" \ + "remove_diacritics 0" \ + "hello\u0301world \u0301helloworld" \ + "hello\u0301world hello\u0301world helloworld helloworld" + +do_unicode_token_test3 5.11 "tokenchars \u0301" \ + "remove_diacritics 0" \ + "hello\u0301world \u0301helloworld" \ + "hello\u0301world hello\u0301world helloworld helloworld" + +#------------------------------------------------------------------------- + +proc do_tokenize {tokenizer txt} { + set res [list] + foreach {b c} [sqlite3_fts5_tokenize -subst db $tokenizer $txt] { + lappend res $b + } + set res +} + +# Argument $lCodepoint must be a list of codepoints (integers) that +# correspond to whitespace characters. This command creates a string +# $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" +# using tokenizer $tokenizer. The test passes if the tokenizer successfully +# extracts the two 5 character tokens. +# +proc do_isspace_test {tn tokenizer lCp} { + set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] + set txt "${whitespace}hello${whitespace}world${whitespace}" + uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}] +} + +set tokenizers [list unicode61] +#ifcapable icu { lappend tokenizers icu } + +# Some tests to check that the tokenizers can both identify white-space +# codepoints. All codepoints tested below are of type "Zs" in the +# UnicodeData.txt file. +foreach T $tokenizers { + do_isspace_test 6.$T.1 $T 32 + do_isspace_test 6.$T.2 $T 160 + do_isspace_test 6.$T.3 $T 5760 + do_isspace_test 6.$T.4 $T 6158 + do_isspace_test 6.$T.5 $T 8192 + do_isspace_test 6.$T.6 $T 8193 + do_isspace_test 6.$T.7 $T 8194 + do_isspace_test 6.$T.8 $T 8195 + do_isspace_test 6.$T.9 $T 8196 + do_isspace_test 6.$T.10 $T 8197 + do_isspace_test 6.$T.11 $T 8198 + do_isspace_test 6.$T.12 $T 8199 + do_isspace_test 6.$T.13 $T 8200 + do_isspace_test 6.$T.14 $T 8201 + do_isspace_test 6.$T.15 $T 8202 + do_isspace_test 6.$T.16 $T 8239 + do_isspace_test 6.$T.17 $T 8287 + do_isspace_test 6.$T.18 $T 12288 + + do_isspace_test 6.$T.19 $T {32 160 5760 6158} + do_isspace_test 6.$T.20 $T {8192 8193 8194 8195} + do_isspace_test 6.$T.21 $T {8196 8197 8198 8199} + do_isspace_test 6.$T.22 $T {8200 8201 8202 8239} + do_isspace_test 6.$T.23 $T {8287 12288} +} + + +#------------------------------------------------------------------------- +# Test that the private use ranges are treated as alphanumeric. +# +foreach {tn1 c} { + 1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff +} { + foreach {tn2 config res} { + 1 "" "hello*world hello*world" + 2 "separators *" "hello hello world world" + } { + set config [string map [list * $c] $config] + set input [string map [list * $c] "hello*world"] + set output [string map [list * $c] $res] + do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output + } +} + +#------------------------------------------------------------------------- +# Cursory test of remove_diacritics=0. +# +# 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS +# 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS +# 00E4;LATIN SMALL LETTER A WITH DIAERESIS +# 00F6;LATIN SMALL LETTER O WITH DIAERESIS +# +do_execsql_test 8.1.1 " + CREATE VIRTUAL TABLE t3 USING fts5( + content, tokenize='unicode61 remove_diacritics 1' + ); + INSERT INTO t3 VALUES('o'); + INSERT INTO t3 VALUES('a'); + INSERT INTO t3 VALUES('O'); + INSERT INTO t3 VALUES('A'); + INSERT INTO t3 VALUES('\xD6'); + INSERT INTO t3 VALUES('\xC4'); + INSERT INTO t3 VALUES('\xF6'); + INSERT INTO t3 VALUES('\xE4'); +" +do_execsql_test 8.1.2 { + SELECT rowid FROM t3 WHERE t3 MATCH 'o' ORDER BY rowid ASC; +} {1 3 5 7} +do_execsql_test 8.1.3 { + SELECT rowid FROM t3 WHERE t3 MATCH 'a' ORDER BY rowid ASC; +} {2 4 6 8} +do_execsql_test 8.2.1 { + CREATE VIRTUAL TABLE t4 USING fts5( + content, tokenize='unicode61 remove_diacritics 0' + ); + INSERT INTO t4 SELECT * FROM t3 ORDER BY rowid ASC; +} +do_execsql_test 8.2.2 { + SELECT rowid FROM t4 WHERE t4 MATCH 'o' ORDER BY rowid ASC; +} {1 3} +do_execsql_test 8.2.3 { + SELECT rowid FROM t4 WHERE t4 MATCH 'a' ORDER BY rowid ASC; +} {2 4} + +#------------------------------------------------------------------------- + +foreach {tn val bErr} { + 1 0 0 + 2 1 0 + 3 2 0 + 4 3 1 + 5 11 1 +} { + reset_db + set aRes(0) {0 {}} + set aRes(1) {1 {error in tokenizer constructor}} + set res $aRes($bErr) + do_catchsql_test 9.1.$tn " + CREATE VIRTUAL TABLE bl USING fts5( + s, tokenize='trigram remove_diacritics $val' + ); + " $res +} + +finish_test diff --git a/ext/fts5/test/fts5unicode3.test b/ext/fts5/test/fts5unicode3.test new file mode 100644 index 0000000000000000000000000000000000000000..ddb61a9997d1d511c645620698cc7faa0b176740 --- /dev/null +++ b/ext/fts5/test/fts5unicode3.test @@ -0,0 +1,140 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the fts5 tokenizers +# + +source [file join [file dirname [info script]] fts5_common.tcl] + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc fts3_unicode_path {file} { + file join .. [file dirname [info script]] .. .. fts3 unicode $file +} + +source [fts3_unicode_path parseunicode.tcl] +set testprefix fts5unicode3 + +set CF [fts3_unicode_path CaseFolding.txt] +set UD [fts3_unicode_path UnicodeData.txt] + +tl_load_casefolding_txt $CF +foreach x [an_load_unicodedata_text $UD] { + set aNotAlnum($x) 1 +} + +foreach {y} [rd_load_unicodedata_text $UD] { + foreach {code ascii f} $y {} + if {$ascii==""} { + set int 0 + } else { + binary scan $ascii c int + } + set aDiacritic($code,$f) $int + if {$f==0} { set aDiacritic($code,1) $int } +} + +proc tcl_fold {i {bRemoveDiacritic 0}} { + global tl_lookup_table + global aDiacritic + set f [expr $bRemoveDiacritic==2] + + if {[info exists tl_lookup_table($i)]} { + set i $tl_lookup_table($i) + } + if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} { + set i $aDiacritic($i,$f) + } + expr $i +} +db func tcl_fold tcl_fold + +proc tcl_isalnum {i} { + global aNotAlnum + expr {![info exists aNotAlnum($i)]} +} +db func tcl_isalnum tcl_isalnum + + +do_catchsql_test 1.0.1 { + SELECT fts5_isalnum(1, 2, 3); +} {1 {wrong number of arguments to function fts5_isalnum}} +do_catchsql_test 1.0.2 { + SELECT fts5_fold(); +} {1 {wrong number of arguments to function fts5_fold}} +do_catchsql_test 1.0.3 { + SELECT fts5_fold(1,2,3); +} {1 {wrong number of arguments to function fts5_fold}} + +do_execsql_test 1.1 { + WITH ii(i) AS ( + SELECT -1 + UNION ALL + SELECT i+1 FROM ii WHERE i<100000 + ) + SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int); +} {0 {}} + +do_execsql_test 1.2.1 { + WITH ii(i) AS ( + SELECT -1 + UNION ALL + SELECT i+1 FROM ii WHERE i<100000 + ) + SELECT count(*), min(i) FROM ii + WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int); +} {0 {}} + +do_execsql_test 1.2.2 { + WITH ii(i) AS ( + SELECT -1 + UNION ALL + SELECT i+1 FROM ii WHERE i<100000 + ) + SELECT count(*), min(i) FROM ii + WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int); +} {0 {}} + +do_execsql_test 1.3 { + WITH ii(i) AS ( + SELECT -1 + UNION ALL + SELECT i+1 FROM ii WHERE i<100000 + ) + SELECT count(*), min(i) FROM ii + WHERE fts5_isalnum(i)!=CAST(tcl_isalnum(i) AS int); +} {0 {}} + +do_test 1.4 { + set str {CREATE VIRTUAL TABLE f3 USING fts5(a, tokenize=} + append str {"unicode61 separators '} + for {set i 700} {$i<900} {incr i} { + append str [format %c $i] + } + append str {'");} + execsql $str +} {} +do_test 1.5 { + set str {CREATE VIRTUAL TABLE f5 USING fts5(a, tokenize=} + append str {"unicode61 tokenchars '} + for {set i 700} {$i<900} {incr i} { + append str [format %c $i] + } + append str {'");} + execsql $str +} {} + + +finish_test diff --git a/ext/fts5/test/fts5unicode4.test b/ext/fts5/test/fts5unicode4.test new file mode 100644 index 0000000000000000000000000000000000000000..dc225cb5e28f389930fb1b54da5e425e06a61af4 --- /dev/null +++ b/ext/fts5/test/fts5unicode4.test @@ -0,0 +1,31 @@ +# 2018 July 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5unicode4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE sss USING fts5(a, prefix=3); +} + +do_execsql_test 1.1 { + INSERT INTO sss VALUES('まりや'); +} + +finish_test diff --git a/ext/fts5/test/fts5unindexed.test b/ext/fts5/test/fts5unindexed.test new file mode 100644 index 0000000000000000000000000000000000000000..5099a896937338bbd0471f8c7c0f1ee59c68902f --- /dev/null +++ b/ext/fts5/test/fts5unindexed.test @@ -0,0 +1,78 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on "unindexed" columns. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5unindexed + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b UNINDEXED); + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('g h i', 'j k l'); +} {} + +do_execsql_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b' } {1} +do_execsql_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'e' } {} + +do_execsql_test 1.4 { INSERT INTO t1(t1) VALUES('integrity-check') } {} +do_execsql_test 1.5 { INSERT INTO t1(t1) VALUES('rebuild') } {} +do_execsql_test 1.6 { INSERT INTO t1(t1) VALUES('integrity-check') } {} + +do_execsql_test 1.7 { SELECT rowid FROM t1 WHERE t1 MATCH 'b' } {1} +do_execsql_test 1.8 { SELECT rowid FROM t1 WHERE t1 MATCH 'e' } {} + +do_execsql_test 1.9 { DELETE FROM t1 WHERE t1 MATCH 'b' } {} + +do_execsql_test 1.10 { INSERT INTO t1(t1) VALUES('integrity-check') } {} +do_execsql_test 1.11 { INSERT INTO t1(t1) VALUES('rebuild') } {} +do_execsql_test 1.12 { INSERT INTO t1(t1) VALUES('integrity-check') } {} + +do_execsql_test 1.13 { SELECT rowid FROM t1 WHERE t1 MATCH 'i' } {2} +do_execsql_test 1.14 { SELECT rowid FROM t1 WHERE t1 MATCH 'l' } {} + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t2 USING fts5(a UNINDEXED, b UNINDEXED); + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('g h i', 'j k l'); + SELECT rowid FROM t2_data; +} {1 10} +do_execsql_test 2.2 { + INSERT INTO t2(t2) VALUES('rebuild'); + INSERT INTO t2(t2) VALUES('integrity-check'); + SELECT rowid FROM t2_data; +} {1 10} + +do_execsql_test 3.1 { + CREATE TABLE x4(i INTEGER PRIMARY KEY, a, b, c); + CREATE VIRTUAL TABLE t4 USING fts5(a, b UNINDEXED, c, content=x4); + INSERT INTO x4 VALUES(10, 'a b c', 'd e f', 'g h i'); + INSERT INTO x4 VALUES(20, 'j k l', 'm n o', 'p q r'); + INSERT INTO t4(t4) VALUES('rebuild'); + INSERT INTO t4(t4) VALUES('integrity-check'); +} {} + +do_execsql_test 3.2 { + INSERT INTO t4(t4, rowid, a, b, c) VALUES('delete', 20, 'j k l', '', 'p q r'); + DELETE FROM x4 WHERE rowid=20; + INSERT INTO t4(t4) VALUES('integrity-check'); +} {} + + +finish_test diff --git a/ext/fts5/test/fts5unindexed2.test b/ext/fts5/test/fts5unindexed2.test new file mode 100644 index 0000000000000000000000000000000000000000..c0abfc39801e3690067b178b3ebee37ed2184445 --- /dev/null +++ b/ext/fts5/test/fts5unindexed2.test @@ -0,0 +1,297 @@ +# 2024 Sep 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on "unindexed" columns in contentless +# tables. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5unindexed2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts5( + a, b UNINDEXED, content=, contentless_unindexed=1 + ); +} {} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('abc def', 'ghi jkl'); +} + +do_execsql_test 1.3 { + SELECT rowid, a, b FROM t1 +} {1 {} {ghi jkl}} + +do_execsql_test 1.4 { + INSERT INTO t1(rowid, a, b) VALUES(11, 'hello world', 'one two three'); +} + +do_execsql_test 1.5 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'abc def', 'ghi jkl'); +} + +do_execsql_test 1.6 { + SELECT rowid, a, b FROM t1 +} { + 11 {} {one two three} +} + +do_execsql_test 1.7 { + PRAGMA integrity_check +} {ok} + +do_execsql_test 1.8 { + INSERT INTO t1(rowid, a, b) VALUES(12, 'abc def', 'ghi jkl'); +} + +do_execsql_test 1.9 { + SELECT rowid, a, b FROM t1('def') +} {12 {} {ghi jkl}} + +do_execsql_test 1.10 { + SELECT rowid, a, b FROM t1('def OR hello') ORDER BY rank +} {11 {} {one two three} 12 {} {ghi jkl}} + +do_execsql_test 1.11 { + SELECT rowid, a, b FROM t1 WHERE rowid=11 +} {11 {} {one two three}} + +do_execsql_test 1.12 { + SELECT rowid, a, b FROM t1 +} {11 {} {one two three} 12 {} {ghi jkl}} + + +fts5_aux_test_functions db +do_execsql_test 1.12.2 { + SELECT rowid, fts5_test_columntext(t1) FROM t1('def OR hello') +} {11 {{} {one two three}} 12 {{} {ghi jkl}}} + +do_execsql_test 1.13 { + INSERT INTO t1(t1) VALUES('delete-all'); +} + +do_execsql_test 1.14 { + SELECT rowid, a, b FROM t1 +} + +do_execsql_test 1.15 { + PRAGMA integrity_check +} {ok} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t4 USING fts5( + x, y UNINDEXED, z, columnsize=0, content='', contentless_unindexed=1 + ); +} + +do_execsql_test 2.1 { + INSERT INTO t4(rowid, x, y, z) VALUES(1, 'a a', 'b b b', 'c'); +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5( + a UNINDEXED, b, c UNINDEXED, d, content=, contentless_delete=1, + contentless_unindexed=1 + ); +} + +do_execsql_test 3.1 { + INSERT INTO x1(rowid, a, b, c, d) VALUES(131, 'aaa', 'bbb', 'ccc', 'ddd'); +} + +do_execsql_test 3.2 { + SELECT * FROM x1 +} {aaa {} ccc {}} + +do_execsql_test 3.3 { + INSERT INTO x1(rowid, a, b, c, d) VALUES(1000, 'AAA', 'BBB', 'CCC', 'DDD'); +} + +do_execsql_test 3.4 { + SELECT rowid, * FROM x1 +} { + 131 aaa {} ccc {} + 1000 AAA {} CCC {} +} + +do_execsql_test 3.5 { + DELETE FROM x1 WHERE rowid=131; + SELECT rowid, * FROM x1 +} { + 1000 AAA {} CCC {} +} + +do_execsql_test 3.6 { + INSERT INTO x1(rowid, a, b, c, d) VALUES(112, 'aaa', 'bbb', 'ccc', 'ddd'); + SELECT rowid, * FROM x1 +} { + 112 aaa {} ccc {} + 1000 AAA {} CCC {} +} + +do_execsql_test 3.7 { + UPDATE x1 SET b='hello', d='world', rowid=1120 WHERE rowid=112 +} + +do_execsql_test 3.8 { + SELECT rowid, * FROM x1 +} { + 1000 AAA {} CCC {} + 1120 aaa {} ccc {} +} + +do_execsql_test 3.9 { + SELECT rowid, * FROM x1('hello'); +} { + 1120 aaa {} ccc {} +} + +do_execsql_test 3.9 { + SELECT rowid, * FROM x1('bbb'); +} { + 1000 AAA {} CCC {} +} + +fts5_aux_test_functions db +do_execsql_test 3.10 { + SELECT rowid, fts5_test_columntext(x1) FROM x1('b*') +} {1000 {AAA {} CCC {}}} + +#------------------------------------------------------------------------- +# Check that if contentless_unindexed=1 is not specified, the values +# of UNINDEXED columns are not stored in the database. +# +# Also check that contentless_unindexed=1 is not allowed unless the table +# is actually contentless. +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, c UNINDEXED, content=''); + INSERT INTO ft VALUES('one', 'two', 'three'); + SELECT rowid, * FROM ft; +} {1 {} {} {}} + +do_execsql_test 4.1 { + SELECT name FROM sqlite_schema ORDER BY 1 +} { + ft ft_config ft_data ft_docsize ft_idx +} + +do_catchsql_test 4.2 { + CREATE VIRTUAL TABLE ft2 USING fts5( + a, b, c UNINDEXED, contentless_unindexed=1 + ); +} {1 {contentless_unindexed=1 requires a contentless table}} + +do_catchsql_test 4.3 { + DELETE FROM ft WHERE rowid=1 +} {1 {cannot DELETE from contentless fts5 table: ft}} + +#------------------------------------------------------------------------- +# Check that the usual restrictions on contentless tables apply to +# contentless_unindexed=1 tables. +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ft USING fts5( + a, b UNINDEXED, c, content='', contentless_unindexed=1 + ); + INSERT INTO ft VALUES('one', 'two', 'three'); + INSERT INTO ft VALUES('four', 'five', 'six'); + INSERT INTO ft VALUES('seven', 'eight', 'nine'); + SELECT rowid, * FROM ft; +} { + 1 {} two {} + 2 {} five {} + 3 {} eight {} +} + +do_execsql_test 5.1 { + PRAGMA integrity_check +} {ok} + +do_catchsql_test 5.2 { + DELETE FROM ft WHERE rowid=2 +} {1 {cannot DELETE from contentless fts5 table: ft}} + +do_execsql_test 5.3 { + SELECT rowid, * FROM ft('six') +} { + 2 {} five {} +} + +do_catchsql_test 5.4 { + UPDATE ft SET a='x', b='y', c='z' WHERE rowid=3 +} {1 {cannot UPDATE contentless fts5 table: ft}} + +fts5_aux_test_functions db + +do_execsql_test 5.5 { + SELECT fts5_test_columntext(ft) FROM ft WHERE rowid=3 +} { + {{} eight {}} +} +do_execsql_test 5.6 { + SELECT fts5_test_columntext(ft) FROM ft('three'); +} { + {{} two {}} +} + +#------------------------------------------------------------------------- +# Check that it is possible to UPDATE a contentless_unindexed=1 table +# if the only columns being modified are UNINDEXED. +# +# If the contentless_unindexed=1 table is also contentless_delete=1, then +# it is also possible to update indexed columns - but only if *all* indexed +# columns are updated. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, c UNINDEXED, d, + contentless_unindexed=1, content='' + ); + + INSERT INTO ft1(rowid, a, b, c, d) VALUES + (100, 'x y', 'b1', 'c1', 'a b'), + (200, 'c d', 'b2', 'c2', 'a b'), + (300, 'e f', 'b3', 'c3', 'a b'); +} + +do_execsql_test 6.1 { + UPDATE ft1 SET b='b1.1', c='c1.1' WHERE rowid=100; +} +do_execsql_test 6.2 { + UPDATE ft1 SET b='b2.1' WHERE rowid=200; +} +do_execsql_test 6.3 { + UPDATE ft1 SET c='c3.1' WHERE rowid=300; +} + +do_execsql_test 6.4 { + SELECT rowid, a, b, c, d FROM ft1 +} { + 100 {} b1.1 c1.1 {} + 200 {} b2.1 c2 {} + 300 {} b3 c3.1 {} +} + +finish_test + diff --git a/ext/fts5/test/fts5update.test b/ext/fts5/test/fts5update.test new file mode 100644 index 0000000000000000000000000000000000000000..75598b47e5b339c60a6957274f97c7fd207e1778 --- /dev/null +++ b/ext/fts5/test/fts5update.test @@ -0,0 +1,157 @@ +# 2016 Jan 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5update + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set docs { + "eight zero iv eight 7" "ix one 8 one three ii one" + "1 9 9 three viii" "5 zero ii 6 nine ix 3" + "3 zero 5 2 seven nine" "two eight viii eight 1" + "4 six two 5 9 vii" "viii ii four 8 i i iv" + "vii 0 iv seven 7 viii" "five 1 nine vi seven" + "1 zero zero iii 1" "one one six 6 nine seven" + "one v 4 zero 4 iii ii" "2 3 eight six ix" + "six iv 7 three 5" "ix zero 0 8 ii 7 3" + "four six nine 2 vii 3" "five viii 5 8 0 7" +} + +foreach_detail_mode $::testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); +} {} + +do_test 1.1 { + foreach {a b} $docs { + execsql {INSERT INTO t1 VALUES($a, $b)} + } +} {} + +proc update {iRowid iA iB} { + set a [lindex $::docs $iA] + set b [lindex $::docs $iB] + execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid } +} + +set nDoc [llength $::docs] +foreach n {1 5 10 50 100} { + do_test 1.2.$n { + execsql BEGIN + for {set i 1} {$i <= 1000} {incr i} { + set iRowid [expr {int(rand() * ($nDoc/2)) + 1}] + set iA [expr {int(rand() * $nDoc)}] + set iB [expr {int(rand() * $nDoc)}] + update $iRowid $iA $iB + + if {($i % $n)==0} { + execsql { COMMIT; BEGIN } + } + + if {($i % $n)==100} { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } + } + execsql COMMIT + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} +} + +do_execsql_test 1.3 { + UPDATE t1 SET a=a AND b=b; + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_test 1.4 { + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } + for {set i 0} {$i < 50} {incr i} { + execsql { UPDATE t1 SET a=a AND b=b } + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } +} {} + +#------------------------------------------------------------------------- +# Lots of deletes/inserts of the same document with the same rowid. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x2 USING fts5(x, detail=%DETAIL%); + INSERT INTO x2(x2, rank) VALUES('crisismerge', 2); + INSERT INTO x2 VALUES('a b c'); + INSERT INTO x2 VALUES('a b c'); +} +do_test 2.1 { + for {set i 0} {$i < 1000} {incr i} { + execsql { DELETE FROM x2 WHERE rowid = 2 } + execsql { INSERT INTO x2(rowid, x) VALUES(2, 'a b c') } + } +} {} +do_execsql_test 2.1.integrity { + INSERT INTO x2(x2) VALUES('integrity-check'); +} + +do_test 2.2 { + for {set i 0} {$i < 1000} {incr i} { + execsql { UPDATE x2 SET x=x WHERE rowid=2 } + } +} {} +do_execsql_test 2.2.integrity { + INSERT INTO x2(x2) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); + INSERT INTO x3 VALUES('one'); + INSERT INTO x3 VALUES('two'); + INSERT INTO x3 VALUES('one'); + INSERT INTO x3 VALUES('two'); + INSERT INTO x3 VALUES('one'); +} + +do_test 3.1 { + db eval { SELECT * FROM x3('one') } { + db eval { + INSERT INTO x3(x3) VALUES('optimize'); + } + } +} {} + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x4 USING fts5(a, detail=%DETAIL%); + INSERT INTO x4 VALUES('one two three'); + INSERT INTO x4(rowid, a) VALUES('2', 'one two three'); + INSERT INTO x4(rowid, a) VALUES('3.0', 'one two three'); +} +do_catchsql_test 4.1 { + INSERT INTO x4(rowid, a) VALUES('four', 'one two three'); +} {1 {datatype mismatch}} + +do_catchsql_test 4.2 { + UPDATE x4 SET rowid = 'four' WHERE rowid=1; +} {1 {datatype mismatch}} + + +} + +reset_db +do_catchsql_test 4.0 { CREATE VIRTUAL TABLE t1 USING fts5(a,b,c); } {0 {}} +do_catchsql_test 4.1 { DELETE FROM t1 WHERE t1 MATCH 'f*'; } {0 {}} +finish_test diff --git a/ext/fts5/test/fts5update2.test b/ext/fts5/test/fts5update2.test new file mode 100644 index 0000000000000000000000000000000000000000..d04af4800d6a064b091d87b7f184be4d2237e8c7 --- /dev/null +++ b/ext/fts5/test/fts5update2.test @@ -0,0 +1,177 @@ +# 2024 Sep 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5update2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +#------------------------------------------------------------------------- +# Test that the various types of UPDATE statement are handled correctly +# by different table types. +# +foreach_detail_mode $testprefix { +foreach {tn cu} { + 1 0 + 2 1 +} { + reset_db + do_execsql_test 1.$tn.1 " + CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, c UNINDEXED, d, + content='', + contentless_unindexed=$cu, + detail=%DETAIL% + ); + CREATE VIRTUAL TABLE ft2 USING fts5(a, b UNINDEXED, c UNINDEXED, d, + content='', + contentless_unindexed=$cu, contentless_delete=1, + detail=%DETAIL% + ); + " + + do_execsql_test 1.$tn.2 { + INSERT INTO ft1(rowid, a, b, c, d) VALUES(1, 'a1', 'b1', 'c1', 'd1'); + INSERT INTO ft1(rowid, a, b, c, d) VALUES(2, 'a2', 'b2', 'c2', 'd2'); + INSERT INTO ft1(rowid, a, b, c, d) VALUES(3, 'a3', 'b3', 'c3', 'd3'); + + INSERT INTO ft2(rowid, a, b, c, d) VALUES(1, 'a1', 'b1', 'c1', 'd1'); + INSERT INTO ft2(rowid, a, b, c, d) VALUES(2, 'a2', 'b2', 'c2', 'd2'); + INSERT INTO ft2(rowid, a, b, c, d) VALUES(3, 'a3', 'b3', 'c3', 'd3'); + } + + # It should be possible to update a subset of the UNINDEXED columns of + # a contentless table. Regardless of whether or not contentless_unindexed=1 + # or contentless_delete=1 is set. + do_execsql_test 1.$tn.3 { + UPDATE ft1 SET b=b||'.1'; + UPDATE ft2 SET b=b||'.1'; + } + do_execsql_test 1.$tn.4 { + UPDATE ft1 SET b=b||'.2', c=c||'.2'; + UPDATE ft2 SET b=b||'.2', c=c||'.2'; + } + + set res(0) { + 1 {} {} {} {} + 2 {} {} {} {} + 3 {} {} {} {} + } + set res(1) { + 1 {} b1.1.2 c1.2 {} + 2 {} b2.1.2 c2.2 {} + 3 {} b3.1.2 c3.2 {} + } + + do_execsql_test 1.$tn.5 { + SELECT rowid, * FROM ft2 + } $res($cu) + + do_execsql_test 1.6.1 { SELECT rowid FROM ft1('a2') } {2} + do_execsql_test 1.6.2 { SELECT rowid FROM ft2('a2') } {2} + + # It should be possible to update all indexed columns (but no other subset) + # if the contentless_delete=1 option is set, as it is for "ft2". + do_execsql_test 1.$tn.7 { + UPDATE ft2 SET a='a22', d='d22' WHERE rowid=2; + } + do_execsql_test 1.$tn.8 { SELECT rowid FROM ft2('a22 AND d22') } {2} + + do_execsql_test 1.$tn.9 { + UPDATE ft2 SET a='a33', d='d33', b='b3' WHERE rowid=3; + } + + set res(1) { + 1 {} b1.1.2 c1.2 {} + 2 {} b2.1.2 c2.2 {} + 3 {} b3 c3.2 {} + } + do_execsql_test 1.$tn.10 { + SELECT rowid, * FROM ft2 + } $res($cu) + + do_catchsql_test 1.$tn.11 { + UPDATE ft2 SET a='a11' WHERE rowid=1 + } {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: ft2}} + do_catchsql_test 1.$tn.12 { + UPDATE ft2 SET d='d11' WHERE rowid=1 + } {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: ft2}} + + # It is not possible to update the values of indexed columns if + # contentless_delete=1 is not set. + do_catchsql_test 1.$tn.13 { + UPDATE ft1 SET a='a11' WHERE rowid=1 + } {1 {cannot UPDATE contentless fts5 table: ft1}} + do_catchsql_test 1.$tn.14 { + UPDATE ft1 SET d='d11' WHERE rowid=1 + } {1 {cannot UPDATE contentless fts5 table: ft1}} + + # It should be possible to update the rowid if contentless_delete=1 is + # set and all indexed columns are updated. + do_execsql_test 1.$tn.15 { + UPDATE ft2 SET a='aXone', d='dXone', rowid=11 WHERE rowid=1 + } + + set res(0) { + 2 {} {} {} {} + 3 {} {} {} {} + 11 {} {} {} {} + } + set res(1) { + 2 {} b2.1.2 c2.2 {} + 3 {} b3 c3.2 {} + 11 {} b1.1.2 c1.2 {} + } + do_execsql_test 1.$tn.16 { + SELECT rowid, * FROM ft2 + } $res($cu) + + # Should not be possible to update the rowid of a contentless_delete=1 + # table if no indexed columns are updated. + do_catchsql_test 1.$tn.17 { + UPDATE ft2 SET rowid=12 WHERE rowid=11 + } {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: ft2}} + do_catchsql_test 1.$tn.18 { + UPDATE ft1 SET rowid=12 WHERE rowid=1 + } {1 {cannot UPDATE contentless fts5 table: ft1}} + + do_execsql_test 1.$tn.19 { + UPDATE ft2 SET a='aXtwo', d='dXtwo', c='newval', rowid=12 WHERE rowid=2 + } {} + + set res(0) { + 3 {} {} {} {} + 11 {} {} {} {} + 12 {} {} {} {} + } + set res(1) { + 3 {} b3 c3.2 {} + 11 {} b1.1.2 c1.2 {} + 12 {} b2.1.2 newval {} + } + do_execsql_test 1.$tn.20 { + SELECT rowid, * FROM ft2 + } $res($cu) + + do_execsql_test 1.$tn.21 { + SELECT rowid, * FROM ft2('aXtwo AND dXtwo') + } [lrange $res($cu) 10 end] + +}} ;# end of [foreach_detail_mode] loop + +finish_test diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test new file mode 100644 index 0000000000000000000000000000000000000000..a92c0dc9f48b93abf015bd3c076e2575404b5a55 --- /dev/null +++ b/ext/fts5/test/fts5version.test @@ -0,0 +1,133 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on testing that unrecognized file-format +# versions are detected and reported. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5version + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(one); + INSERT INTO t1 VALUES('a b c d'); +} {} + +do_execsql_test 1.2 { + SELECT * FROM t1_config WHERE k='version' +} {version 4} + +do_execsql_test 1.3 { + SELECT rowid FROM t1 WHERE t1 MATCH 'a'; +} {1} + +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 1.4 { + UPDATE t1_config set v=6 WHERE k='version'; +} + +do_test 1.5 { + db close + sqlite3 db test.db + catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } +} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}} + +do_test 1.6 { + db close + sqlite3 db test.db + catchsql { INSERT INTO t1 VALUES('x y z') } +} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}} + +do_test 1.7 { + sqlite3_db_config db DEFENSIVE 0 + execsql { DELETE FROM t1_config WHERE k='version' } + db close + sqlite3 db test.db + catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} + +do_test 1.8 { + sqlite3_db_config db DEFENSIVE 0 + execsql { INSERT INTO t1_config VALUES('version', 4) } + execsql { INSERT INTO t1(t1, rank) VALUES('secure-delete', 1) } +} {} + +do_execsql_test 1.10 { + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +do_execsql_test 1.11 { + INSERT INTO t1(rowid, one) VALUES(123, 'one two three'); + DELETE FROM t1 WHERE rowid=123; + SELECT * FROM t1_config +} {secure-delete 1 version 5} + +do_execsql_test 1.11 { + INSERT INTO t1(t1) VALUES('rebuild'); + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +do_execsql_test 1.12 { + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5(x); + INSERT INTO xyz(rowid, x) VALUES + (1, 'one document'), + (2, 'two document'), + (3, 'three document'), + (4, 'four document'), + (5, 'five document'), + (6, 'six document'); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + SELECT v FROM xyz_config WHERE k='version'; +} {4} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO xyz(rowid, x) VALUES(7, 'seven document'); + SAVEPOINT one; + DELETE FROM xyz WHERE rowid = 4; +} + +do_execsql_test 2.2 { + SELECT v FROM xyz_config WHERE k='version'; +} {5} + +do_execsql_test 2.3 { + ROLLBACK TO one; + SELECT v FROM xyz_config WHERE k='version'; +} {4} + + +do_execsql_test 2.4 { + DELETE FROM xyz WHERE rowid = 3; + COMMIT; + SELECT v FROM xyz_config WHERE k='version'; +} {5} + + + + +finish_test + diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test new file mode 100644 index 0000000000000000000000000000000000000000..b1644527eafbb3e75df6fef48a7759576daf2e65 --- /dev/null +++ b/ext/fts5/test/fts5vocab.test @@ -0,0 +1,558 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on testing the fts5vocab module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5vocab + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +proc null_list_entries {iFirst nInterval L} { + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { + lset L $i {} + } + return $L +} + +proc star_from_row {L} { + if {[detail_is_full]==0} { + set L [null_list_entries 2 3 $L] + } + return $L +} + +proc star_from_col {L} { + if {[detail_is_col]} { + set L [null_list_entries 3 4 $L] + } + if {[detail_is_none]} { + set L [null_list_entries 1 4 $L] + set L [null_list_entries 3 4 $L] + } + return $L +} + +proc row_to_col {L} { + if {[detail_is_none]==0} { error "this is for detail=none mode" } + set ret [list] + foreach {a b c} $L { + lappend ret $a {} $b {} + } + set ret +} + +if 1 { + +do_execsql_test 1.1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); + PRAGMA table_info = v1; +} { + 0 term {} 0 {} 0 + 1 doc {} 0 {} 0 + 2 cnt {} 0 {} 0 +} + +do_execsql_test 1.1.2 { + CREATE VIRTUAL TABLE v2 USING fts5vocab(t1, 'col'); + PRAGMA table_info = v2; +} { + 0 term {} 0 {} 0 + 1 col {} 0 {} 0 + 2 doc {} 0 {} 0 + 3 cnt {} 0 {} 0 +} + +do_execsql_test 1.2.1 { SELECT * FROM v1 } {} +do_execsql_test 1.2.2 { SELECT * FROM v2 } {} + +do_execsql_test 1.3 { + INSERT INTO t1 VALUES('x y z'); + INSERT INTO t1 VALUES('x x x'); +} + +do_execsql_test 1.4.1 { + SELECT * FROM v1; +} [star_from_row {x 2 4 y 1 1 z 1 1}] + +do_execsql_test 1.4.2 { + SELECT * FROM v2; +} [star_from_col {x one 2 4 y one 1 1 z one 1 1}] + +do_execsql_test 1.5.1 { + BEGIN; + INSERT INTO t1 VALUES('a b c'); + SELECT * FROM v1 WHERE term<'d'; +} [star_from_row {a 1 1 b 1 1 c 1 1}] + +do_execsql_test 1.5.2 { + SELECT * FROM v2 WHERE term<'d'; + COMMIT; +} [star_from_col {a one 1 1 b one 1 1 c one 1 1}] + +do_execsql_test 1.6 { + DELETE FROM t1 WHERE one = 'a b c'; + SELECT * FROM v1; +} [star_from_row {x 2 4 y 1 1 z 1 1}] + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%); + INSERT INTO tt VALUES('d g b f d f', 'f c e c d a'); + INSERT INTO tt VALUES('f a e a a b', 'e d c f d d'); + INSERT INTO tt VALUES('b c a a a b', 'f f c c b c'); + INSERT INTO tt VALUES('f d c a c e', 'd g d e g d'); + INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b'); + INSERT INTO tt VALUES('g c f b c g', 'a g f d c b'); + INSERT INTO tt VALUES('c e c f g b', 'f e d b g a'); + INSERT INTO tt VALUES('g d e f d e', 'a c d b a g'); + INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y'); + INSERT INTO tt VALUES('c c a a c f', 'd g a e b g'); +} + +set res_row [star_from_row { + a 10 20 b 9 14 c 9 20 d 9 19 + e 8 13 f 10 20 g 7 14 x 1 1 + y 1 1 +}] +set res_col [star_from_col { + a a 6 11 a b 7 9 + b a 6 7 b b 7 7 + c a 6 12 c b 5 8 + d a 4 6 d b 9 13 + e a 6 7 e b 6 6 + f a 9 10 f b 7 10 + g a 5 7 g b 5 7 + x a 1 1 y b 1 1 +}] +if {[detail_is_none]} { + set res_col [row_to_col $res_row] +} + +foreach {tn tbl resname} { + 1 "fts5vocab(tt, 'col')" res_col + 2 "fts5vocab(tt, 'row')" res_row + 3 "fts5vocab(tt, \"row\")" res_row + 4 "fts5vocab(tt, [row])" res_row + 5 "fts5vocab(tt, `row`)" res_row + + 6 "fts5vocab('tt', 'row')" res_row + 7 "fts5vocab(\"tt\", \"row\")" res_row + 8 "fts5vocab([tt], [row])" res_row + 9 "fts5vocab(`tt`, `row`)" res_row +} { + do_execsql_test 2.$tn " + DROP TABLE IF EXISTS tv; + CREATE VIRTUAL TABLE tv USING $tbl; + SELECT * FROM tv; + " [set $resname] +} + +#------------------------------------------------------------------------- +# Test errors in the CREATE VIRTUAL TABLE statement. +# +foreach {tn sql} { + 1 { CREATE VIRTUAL TABLE aa USING fts5vocab() } + 2 { CREATE VIRTUAL TABLE aa USING fts5vocab(x) } + 3 { CREATE VIRTUAL TABLE aa USING fts5vocab(x,y,z) } + 4 { CREATE VIRTUAL TABLE temp.aa USING fts5vocab(x,y,z,y) } +} { + do_catchsql_test 3.$tn $sql {1 {wrong number of vtable arguments}} +} + +do_catchsql_test 4.0 { + CREATE VIRTUAL TABLE cc USING fts5vocab(tbl, unknown); +} {1 {fts5vocab: unknown table type: 'unknown'}} + +do_catchsql_test 4.1 { + ATTACH 'test.db' AS aux; + CREATE VIRTUAL TABLE aux.cc USING fts5vocab(main, tbl, row); +} {1 {wrong number of vtable arguments}} + +#------------------------------------------------------------------------- +# Test fts5vocab tables created in the temp schema. +# +reset_db +forcedelete test.db2 +do_execsql_test 5.0 { + ATTACH 'test.db2' AS aux; + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%); + + INSERT INTO main.t1 VALUES('a b c'); + INSERT INTO main.t1 VALUES('d e f'); + INSERT INTO main.t1 VALUES('a e c'); + + INSERT INTO temp.t1 VALUES('1 2 3'); + INSERT INTO temp.t1 VALUES('4 5 6'); + INSERT INTO temp.t1 VALUES('1 5 3'); + + INSERT INTO aux.t1 VALUES('x y z'); + INSERT INTO aux.t1 VALUES('m n o'); + INSERT INTO aux.t1 VALUES('x n z'); +} + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE temp.vm USING fts5vocab(main, t1, row); + CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row); + CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row); + CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); +} + +do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row { + a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1 +}] +do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row { + 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 +}] +do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row { + 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 +}] +do_execsql_test 5.5 { SELECT * FROM va } [star_from_row { + m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2 +}] + +#------------------------------------------------------------------------- +# +do_execsql_test 6.0 { + CREATE TABLE iii(iii); + CREATE TABLE jjj(x); +} + +do_catchsql_test 6.1 { + CREATE VIRTUAL TABLE vocab1 USING fts5vocab(iii, row); + SELECT * FROM vocab1; +} {1 {no such fts5 table: main.iii}} + +do_catchsql_test 6.2 { + CREATE VIRTUAL TABLE vocab2 USING fts5vocab(jjj, row); + SELECT * FROM vocab2; +} {1 {no such fts5 table: main.jjj}} + +do_catchsql_test 6.2 { + CREATE VIRTUAL TABLE vocab3 USING fts5vocab(lll, row); + SELECT * FROM vocab3; +} {1 {no such fts5 table: main.lll}} + +#------------------------------------------------------------------------- +# Test single term queries on fts5vocab tables (i.e. those with term=? +# constraints in the WHERE clause). +# +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%); + INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); + INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); + INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); + INSERT INTO tx VALUES('e bb h jjj ii gg', 'e aa e f c fff'); + INSERT INTO tx VALUES('j ff aa a h', 'h a j bbb bb'); + INSERT INTO tx VALUES('cc i ff c d f', 'dd ii fff f c cc d'); + INSERT INTO tx VALUES('jjj g i bb cc eee', 'hhh iii aaa b bbb aaa'); + INSERT INTO tx VALUES('hhh hhh hhh bb fff f', 'fff gg aa ii h a'); + INSERT INTO tx VALUES('b c cc aaa iii ggg f', 'iii ff ee a ff c cc'); + INSERT INTO tx VALUES('hhh b hhh aaa j i i', 'dd ee ee aa bbb iii'); + INSERT INTO tx VALUES('hh dd h b g ff i', 'ccc bb cc ccc f a d'); + INSERT INTO tx VALUES('g d b ggg jj', 'fff jj ff jj g gg ee'); + INSERT INTO tx VALUES('g ee ggg ggg cc bb eee', 'aa j jjj bbb dd eee ff'); + INSERT INTO tx VALUES('c jjj hh ddd dd h', 'e aaa h jjj gg'); + + CREATE VIRTUAL TABLE txr USING fts5vocab(tx, row); + CREATE VIRTUAL TABLE txc USING fts5vocab(tx, col); +} + +proc cont {L elem} { + set n 0 + foreach e $L { if {$elem==$e} {incr n} } + set n +} +db func cont cont + +foreach {term} { + a aa aaa + b bb bbb + c cc ccc + d dd ddd + e ee eee + f ff fff + g gg ggg + h hh hhh + i ii iii + j jj jjj +} { + set resr [db eval { + SELECT $term, + sum(cont(one || ' ' || two, $term) > 0), + sum(cont(one || ' ' || two, $term)) + FROM tx + }] + if {[lindex $resr 1]==0} {set resr [list]} + + set r1 [db eval { + SELECT $term, 'one', sum(cont(one, $term)>0), sum(cont(one, $term)) FROM tx + }] + if {[lindex $r1 2]==0} {set r1 [list]} + + set r2 [db eval { + SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx + }] + if {[lindex $r2 2]==0} {set r2 [list]} + + set resc [concat $r1 $r2] + + set resc [star_from_col $resc] + set resr [star_from_row $resr] + if {[detail_is_none]} { set resc [row_to_col $resr] } + do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc + do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr +} + +do_execsql_test 7.1 { + CREATE TABLE txr_c AS SELECT * FROM txr; + CREATE TABLE txc_c AS SELECT * FROM txc; +} + +# Test range queries on the fts5vocab tables created above. +# +foreach {tn a b} { + 1 a jjj + 2 bb j + 3 ccc ddd + 4 dd xyz + 5 xzy dd + 6 h hh +} { + do_execsql_test 7.2.$tn.1 { + SELECT * FROM txr WHERE term>=$a + } [db eval {SELECT * FROM txr_c WHERE term>=$a}] + do_execsql_test 7.2.$tn.2 { + SELECT * FROM txr WHERE term<=$b + } [db eval {SELECT * FROM txr_c WHERE term <=$b}] + do_execsql_test 7.2.$tn.3 { + SELECT * FROM txr WHERE term>=$a AND term<=$b + } [db eval {SELECT * FROM txr_c WHERE term>=$a AND term <=$b}] + + do_execsql_test 7.2.$tn.4 { + SELECT * FROM txc WHERE term>=$a + } [db eval {SELECT * FROM txc_c WHERE term>=$a}] + do_execsql_test 7.2.$tn.5 { + SELECT * FROM txc WHERE term<=$b + } [db eval {SELECT * FROM txc_c WHERE term <=$b}] + do_execsql_test 7.2.$tn.6 { + SELECT * FROM txc WHERE term>=$a AND term<=$b + } [db eval {SELECT * FROM txc_c WHERE term>=$a AND term <=$b}] + + do_execsql_test 7.2.$tn.7 { + SELECT * FROM txr WHERE term>$a + } [db eval {SELECT * FROM txr_c WHERE term>$a}] + do_execsql_test 7.2.$tn.8 { + SELECT * FROM txr WHERE term<$b + } [db eval {SELECT * FROM txr_c WHERE term<$b}] + do_execsql_test 7.2.$tn.9 { + SELECT * FROM txr WHERE term>$a AND term<$b + } [db eval {SELECT * FROM txr_c WHERE term>$a AND term <$b}] + + do_execsql_test 7.2.$tn.10 { + SELECT * FROM txc WHERE term>$a + } [db eval {SELECT * FROM txc_c WHERE term>$a}] + do_execsql_test 7.2.$tn.11 { + SELECT * FROM txc WHERE term<$b + } [db eval {SELECT * FROM txc_c WHERE term<$b}] + do_execsql_test 7.2.$tn.12 { + SELECT * FROM txc WHERE term>$a AND term<$b + } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}] +} + +do_execsql_test 7.3.1 { + SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; +} {30} + +if {![detail_is_none]} { + do_execsql_test 7.3.2 { + SELECT count(*) FROM txc, txc_c + WHERE txc.term = txc_c.term AND txc.col=txc_c.col; + } {57} +} + +} + +#------------------------------------------------------------------------- +# Test the fts5vocab tables response to a specific types of corruption: +# where the fts5 index contains hits for columns that do not exist. +# +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i'); + INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f'); + INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c'); + CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row); + CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col); +} + +set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}] +set resc [star_from_col { + a a 1 1 a b 1 1 a c 1 1 b a 1 1 + b b 1 1 b c 1 1 c a 1 1 c b 1 1 + c c 1 1 d a 1 1 d b 1 1 d c 1 1 + e a 1 1 e b 1 1 e c 1 1 f a 1 1 + f b 1 1 f c 1 1 g a 1 1 g b 1 1 + g c 1 1 h a 1 1 h b 1 1 h c 1 1 + i a 1 1 i b 1 1 i c 1 1 +}] +if {[detail_is_none]} { set resc [row_to_col $resr] } + +do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr +do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc + +sqlite3_db_config db DEFENSIVE 0 +do_execsql_test 8.2 { + PRAGMA writable_schema = 1; + UPDATE sqlite_master + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)' + WHERE name = 'x1'; +} +db close +sqlite3 db test.db +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr + +if {[detail_is_none]} { + do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc +} else { + do_catchsql_test 8.2.2 { + SELECT * FROM x1_c + } {1 {database disk image is malformed}} +} + +sqlite3_fts5_may_be_corrupt 0 +} + +#------------------------------------------------------------------------- +# Test that both "ORDER BY term" and "ORDER BY term DESC" work. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1 VALUES('def ABC ghi'); + INSERT INTO x1 VALUES('DEF abc GHI'); +} + +do_execsql_test 9.2 { + CREATE VIRTUAL TABLE rrr USING fts5vocab(x1, row); + SELECT * FROM rrr +} { + abc 2 2 def 2 2 ghi 2 2 +} +do_execsql_test 9.3 { + SELECT * FROM rrr ORDER BY term ASC +} { + abc 2 2 def 2 2 ghi 2 2 +} +do_execsql_test 9.4 { + SELECT * FROM rrr ORDER BY term DESC +} { + ghi 2 2 def 2 2 abc 2 2 +} +do_test 9.5 { + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term ASC }] + expr [lsearch $e2 SorterSort]<0 +} 1 +do_test 9.6 { + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }] + expr [lsearch $e2 SorterSort]<0 +} 0 + +#------------------------------------------------------------------------- +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, c); + CREATE VIRTUAL TABLE t2 USING fts5vocab('ft','row'); + CREATE VIRTUAL TABLE t3 USING fts5vocab('ft','row'); +} + +do_execsql_test 10.1 { + BEGIN; + INSERT INTO ft(b) VALUES('x y'); +} + +do_execsql_test 10.2 { + SELECT t2.term FROM t2; +} {x y} + +do_execsql_test 10.3 { + SELECT t2.term, t3.term FROM t2, t3; +} {x x x y y x y y} + +do_execsql_test 10.4 { + COMMIT; +} + +do_execsql_test 10.5 { + BEGIN; + INSERT INTO ft(a) VALUES('1 2 3'); + INSERT INTO ft(a) VALUES('4 5 6'); + INSERT INTO ft(a) VALUES('1 2 3'); + INSERT INTO ft(a) VALUES('4 5 6'); + INSERT INTO ft(a) VALUES('1 2 3'); + INSERT INTO ft(a) VALUES('4 5 6'); +} + +unset -nocomplain x res +do_test 10.6 { + set res [list] + db eval { SELECT rowid FROM ft('4') } x { + db eval { SELECT * FROM t2 } + lappend res $x(rowid) + } + db eval COMMIT + set res +} {3 5 7} + +do_execsql_test 10.6.1 { + SELECT * FROM t2 WHERE termNULL; +} +do_execsql_test 10.6.3 { + SELECT * FROM t2 WHERE term=NULL; +} +do_execsql_test 10.7.1 { + SELECT * FROM t2 WHERE term?; +} +do_execsql_test 10.7.3 { + SELECT * FROM t2 WHERE term=?; +} + +# 2020-02-16 Detect recursively define fts5vocab() tables. +# Error found by dbsqlfuzz. +# +reset_db +do_execsql_test 11.100 { + CREATE VIRTUAL TABLE t3 USING fts5vocab(rowid , 'col'); + CREATE VIRTUAL TABLE rowid USING fts5vocab(rowid , 'instance'); +} {} +do_catchsql_test 11.110 { + SELECT rowid+1,rowid, * FROM t3 WHERE null>rowid ; +} {1 {SQL logic error}} + +finish_test diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test new file mode 100644 index 0000000000000000000000000000000000000000..7b3c3b0d6a975878e3f35e11a53e849f5be8fbe9 --- /dev/null +++ b/ext/fts5/test/fts5vocab2.test @@ -0,0 +1,310 @@ +# 2017 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on testing the fts5vocab module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5vocab2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 1.1 { + SELECT * FROM v1; +} { + five 2 b 1 + five 2 b 2 + five 2 b 3 + four 2 a 1 + four 2 b 0 + one 1 a 0 + three 1 b 1 + three 2 a 0 + two 1 a 1 + two 1 b 0 +} + +do_execsql_test 1.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 b 1 + three 2 a 0 +} + +do_execsql_test 1.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 a 0 + three 1 b 1 + two 1 a 1 + two 1 b 0 +} + +do_execsql_test 1.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 b 1 + five 2 b 2 + five 2 b 3 + four 2 a 1 + four 2 b 0 + three 2 a 0 +} + +do_execsql_test 1.5 { + DELETE FROM t1; + SELECT * FROM v1; +} {} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS v1; + + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=column); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 2.1 { + SELECT * FROM v1; +} { + five 2 b {} + four 2 a {} + four 2 b {} + one 1 a {} + three 1 b {} + three 2 a {} + two 1 a {} + two 1 b {} +} + +do_execsql_test 2.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 b {} + three 2 a {} +} + +do_execsql_test 2.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 a {} + three 1 b {} + two 1 a {} + two 1 b {} +} + +do_execsql_test 2.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 b {} + four 2 a {} + four 2 b {} + three 2 a {} +} + +do_execsql_test 2.5 { + DELETE FROM t1; + SELECT * FROM v1; +} {} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS v1; + + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 3.1 { + SELECT * FROM v1; +} { + five 2 {} {} + four 2 {} {} + one 1 {} {} + three 1 {} {} + three 2 {} {} + two 1 {} {} +} + +do_execsql_test 3.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 {} {} + three 2 {} {} +} + +do_execsql_test 3.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 {} {} + three 1 {} {} + two 1 {} {} +} + +do_execsql_test 3.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 {} {} + four 2 {} {} + three 2 {} {} +} + +do_execsql_test 3.5 { + DELETE FROM t1; + SELECT * FROM v1; +} {} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE v1 USING fts5vocab(nosuchtable, col); +} + +do_catchsql_test 4.1 { + SELECT * FROM v1 WHERE term=='nosuchterm'; +} {1 {no such fts5 table: main.nosuchtable}} + +do_execsql_test 4.2.1 { + CREATE TABLE nosuchtable(nosuchtable, y, z); +} +do_catchsql_test 4.2.2 { + SELECT * FROM v1 WHERE term=='nosuchterm'; +} {1 {no such fts5 table: main.nosuchtable}} + +ifcapable fts3 { + do_execsql_test 4.3.1 { + DROP TABLE nosuchtable; + CREATE VIRTUAL TABLE nosuchtable USING fts3(a, b); + } {} + do_catchsql_test 4.3.2 { + SELECT * FROM v1 WHERE term=='nosuchterm'; + } {1 {no such fts5 table: main.nosuchtable}} + do_catchsql_test 4.3.3 { + INSERT INTO nosuchtable VALUES('id', '*id'); + SELECT * FROM v1 WHERE term=='nosuchterm'; + } {1 {no such fts5 table: main.nosuchtable}} +} + +#------------------------------------------------------------------------- +# Check that the fts5 table cannot be written while there are vocab +# cursors open. +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + INSERT INTO t1 VALUES('one'), ('two'), ('three'), ('four'); +} + +do_test 5.1 { + list [catch { + db eval { SELECT * FROM v1 } { + db eval {INSERT INTO t1 VALUES('five')} + } + } msg] $msg +} {1 {query aborted}} + +do_execsql_test 5.2 { + SELECT * FROM t1 +} {one two three four five} + +#------------------------------------------------------------------------- +# Check that the fts5 table cannot be written while there are vocab +# cursors open. +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO t1 SELECT + 'State Emergency Service (SES), Rural Fire Service (RFS) and Volunteers' + FROM s; +} + +do_catchsql_test 5.1 { + INSERT INTO t1 SELECT rowid FROM v1 +} {1 {query aborted}} + +do_catchsql_test 5.2 { + DELETE FROM t1 WHERE rowid>100; + INSERT INTO t1 SELECT randomblob(3000) FROM v1 +} {1 {query aborted}} + +#------------------------------------------------------------------------- +reset_db +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 6.0 { + BEGIN TRANSACTION; + CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1); + REPLACE INTO t1_data VALUES(1,X'03090009'); + REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101'); + REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808'); + REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909'); + REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808'); + COMMIT; +} + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row'); +} + +do_catchsql_test 6.2 { + SELECT * FROM t3; +} {1 {database disk image is malformed}} + +sqlite3_fts5_may_be_corrupt 0 + +finish_test + + diff --git a/ext/fts5/tool/fts5speed.tcl b/ext/fts5/tool/fts5speed.tcl new file mode 100644 index 0000000000000000000000000000000000000000..0f38638c26ae553df7812b1ffb3ffc1b26ff7714 --- /dev/null +++ b/ext/fts5/tool/fts5speed.tcl @@ -0,0 +1,64 @@ + + +set Q { + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"} + {25 "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"} + {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"} + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"} + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"} + + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} + {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:the'"} + + {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes OR e:holmes OR f:holmes OR g:holmes'" } + {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes AND e:holmes AND f:holmes AND g:holmes'" } + {4 "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes NOT e:holmes'" } +} + +proc usage {} { + global Q + puts stderr "Usage: $::argv0 DATABASE QUERY" + puts stderr "" + for {set i 1} {$i <= [llength $Q]} {incr i} { + puts stderr " $i. [lindex $Q [expr $i-1]]" + } + puts stderr "" + exit -1 +} + + +set nArg [llength $argv] +if {$nArg!=2 && $nArg!=3} usage +set database [lindex $argv 0] +set iquery [lindex $argv 1] +if {$iquery<1 || $iquery>[llength $Q]} usage +set nRepeat 0 +if {$nArg==3} { set nRepeat [lindex $argv 2] } + + +sqlite3 db $database +catch { load_static_extension db fts5 } + +incr iquery -1 +set sql [lindex $Q $iquery 1] +if {$nRepeat==0} { + set nRepeat [lindex $Q $iquery 0] +} + +puts "sql: $sql" +puts "nRepeat: $nRepeat" +if {[regexp matchinfo $sql]} { + sqlite3_fts5_register_matchinfo db + db eval $sql +} else { + puts "result: [db eval $sql]" +} + +for {set i 1} {$i < $nRepeat} {incr i} { + db eval $sql +} + + diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl new file mode 100644 index 0000000000000000000000000000000000000000..1996b2c313693cbfd67cb714cd19c892e0171c1c --- /dev/null +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -0,0 +1,231 @@ +########################################################################## +# 2016 Jan 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +proc process_cmdline {} { + cmdline::process ::A $::argv { + {fts5 "use fts5 (this is the default)"} + {fts4 "use fts4"} + {trigram "Use tokenize=trigram"} + {colsize "10 10 10" "list of column sizes"} + {tblname "t1" "table name to create"} + {detail "full" "Fts5 detail mode to use"} + {repeat 1 "Load each file this many times"} + {prefix "" "Fts prefix= option"} + {trans 1 "True to use a transaction"} + database + file... + } { + This script is designed to create fts4/5 tables with more than one column. + The -colsize option should be set to a Tcl list of integer values, one for + each column in the table. Each value is the number of tokens that will be + inserted into the column value for each row. For example, setting the -colsize + option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10 + tokens per row in each, respectively. + + Each "FILE" argument should be a text file. The contents of these text files + is split on whitespace characters to form a list of tokens. The first N1 + tokens are used for the first column of the first row, where N1 is the first + element of the -colsize list. The next N2 are used for the second column of + the first row, and so on. Rows are added to the table until the entire list + of tokens is exhausted. + } +} + +########################################################################### +########################################################################### +# Command line options processor. This is generic code that can be copied +# between scripts. +# +namespace eval cmdline { + proc cmdline_error {O E {msg ""}} { + if {$msg != ""} { + puts stderr "Error: $msg" + puts stderr "" + } + + set L [list] + foreach o $O { + if {[llength $o]==1} { + lappend L [string toupper $o] + } + } + + puts stderr "Usage: $::argv0 ?SWITCHES? $L" + puts stderr "" + puts stderr "Switches are:" + foreach o $O { + if {[llength $o]==3} { + foreach {a b c} $o {} + puts stderr [format " -%-15s %s (default \"%s\")" "$a VAL" $c $b] + } elseif {[llength $o]==2} { + foreach {a b} $o {} + puts stderr [format " -%-15s %s" $a $b] + } + } + puts stderr "" + puts stderr $E + exit -1 + } + + proc process {avar lArgs O E} { + upvar $avar A + set zTrailing "" ;# True if ... is present in $O + set lPosargs [list] + + # Populate A() with default values. Also, for each switch in the command + # line spec, set an entry in the idx() array as follows: + # + # {tblname t1 "table name to use"} + # -> [set idx(-tblname) {tblname t1 "table name to use"} + # + # For each position parameter, append its name to $lPosargs. If the ... + # specifier is present, set $zTrailing to the name of the prefix. + # + foreach o $O { + set nm [lindex $o 0] + set nArg [llength $o] + switch -- $nArg { + 1 { + if {[string range $nm end-2 end]=="..."} { + set zTrailing [string range $nm 0 end-3] + } else { + lappend lPosargs $nm + } + } + 2 { + set A($nm) 0 + set idx(-$nm) $o + } + 3 { + set A($nm) [lindex $o 1] + set idx(-$nm) $o + } + default { + error "Error in command line specification" + } + } + } + + # Set explicitly specified option values + # + set nArg [llength $lArgs] + for {set i 0} {$i < $nArg} {incr i} { + set opt [lindex $lArgs $i] + if {[string range $opt 0 0]!="-" || $opt=="--"} break + set c [array names idx "${opt}*"] + if {[llength $c]==0} { cmdline_error $O $E "Unrecognized option: $opt"} + if {[llength $c]>1} { cmdline_error $O $E "Ambiguous option: $opt"} + + if {[llength $idx($c)]==3} { + if {$i==[llength $lArgs]-1} { + cmdline_error $O $E "Option requires argument: $c" + } + incr i + set A([lindex $idx($c) 0]) [lindex $lArgs $i] + } else { + set A([lindex $idx($c) 0]) 1 + } + } + + # Deal with position arguments. + # + set nPosarg [llength $lPosargs] + set nRem [expr $nArg - $i] + if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} { + cmdline_error $O $E + } + for {set j 0} {$j < $nPosarg} {incr j} { + set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]] + } + if {$zTrailing!=""} { + set A($zTrailing) [lrange $lArgs [expr $j+$i] end] + } + } +} ;# namespace eval cmdline +# End of command line options processor. +########################################################################### +########################################################################### + +process_cmdline + +# If -fts4 was specified, use fts4. Otherwise, fts5. +if {$A(fts4)} { + set A(fts) fts4 +} else { + set A(fts) fts5 +} + +sqlite3 db $A(database) + +# Create the FTS table in the db. Return a list of the table columns. +# +proc create_table {} { + global A + set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + + set nCol [llength $A(colsize)] + set cols [lrange $cols 0 [expr $nCol-1]] + + set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $A(tblname) USING $A(fts) (" + append sql [join $cols ,] + if {$A(fts)=="fts5"} { append sql ",detail=$A(detail)" } + if {$A(trigram)} { append sql ",tokenize=trigram" } + append sql ", prefix='$A(prefix)');" + + db eval $sql + return $cols +} + +# Return a list of tokens from the named file. +# +proc readfile {file} { + set fd [open $file] + set data [read $fd] + close $fd + split $data +} + +proc repeat {L n} { + set res [list] + for {set i 0} {$i < $n} {incr i} { + set res [concat $res $L] + } + set res +} + + +# Load all the data into a big list of tokens. +# +set tokens [list] +foreach f $A(file) { + set tokens [concat $tokens [repeat [readfile $f] $A(repeat)]] +} + +set N [llength $tokens] +set i 0 +set cols [create_table] +set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])" +foreach c [lrange $cols 1 end] { + append sql ", \$R($c)" +} +append sql ")" + +if {$A(trans)} { db eval BEGIN } + while {$i < $N} { + foreach c $cols s $A(colsize) { + set R($c) [lrange $tokens $i [expr $i+$s-1]] + incr i $s + } + db eval $sql + } +if {$A(trans)} { db eval COMMIT } + + + diff --git a/ext/fts5/tool/loadfts5.tcl b/ext/fts5/tool/loadfts5.tcl new file mode 100644 index 0000000000000000000000000000000000000000..96fd69260ec2aaf86451a4a1dad8b33f30779830 --- /dev/null +++ b/ext/fts5/tool/loadfts5.tcl @@ -0,0 +1,172 @@ + + +proc loadfile {f} { + set fd [open $f] + set data [read $fd] + close $fd + return $data +} + +set ::nRow 0 +set ::nRowPerDot 1000 + +proc load_hierachy {dir} { + foreach f [glob -nocomplain -dir $dir *] { + if {$::O(limit) && $::nRow>=$::O(limit)} break + if {[file isdir $f]} { + load_hierachy $f + } else { + db eval { INSERT INTO t1 VALUES($f, loadfile($f)) } + incr ::nRow + + if {$::O(trans) && ($::nRow % $::O(trans))==0} { + db eval { COMMIT } + db eval { INSERT INTO t1(t1) VALUES('integrity-check') } + db eval { BEGIN } + } + + if {($::nRow % $::nRowPerDot)==0} { + puts -nonewline . + if {($::nRow % (65*$::nRowPerDot))==0} { puts "" } + flush stdout + } + + } + } +} + +proc usage {} { + puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE PATH" + puts stderr "" + puts stderr "Switches are:" + puts stderr " -fts4 (use fts4 instead of fts5)" + puts stderr " -fts5 (use fts5)" + puts stderr " -porter (use porter tokenizer)" + puts stderr " -delete (delete the database file before starting)" + puts stderr " -limit N (load no more than N documents)" + puts stderr " -automerge N (set the automerge parameter to N)" + puts stderr " -crisismerge N (set the crisismerge parameter to N)" + puts stderr " -prefix PREFIX (comma separated prefix= argument)" + puts stderr " -trans N (commit after N inserts - 0 == never)" + puts stderr " -hashsize N (set the fts5 hashsize parameter to N)" + puts stderr " -detail MODE (detail mode for fts5 tables)" + exit 1 +} + +set O(vtab) fts5 +set O(tok) "" +set O(limit) 0 +set O(delete) 0 +set O(automerge) -1 +set O(crisismerge) -1 +set O(prefix) "" +set O(trans) 0 +set O(hashsize) -1 +set O(detail) full + +if {[llength $argv]<2} usage +set nOpt [expr {[llength $argv]-2}] +for {set i 0} {$i < $nOpt} {incr i} { + set arg [lindex $argv $i] + switch -- [lindex $argv $i] { + -fts4 { + set O(vtab) fts4 + } + + -fts5 { + set O(vtab) fts5 + } + + -porter { + set O(tok) ", tokenize=porter" + } + + -delete { + set O(delete) 1 + } + + -limit { + if { [incr i]>=$nOpt } usage + set O(limit) [lindex $argv $i] + } + + -trans { + if { [incr i]>=$nOpt } usage + set O(trans) [lindex $argv $i] + } + + -automerge { + if { [incr i]>=$nOpt } usage + set O(automerge) [lindex $argv $i] + } + + -crisismerge { + if { [incr i]>=$nOpt } usage + set O(crisismerge) [lindex $argv $i] + } + + -prefix { + if { [incr i]>=$nOpt } usage + set O(prefix) [lindex $argv $i] + } + + -hashsize { + if { [incr i]>=$nOpt } usage + set O(hashsize) [lindex $argv $i] + } + + -detail { + if { [incr i]>=$nOpt } usage + set O(detail) [lindex $argv $i] + } + + default { + usage + } + } +} + +set dbfile [lindex $argv end-1] +if {$O(delete)} { file delete -force $dbfile } +sqlite3 db $dbfile +catch { load_static_extension db fts5 } +db func loadfile loadfile +db eval "PRAGMA page_size=4096" + +db eval BEGIN + set pref "" + if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } + if {$O(vtab)=="fts5"} { + append pref ", detail=$O(detail)" + } + catch { + db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)" + db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);" + } + + if {$O(hashsize)>=0} { + catch { + db eval "INSERT INTO t1(t1, rank) VALUES('hashsize', $O(hashsize));" + } + } + + + if {$O(automerge)>=0} { + if {$O(vtab) == "fts5"} { + db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } + } else { + db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } + } + } + if {$O(crisismerge)>=0} { + if {$O(vtab) == "fts5"} { + db eval {INSERT INTO t1(t1, rank) VALUES('crisismerge', $O(crisismerge))} + } else { + } + } + load_hierachy [lindex $argv end] +db eval COMMIT +puts "" + + + diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl new file mode 100644 index 0000000000000000000000000000000000000000..6f20a0cd734169fb693b2d4f0e1dbcfa4fc0ee23 --- /dev/null +++ b/ext/fts5/tool/mkfts5c.tcl @@ -0,0 +1,140 @@ +#!/bin/sh +# restart with tclsh \ +exec tclsh "$0" "$@" + +set srcdir [file dirname [file dirname [file normalize [info script]]]] +set G(src) [string map [list %dir% $srcdir] { + %dir%/fts5.h + %dir%/fts5Int.h + fts5parse.h + fts5parse.c + %dir%/fts5_aux.c + %dir%/fts5_buffer.c + %dir%/fts5_config.c + %dir%/fts5_expr.c + %dir%/fts5_hash.c + %dir%/fts5_index.c + %dir%/fts5_main.c + %dir%/fts5_storage.c + %dir%/fts5_tokenize.c + %dir%/fts5_unicode2.c + %dir%/fts5_varint.c + %dir%/fts5_vocab.c +}] + +set G(hdr) { +/* +** This, the "fts5.c" source file, is a composite file that is itself +** assembled from the following files: +** +** fts5.h +** fts5Int.h +** fts5parse.h <--- Generated from fts5parse.y by Lemon +** fts5parse.c <--- Generated from fts5parse.y by Lemon +** fts5_aux.c +** fts5_buffer.c +** fts5_config.c +** fts5_expr.c +** fts5_hash.c +** fts5_index.c +** fts5_main.c +** fts5_storage.c +** fts5_tokenize.c +** fts5_unicode2.c +** fts5_varint.c +** fts5_vocab.c +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +#ifdef HAVE_STDINT_H +#include +#endif +#ifdef HAVE_INTTYPES_H +#include +#endif +} + +set G(footer) { +/* Here ends the fts5.c composite file. */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ +} + +#------------------------------------------------------------------------- +# Read and return the entire contents of text file $zFile from disk. +# +proc readfile {zFile} { + set fd [open $zFile] + set data [read $fd] + close $fd + return $data +} + +#------------------------------------------------------------------------- +# This command returns a string identifying the current sqlite version - +# the equivalent of the SQLITE_SOURCE_ID string. +# +proc fts5_source_id {zDir} { + set top [file dirname [file dirname $zDir]] + set uuid [string trim [readfile [file join $top manifest.uuid]]] + + set L [split [readfile [file join $top manifest]]] + set date [lindex $L [expr [lsearch -exact $L D]+1]] + set idx [expr {[string last . $date]-1}] + set date [string range $date 0 $idx] + set date [string map {T { }} $date] + + return "fts5: $date $uuid" +} + +proc fts5c_init {zOut} { + global G + set G(fd) stdout + set G(fd) [open $zOut w] + + puts -nonewline $G(fd) $G(hdr) +} + +proc fts5c_printfile {zIn} { + global G + set data [readfile $zIn] + set zTail [file tail $zIn] + puts $G(fd) "#line 1 \"$zTail\"" + + set sub_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]] + if {$zTail=="fts5parse.c"} { + lappend sub_map yy fts5yy YY fts5YY TOKEN FTS5TOKEN + } + + foreach line [split $data "\n"] { + if {[regexp {^#include.*fts5} $line]} { + set line "/* $line */" + } elseif { + ![regexp { sqlite3Fts5Init\(} $line] + && [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line] + } { + set line "static $line" + } + set line [string map $sub_map $line] + puts $G(fd) $line + } +} + +proc fts5c_close {} { + global G + puts -nonewline $G(fd) $G(footer) + if {$G(fd)!="stdout"} { + close $G(fd) + } +} + + +fts5c_init fts5.c +foreach f $G(src) { fts5c_printfile $f } +fts5c_close diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl new file mode 100644 index 0000000000000000000000000000000000000000..75ac0f1c8f809dca172b1130b3a0452cc2c4d127 --- /dev/null +++ b/ext/fts5/tool/showfts5.tcl @@ -0,0 +1,97 @@ + + + +#------------------------------------------------------------------------- +# Process command line arguments. +# +proc usage {} { + puts stderr "usage: $::argv0 ?OPTIONS? database table" + puts stderr "" + puts stderr " -nterm (count number of terms in each segment)" + puts stderr " -segments (output segment contents)" + puts stderr "" + exit 1 +} + +set O(nterm) 0 +set O(segments) 0 + +if {[llength $argv]<2} usage +foreach a [lrange $argv 0 end-2] { + switch -- $a { + -nterm { + set O(nterm) 1 + } + + -segments { + set O(segments) 1 + } + + default { + usage + } + } +} + +set database [lindex $argv end-1] +set tbl [lindex $argv end] + + +#------------------------------------------------------------------------- +# Count the number of terms in each segment of fts5 table $tbl. Store the +# counts in the array variable in the parent context named by parameter +# $arrayname, indexed by segment-id. Example: +# +# count_terms fts_tbl A +# foreach {k v} [array get A] { puts "segid=$k nTerm=$v" } +# +proc count_terms {tbl arrayname} { + upvar A $arrayname + array unset A + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data" { + set desc [lindex $d 0] + if {[regexp {^segid=([0-9]*)} $desc -> id]} { + foreach i [lrange $d 1 end] { + if {[string match {term=*} $i]} { incr A($id) } + } + } + } +} + + +#------------------------------------------------------------------------- +# Start of main program. +# +sqlite3 db $database +catch { load_static_extension db fts5 } + +if {$O(nterm)} { count_terms $tbl A } + +db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" { + foreach lvl [lrange $d 1 end] { + puts [lrange $lvl 0 2] + + foreach seg [lrange $lvl 3 end] { + if {$::O(nterm)} { + regexp {^id=([0-9]*)} $seg -> id + set nTerm 0 + catch { set nTerm $A($id) } + puts [format " % -28s nTerm=%d" $seg $nTerm] + } else { + puts [format " % -28s" $seg] + } + } + } +} + +if {$O(segments)} { + puts "" + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id>10" { + puts $d + } +} + + + + + diff --git a/ext/icu/README.txt b/ext/icu/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..40def24662ad09ae169a0fb83947afe35faad3bc --- /dev/null +++ b/ext/icu/README.txt @@ -0,0 +1,168 @@ +This directory contains source code for the SQLite "ICU" extension, an +integration of the "International Components for Unicode" library with +SQLite. Documentation follows. + + 1. Features + + 1.1 SQL Scalars upper() and lower() + 1.2 Unicode Aware LIKE Operator + 1.3 ICU Collation Sequences + 1.4 SQL REGEXP Operator + + 2. Compilation and Usage + + 3. Bugs, Problems and Security Issues + + 3.1 The "case_sensitive_like" Pragma + 3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro + 3.3 Collation Sequence Security Issue + + +1. FEATURES + + 1.1 SQL Scalars upper() and lower() + + SQLite's built-in implementations of these two functions only + provide case mapping for the 26 letters used in the English + language. The ICU based functions provided by this extension + provide case mapping, where defined, for the full range of + unicode characters. + + ICU provides two types of case mapping, "general" case mapping and + "language specific". Refer to ICU documentation for the differences + between the two. Specifically: + + http://www.icu-project.org/userguide/caseMappings.html + http://www.icu-project.org/userguide/posix.html#case_mappings + + To utilise "general" case mapping, the upper() or lower() scalar + functions are invoked with one argument: + + upper('abc') -> 'ABC' + lower('ABC') -> 'abc' + + To access ICU "language specific" case mapping, upper() or lower() + should be invoked with two arguments. The second argument is the name + of the locale to use. Passing an empty string ("") or SQL NULL value + as the second argument is the same as invoking the 1 argument version + of upper() or lower(): + + lower('I', 'en_us') -> 'i' + lower('I', 'tr_tr') -> 'ı' (small dotless i) + + 1.2 Unicode Aware LIKE Operator + + Similarly to the upper() and lower() functions, the built-in SQLite LIKE + operator understands case equivalence for the 26 letters of the English + language alphabet. The implementation of LIKE included in this + extension uses the ICU function u_foldCase() to provide case + independent comparisons for the full range of unicode characters. + + The U_FOLD_CASE_DEFAULT flag is passed to u_foldCase(), meaning the + dotless 'I' character used in the Turkish language is considered + to be in the same equivalence class as the dotted 'I' character + used by many languages (including English). + + 1.3 ICU Collation Sequences + + A special SQL scalar function, icu_load_collation() is provided that + may be used to register ICU collation sequences with SQLite. It + is always called with exactly two arguments, the ICU locale + identifying the collation sequence to ICU, and the name of the + SQLite collation sequence to create. For example, to create an + SQLite collation sequence named "turkish" using Turkish language + sorting rules, the SQL statement: + + SELECT icu_load_collation('tr_TR', 'turkish'); + + Or, for Australian English: + + SELECT icu_load_collation('en_AU', 'australian'); + + The identifiers "turkish" and "australian" may then be used + as collation sequence identifiers in SQL statements: + + CREATE TABLE aust_turkish_penpals( + australian_penpal_name TEXT COLLATE australian, + turkish_penpal_name TEXT COLLATE turkish + ); + + 1.4 SQL REGEXP Operator + + This extension provides an implementation of the SQL binary + comparision operator "REGEXP", based on the regular expression functions + provided by the ICU library. The syntax of the operator is as described + in SQLite documentation: + + REGEXP + + This extension uses the ICU defaults for regular expression matching + behavior. Specifically, this means that: + + * Matching is case-sensitive, + * Regular expression comments are not allowed within patterns, and + * The '^' and '$' characters match the beginning and end of the + argument, not the beginning and end of lines within + the argument. + + Even more specifically, the value passed to the "flags" parameter + of ICU C function uregex_open() is 0. + + +2 COMPILATION AND USAGE + + The easiest way to compile and use the ICU extension is to build + and use it as a dynamically loadable SQLite extension. To do this + using gcc on *nix: + + gcc -fPIC -shared icu.c `pkg-config --libs --cflags icu-io` \ + -o libSqliteIcu.so + + You may need to add "-I" flags so that gcc can find sqlite3ext.h + and sqlite3.h. The resulting shared lib, libSqliteIcu.so, may be + loaded into sqlite in the same way as any other dynamically loadable + extension. + + As of version 3.48, it can be enabled in the canonical build process + by passing one of --with-icu-config or --with-icu-ldflags to the + configure script, optionally together with --enable-icu-collations. + See the configure --help for more details. + + +3 BUGS, PROBLEMS AND SECURITY ISSUES + + 3.1 The "case_sensitive_like" Pragma + + This extension does not work well with the "case_sensitive_like" + pragma. If this pragma is used before the ICU extension is loaded, + then the pragma has no effect. If the pragma is used after the ICU + extension is loaded, then SQLite ignores the ICU implementation and + always uses the built-in LIKE operator. + + The ICU extension LIKE operator is always case insensitive. + + 3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro + + Passing very long patterns to the built-in SQLite LIKE operator can + cause excessive CPU usage. To curb this problem, SQLite defines the + SQLITE_MAX_LIKE_PATTERN_LENGTH macro as the maximum length of a + pattern in bytes (irrespective of encoding). The default value is + defined in internal header file "limits.h". + + The ICU extension LIKE implementation suffers from the same + problem and uses the same solution. However, since the ICU extension + code does not include the SQLite file "limits.h", modifying + the default value therein does not affect the ICU extension. + The default value of SQLITE_MAX_LIKE_PATTERN_LENGTH used by + the ICU extension LIKE operator is 50000, defined in source + file "icu.c". + + 3.3 Collation Sequence Security + + Internally, SQLite assumes that indices stored in database files + are sorted according to the collation sequence indicated by the + SQL schema. Changing the definition of a collation sequence after + an index has been built is therefore equivalent to database + corruption. The SQLite library is well tested for robustness in + the fact of database corruption. Database corruption may well + lead to incorrect answers, but should not cause memory errors. diff --git a/ext/icu/icu.c b/ext/icu/icu.c new file mode 100644 index 0000000000000000000000000000000000000000..50110072b5fd705136f275522b087de898ed7d18 --- /dev/null +++ b/ext/icu/icu.c @@ -0,0 +1,588 @@ +/* +** 2007 May 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +** +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: +** +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. +** +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. +** +** * Integration of ICU and SQLite collation sequences. +** +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. +*/ + +#if !defined(SQLITE_CORE) \ + || defined(SQLITE_ENABLE_ICU) \ + || defined(SQLITE_ENABLE_ICU_COLLATIONS) + +/* Include ICU headers */ +#include +#include +#include +#include + +#include + +#ifndef SQLITE_CORE + #include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#else + #include "sqlite3.h" +#endif + +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); +} + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + +/* +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. +*/ +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const uint32_t MATCH_ONE = (uint32_t)'_'; + static const uint32_t MATCH_ALL = (uint32_t)'%'; + + int prevEscape = 0; /* True if the previous character was uEsc */ + + while( 1 ){ + + /* Read (and consume) the next character from the input pattern. */ + uint32_t uPattern; + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; + + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + } + zPattern++; + } + + if( *zPattern==0 ) return 1; + + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ + return 1; + } + SQLITE_ICU_SKIP_UTF8(zString); + } + return 0; + + }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 2. */ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + + }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ + /* Case 3. */ + prevEscape = 1; + + }else{ + /* Case 4. */ + uint32_t uString; + SQLITE_ICU_READ_UTF8(zString, uString); + uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); + uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; + } + prevEscape = 0; + } + } + + return *zString==0; +} + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). +*/ +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } +} + +/* +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). +*/ +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); +} + +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); + + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ + return; + } + + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + pExpr = sqlite3_get_auxdata(p, 0); + } + if( !pExpr ){ + icuFunctionError(p, "uregex_open", status); + return; + } + } + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; + } + + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; + } + + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); +} + +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). +** +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. +** +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> '\u131' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings +*/ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; + const char *zLocale = 0; + + assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } + + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); + return; + } + + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( bToUpper ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } + + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } + return; + } + assert( 0 ); /* Unreachable */ +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + +/* +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); +} + +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight +){ + UCollationResult res; + UCollator *p = (UCollator *)pCtx; + res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); + switch( res ){ + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} + +/* +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. +*/ +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg +){ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg==2 || nArg==3); + (void)nArg; /* Unused parameter */ + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); + + if( !zLocale || !zName ){ + return; + } + + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; izName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 + ); + } + + return rc; +} + +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_icu_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); +} +#endif + +#endif diff --git a/ext/icu/sqliteicu.h b/ext/icu/sqliteicu.h new file mode 100644 index 0000000000000000000000000000000000000000..24a4d623bdf058644971947cc2739d5c437cfd13 --- /dev/null +++ b/ext/icu/sqliteicu.h @@ -0,0 +1,26 @@ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +int sqlite3IcuInit(sqlite3 *db); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test new file mode 100644 index 0000000000000000000000000000000000000000..187132f766c67e88f706ca7a2421fa59a22a0163 --- /dev/null +++ b/ext/intck/intck1.test @@ -0,0 +1,332 @@ +# 2008 Feb 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the incremental integrity check +# (intck) extension. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intck1 +return_if_no_intck + +foreach {tn sql} { + 1 "CREATE TABLE t1(a PRIMARY KEY, b)" + 2 "CREATE TABLE t2(a PRIMARY KEY, b) WITHOUT ROWID " + 3 "CREATE TABLE t3(a PRIMARY KEY, b) WITHOUT rowID;" + 4 "CREATE TABLE t4(a PRIMARY KEY, ROWID)" + 5 {CREATE TABLE t5(a PRIMARY KEY, ROWID) WITHOUT ROWID + } +} { + do_test 1.1.$tn { + db eval $sql + set {} {} + } {} +} + +set space " \n\v\t\r\f" + +do_execsql_test 1.2 { + SELECT name, (rtrim(sql, $space) LIKE '%rowid') + FROM sqlite_schema WHERE type='table' + ORDER BY 1 +} { + t1 0 + t2 1 + t3 1 + t4 0 + t5 1 +} + +do_execsql_test 1.3 { + CREATE TABLE x1(a COLLATE nocase, b INTEGER, c BLOB); + INSERT INTO x1 VALUES('lEtTeRs', 1234, 1234); +} +do_execsql_test 1.3.1 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c1='letters'; +} {lEtTeRs 1234 1234} +do_execsql_test 1.3.2 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c2='1234'; +} {lEtTeRs 1234 1234} +do_execsql_test 1.3.2 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c3='1234'; +} {} + +do_execsql_test 1.4 { + CREATE TABLE z1(a, b); + CREATE INDEX z1ab ON z1(a+b COLLATE nocase); +} +do_execsql_test 1.4.1 { + SELECT * FROM z1 INDEXED BY z1ab +} + +do_catchsql_test 1.5.1 { + CREATE INDEX z1b ON z1(b ASC NULLS LAST); +} {1 {unsupported use of NULLS LAST}} +do_catchsql_test 1.5.2 { + CREATE INDEX z1b ON z1(b DESC NULLS LAST); +} {1 {unsupported use of NULLS LAST}} +do_catchsql_test 1.5.3 { + CREATE INDEX z1b ON z1(b ASC NULLS FIRST); +} {1 {unsupported use of NULLS FIRST}} +do_catchsql_test 1.5.4 { + CREATE INDEX z1b ON z1(b DESC NULLS FIRST); +} {1 {unsupported use of NULLS FIRST}} + + +reset_db +do_execsql_test 1.6.1 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + ANALYZE; + INSERT INTO sqlite_stat1 VALUES('t1', 'i1', '10000 10000'); + ANALYZE sqlite_schema; +} {} +do_eqp_test 1.6.2 { + SELECT 1 FROM t1 INDEXED BY i1 WHERE (b, i) IS (?, ?); +} {SEARCH} + + + +#------------------------------------------------------------------------- +reset_db + +do_test 2.0 { + set ic [sqlite3_intck db main] + $ic close +} {} + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + + CREATE INDEX i1 ON t1(a COLLATE nocase); + CREATE INDEX i2 ON t1(b, a); + CREATE INDEX i3 ON t1(b + a COLLATE nocase) WHERE a!=1; +} + +do_intck_test 2.2 { +} + +# Delete a row from each of the i1 and i2 indexes using the imposter +# table interface. +# +do_test 2.3 { + db eval {SELECT name, rootpage FROM sqlite_schema} { + set R($name) $rootpage + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i1) + db eval { CREATE TABLE imp1(a PRIMARY KEY, rowid) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i2) + db eval { CREATE TABLE imp2(b, a, rowid, PRIMARY KEY(b, a)) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db eval { + DELETE FROM imp1 WHERE rowid=1; + DELETE FROM imp2 WHERE rowid=2; + } + + db close + sqlite3 db test.db +} {} + +do_intck_test 2.4 { + {entry (1,1) missing from index i1} + {entry (4,3,2) missing from index i2} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(c, b)) WITHOUT ROWID; + CREATE INDEX x1a ON x1(a COLLATE nocase); + + INSERT INTO x1 VALUES(1, 2, 'three'); + INSERT INTO x1 VALUES(4, 5, 'six'); + INSERT INTO x1 VALUES(7, 8, 'nine'); +} + +do_intck_test 3.1 { } + +do_test 3.2 { + db eval {SELECT name, rootpage FROM sqlite_schema} { + set R($name) $rootpage + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(x1a) + db eval { CREATE TABLE imp1(c, b, a, PRIMARY KEY(c, b)) WITHOUT ROWID } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db eval { + DELETE FROM imp1 WHERE a=5; + } + execsql_pp { + } + + db close + sqlite3 db test.db +} {} + +do_intck_test 3.3 { + {entry (4,'six',5) missing from index x1a} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE www(x, y, z); + CREATE INDEX w1 ON www( (x+1), z ); + INSERT INTO www VALUES(1, 1, 1), (2, 2, 2); +} + +do_intck_test 4.1 { } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a COLLATE NOCASE); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); +} + +do_test 5.1 { + set ic [sqlite3_intck db nosuchdb] + $ic step +} {SQLITE_ERROR} + +do_test 5.2 { + $ic close + set ic [sqlite3_intck db {}] + while {[$ic step]=="SQLITE_OK"} {} + set res [$ic error] + $ic close + set res +} {SQLITE_OK {}} + +do_test 5.3 { test_do_intck db "main" } {} + +do_test 5.4 { + set ret {} + set ic [sqlite3_intck db main] + db eval [$ic test_sql t1] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +do_test 5.5 { + set ret {} + set ic [sqlite3_intck db main] + db eval [$ic test_sql {}] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +db cache flush + +do_test 5.6 { + set ret {} + set ic [sqlite3_intck db main] + $ic step + db eval [$ic test_sql {}] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 6.0 { + CREATE TABLE t1(x, y, PRIMARY KEY(x)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(y, x); + INSERT INTO t1 VALUES(X'0000', X'1111'); +} + +do_intck_test 6.1 {} + +do_execsql_test 6.2.1 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1' WHERE name='i1'; +} {} +do_intck_test 6.2.2 {} + +do_execsql_test 6.3.1 { + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1(y' WHERE name='i1'; +} {} +do_intck_test 6.3.2 {} + +do_execsql_test 6.4.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1(y) hello world' + WHERE name='i1'; +} {} +do_intck_test 6.4.2 {} + +do_execsql_test 6.5.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1(y, x) WHERE 1 ' + WHERE name='i1'; +} {} +do_intck_test 6.5.2 {} + +do_execsql_test 6.6.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1( , ) WHERE 1 ' + WHERE name='i1'; +} {} + +do_test 6.7.2 { + set ic [sqlite3_intck db main] + $ic step +} {SQLITE_ERROR} +do_test 6.5.3 { + $ic error +} {SQLITE_ERROR {near "AS": syntax error}} +$ic close + +do_execsql_test 6.6.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1([y' + WHERE name='i1'; +} {} +do_intck_test 6.6.2 {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE x1("1", "22", "3333", four); + CREATE INDEX i1 ON x1( "1" , "22", NULL); + INSERT INTO x1 VALUES(1, 22, 3333, NULL); + INSERT INTO x1 VALUES(1, 22, 3333, NULL); +} +do_execsql_test 7.1 " CREATE INDEX i2 ON x1( \"1\"\r\n\t ) " +do_execsql_test 7.2 { CREATE INDEX i3 ON x1( "22" || 'abc''def' || `1` ) } +do_execsql_test 7.3 { CREATE INDEX i4 ON x1( [22] + [1] ) } +do_execsql_test 7.4 { CREATE INDEX i5 ON x1( four||'hello' ) } + +do_intck_test 7.5 {} + + +finish_test diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test new file mode 100644 index 0000000000000000000000000000000000000000..23b241b5a92058b056b68138f5ea964d2c1a667b --- /dev/null +++ b/ext/intck/intck2.test @@ -0,0 +1,177 @@ +# 2024 Feb 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the incremental integrity check +# (intck) extension. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intck2 +return_if_no_intck + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + CREATE INDEX i1 ON t1(b); +} + +proc imposter_edit {obj create sql} { + sqlite3 xdb test.db + set pgno [xdb one {SELECT rootpage FROM sqlite_schema WHERE name=$obj}] + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 1 $pgno + xdb eval $create + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 0 0 + xdb eval $sql + xdb close +} + +imposter_edit i1 { + CREATE TABLE imp(b, a, PRIMARY KEY(b)) WITHOUT ROWID; +} { + DELETE FROM imp WHERE b='two'; + INSERT INTO imp(b, a) VALUES('four', 4); +} + +do_intck_test 1.1 { + {surplus entry ('four',4) in index i1} + {entry ('two',2) missing from index i1} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(a, b, "c d"); + CREATE INDEX x1a ON x1(a COLLATE nocase DESC , b ASC); + CREATE INDEX x1b ON x1( a || b || ' "''" ' COLLATE binary ASC ); + CREATE INDEX x1c ON x1( format('%s', a)ASC, format('%d', "c d" ) ); + INSERT INTO x1 VALUES('one', 2, 3); + INSERT INTO x1 VALUES('One', 4, 5); + INSERT INTO x1 VALUES('ONE', 6, 7); + INSERT INTO x1 VALUES(NULL, NULL, NULL); +} + +do_intck_test 2.1 {} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE c=7; +} +do_intck_test 2.2 { + {surplus entry ('ONE',6,3) in index x1a} + {surplus entry ('ONE6 "''" ',3) in index x1b} + {surplus entry ('ONE','7',3) in index x1c} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c); + CREATE INDEX x1all ON x1(a DESC, b ASC, c DESC); + INSERT INTO x1 VALUES(2, 1, 2); + INSERT INTO x1 VALUES(2, 1, 1); + INSERT INTO x1 VALUES(2, 2, 2); + INSERT INTO x1 VALUES(2, 2, 1); + INSERT INTO x1 VALUES(1, 1, 2); + INSERT INTO x1 VALUES(1, 1, 1); + INSERT INTO x1 VALUES(1, 2, 2); + INSERT INTO x1 VALUES(1, 2, 1); +} + +do_intck_test 3.1 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; +} + +db close +sqlite3 db test.db + +do_intck_test 3.2 { + {surplus entry (2,1,2,1) in index x1all} + {surplus entry (2,1,1,2) in index x1all} + {surplus entry (2,2,2,3) in index x1all} + {surplus entry (2,2,1,4) in index x1all} + {surplus entry (1,1,2,5) in index x1all} + {surplus entry (1,1,1,6) in index x1all} + {surplus entry (1,2,2,7) in index x1all} + {surplus entry (1,2,1,8) in index x1all} +} + +do_execsql_test 3.3 { + DELETE FROM x1; + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); +} + +do_intck_test 3.4 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; + INSERT INTO imp(rowid) VALUES(-123); + INSERT INTO imp(rowid) VALUES(456); +} + +db close +sqlite3 db test.db + +do_intck_test 3.5 { + {entry (NULL,NULL,NULL,-123) missing from index x1all} + {entry (NULL,NULL,NULL,456) missing from index x1all} + {surplus entry (NULL,NULL,NULL,1) in index x1all} + {surplus entry (NULL,NULL,NULL,2) in index x1all} + {surplus entry (NULL,NULL,NULL,3) in index x1all} + {surplus entry (NULL,NULL,NULL,4) in index x1all} +} + +reset_db + +do_execsql_test 3.6 { + CREATE TABLE w1(a PRIMARY KEY, b, c); + INSERT INTO w1 VALUES(1.0, NULL, NULL); + INSERT INTO w1 VALUES(33.0, NULL, NULL); + INSERT INTO w1 VALUES(100.0, NULL, NULL); + CREATE INDEX w1bc ON w1(b, c); +} + +do_intck_test 3.7 { +} + +imposter_edit w1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE a=33; + INSERT INTO imp(a) VALUES(1234.5); + INSERT INTO imp(a) VALUES(-1234.5); +} + +do_intck_test 3.8 { + {surplus entry (33.0,2) in index sqlite_autoindex_w1_1} + {entry (1234.5,4) missing from index sqlite_autoindex_w1_1} + {entry (NULL,NULL,4) missing from index w1bc} + {entry (-1234.5,5) missing from index sqlite_autoindex_w1_1} + {entry (NULL,NULL,5) missing from index w1bc} + {surplus entry (NULL,NULL,2) in index w1bc} +} + +finish_test diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl new file mode 100644 index 0000000000000000000000000000000000000000..1e216b59f3e23bb96aff3a99a4602fd00fcdbb7d --- /dev/null +++ b/ext/intck/intck_common.tcl @@ -0,0 +1,66 @@ +# 2024 Feb 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +ifcapable !vtab||!pragma { + proc return_if_no_intck {} { + finish_test + return -code return + } + return +} else { + proc return_if_no_intck {} {} +} + +proc do_intck {db {bSuspend 0}} { + set ic [sqlite3_intck $db main] + + set ret [list] + while {"SQLITE_OK"==[$ic step]} { + set msg [$ic message] + if {$msg!=""} { + lappend ret $msg + } + if {$bSuspend} { + $ic unlock + #puts "SQL: [$ic test_sql {}]" + #execsql_pp "EXPLAIN query plan [$ic test_sql {}]" + #explain_i [$ic test_sql {}] + } + } + + set err [$ic error] + if {[lindex $err 0]!="SQLITE_OK"} { + error $err + } + $ic close + + return $ret +} + +proc intck_sql {db tbl} { + set ic [sqlite3_intck $db main] + set sql [$ic test_sql $tbl] + $ic close + return $sql +} + +proc do_intck_test {tn expect} { + uplevel [list do_test $tn.a [list do_intck db] [list {*}$expect]] + uplevel [list do_test $tn.b [list do_intck db 1] [list {*}$expect]] +} + + diff --git a/ext/intck/intckbusy.test b/ext/intck/intckbusy.test new file mode 100644 index 0000000000000000000000000000000000000000..7c65b686bb79328d95f399829ce8d3cdf6b34f1b --- /dev/null +++ b/ext/intck/intckbusy.test @@ -0,0 +1,49 @@ +# 2024 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckbusy +return_if_no_intck + + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 'two', 'three'); + INSERT INTO t1 VALUES(3, NULL, NULL); + CREATE INDEX i1 ON t1(b, c); +} + +sqlite3 db2 test.db + +do_execsql_test -db db2 1.1 { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_test 1.2 { + set ic [sqlite3_intck db main] + $ic step +} {SQLITE_BUSY} +do_test 1.3 { + $ic unlock +} {SQLITE_BUSY} +do_test 1.4 { + $ic error +} {SQLITE_BUSY {database is locked}} +do_test 1.4 { + $ic close +} {} + +finish_test + diff --git a/ext/intck/intckcorrupt.test b/ext/intck/intckcorrupt.test new file mode 100644 index 0000000000000000000000000000000000000000..eee63b32f0eb1d1f0a1c195abef972edd1b16dfd --- /dev/null +++ b/ext/intck/intckcorrupt.test @@ -0,0 +1,236 @@ +# 2024 Feb 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the intck extensions response +# to corruption at the b-tree level. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckcorrupt +return_if_no_intck + +#------------------------------------------------------------------------- +reset_db +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 356352 pagesize 4096 filename crash-acaae0347204ae.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 d0 00 00 00 .....@ ........ +| 32: 40 00 ea 00 00 00 00 00 00 40 00 00 00 40 00 00 @........@...@.. +| 96: 00 00 00 00 0d 00 00 00 04 0e 9c 00 0f ad 0f 4f ...............O +| 112: 0e fc 0e 9c 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3728: 00 00 00 00 00 00 00 00 00 00 00 00 5e 04 07 17 ............^... +| 3744: 1f 1f 01 81 0b 74 61 62 6c 65 74 31 5f 70 61 72 .....tablet1_par +| 3760: 65 6e 74 74 31 5f 70 61 72 65 6e 74 04 43 52 45 entt1_parent.CRE +| 3776: 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 70 61 ATE TABLE .t1_pa +| 3792: 72 65 6e 74 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 rent.(nodeno INT +| 3808: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3824: 2c 70 61 72 65 6e 74 6e 6f 64 65 29 51 03 06 17 ,parentnode)Q... +| 3840: 1b 1b 01 7b 74 61 62 6c 65 74 31 5f 6e 6f 64 65 ....tablet1_node +| 3856: 74 31 5f 6e 6f 64 65 03 43 52 45 41 54 45 20 54 t1_node.CREATE T +| 3872: 41 42 4c 45 20 22 74 31 5f 6e 6f 64 65 22 28 6e ABLE .t1_node.(n +| 3888: 6f 64 65 6e 6f 20 49 4e 54 45 47 45 52 20 50 52 odeno INTEGER PR +| 3904: 49 4d 41 52 59 20 4b 45 59 2c 64 61 74 61 29 5c IMARY KEY,data). +| 3920: 02 07 17 1d 1d 01 81 0b 74 61 62 6c 65 74 31 5f ........tablet1_ +| 3936: 72 6f 77 69 64 74 31 5f 72 6f 77 69 64 02 43 52 rowidt1_rowid.CR +| 3952: 45 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 72 EATE TABLE .t1_r +| 3968: 6f 77 69 64 22 28 72 6f 77 69 64 20 49 4e 54 45 owid.(rowid INTE +| 3984: 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c GER PRIMARY KEY, +| 4000: 6e 6f 64 65 6e 6f 2c 61 30 2c 61 31 29 51 01 07 nodeno,a0,a1)Q.. +| 4016: 17 11 11 08 81 0f 74 61 62 6c 65 74 31 74 31 43 ......tablet1t1C +| 4032: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 4048: 42 4c 45 20 74 31 20 55 53 49 4e 47 20 72 74 72 BLE t1 USING rtr +| 4064: 65 65 28 69 64 2c 78 30 20 50 52 49 4d 41 52 59 ee(id,x0 PRIMARY +| 4080: 20 4b 45 59 2c 70 61 72 65 6e 74 6e 6f 64 65 29 KEY,parentnode) +| page 2 offset 4096 +| 0: 51 03 06 17 1b 1b 01 7b 74 61 62 6c 65 74 31 5f Q.......tablet1_ +| 16: 6e 6f 64 65 74 31 5f 6e 6f 64 65 03 43 52 45 41 nodet1_node.CREA +| 32: 54 45 20 54 41 42 4c 45 20 22 74 31 5f 6e 6f 64 TE TABLE .t1_nod +| 48: 65 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 e.(nodeno INTEGE +| 64: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da +| 80: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl +| 96: 65 74 31 5f 72 6f 77 69 64 74 31 5f 72 6f 77 69 et1_rowidt1_rowi +| 112: 64 02 43 52 45 41 54 45 20 54 41 42 4c 45 00 00 d.CREATE TABLE.. +| 128: 01 0a 02 00 00 00 01 0e 0d 00 00 00 00 24 0e 0d .............$.. +| 144: 0c 1a 06 85 50 46 60 27 70 08 00 00 00 00 00 00 ....PF`'p....... +| 3824: 00 00 00 00 00 00 00 0d 0e 05 00 09 1d 00 74 6f ..............to +| 3840: 79 20 68 61 6c 66 10 0d 05 00 09 23 00 62 6f 74 y half.....#.bot +| 3856: 74 6f 6d 20 68 61 6c 66 0f 0c 05 00 09 21 00 72 tom half.....!.r +| 3872: 69 67 68 74 20 68 61 6c 66 0e 0b 05 00 09 1f 00 ight half....... +| 3888: 6c 65 66 74 20 43 15 f6 e6 f6 46 50 34 35 24 54 left C....FP45$T +| 3904: 15 44 52 05 44 14 24 c4 52 02 27 43 15 f6 e6 f6 .DR.D.$.R.'C.... +| 3920: 46 52 22 8e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 FR..odeno INTEGE +| 3936: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da +| 3952: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl +| 3968: 65 74 31 5f 72 6f 74 74 6f 6d 20 65 64 67 65 0f et1_rottom edge. +| 3984: 07 05 00 09 21 00 72 69 67 68 74 20 65 64 67 65 ....!.right edge +| 4000: 0e 06 05 00 09 1f 00 6c 65 66 74 20 65 64 67 65 .......left edge +| 4016: 0b 05 05 00 09 19 00 63 65 6e 74 65 72 17 04 05 .......center... +| 4032: 00 09 31 00 75 70 70 65 72 2d 72 69 67 68 74 20 ..1.upper-right +| 4048: 63 6f 72 6e 65 72 17 03 05 00 09 31 00 6c 6f 77 corner.....1.low +| 4064: 65 72 2d 72 69 67 68 74 20 63 6f 72 6e 65 72 16 er-right corner. +| 4080: 02 05 00 09 2f 00 75 70 70 65 72 2d 6c 65 66 74 ..../.upper-left +| page 3 offset 8192 +| 0: 20 63 6f 72 6e 65 72 16 01 05 00 09 2f 01 8c 6f corner...../..o +| 16: 77 65 72 2d 6c 53 51 4c 69 74 65 20 66 6f 72 6d wer-lSQLite form +| 32: 61 74 20 33 00 10 00 01 01 00 40 20 20 00 00 00 at 3......@ ... +| 48: 00 00 00 00 2f 00 00 0d eb 13 00 00 00 03 00 00 ..../........... +| 64: 00 04 00 00 00 00 00 00 00 06 00 00 00 01 00 00 ................ +| 80: 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................ +| page 6 offset 20480 +| 128: 00 00 00 00 00 00 00 00 97 3d 04 ae 7c 01 00 00 .........=..|... +| 624: 00 00 00 00 00 00 21 97 3d 04 ae 7c 01 00 00 00 ......!.=..|.... +| 1120: 00 00 00 00 00 20 97 3d 04 ae 7c 01 00 00 00 00 ..... .=..|..... +| 1616: 00 00 00 00 1f 97 3d 04 ae 7c 01 00 00 00 00 00 ......=..|...... +| 2112: 00 00 00 1e 97 3d 04 ae 7c 01 00 00 00 00 00 00 .....=..|....... +| 2608: 00 00 1d 97 d3 d0 4a e7 c0 00 00 00 00 00 00 00 ......J......... +| 3088: 00 00 00 00 00 00 00 00 00 00 00 00 01 f3 00 00 ................ +| 3600: 23 97 3d 04 ae 7c 01 00 00 00 00 00 00 00 00 00 #.=..|.......... +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 26 ...............& +| page 8 offset 28672 +| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0...... +| 1072: 97 4d 1e 14 00 ae 7c 00 00 00 00 00 00 00 00 00 .M....|......... +| 1088: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| page 10 offset 36864 +| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0...... +| 1072: 9a ee c1 80 fd 78 1f ce 1b ae eb b4 00 00 00 00 .....x.......... +| 1088: 13 20 ff 20 00 70 00 00 00 60 50 00 00 00 11 e0 . . .p...`P..... +| 1104: 00 00 00 70 00 00 00 60 50 05 35 14 c6 97 46 52 ...p...`P.5...FR +| 1120: 06 66 f7 26 d6 17 42 03 30 01 00 00 10 10 04 02 .f.&..B.0....... +| 1136: 02 00 00 00 00 00 00 00 00 40 00 00 00 00 00 00 .........@...... +| 1152: 00 00 00 00 00 40 00 00 00 40 00 00 00 00 00 00 .....@...@...... +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 ................ +| page 12 offset 45056 +| 0: 0d 00 00 00 01 04 30 00 04 30 e1 b4 30 97 4d 46 ......0..0..0.MF +| 16: 14 00 ae 7c 00 00 00 00 00 00 00 03 00 00 43 00 ...|..........C. +| page 47 offset 188416 +| 2512: 00 00 00 00 00 00 00 00 be 00 00 00 00 00 00 00 ................ +| page 87 offset 352256 +| 2512: 00 00 00 00 00 00 00 00 aa 00 00 00 00 00 00 00 ................ +| end crash-acaae0347204ae.db +}]} {} + +do_intck_test 1.1 { + {corruption found while reading database schema} +} + +#------------------------------------------------------------------------- +reset_db +do_test 2.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-3afa1ca9e9c1bd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 88 00 0f b8 0f 6d ...............m +| 112: 0f 3a 0f 0b 0e d5 0e 88 01 00 00 00 00 00 00 00 .:.............. +| 3712: 00 00 00 00 00 00 00 00 4b 06 06 17 25 25 01 5b ........K...%%.[ +| 3728: 74 61 62 6c 65 73 71 6c 69 74 65 5f 73 74 61 74 tablesqlite_stat +| 3744: 31 73 71 6c 69 74 65 5f 73 74 61 74 31 07 43 52 1sqlite_stat1.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 73 71 6c 69 74 EATE TABLE sqlit +| 3776: 65 5f 73 74 61 74 31 28 74 62 6c 2c 69 64 78 2c e_stat1(tbl,idx, +| 3792: 73 74 61 74 29 34 05 06 17 13 11 01 53 69 6e 64 stat)4......Sind +| 3808: 65 78 63 31 63 63 31 06 43 52 45 41 54 45 20 55 exc1cc1.CREATE U +| 3824: 4e 49 51 55 45 20 49 4e 44 45 58 20 63 31 63 20 NIQUE INDEX c1c +| 3840: 4f 4e 20 63 31 28 63 2c 20 62 29 2d 04 06 17 13 ON c1(c, b)-.... +| 3856: 11 01 45 69 6e 64 65 78 63 31 64 63 31 05 43 52 ..Eindexc1dc1.CR +| 3872: 45 41 54 45 20 49 4e 44 45 58 20 63 31 64 20 4f EATE INDEX c1d O +| 3888: 4e 20 63 31 28 64 2c 20 62 29 31 03 06 17 13 11 N c1(d, b)1..... +| 3904: 01 4d 69 6e 64 65 78 62 31 63 62 31 05 43 52 45 .Mindexb1cb1.CRE +| 3920: 41 54 45 20 55 4e 49 51 55 45 20 49 4e 44 45 58 ATE UNIQUE INDEX +| 3936: 20 62 31 63 20 4f 4e 20 62 31 28 63 29 49 02 06 b1c ON b1(c)I.. +| 3952: 17 11 11 0f 7f 74 61 62 6c 65 63 31 63 31 03 43 .....tablec1c1.C +| 3968: 52 45 41 54 45 20 54 41 42 4c 45 20 63 31 28 61 REATE TABLE c1(a +| 3984: 20 49 4e 54 20 50 52 49 4d 41 52 59 20 4b 45 59 INT PRIMARY KEY +| 4000: 2c 20 62 2c 20 63 2c 20 64 29 20 57 49 54 48 4f , b, c, d) WITHO +| 4016: 55 54 20 52 4f 57 49 44 46 01 06 17 11 11 01 79 UT ROWIDF......y +| 4032: 74 61 62 6c 65 62 31 62 31 02 43 52 45 41 54 45 tableb1b1.CREATE +| 4048: 20 54 41 42 4c 45 20 62 31 28 61 20 49 4e 54 20 TABLE b1(a INT +| 4064: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 2c 20 PRIMARY KEY, b, +| 4080: 63 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 c) WITHOUT ROWID +| page 2 offset 4096 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f e2 ................ +| 16: 0f da 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 06 ................ +| 4048: 67 07 07 04 01 0f 01 06 66 06 07 04 01 0f 01 05 g.......f....... +| 4064: 65 05 07 04 01 0f 01 04 64 04 07 04 01 0f 01 03 e.......d....... +| 4080: 63 03 07 04 01 0f 01 02 62 0f 05 04 09 0f 09 61 c.......b......a +| page 3 offset 8192 +| 0: 0a 00 00 00 07 0f bd 00 0f f9 0f ef 0f e5 0f db ................ +| 16: 0f d1 0f c7 0f bd 00 00 00 00 01 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 09 05 01 ................ +| 4032: 0f 01 01 07 61 07 07 09 05 01 0f 01 01 06 61 06 ....a.........a. +| 4048: 06 09 05 01 0f 01 01 05 61 05 05 09 05 01 0f 01 ........a....... +| 4064: 01 04 61 04 04 09 05 01 0f 01 01 03 61 03 03 09 ..a.........a... +| 4080: 05 01 0f 01 01 02 61 0f 02 06 05 09 0f 09 09 61 ......a........a +| page 4 offset 12288 +| 0: 0a 00 00 00 07 0f d8 00 0f fc 0f f0 0f ea 0f e4 ................ +| 16: 0f de 0f d8 0f f6 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 05 03 01 01 07 07 05 03 ................ +| 4064: 01 01 06 06 05 03 01 01 05 05 05 03 01 01 04 04 ................ +| 4080: 05 03 01 01 03 03 05 03 01 01 0f 02 03 03 09 09 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................ +| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a....... +| 4064: 61 05 07 04 01 1f 01 04 61 04 07 04 01 0f 01 03 a.......a....... +| 4080: 61 03 07 04 01 0f 01 02 61 02 05 04 09 0f 09 61 a.......a......a +| page 6 offset 20480 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f ea 0f e2 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................ +| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a....... +| 4064: 61 05 07 04 01 0f 01 04 61 04 07 04 01 0f 01 03 a.......a....... +| 4080: 61 03 07 04 01 0f 01 0f 61 02 05 04 09 0f 09 61 a.......a......a +| page 7 offset 24576 +| 0: 0d 00 00 00 05 0f 1c 00 0f f0 0f e0 0f d3 0f c5 ................ +| 16: 0f b8 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 0b 05 04 11 11 13 62 31 ..............b1 +| 4032: 62 31 37 20 31 0c 04 04 11 13 13 62 31 62 31 63 b17 1......b1b1c +| 4048: 37 20 31 0b 03 04 11 11 13 63 31 63 31 37 20 31 7 1......c1c17 1 +| 4064: 0e 02 04 11 13 07 63 31 63 31 64 37 20 31 20 31 ......c1c1d7 1 1 +| 4080: 0e 01 04 11 13 17 63 31 63 31 63 37 20 31 00 00 ......c1c1c7 1.. +| end crash-3afa1ca9e9c1bd.db +}]} {} + +do_intck_test 2.1 { + {corruption found while reading database schema} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3); +} + +do_test 3.1 { + set pgno [db one {SELECT rootpage FROM sqlite_schema WHERE name='t1'}] + db close + hexio_write test.db [expr ($pgno-1)*1024] 0000 +} {2} + +sqlite3 db test.db +do_intck_test 3.2 { + {corruption found while scanning database object i1} + {corruption found while scanning database object t1} +} + +finish_test + + diff --git a/ext/intck/intckfault.test b/ext/intck/intckfault.test new file mode 100644 index 0000000000000000000000000000000000000000..0bc06e584a2d814e0433586943035019b9d1411f --- /dev/null +++ b/ext/intck/intckfault.test @@ -0,0 +1,42 @@ +# 2024 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckfault +return_if_no_intck + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 'two', 'three'); + INSERT INTO t1 VALUES(3, NULL, NULL); + CREATE INDEX i1 ON t1(b, c); +} + +do_faultsim_test 1 -faults oom-t* -prep { +} -body { + set ::ic [sqlite3_intck db main] + set nStep 0 + while {"SQLITE_OK"==[$::ic step]} { + incr nStep + if {$nStep==3} { $::ic unlock } + } + set res [$::ic error] + $::ic close + set res +} -test { + catch { $::ic close } + faultsim_test_result {0 {SQLITE_OK {}}} {0 {SQLITE_NOMEM {}}} {0 {SQLITE_NOMEM {out of memory}}} +} + +finish_test + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c new file mode 100644 index 0000000000000000000000000000000000000000..ed169a26643b5dcec0bab8e2df36da8c24619055 --- /dev/null +++ b/ext/intck/sqlite3intck.c @@ -0,0 +1,940 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqlite3intck.h" +#include +#include + +#include +#include + +/* +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. +*/ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + return zRet; +} + +/* +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. +*/ +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); + } + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +/* +** Return true if argument c is an ascii whitespace character. +*/ +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); +*/ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" + + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } + } + intckFinalize(p, pStmt); + + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); + return zRet; +} + +/* +** Open a new integrity-check object. +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } + } + + *ppOut = pNew; + return rc; +} + +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } +} + +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; + } + } + + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } + } + } + } + + return p->rc; +} + +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +/* +** Return the error code and message. +*/ +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; +} + +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; +} diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h new file mode 100644 index 0000000000000000000000000000000000000000..e08a86f289e3fab83f9e3ab97b17ffc1305b1222 --- /dev/null +++ b/ext/intck/sqlite3intck.h @@ -0,0 +1,171 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ +typedef struct sqlite3_intck sqlite3_intck; + +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); + +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); + +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_step(sqlite3_intck *pCk); + +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); + +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c new file mode 100644 index 0000000000000000000000000000000000000000..d3a619b503845c46d94a5e763cf9b9c33f75759c --- /dev/null +++ b/ext/intck/test_intck.c @@ -0,0 +1,233 @@ +/* +** 2010 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Code for testing all sorts of SQLite interfaces. This code +** is not included in the SQLite library. +*/ + +#include "sqlite3.h" +#include "sqlite3intck.h" +#include "tclsqlite.h" +#include +#include + +/* In test1.c */ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +const char *sqlite3ErrName(int); + +typedef struct TestIntck TestIntck; +struct TestIntck { + sqlite3_intck *intck; +}; + +static int testIntckCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct Subcmd { + const char *zName; + int nArg; + const char *zExpect; + } aCmd[] = { + {"close", 0, ""}, /* 0 */ + {"step", 0, ""}, /* 1 */ + {"message", 0, ""}, /* 2 */ + {"error", 0, ""}, /* 3 */ + {"unlock", 0, ""}, /* 4 */ + {"test_sql", 1, ""}, /* 5 */ + {0 , 0} + }; + int rc = TCL_OK; + int iIdx = -1; + TestIntck *p = (TestIntck*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); + return TCL_ERROR; + } + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aCmd, sizeof(aCmd[0]), "SUB-COMMAND", 0, &iIdx + ); + if( rc ) return rc; + + if( objc!=2+aCmd[iIdx].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aCmd[iIdx].zExpect); + return TCL_ERROR; + } + + switch( iIdx ){ + case 0: assert( 0==strcmp("close", aCmd[iIdx].zName) ); { + Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0)); + break; + } + + case 1: assert( 0==strcmp("step", aCmd[iIdx].zName) ); { + rc = sqlite3_intck_step(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 2: assert( 0==strcmp("message", aCmd[iIdx].zName) ); { + const char *z = sqlite3_intck_message(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(z ? z : "", -1)); + break; + } + + case 3: assert( 0==strcmp("error", aCmd[iIdx].zName) ); { + const char *zErr = 0; + Tcl_Obj *pRes; + rc = sqlite3_intck_error(p->intck, 0); + pRes = Tcl_NewObj(); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(sqlite3ErrName(rc), -1) + ); + sqlite3_intck_error(p->intck, &zErr); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(zErr ? zErr : 0, -1) + ); + Tcl_SetObjResult(interp, pRes); + break; + } + + case 4: assert( 0==strcmp("unlock", aCmd[iIdx].zName) ); { + rc = sqlite3_intck_unlock(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 5: assert( 0==strcmp("test_sql", aCmd[iIdx].zName) ); { + const char *zObj = Tcl_GetString(objv[2]); + const char *zSql = sqlite3_intck_test_sql(p->intck, zObj[0] ? zObj : 0); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zSql, -1)); + break; + } + } + + return TCL_OK; +} + +/* +** Destructor for commands created by test_sqlite3_intck(). +*/ +static void testIntckFree(void *clientData){ + TestIntck *p = (TestIntck*)clientData; + sqlite3_intck_close(p->intck); + ckfree(p); +} + +/* +** tclcmd: sqlite3_intck DB DBNAME +*/ +static int test_sqlite3_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char zName[64]; + int iName = 0; + Tcl_CmdInfo info; + TestIntck *p = 0; + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + + p = (TestIntck*)ckalloc(sizeof(TestIntck)); + memset(p, 0, sizeof(TestIntck)); + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = 0; + + rc = sqlite3_intck_open(db, zDb, &p->intck); + if( rc!=SQLITE_OK ){ + ckfree(p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errstr(rc), -1)); + return TCL_ERROR; + } + + do { + sprintf(zName, "intck%d", iName++); + }while( Tcl_GetCommandInfo(interp, zName, &info)!=0 ); + Tcl_CreateObjCommand(interp, zName, testIntckCmd, (void*)p, testIntckFree); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zName, -1)); + + return TCL_OK; +} + +/* +** tclcmd: test_do_intck DB DBNAME +*/ +static int test_do_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + sqlite3_intck *pCk = 0; + Tcl_Obj *pRet = 0; + const char *zErr = 0; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + rc = sqlite3_intck_open(db, zDb, &pCk); + if( rc==SQLITE_OK ){ + while( sqlite3_intck_step(pCk)==SQLITE_OK ){ + const char *zMsg = sqlite3_intck_message(pCk); + if( zMsg ){ + Tcl_ListObjAppendElement(interp, pRet, Tcl_NewStringObj(zMsg, -1)); + } + } + rc = sqlite3_intck_error(pCk, &zErr); + } + if( rc!=SQLITE_OK ){ + if( zErr ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + }else{ + Tcl_SetObjResult(interp, pRet); + } + Tcl_DecrRefCount(pRet); + sqlite3_intck_close(pCk); + sqlite3_intck_close(0); + return rc ? TCL_ERROR : TCL_OK; +} + +int Sqlitetestintck_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_intck", test_sqlite3_intck, 0, 0); + Tcl_CreateObjCommand(interp, "test_do_intck", test_do_intck, 0, 0); + return TCL_OK; +} diff --git a/ext/jni/GNUmakefile b/ext/jni/GNUmakefile new file mode 100644 index 0000000000000000000000000000000000000000..26a38dad9a13c4665a8d8dd86240c3c41fb380b8 --- /dev/null +++ b/ext/jni/GNUmakefile @@ -0,0 +1,505 @@ +# Quick-and-dirty makefile to bootstrap the sqlite3-jni project. This +# build assumes a Linux-like system. +default: all + +JAVA_HOME ?= $(HOME)/jdk/current +# e.g. /usr/lib/jvm/default-javajava-19-openjdk-amd64 +JDK_HOME ?= $(JAVA_HOME) +# ^^^ JDK_HOME is not as widely used as JAVA_HOME +bin.jar := $(JDK_HOME)/bin/jar +bin.java := $(JDK_HOME)/bin/java +bin.javac := $(JDK_HOME)/bin/javac +bin.javadoc := $(JDK_HOME)/bin/javadoc +ifeq (,$(wildcard $(JDK_HOME))) +$(error set JDK_HOME to the top-most dir of your JDK installation.) +endif +MAKEFILE := $(lastword $(MAKEFILE_LIST)) +$(MAKEFILE): + +package.jar := sqlite3-jni.jar + +dir.top := ../.. +dir.tool := ../../tool +dir.jni := $(patsubst %/,%,$(dir $(MAKEFILE))) +dir.src := $(dir.jni)/src +dir.src.c := $(dir.src)/c +dir.bld := $(dir.jni)/bld +dir.bld.c := $(dir.bld) +dir.src.jni := $(dir.src)/org/sqlite/jni +dir.src.capi := $(dir.src.jni)/capi +dir.src.fts5 := $(dir.src.jni)/fts5 +dir.tests := $(dir.src)/tests +mkdir ?= mkdir -p +$(dir.bld.c): + $(mkdir) $@ + +javac.flags ?= -Xlint:unchecked -Xlint:deprecation +java.flags ?= +javac.flags += -encoding utf8 +# -------------^^^^^^^^^^^^^^ required for Windows builds +jnicheck ?= 1 +ifeq (1,$(jnicheck)) + java.flags += -Xcheck:jni +endif + +classpath := $(dir.src) +CLEAN_FILES := $(package.jar) +DISTCLEAN_FILES := $(dir.jni)/*~ $(dir.src.c)/*~ $(dir.src.jni)/*~ + +sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h +.NOTPARALLEL: $(sqlite3-jni.h) +CApi.java := $(dir.src.capi)/CApi.java +SQLTester.java := $(dir.src.capi)/SQLTester.java +CApi.class := $(CApi.java:.java=.class) +SQLTester.class := $(SQLTester.java:.java=.class) + +######################################################################## +# The future of FTS5 customization in this API is as yet unclear. +# The pieces are all in place, and are all thin proxies so not much +# complexity, but some semantic changes were required in porting +# which are largely untested. +# +# Reminder: this flag influences the contents of $(sqlite3-jni.h), +# which is checked in. Please do not check in changes to that file in +# which the fts5 APIs have been stripped unless that feature is +# intended to be stripped for good. +enable.fts5 ?= 1 + +ifeq (,$(wildcard $(dir.tests)/*)) + enable.tester := 0 +else + enable.tester := 1 +endif + +# bin.version-info = binary to output various sqlite3 version info +# building the distribution zip file. +bin.version-info := $(dir.top)/version-info +.NOTPARALLEL: $(bin.version-info) +$(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile + $(MAKE) -C $(dir.top) version-info + +# Be explicit about which Java files to compile so that we can work on +# in-progress files without requiring them to be in a compilable statae. +JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\ + Experimental.java \ + NotNull.java \ + Nullable.java \ +) $(patsubst %,$(dir.src.capi)/%,\ + AbstractCollationCallback.java \ + AggregateFunction.java \ + AuthorizerCallback.java \ + AutoExtensionCallback.java \ + BusyHandlerCallback.java \ + CollationCallback.java \ + CollationNeededCallback.java \ + CommitHookCallback.java \ + ConfigLogCallback.java \ + ConfigSqlLogCallback.java \ + NativePointerHolder.java \ + OutputPointer.java \ + PrepareMultiCallback.java \ + PreupdateHookCallback.java \ + ProgressHandlerCallback.java \ + ResultCode.java \ + RollbackHookCallback.java \ + ScalarFunction.java \ + SQLFunction.java \ + CallbackProxy.java \ + CApi.java \ + TableColumnMetadata.java \ + TraceV2Callback.java \ + UpdateHookCallback.java \ + ValueHolder.java \ + WindowFunction.java \ + XDestroyCallback.java \ + sqlite3.java \ + sqlite3_blob.java \ + sqlite3_context.java \ + sqlite3_stmt.java \ + sqlite3_value.java \ +) $(patsubst %,$(dir.src.jni)/wrapper1/%,\ + AggregateFunction.java \ + ScalarFunction.java \ + SqlFunction.java \ + Sqlite.java \ + SqliteException.java \ + ValueHolder.java \ + WindowFunction.java \ +) + +JAVA_FILES.unittest := $(patsubst %,$(dir.src.jni)/%,\ + capi/Tester1.java \ + wrapper1/Tester2.java \ +) +ifeq (1,$(enable.fts5)) + JAVA_FILES.unittest += $(patsubst %,$(dir.src.fts5)/%,\ + TesterFts5.java \ + ) + JAVA_FILES.main += $(patsubst %,$(dir.src.fts5)/%,\ + fts5_api.java \ + fts5_extension_function.java \ + fts5_tokenizer.java \ + Fts5.java \ + Fts5Context.java \ + Fts5ExtensionApi.java \ + Fts5PhraseIter.java \ + Fts5Tokenizer.java \ + XTokenizeCallback.java \ + ) +endif +JAVA_FILES.tester := $(SQLTester.java) +JAVA_FILES.package.info := \ + $(dir.src.jni)/package-info.java \ + $(dir.src.jni)/annotation/package-info.java + +CLASS_FILES.main := $(JAVA_FILES.main:.java=.class) +CLASS_FILES.unittest := $(JAVA_FILES.unittest:.java=.class) +CLASS_FILES.tester := $(JAVA_FILES.tester:.java=.class) + +JAVA_FILES += $(JAVA_FILES.main) $(JAVA_FILES.unittest) +ifeq (1,$(enable.tester)) + JAVA_FILES += $(JAVA_FILES.tester) +endif + +CLASS_FILES := +define CLASSFILE_DEPS +all: $(1).class +$(1).class: $(1).java +CLASS_FILES += $(1).class +endef +$(foreach B,$(basename \ + $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.tester)),\ + $(eval $(call CLASSFILE_DEPS,$(B)))) +$(CLASS_FILES): $(MAKEFILE) + $(bin.javac) $(javac.flags) -h $(dir.bld.c) -cp $(classpath) $(JAVA_FILES) + +#.PHONY: classfiles + +######################################################################## +# Set up sqlite3.c and sqlite3.h... +# +# To build with SEE (https://sqlite.org/see), either put sqlite3-see.c +# in the top of this build tree or pass +# sqlite3.c=PATH_TO_sqlite3-see.c to the build. Note that only +# encryption modules with no 3rd-party dependencies will currently +# work here: AES256-OFB, AES128-OFB, and AES128-CCM. Not +# coincidentally, those 3 modules are included in the sqlite3-see.c +# bundle. +# +# A custom sqlite3.c must not have any spaces in its name. +# $(sqlite3.canonical.c) must point to the sqlite3.c in +# the sqlite3 canonical source tree, as that source file +# is required for certain utility and test code. +sqlite3.canonical.c := $(firstword $(wildcard $(dir.src.c)/sqlite3.c) $(dir.top)/sqlite3.c) +sqlite3.canonical.h := $(firstword $(wildcard $(dir.src.c)/sqlite3.h) $(dir.top)/sqlite3.h) +sqlite3.c := $(sqlite3.canonical.c) +sqlite3.h := $(sqlite3.canonical.h) +#ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c) 2>/dev/null)) +# SQLITE_C_IS_SEE := 0 +#else +# SQLITE_C_IS_SEE := 1 +# $(info This is an SEE build.) +#endif + +.NOTPARALLEL: $(sqlite3.h) +$(sqlite3.h): + $(MAKE) -C $(dir.top) sqlite3.c +$(sqlite3.c): $(sqlite3.h) + +opt.threadsafe ?= 1 +opt.fatal-oom ?= 1 +opt.debug ?= 1 +opt.metrics ?= 1 +SQLITE_OPT = \ + -DSQLITE_THREADSAFE=$(opt.threadsafe) \ + -DSQLITE_TEMP_STORE=2 \ + -DSQLITE_USE_URI=1 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_C=$(sqlite3.c) \ + -DSQLITE_JNI_FATAL_OOM=$(opt.fatal-oom) \ + -DSQLITE_JNI_ENABLE_METRICS=$(opt.metrics) + +opt.extras ?= 1 +ifeq (1,$(opt.extras)) +SQLITE_OPT += -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_NORMALIZE \ + -DSQLITE_ENABLE_SQLLOG \ + -DSQLITE_ENABLE_COLUMN_METADATA +endif + +ifeq (1,$(opt.debug)) + SQLITE_OPT += -DSQLITE_DEBUG -g -DDEBUG -UNDEBUG +else + SQLITE_OPT += -Os +endif + +ifeq (1,$(enable.fts5)) + SQLITE_OPT += -DSQLITE_ENABLE_FTS5 +endif + +sqlite3-jni.c := $(dir.src.c)/sqlite3-jni.c +sqlite3-jni.o := $(dir.bld.c)/sqlite3-jni.o +sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h +package.dll := $(dir.bld.c)/libsqlite3-jni.so +# All javac-generated .h files must be listed in $(sqlite3-jni.h.in): +sqlite3-jni.h.in := +# $(java.with.jni) lists all Java files which contain JNI decls: +java.with.jni := +define ADD_JNI_H +sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h +java.with.jni += $(1)/$(2).java +$$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h: $(1)/$(2).java +endef +# Invoke ADD_JNI_H once for each Java file which includes JNI +# declarations: +$(eval $(call ADD_JNI_H,$(dir.src.capi),CApi,_capi)) +$(eval $(call ADD_JNI_H,$(dir.src.capi),SQLTester,_capi)) +ifeq (1,$(enable.fts5)) + $(eval $(call ADD_JNI_H,$(dir.src.fts5),Fts5ExtensionApi,_fts5)) + $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_api,_fts5)) + $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_tokenizer,_fts5)) +endif +$(sqlite3-jni.h.in): $(dir.bld.c) + +#package.dll.cfiles := +package.dll.cflags = \ + -std=c99 \ + -fPIC \ + -I. \ + -I$(dir $(sqlite3.h)) \ + -I$(dir.src.c) \ + -I$(JDK_HOME)/include \ + $(patsubst %,-I%,$(patsubst %.h,,$(wildcard $(JDK_HOME)/include/*))) \ + -Wall +# The gross $(patsubst...) above is to include the platform-specific +# subdir which lives under $(JDK_HOME)/include and is a required +# include path for client-level code. +# +# Using (-Wall -Wextra) triggers an untennable number of +# gcc warnings from sqlite3.c for mundane things like +# unused parameters. +######################################################################## +ifeq (1,$(enable.tester)) + package.dll.cflags += -DSQLITE_JNI_ENABLE_SQLTester +endif + +$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE) + @cat $(sqlite3-jni.h.in) > $@.tmp + @if cmp $@ $@.tmp >/dev/null; then \ + rm -f $@.tmp; \ + echo "$@ not modified"; \ + else \ + mv $@.tmp $@; \ + echo "Updated $@"; \ + fi + @if [ x1 != x$(enable.fts5) ]; then \ + echo "*** REMINDER:"; \ + echo "*** enable.fts5=0, so please do not check in changes to $@."; \ + fi + +$(package.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h) +$(package.dll): $(sqlite3-jni.c) $(MAKEFILE) + $(CC) $(package.dll.cflags) $(SQLITE_OPT) \ + $(sqlite3-jni.c) -shared -o $@ +all: $(package.dll) + +.PHONY: test test-one +Tester1.flags ?= +Tester2.flags ?= +test.flags.jvm = -ea -Djava.library.path=$(dir.bld.c) \ + $(java.flags) -cp $(classpath) +test.deps := $(CLASS_FILES) $(package.dll) +test-one: $(test.deps) + $(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) + $(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 $(Tester2.flags) +test-sqllog: $(test.deps) + @echo "Testing with -sqllog..." + $(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) -sqllog +test-mt: $(test.deps) + @echo "Testing in multi-threaded mode:"; + $(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 \ + -t 7 -r 50 -shuffle $(Tester1.flags) + $(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 \ + -t 7 -r 50 -shuffle $(Tester2.flags) + +test: test-one test-mt +tests: test test-sqllog + +tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test)) +tester.flags ?= # --verbose +.PHONY: tester tester-local tester-ext +ifeq (1,$(enable.tester)) +tester-local: $(CLASS_FILES.tester) $(package.dll) + $(bin.java) -ea -Djava.library.path=$(dir.bld.c) \ + $(java.flags) -cp $(classpath) \ + org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.scripts) +tester: tester-local +else +tester: + @echo "SQLTester support is disabled." +endif + +tester.extdir.default := $(dir.tests)/ext +tester.extdir ?= $(tester.extdir.default) +tester.extern-scripts := $(wildcard $(tester.extdir)/*.test) +ifneq (,$(tester.extern-scripts)) +tester-ext: + $(bin.java) -ea -Djava.library.path=$(dir.bld.c) \ + $(java.flags) -cp $(classpath) \ + org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.extern-scripts) +else +tester-ext: + @echo "******************************************************"; \ + echo "*** Include the out-of-tree test suite in the 'tester'"; \ + echo "*** target by either symlinking its directory to"; \ + echo "*** $(tester.extdir.default) or passing it to make"; \ + echo "*** as tester.extdir=/path/to/that/dir."; \ + echo "******************************************************"; +endif + +tester-ext: tester-local +tester: tester-ext +tests: tester +######################################################################## +# Build each SQLITE_THREADMODE variant and run all tests against them. +multitest: clean +define MULTIOPT +multitest: multitest-$(1) +multitest-$(1): + $$(MAKE) opt.debug=$$(opt.debug) $(patsubst %,opt.%,$(2)) \ + tests clean enable.fts5=1 +endef + +$(eval $(call MULTIOPT,01,threadsafe=0 oom=1)) +$(eval $(call MULTIOPT,00,threadsafe=0 oom=0)) +$(eval $(call MULTIOPT,11,threadsafe=1 oom=1)) +$(eval $(call MULTIOPT,10,threadsafe=1 oom=0)) +$(eval $(call MULTIOPT,21,threadsafe=2 oom=1)) +$(eval $(call MULTIOPT,20,threadsafe=2 oom=0)) + + +######################################################################## +# jar bundle... +package.jar.in := $(abspath $(dir.src)/jar.in) +CLEAN_FILES += $(package.jar.in) +JAVA_FILES.jar := $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.package.info) +CLASS_FILES.jar := $(filter-out %/package-info.class,$(JAVA_FILES.jar:.java=.class)) +$(package.jar.in): $(package.dll) $(MAKEFILE) + ls -1 \ + $(dir.src.jni)/*/*.java $(dir.src.jni)/*/*.class \ + | sed -e 's,^$(dir.src)/,,' | sort > $@ + +$(package.jar): $(CLASS_FILES.jar) $(MAKEFILE) $(package.jar.in) + @rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~ + cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.capi.Tester1 @$(package.jar.in) + @ls -la $@ + @echo "To use this jar you will need the -Djava.library.path=DIR/CONTAINING/libsqlite3-jni.so flag." + @echo "e.g. java -Djava.library.path=bld -jar $@" + +jar: $(package.jar) +run-jar: $(package.jar) $(package.dll) + $(bin.java) -Djava.library.path=$(dir.bld) -jar $(package.jar) $(run-jar.flags) + +######################################################################## +# javadoc... +dir.doc := $(dir.jni)/javadoc +doc.index := $(dir.doc)/index.html +javadoc.exclude := -exclude org.sqlite.jni.fts5 +# ^^^^ 2023-09-13: elide the fts5 parts from the public docs for +# the time being, as it's not clear where the Java bindings for +# those bits are going. +# javadoc.exclude += -exclude org.sqlite.jni.capi +# ^^^^ exclude the capi API only for certain builds (TBD) +$(doc.index): $(JAVA_FILES.main) $(MAKEFILE) + @if [ -d $(dir.doc) ]; then rm -fr $(dir.doc)/*; fi + $(bin.javadoc) -cp $(classpath) -d $(dir.doc) -quiet \ + -subpackages org.sqlite.jni $(javadoc.exclude) + @echo "javadoc output is in $@" + +.PHONY: doc javadoc docserve +.FORCE: doc +doc: $(doc.index) +javadoc: $(doc.index) +# Force rebild of docs +redoc: + @rm -f $(doc.index) + @$(MAKE) doc +docserve: $(doc.index) + cd $(dir.doc) && althttpd -max-age 1 -page index.html +######################################################################## +# Clean up... +CLEAN_FILES += $(dir.bld.c)/* \ + $(dir.src.jni)/*.class \ + $(dir.src.jni)/*/*.class \ + $(package.dll) \ + hs_err_pid*.log + +.PHONY: clean distclean +clean: + -rm -f $(CLEAN_FILES) +distclean: clean + -rm -f $(DISTCLEAN_FILES) + -rm -fr $(dir.bld.c) $(dir.doc) + +######################################################################## +# disttribution bundle rules... + +ifeq (,$(filter snapshot,$(MAKECMDGOALS))) +dist-name-prefix := sqlite-jni +else +dist-name-prefix := sqlite-jni-snapshot-$(shell /usr/bin/date +%Y%m%d) +endif +dist-name := $(dist-name-prefix)-TEMP + + +dist-dir.top := $(dist-name) +dist-dir.src := $(dist-dir.top)/src +dist.top.extras := \ + README.md + +.PHONY: dist snapshot + +dist: \ + $(bin.version-info) $(sqlite3.canonical.c) \ + $(package.jar) $(MAKEFILE) + @echo "Making end-user deliverables..." + @echo "****************************************************************************"; \ + echo "*** WARNING: be sure to build this with JDK8 (javac 1.8) for compatibility."; \ + echo "*** reasons!"; $$($(bin.javac) -version); \ + echo "****************************************************************************" + @rm -fr $(dist-dir.top) + @mkdir -p $(dist-dir.src) + @cp -p $(dist.top.extras) $(dist-dir.top)/. + @cp -p jar-dist.make $(dist-dir.top)/Makefile + @cp -p $(dir.src.c)/*.[ch] $(dist-dir.src)/. + @cp -p $(sqlite3.canonical.c) $(sqlite3.canonical.h) $(dist-dir.src)/. + @set -e; \ + vnum=$$($(bin.version-info) --download-version); \ + vjar=$$($(bin.version-info) --version); \ + vdir=$(dist-name-prefix)-$$vnum; \ + arczip=$$vdir.zip; \ + cp -p $(package.jar) $(dist-dir.top)/sqlite3-jni-$${vjar}.jar; \ + echo "Making $$arczip ..."; \ + rm -fr $$arczip $$vdir; \ + mv $(dist-dir.top) $$vdir; \ + zip -qr $$arczip $$vdir; \ + rm -fr $$vdir; \ + ls -la $$arczip; \ + set +e; \ + unzip -lv $$arczip || echo "Missing unzip app? Not fatal." + +snapshot: dist + +.PHONY: dist-clean +clean: dist-clean +dist-clean: + rm -fr $(dist-name) $(wildcard sqlite-jni-*.zip) diff --git a/ext/jni/README.md b/ext/jni/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fc7b5f761195d39412e0bb6045e6e8cb3840b4f8 --- /dev/null +++ b/ext/jni/README.md @@ -0,0 +1,316 @@ +SQLite3 via JNI +======================================================================== + +This directory houses a Java Native Interface (JNI) binding for the +sqlite3 API. If you are reading this from the distribution ZIP file, +links to resources in the canonical source tree will note work. The +canonical copy of this file can be browsed at: + + + +Technical support is available in the forum: + + + + +> **FOREWARNING:** this subproject is very much in development and + subject to any number of changes. Please do not rely on any + information about its API until this disclaimer is removed. The JNI + bindings released with version 3.43 are a "tech preview." Once + finalized, strong backward compatibility guarantees will apply. + +Project goals/requirements: + +- A [1-to-1(-ish) mapping of the C API](#1to1ish) to Java via JNI, + insofar as cross-language semantics allow for. A closely-related + goal is that [the C documentation](https://sqlite.org/c3ref/intro.html) + should be usable as-is, insofar as possible, for the JNI binding. + +- Support Java as far back as version 8 (2014). + +- Environment-independent. Should work everywhere both Java + and SQLite3 do. + +- No 3rd-party dependencies beyond the JDK. That includes no + build-level dependencies for specific IDEs and toolchains. We + welcome the addition of build files for arbitrary environments + insofar as they neither interfere with each other nor become + a maintenance burden for the sqlite developers. + +Non-goals: + +- Creation of high-level OO wrapper APIs. Clients are free to create + them off of the C-style API. + +- Virtual tables are unlikely to be supported due to the amount of + glue code needed to fit them into Java. + +- Support for mixed-mode operation, where client code accesses SQLite + both via the Java-side API and the C API via their own native + code. Such cases would be a minefield of potential mis-interactions + between this project's JNI bindings and mixed-mode client code. + + +Hello World +----------------------------------------------------------------------- + +```java +import org.sqlite.jni.*; +import static org.sqlite.jni.CApi.*; + +... + +final sqlite3 db = sqlite3_open(":memory:"); +try { + final int rc = sqlite3_errcode(db); + if( 0 != rc ){ + if( null != db ){ + System.out.print("Error opening db: "+sqlite3_errmsg(db)); + }else{ + System.out.print("Error opening db: rc="+rc); + } + ... handle error ... + } + // ... else use the db ... +}finally{ + // ALWAYS close databases using sqlite3_close() or sqlite3_close_v2() + // when done with them. All of their active statement handles must + // first have been passed to sqlite3_finalize(). + sqlite3_close_v2(db); +} +``` + + +Building +======================================================================== + +The canonical builds assumes a Linux-like environment and requires: + +- GNU Make +- A JDK supporting Java 8 or higher +- A modern C compiler. gcc and clang should both work. + +Put simply: + +```console +$ export JAVA_HOME=/path/to/jdk/root +$ make +$ make test +$ make clean +``` + +The jar distribution can be created with `make jar`, but note that it +does not contain the binary DLL file. A different DLL is needed for +each target platform. + + + +One-to-One(-ish) Mapping to C +======================================================================== + +This JNI binding aims to provide as close to a 1-to-1 experience with +the C API as cross-language semantics allow. Interface changes are +necessarily made where cross-language semantics do not allow a 1-to-1, +and judiciously made where a 1-to-1 mapping would be unduly cumbersome +to use in Java. In all cases, this binding makes every effort to +provide semantics compatible with the C API documentation even if the +interface to those semantics is slightly different. Any cases which +deviate from those semantics (either removing or adding semantics) are +clearly documented. + +Where it makes sense to do so for usability, Java-side overloads are +provided which accept or return data in alternative forms or provide +sensible default argument values. In all such cases they are thin +proxies around the corresponding C APIs and do not introduce new +semantics. + +In a few cases, Java-specific capabilities have been added in +new APIs, all of which have "_java" somewhere in their names. +Examples include: + +- `sqlite3_result_java_object()` +- `sqlite3_column_java_object()` +- `sqlite3_value_java_object()` + +which, as one might surmise, collectively enable the passing of +arbitrary Java objects from user-defined SQL functions through to the +caller. + + +Golden Rule: Garbage Collection Cannot Free SQLite Resources +------------------------------------------------------------------------ + +It is important that all databases and prepared statement handles get +cleaned up by client code. A database cannot be closed if it has open +statement handles. `sqlite3_close()` fails if the db cannot be closed +whereas `sqlite3_close_v2()` recognizes that case and marks the db as +a "zombie," pending finalization when the library detects that all +pending statements have been closed. Be aware that Java garbage +collection _cannot_ close a database or finalize a prepared statement. +Those things require explicit API calls. + +Classes for which it is sensible support Java's `AutoCloseable` +interface so can be used with try-with-resources constructs. + + +Golden Rule #2: _Never_ Throw from Callbacks (Unless...) +------------------------------------------------------------------------ + +All routines in this API, barring explicitly documented exceptions, +retain C-like semantics. For example, they are not permitted to throw +or propagate exceptions and must return error information (if any) via +result codes or `null`. The only cases where the C-style APIs may +throw is through client-side misuse, e.g. passing in a null where it +may cause a `NullPointerException`. The APIs clearly mark function +parameters which should not be null, but does not generally actively +defend itself against such misuse. Some C-style APIs explicitly accept +`null` as a no-op for usability's sake, and some of the JNI APIs +deliberately return an error code, instead of segfaulting, when passed +a `null`. + +Client-defined callbacks _must never throw exceptions_ unless _very +explitly documented_ as being throw-safe. Exceptions are generally +reserved for higher-level bindings which are constructed to +specifically deal with them and ensure that they do not leak C-level +resources. In some cases, callback handlers are permitted to throw, in +which cases they get translated to C-level result codes and/or +messages. If a callback which is not permitted to throw throws, its +exception may trigger debug output but will otherwise be suppressed. + +The reason some callbacks are permitted to throw and others not is +because all such callbacks act as proxies for C function callback +interfaces and some of those interfaces have no error-reporting +mechanism. Those which are capable of propagating errors back through +the library convert exceptions from callbacks into corresponding +C-level error information. Those which cannot propagate errors +necessarily suppress any exceptions in order to maintain the C-style +semantics of the APIs. + + +Unwieldy Constructs are Re-mapped +------------------------------------------------------------------------ + +Some constructs, when modelled 1-to-1 from C to Java, are unduly +clumsy to work with in Java because they try to shoehorn C's way of +doing certain things into Java's wildly different ways. The following +subsections cover those, starting with a verbose explanation and +demonstration of where such changes are "really necessary"... + +### Custom Collations + +A prime example of where interface changes for Java are necessary for +usability is [registration of a custom +collation](https://sqlite.org/c3ref/create_collation.html): + +```c +// C: +int sqlite3_create_collation(sqlite3 * db, const char * name, int eTextRep, + void *pUserData, + int (*xCompare)(void*,int,void const *,int,void const *)); + +int sqlite3_create_collation_v2(sqlite3 * db, const char * name, int eTextRep, + void *pUserData, + int (*xCompare)(void*,int,void const *,int,void const *), + void (*xDestroy)(void*)); +``` + +The `pUserData` object is optional client-defined state for the +`xCompare()` and/or `xDestroy()` callback functions, both of which are +passed that object as their first argument. That data is passed around +"externally" in C because that's how C models the world. If we were to +bind that part as-is to Java, the result would be awkward to use (^Yes, +we tried this.): + +```java +// Java: +int sqlite3_create_collation(sqlite3 db, String name, int eTextRep, + Object pUserData, xCompareType xCompare); + +int sqlite3_create_collation_v2(sqlite3 db, String name, int eTextRep, + Object pUserData, + xCompareType xCompare, xDestroyType xDestroy); +``` + +The awkwardness comes from (A) having two distinctly different objects +for callbacks and (B) having their internal state provided separately, +which is ill-fitting in Java. For the sake of usability, C APIs which +follow that pattern use a slightly different Java interface: + +```java +int sqlite3_create_collation(sqlite3 db, String name, int eTextRep, + SomeCallbackType collation); +``` + +Where the `Collation` class has an abstract `call()` method and +no-op `xDestroy()` method which can be overridden if needed, leading to +a much more Java-esque usage: + +```java +int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new SomeCallbackType(){ + + // Required comparison function: + @Override public int call(byte[] lhs, byte[] rhs){ ... } + + // Optional finalizer function: + @Override public void xDestroy(){ ... } + + // Optional local state: + private String localState1 = + "This is local state. There are many like it, but this one is mine."; + private MyStateType localState2 = new MyStateType(); + ... +}); +``` + +Noting that: + +- It is possible to bind in call-scope-local state via closures, if + desired, as opposed to packing it into the Collation object. + +- No capabilities of the C API are lost or unduly obscured via the + above API reshaping, so power users need not make any compromises. + +- In the specific example above, `sqlite3_create_collation_v2()` + becomes superfluous because the provided interface effectively + provides both the v1 and v2 interfaces, the difference being that + overriding the `xDestroy()` method effectively gives it v2 + semantics. + + +### User-defined SQL Functions (a.k.a. UDFs) + +The [`sqlite3_create_function()`](https://sqlite.org/c3ref/create_function.html) +family of APIs make heavy use of function pointers to provide +client-defined callbacks, necessitating interface changes in the JNI +binding. The Java API has only one core function-registration function: + +```java +int sqlite3_create_function(sqlite3 db, String funcName, int nArgs, + int encoding, SQLFunction func); +``` + +> Design question: does the encoding argument serve any purpose in + Java? That's as-yet undetermined. If not, it will be removed. + +`SQLFunction` is not used directly, but is instead instantiated via +one of its three subclasses: + +- `ScalarFunction` implements simple scalar functions using but a + single callback. +- `AggregateFunction` implements aggregate functions using two + callbacks. +- `WindowFunction` implements window functions using four + callbacks. + +Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/capi/Tester1.java) for +`SQLFunction` for how it's used. + +Reminder: see the disclaimer at the top of this document regarding the +in-flux nature of this API. + +### And so on... + +Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and +`sqlite3_update_hook()`, use interfaces similar to those shown above. +Despite the changes in signature, the JNI layer makes every effort to +provide the same semantics as the C API documentation suggests. diff --git a/ext/jni/jar-dist.make b/ext/jni/jar-dist.make new file mode 100644 index 0000000000000000000000000000000000000000..7596c99f3f8aa2be92abf5317beb4293ffc89857 --- /dev/null +++ b/ext/jni/jar-dist.make @@ -0,0 +1,60 @@ +#!/this/is/make +#^^^^ help emacs out +# +# This is a POSIX-make-compatible makefile for building the sqlite3 +# JNI library from "dist" zip file. It must be edited to set the +# proper top-level JDK directory and, depending on the platform, add a +# platform-specific -I directory. It should build as-is with any +# 2020s-era version of gcc or clang. It requires JDK version 8 or +# higher and that JAVA_HOME points to the top-most installation +# directory of that JDK. On Ubuntu-style systems the JDK is typically +# installed under /usr/lib/jvm/java-VERSION-PLATFORM. + +default: all + +JAVA_HOME = /usr/lib/jvm/java-1.8.0-openjdk-amd64 +CFLAGS = \ + -fPIC \ + -Isrc \ + -I$(JAVA_HOME)/include \ + -I$(JAVA_HOME)/include/linux \ + -I$(JAVA_HOME)/include/apple \ + -I$(JAVA_HOME)/include/bsd \ + -Wall + +SQLITE_OPT = \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_THREADSAFE=1 \ + -DSQLITE_TEMP_STORE=2 \ + -DSQLITE_USE_URI=1 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_DEBUG + +sqlite3-jni.dll = libsqlite3-jni.so +$(sqlite3-jni.dll): + @echo "************************************************************************"; \ + echo "*** If this fails to build, be sure to edit this makefile ***"; \ + echo "*** to configure it for your system. ***"; \ + echo "************************************************************************" + $(CC) $(CFLAGS) $(SQLITE_OPT) \ + src/sqlite3-jni.c -shared -o $@ + @echo "Now try running it with: make test" + +test.flags = -Djava.library.path=. sqlite3-jni-*.jar +test: $(sqlite3-jni.dll) + java -jar $(test.flags) + java -jar $(test.flags) -t 7 -r 10 -shuffle + +clean: + -rm -f $(sqlite3-jni.dll) + +all: $(sqlite3-jni.dll) diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c new file mode 100644 index 0000000000000000000000000000000000000000..5deff19ef1fd5e668a12f73640bd0eb4eed07605 --- /dev/null +++ b/ext/jni/src/c/sqlite3-jni.c @@ -0,0 +1,6343 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements the JNI bindings declared in +** org.sqlite.jni.capi.CApi (from which sqlite3-jni.h is generated). +*/ + +/* +** If you found this comment by searching the code for +** CallStaticObjectMethod because it appears in console output then +** you're probably the victim of an OpenJDK bug: +** +** https://bugs.openjdk.org/browse/JDK-8130659 +** +** It's known to happen with OpenJDK v8 but not with v19. It was +** triggered by this code long before it made any use of +** CallStaticObjectMethod(). +*/ + +/* +** Define any SQLITE_... config defaults we want if they aren't +** overridden by the builder. Please keep these alphabetized. +*/ + +/**********************************************************************/ +/* SQLITE_D... */ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +# define SQLITE_DEFAULT_CACHE_SIZE -16384 +#endif +#if !defined(SQLITE_DEFAULT_PAGE_SIZE) +# define SQLITE_DEFAULT_PAGE_SIZE 8192 +#endif +#ifndef SQLITE_DQS +# define SQLITE_DQS 0 +#endif + +/**********************************************************************/ +/* SQLITE_ENABLE_... */ +/* +** Unconditionally enable API_ARMOR in the JNI build. It ensures that +** public APIs behave predictable in the face of passing illegal NULLs +** or ranges which might otherwise invoke undefined behavior. +*/ +#undef SQLITE_ENABLE_API_ARMOR +#define SQLITE_ENABLE_API_ARMOR 1 + +#ifndef SQLITE_ENABLE_BYTECODE_VTAB +# define SQLITE_ENABLE_BYTECODE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBPAGE_VTAB +# define SQLITE_ENABLE_DBPAGE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBSTAT_VTAB +# define SQLITE_ENABLE_DBSTAT_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif +#ifndef SQLITE_ENABLE_MATH_FUNCTIONS +# define SQLITE_ENABLE_MATH_FUNCTIONS 1 +#endif +#ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC +# define SQLITE_ENABLE_OFFSET_SQL_FUNC 1 +#endif +#ifndef SQLITE_ENABLE_RTREE +# define SQLITE_ENABLE_RTREE 1 +#endif +//#ifndef SQLITE_ENABLE_SESSION +//# define SQLITE_ENABLE_SESSION 1 +//#endif +#ifndef SQLITE_ENABLE_STMTVTAB +# define SQLITE_ENABLE_STMTVTAB 1 +#endif +//#ifndef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +//# define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +//#endif + +/**********************************************************************/ +/* SQLITE_J... */ +#ifdef SQLITE_JNI_FATAL_OOM +#if !SQLITE_JNI_FATAL_OOM +#undef SQLITE_JNI_FATAL_OOM +#endif +#endif + +/**********************************************************************/ +/* SQLITE_O... */ +#ifndef SQLITE_OMIT_DEPRECATED +# define SQLITE_OMIT_DEPRECATED 1 +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE +# define SQLITE_OMIT_SHARED_CACHE 1 +#endif +#ifdef SQLITE_OMIT_UTF16 +/* UTF16 is required for java */ +# undef SQLITE_OMIT_UTF16 1 +#endif + +/**********************************************************************/ +/* SQLITE_S... */ +#ifndef SQLITE_STRICT_SUBTYPE +# define SQLITE_STRICT_SUBTYPE 1 +#endif + +/**********************************************************************/ +/* SQLITE_T... */ +#ifndef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 2 +#endif +#ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 1 +#endif + +/**********************************************************************/ +/* SQLITE_USE_... */ +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 1 +#endif + + +/* +** Which sqlite3.c we're using needs to be configurable to enable +** building against a custom copy, e.g. the SEE variant. We have to +** include sqlite3.c, as opposed to sqlite3.h, in order to get access +** to some interal details like SQLITE_MAX_... and friends. This +** increases the rebuild time considerably but we need this in order +** to access some internal functionality and keep the to-Java-exported +** values of SQLITE_MAX_... and SQLITE_LIMIT_... in sync with the C +** build. +*/ +#ifndef SQLITE_C +# define SQLITE_C sqlite3.c +#endif +#define INC__STRINGIFY_(f) #f +#define INC__STRINGIFY(f) INC__STRINGIFY_(f) +#include INC__STRINGIFY(SQLITE_C) +#undef INC__STRINGIFY_ +#undef INC__STRINGIFY +#undef SQLITE_C + +/* +** End of the sqlite3 lib setup. What follows is JNI-specific. +*/ + +#include "sqlite3-jni.h" +#include +#include /* only for testing/debugging */ +#include /* intptr_t for 32-bit builds */ + +/* Only for debugging */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/* +** Creates a verbose JNI function name. Suffix must be +** the JNI-mangled form of the function's name, minus the +** prefix seen in this macro. +*/ +#define JniFuncName(Suffix) \ + Java_org_sqlite_jni_capi_CApi_sqlite3_ ## Suffix + +/* Prologue for JNI function declarations and definitions. */ +#define JniDecl(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL JniFuncName(Suffix) + +/* +** S3JniApi's intent is that CFunc be the C API func(s) the +** being-declared JNI function is wrapping, making it easier to find +** that function's JNI-side entry point. The other args are for JniDecl. +** See the many examples in this file. +*/ +#define S3JniApi(CFunc,ReturnType,Suffix) JniDecl(ReturnType,Suffix) + +/* +** S3JniCast_L2P and P2L cast jlong (64-bit) to/from pointers. This is +** required for casting warning-free on 32-bit builds, where we +** otherwise get complaints that we're casting between different-sized +** int types. +** +** This use of intptr_t is the _only_ reason we require +** which, in turn, requires building with -std=c99 (or later). +** +** See also: the notes for LongPtrGet_T. +*/ +#define S3JniCast_L2P(JLongAsPtr) (void*)((intptr_t)(JLongAsPtr)) +#define S3JniCast_P2L(PTR) (jlong)((intptr_t)(PTR)) + +/* +** Shortcuts for the first 2 parameters to all JNI bindings. +** +** The type of the jSelf arg differs, but no docs seem to mention +** this: for static methods it's of type jclass and for non-static +** it's jobject. jobject actually works for all funcs, in the sense +** that it compiles and runs so long as we don't use jSelf (which is +** only rarely needed in this code), but to be pedantically correct we +** need the proper type in the signature. +** +** https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/design.html#jni_interface_functions_and_pointers +*/ +#define JniArgsEnvObj JNIEnv * env, jobject jSelf +#define JniArgsEnvClass JNIEnv * env, jclass jKlazz +/* +** Helpers to account for -Xcheck:jni warnings about not having +** checked for exceptions. +*/ +#define S3JniIfThrew if( (*env)->ExceptionCheck(env) ) +#define S3JniExceptionClear (*env)->ExceptionClear(env) +#define S3JniExceptionReport (*env)->ExceptionDescribe(env) +#define S3JniExceptionIgnore S3JniIfThrew S3JniExceptionClear +#define S3JniExceptionWarnIgnore \ + S3JniIfThrew {S3JniExceptionReport; S3JniExceptionClear;}(void)0 +#define S3JniExceptionWarnCallbackThrew(STR) \ + MARKER(("WARNING: " STR " MUST NOT THROW.\n")); \ + (*env)->ExceptionDescribe(env) + +/** To be used for cases where we're _really_ not expecting an + exception, e.g. looking up well-defined Java class members. */ +#define S3JniExceptionIsFatal(MSG) S3JniIfThrew {\ + S3JniExceptionReport; S3JniExceptionClear; \ + (*env)->FatalError(env, MSG); \ + } + +/* +** Declares local var env = s3jni_env(). All JNI calls involve a +** JNIEnv somewhere, always named env, and many of our macros assume +** env is in scope. Where it's not, but should be, use this to make it +** so. +*/ +#define S3JniDeclLocal_env JNIEnv * const env = s3jni_env() + +/* Fail fatally with an OOM message. */ +static inline void s3jni_oom(JNIEnv * const env){ + (*env)->FatalError(env, "SQLite3 JNI is out of memory.") /* does not return */; +} + +/* +** sqlite3_malloc() proxy which fails fatally on OOM. This should +** only be used for routines which manage global state and have no +** recovery strategy for OOM. For sqlite3 API which can reasonably +** return SQLITE_NOMEM, s3jni_malloc() should be used instead. +*/ +static void * s3jni_malloc_or_die(JNIEnv * const env, size_t n){ + void * const rv = sqlite3_malloc(n); + if( n && !rv ) s3jni_oom(env); + return rv; +} + +/* +** Works like sqlite3_malloc() unless built with SQLITE_JNI_FATAL_OOM, +** in which case it calls s3jni_oom() on OOM. +*/ +#ifdef SQLITE_JNI_FATAL_OOM +#define s3jni_malloc(SIZE) s3jni_malloc_or_die(env, SIZE) +#else +#define s3jni_malloc(SIZE) sqlite3_malloc(((void)env,(SIZE))) +/* the ((void)env) trickery here is to avoid ^^^^^^ an otherwise + unused arg in at least one place. */ +#endif + +/* +** Works like sqlite3_realloc() unless built with SQLITE_JNI_FATAL_OOM, +** in which case it calls s3jni_oom() on OOM. +*/ +#ifdef SQLITE_JNI_FATAL_OOM +static void * s3jni_realloc_or_die(JNIEnv * const env, void * p, size_t n){ + void * const rv = sqlite3_realloc(p, (int)n); + if( n && !rv ) s3jni_oom(env); + return rv; +} +#define s3jni_realloc(MEM,SIZE) s3jni_realloc_or_die(env, (MEM), (SIZE)) +#else +#define s3jni_realloc(MEM,SIZE) sqlite3_realloc((MEM), ((void)env, (SIZE))) +#endif + +/* Fail fatally if !EXPR. */ +#define s3jni_oom_fatal(EXPR) if( !(EXPR) ) s3jni_oom(env) +/* Maybe fail fatally if !EXPR. */ +#ifdef SQLITE_JNI_FATAL_OOM +#define s3jni_oom_check s3jni_oom_fatal +#else +#define s3jni_oom_check(EXPR) +#endif +//#define S3JniDb_oom(pDb,EXPR) ((EXPR) ? sqlite3OomFault(pDb) : 0) + +#define s3jni_db_oom(pDb) (void)((pDb) ? ((pDb)->mallocFailed=1) : 0) + +/* Helpers for Java value reference management. */ +static jobject s3jni_ref_global(JNIEnv * const env, jobject const v){ + jobject const rv = v ? (*env)->NewGlobalRef(env, v) : NULL; + s3jni_oom_fatal( v ? !!rv : 1 ); + return rv; +} +static jobject s3jni_ref_local(JNIEnv * const env, jobject const v){ + jobject const rv = v ? (*env)->NewLocalRef(env, v) : NULL; + s3jni_oom_fatal( v ? !!rv : 1 ); + return rv; +} +static inline void s3jni_unref_global(JNIEnv * const env, jobject const v){ + if( v ) (*env)->DeleteGlobalRef(env, v); +} +static inline void s3jni_unref_local(JNIEnv * const env, jobject const v){ + if( v ) (*env)->DeleteLocalRef(env, v); +} +#define S3JniRefGlobal(VAR) s3jni_ref_global(env, (VAR)) +#define S3JniRefLocal(VAR) s3jni_ref_local(env, (VAR)) +#define S3JniUnrefGlobal(VAR) s3jni_unref_global(env, (VAR)) +#define S3JniUnrefLocal(VAR) s3jni_unref_local(env, (VAR)) + +/* +** Lookup key type for use with s3jni_nphop() and a cache of a +** frequently-needed Java-side class reference and one or two Java +** class member IDs. +*/ +typedef struct S3JniNphOp S3JniNphOp; +struct S3JniNphOp { + const int index /* index into S3JniGlobal.nph[] */; + const char * const zName /* Full Java name of the class */; + const char * const zMember /* Name of member property */; + const char * const zTypeSig /* JNI type signature of zMember */; + /* + ** klazz is a global ref to the class represented by pRef. + ** + ** According to: + ** + ** https://developer.ibm.com/articles/j-jni/ + ** + ** > ... the IDs returned for a given class don't change for the + ** lifetime of the JVM process. But the call to get the field or + ** method can require significant work in the JVM, because fields + ** and methods might have been inherited from superclasses, making + ** the JVM walk up the class hierarchy to find them. Because the + ** IDs are the same for a given class, you should look them up + ** once and then reuse them. Similarly, looking up class objects + ** can be expensive, so they should be cached as well. + */ + jclass klazz; + volatile jfieldID fidValue /* NativePointerHolder.nativePointer or + ** OutputPointer.T.value */; + volatile jmethodID midCtor /* klazz's no-arg constructor. Used by + ** NativePointerHolder_new(). */; +}; + +/* +** Cache keys for each concrete NativePointerHolder subclasses and +** OutputPointer.T types. The members are to be used with s3jni_nphop() +** and friends, and each one's member->index corresponds to its index +** in the S3JniGlobal.nph[] array. +*/ +static const struct { + const S3JniNphOp sqlite3; + const S3JniNphOp sqlite3_backup; + const S3JniNphOp sqlite3_blob; + const S3JniNphOp sqlite3_context; + const S3JniNphOp sqlite3_stmt; + const S3JniNphOp sqlite3_value; + const S3JniNphOp OutputPointer_Bool; + const S3JniNphOp OutputPointer_Int32; + const S3JniNphOp OutputPointer_Int64; + const S3JniNphOp OutputPointer_sqlite3; + const S3JniNphOp OutputPointer_sqlite3_blob; + const S3JniNphOp OutputPointer_sqlite3_stmt; + const S3JniNphOp OutputPointer_sqlite3_value; + const S3JniNphOp OutputPointer_String; +#ifdef SQLITE_ENABLE_FTS5 + const S3JniNphOp OutputPointer_ByteArray; + const S3JniNphOp Fts5Context; + const S3JniNphOp Fts5ExtensionApi; + const S3JniNphOp fts5_api; + const S3JniNphOp fts5_tokenizer; + const S3JniNphOp Fts5Tokenizer; +#endif +} S3JniNphOps = { +#define MkRef(INDEX, KLAZZ, MEMBER, SIG) \ + { INDEX, "org/sqlite/jni/" KLAZZ, MEMBER, SIG } +/* NativePointerHolder ref */ +#define RefN(INDEX, KLAZZ) MkRef(INDEX, KLAZZ, "nativePointer", "J") +/* OutputPointer.T ref */ +#define RefO(INDEX, KLAZZ, SIG) MkRef(INDEX, KLAZZ, "value", SIG) + RefN(0, "capi/sqlite3"), + RefN(1, "capi/sqlite3_backup"), + RefN(2, "capi/sqlite3_blob"), + RefN(3, "capi/sqlite3_context"), + RefN(4, "capi/sqlite3_stmt"), + RefN(5, "capi/sqlite3_value"), + RefO(6, "capi/OutputPointer$Bool", "Z"), + RefO(7, "capi/OutputPointer$Int32", "I"), + RefO(8, "capi/OutputPointer$Int64", "J"), + RefO(9, "capi/OutputPointer$sqlite3", + "Lorg/sqlite/jni/capi/sqlite3;"), + RefO(10, "capi/OutputPointer$sqlite3_blob", + "Lorg/sqlite/jni/capi/sqlite3_blob;"), + RefO(11, "capi/OutputPointer$sqlite3_stmt", + "Lorg/sqlite/jni/capi/sqlite3_stmt;"), + RefO(12, "capi/OutputPointer$sqlite3_value", + "Lorg/sqlite/jni/capi/sqlite3_value;"), + RefO(13, "capi/OutputPointer$String", "Ljava/lang/String;"), +#ifdef SQLITE_ENABLE_FTS5 + RefO(14, "capi/OutputPointer$ByteArray", "[B"), + RefN(15, "fts5/Fts5Context"), + RefN(16, "fts5/Fts5ExtensionApi"), + RefN(17, "fts5/fts5_api"), + RefN(18, "fts5/fts5_tokenizer"), + RefN(19, "fts5/Fts5Tokenizer") +#endif +#undef MkRef +#undef RefN +#undef RefO +}; + +#define S3JniNph(T) &S3JniNphOps.T + +enum { + /* + ** Size of the NativePointerHolder cache. Need enough space for + ** (only) the library's NativePointerHolder and OutputPointer types, + ** a fixed count known at build-time. This value needs to be + ** exactly the number of S3JniNphOp entries in the S3JniNphOps + ** object. + */ + S3Jni_NphCache_size = sizeof(S3JniNphOps) / sizeof(S3JniNphOp) +}; + +/* +** State for binding C callbacks to Java methods. +*/ +typedef struct S3JniHook S3JniHook; +struct S3JniHook{ + jobject jObj /* global ref to Java instance */; + jmethodID midCallback /* callback method. Signature depends on + ** jObj's type */; + /* We lookup the jObj.xDestroy() method as-needed for contexts which + ** support custom finalizers. Fundamentally we can support them for + ** any Java type, but we only want to expose support for them where + ** the C API does. */ + jobject jExtra /* Global ref to a per-hook-type value */; + int doXDestroy /* If true then S3JniHook_unref() will call + jObj->xDestroy() if it's available. */; + S3JniHook * pNext /* Next entry in S3Global.hooks.aFree */; +}; +/* For clean bitwise-copy init of local instances. */ +static const S3JniHook S3JniHook_empty = {0,0,0,0,0}; + +/* +** Per-(sqlite3*) state for various JNI bindings. This state is +** allocated as needed, cleaned up in sqlite3_close(_v2)(), and +** recycled when possible. +** +** Trivia: vars and parameters of this type are often named "ps" +** because this class used to have a name for which that abbreviation +** made sense. +*/ +typedef struct S3JniDb S3JniDb; +struct S3JniDb { + sqlite3 *pDb /* The associated db handle */; + jobject jDb /* A global ref of the output object which gets + returned from sqlite3_open(_v2)(). We need this in + order to have an object to pass to routines like + sqlite3_collation_needed()'s callback, or else we + have to dynamically create one for that purpose, + which would be fine except that it would be a + different instance (and maybe even a different + class) than the one the user may expect to + receive. */; + char * zMainDbName /* Holds the string allocated on behalf of + SQLITE_DBCONFIG_MAINDBNAME. */; + struct { + S3JniHook busyHandler; + S3JniHook collationNeeded; + S3JniHook commit; + S3JniHook progress; + S3JniHook rollback; + S3JniHook trace; + S3JniHook update; + S3JniHook auth; +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + S3JniHook preUpdate; +#endif + } hooks; +#ifdef SQLITE_ENABLE_FTS5 + /* FTS5-specific state */ + struct { + jobject jApi /* global ref to s3jni_fts5_api_from_db() */; + } fts; +#endif + S3JniDb * pNext /* Next entry in SJG.perDb.aFree */; +}; + +static const char * const S3JniDb_clientdata_key = "S3JniDb"; +#define S3JniDb_from_clientdata(pDb) \ + (pDb ? sqlite3_get_clientdata(pDb, S3JniDb_clientdata_key) : 0) + +/* +** Cache for per-JNIEnv (i.e. per-thread) data. +** +** Trivia: vars and parameters of this type are often named "jc" +** because this class used to have a name for which that abbreviation +** made sense. +*/ +typedef struct S3JniEnv S3JniEnv; +struct S3JniEnv { + JNIEnv *env /* JNIEnv in which this cache entry was created */; + /* + ** pdbOpening is used to coordinate the Java/DB connection of a + ** being-open()'d db in the face of auto-extensions. + ** Auto-extensions run before we can bind the C db to its Java + ** representation, but auto-extensions require that binding to pass + ** on to their Java-side callbacks. We handle this as follows: + ** + ** - In the JNI side of sqlite3_open(), allocate the Java side of + ** that connection and set pdbOpening to point to that + ** object. + ** + ** - Call sqlite3_open(), which triggers the auto-extension + ** handler. That handler uses pdbOpening to connect the native + ** db handle which it receives with pdbOpening. + ** + ** - When sqlite3_open() returns, check whether pdbOpening->pDb is + ** NULL. If it isn't, auto-extension handling set it up. If it + ** is, complete the Java/C binding unless sqlite3_open() returns + ** a NULL db, in which case free pdbOpening. + */ + S3JniDb * pdbOpening; + S3JniEnv * pNext /* Next entry in SJG.envCache.aHead or + SJG.envCache.aFree */; +}; + +/* +** State for proxying sqlite3_auto_extension() in Java. This was +** initially a separate class from S3JniHook and now the older name is +** retained for readability in the APIs which use this, as well as for +** its better code-searchability. +*/ +typedef S3JniHook S3JniAutoExtension; + +/* +** Type IDs for SQL function categories. +*/ +enum UDFType { + UDF_UNKNOWN_TYPE = 0/*for error propagation*/, + UDF_SCALAR, + UDF_AGGREGATE, + UDF_WINDOW +}; + +/* +** State for binding Java-side UDFs. +*/ +typedef struct S3JniUdf S3JniUdf; +struct S3JniUdf { + jobject jObj /* SQLFunction instance */; + char * zFuncName /* Only for error reporting and debug logging */; + enum UDFType type /* UDF type */; + /** Method IDs for the various UDF methods. */ + jmethodID jmidxFunc /* xFunc method (scalar) */; + jmethodID jmidxStep /* xStep method (aggregate/window) */; + jmethodID jmidxFinal /* xFinal method (aggregate/window) */; + jmethodID jmidxValue /* xValue method (window) */; + jmethodID jmidxInverse /* xInverse method (window) */; + S3JniUdf * pNext /* Next entry in SJG.udf.aFree. */; +}; + +#if defined(SQLITE_JNI_ENABLE_METRICS) && 0==SQLITE_JNI_ENABLE_METRICS +# undef SQLITE_JNI_ENABLE_METRICS +#endif + +/* +** If true, modifying S3JniGlobal.metrics is protected by a mutex, +** else it isn't. +*/ +#ifdef SQLITE_DEBUG +# define S3JNI_METRICS_MUTEX SQLITE_THREADSAFE +#else +# define S3JNI_METRICS_MUTEX 0 +#endif +#ifndef SQLITE_JNI_ENABLE_METRICS +# undef S3JNI_METRICS_MUTEX +# define S3JNI_METRICS_MUTEX 0 +#endif + +/* +** Global state, e.g. caches and metrics. +*/ +typedef struct S3JniGlobalType S3JniGlobalType; +struct S3JniGlobalType { + /* + ** According to: https://developer.ibm.com/articles/j-jni/ + ** + ** > A thread can get a JNIEnv by calling GetEnv() using the JNI + ** invocation interface through a JavaVM object. The JavaVM object + ** itself can be obtained by calling the JNI GetJavaVM() method + ** using a JNIEnv object and can be cached and shared across + ** threads. Caching a copy of the JavaVM object enables any thread + ** with access to the cached object to get access to its own + ** JNIEnv when necessary. + */ + JavaVM * jvm; + /* + ** Global mutex. It must not be used for anything which might call + ** back into the JNI layer. + */ + sqlite3_mutex * mutex; + /* + ** Cache of references to Java classes and method IDs for + ** NativePointerHolder subclasses and OutputPointer.T types. + */ + struct { + S3JniNphOp list[S3Jni_NphCache_size]; + sqlite3_mutex * mutex; /* mutex for this->list */ + volatile void const * locker; /* sanity-checking-only context object + for this->mutex */ + } nph; + /* + ** Cache of per-thread state. + */ + struct { + S3JniEnv * aHead /* Linked list of in-use instances */; + S3JniEnv * aFree /* Linked list of free instances */; + sqlite3_mutex * mutex /* mutex for aHead and aFree. */; + volatile void const * locker /* env mutex is held on this + object's behalf. Used only for + sanity checking. */; + } envCache; + /* + ** Per-db state. This can move into the core library once we can tie + ** client-defined state to db handles there. + */ + struct { + S3JniDb * aFree /* Linked list of free instances */; + sqlite3_mutex * mutex /* mutex for aHead and aFree */; + volatile void const * locker + /* perDb mutex is held on this object's behalf. Used only for + sanity checking. Note that the mutex is at the class level, not + instance level. */; + } perDb; + struct { + S3JniUdf * aFree /* Head of the free-item list. Guarded by global + mutex. */; + } udf; + /* + ** Refs to global classes and methods. Obtained during static init + ** and never released. + */ + struct { + jclass cLong /* global ref to java.lang.Long */; + jclass cString /* global ref to java.lang.String */; + jobject oCharsetUtf8 /* global ref to StandardCharset.UTF_8 */; + jmethodID ctorLong1 /* the Long(long) constructor */; + jmethodID ctorStringBA /* the String(byte[],Charset) constructor */; + jmethodID stringGetBytes /* the String.getBytes(Charset) method */; + + /* + ByteBuffer may or may not be supported via JNI on any given + platform: + + https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#nio_support + + We only store a ref to byteBuffer.klazz if JNI support for + ByteBuffer is available (which we determine during static init). + */ + struct { + jclass klazz /* global ref to java.nio.ByteBuffer */; + jmethodID midAlloc /* ByteBuffer.allocateDirect() */; + jmethodID midLimit /* ByteBuffer.limit() */; + } byteBuffer; + } g; + /* + ** The list of Java-side auto-extensions + ** (org.sqlite.jni.capi.AutoExtensionCallback objects). + */ + struct { + S3JniAutoExtension *aExt /* The auto-extension list. It is + maintained such that all active + entries are in the first contiguous + nExt array elements. */; + int nAlloc /* number of entries allocated for aExt, + as distinct from the number of active + entries. */; + int nExt /* number of active entries in aExt, all in the + first nExt'th array elements. */; + sqlite3_mutex * mutex /* mutex for manipulation/traversal of aExt */; + volatile const void * locker /* object on whose behalf the mutex + is held. Only for sanity checking + in debug builds. */; + } autoExt; +#ifdef SQLITE_ENABLE_FTS5 + struct { + volatile jobject jExt /* Global ref to Java singleton for the + Fts5ExtensionApi instance. */; + struct { + jfieldID fidA /* Fts5Phrase::a member */; + jfieldID fidB /* Fts5Phrase::b member */; + } jPhraseIter; + } fts5; +#endif + struct { +#ifdef SQLITE_ENABLE_SQLLOG + S3JniHook sqllog /* sqlite3_config(SQLITE_CONFIG_SQLLOG) callback */; +#endif + S3JniHook configlog /* sqlite3_config(SQLITE_CONFIG_LOG) callback */; + S3JniHook * aFree /* free-item list, for recycling. */; + sqlite3_mutex * mutex /* mutex for aFree */; + volatile const void * locker /* object on whose behalf the mutex + is held. Only for sanity checking + in debug builds. */; + } hook; +#ifdef SQLITE_JNI_ENABLE_METRICS + /* Internal metrics. */ + struct { + volatile unsigned nEnvHit; + volatile unsigned nEnvMiss; + volatile unsigned nEnvAlloc; + volatile unsigned nMutexEnv /* number of times envCache.mutex was entered for + a S3JniEnv operation. */; + volatile unsigned nMutexNph /* number of times SJG.mutex was entered */; + volatile unsigned nMutexHook /* number of times SJG.mutex hooks.was entered */; + volatile unsigned nMutexPerDb /* number of times perDb.mutex was entered */; + volatile unsigned nMutexAutoExt /* number of times autoExt.mutex was entered */; + volatile unsigned nMutexGlobal /* number of times global mutex was entered. */; + volatile unsigned nMutexUdf /* number of times global mutex was entered + for UDFs. */; + volatile unsigned nDestroy /* xDestroy() calls across all types */; + volatile unsigned nPdbAlloc /* Number of S3JniDb alloced. */; + volatile unsigned nPdbRecycled /* Number of S3JniDb reused. */; + volatile unsigned nUdfAlloc /* Number of S3JniUdf alloced. */; + volatile unsigned nUdfRecycled /* Number of S3JniUdf reused. */; + volatile unsigned nHookAlloc /* Number of S3JniHook alloced. */; + volatile unsigned nHookRecycled /* Number of S3JniHook reused. */; + struct { + /* Number of calls for each type of UDF callback. */ + volatile unsigned nFunc; + volatile unsigned nStep; + volatile unsigned nFinal; + volatile unsigned nValue; + volatile unsigned nInverse; + } udf; + unsigned nMetrics /* Total number of mutex-locked + metrics increments. */; +#if S3JNI_METRICS_MUTEX + sqlite3_mutex * mutex; +#endif + } metrics; +#endif /* SQLITE_JNI_ENABLE_METRICS */ +}; +static S3JniGlobalType S3JniGlobal = {}; +#define SJG S3JniGlobal + +/* Increments *p, possibly protected by a mutex. */ +#ifndef SQLITE_JNI_ENABLE_METRICS +#define s3jni_incr(PTR) +#elif S3JNI_METRICS_MUTEX +static void s3jni_incr( volatile unsigned int * const p ){ + sqlite3_mutex_enter(SJG.metrics.mutex); + ++SJG.metrics.nMetrics; + ++(*p); + sqlite3_mutex_leave(SJG.metrics.mutex); +} +#else +#define s3jni_incr(PTR) ++(*(PTR)) +#endif + +/* Helpers for working with specific mutexes. */ +#if SQLITE_THREADSAFE +#define s3jni_mutex_enter2(M, Metric) \ + sqlite3_mutex_enter( M ); \ + s3jni_incr( &SJG.metrics.Metric ) +#define s3jni_mutex_leave2(M) \ + sqlite3_mutex_leave( M ) + +#define s3jni_mutex_enter(M, L, Metric) \ + assert( (void*)env != (void*)L && "Invalid use of " #L); \ + s3jni_mutex_enter2( M, Metric ); \ + L = env +#define s3jni_mutex_leave(M, L) \ + assert( (void*)env == (void*)L && "Invalid use of " #L); \ + L = 0; \ + s3jni_mutex_leave2( M ) + +#define S3JniEnv_mutex_assertLocked \ + assert( 0 != SJG.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) +#define S3JniEnv_mutex_assertLocker \ + assert( (env) == SJG.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) +#define S3JniEnv_mutex_assertNotLocker \ + assert( (env) != SJG.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) + +#define S3JniEnv_mutex_enter \ + s3jni_mutex_enter( SJG.envCache.mutex, SJG.envCache.locker, nMutexEnv ) +#define S3JniEnv_mutex_leave \ + s3jni_mutex_leave( SJG.envCache.mutex, SJG.envCache.locker ) + +#define S3JniAutoExt_mutex_enter \ + s3jni_mutex_enter( SJG.autoExt.mutex, SJG.autoExt.locker, nMutexAutoExt ) +#define S3JniAutoExt_mutex_leave \ + s3jni_mutex_leave( SJG.autoExt.mutex, SJG.autoExt.locker ) +#define S3JniAutoExt_mutex_assertLocker \ + assert( env == SJG.autoExt.locker && "Misuse of S3JniGlobal.autoExt.mutex" ) + +#define S3JniGlobal_mutex_enter \ + s3jni_mutex_enter2( SJG.mutex, nMutexGlobal ) +#define S3JniGlobal_mutex_leave \ + s3jni_mutex_leave2( SJG.mutex ) + +#define S3JniHook_mutex_enter \ + s3jni_mutex_enter( SJG.hook.mutex, SJG.hook.locker, nMutexHook ) +#define S3JniHook_mutex_leave \ + s3jni_mutex_leave( SJG.hook.mutex, SJG.hook.locker ) + +#define S3JniNph_mutex_enter \ + s3jni_mutex_enter( SJG.nph.mutex, SJG.nph.locker, nMutexNph ) +#define S3JniNph_mutex_leave \ + s3jni_mutex_leave( SJG.nph.mutex, SJG.nph.locker ) + +#define S3JniDb_mutex_assertLocker \ + assert( (env) == SJG.perDb.locker && "Misuse of S3JniGlobal.perDb.mutex" ) +#define S3JniDb_mutex_enter \ + s3jni_mutex_enter( SJG.perDb.mutex, SJG.perDb.locker, nMutexPerDb ) +#define S3JniDb_mutex_leave \ + s3jni_mutex_leave( SJG.perDb.mutex, SJG.perDb.locker ) + +#else /* SQLITE_THREADSAFE==0 */ +#define S3JniAutoExt_mutex_assertLocker +#define S3JniAutoExt_mutex_enter +#define S3JniAutoExt_mutex_leave +#define S3JniDb_mutex_assertLocker +#define S3JniDb_mutex_enter +#define S3JniDb_mutex_leave +#define S3JniEnv_mutex_assertLocked +#define S3JniEnv_mutex_assertLocker +#define S3JniEnv_mutex_assertNotLocker +#define S3JniEnv_mutex_enter +#define S3JniEnv_mutex_leave +#define S3JniGlobal_mutex_enter +#define S3JniGlobal_mutex_leave +#define S3JniHook_mutex_enter +#define S3JniHook_mutex_leave +#define S3JniNph_mutex_enter +#define S3JniNph_mutex_leave +#endif + +/* Helpers for jstring and jbyteArray. */ +static const char * s3jni__jstring_to_mutf8(JNIEnv * const env, jstring v ){ + const char *z = v ? (*env)->GetStringUTFChars(env, v, NULL) : 0; + s3jni_oom_check( v ? !!z : !z ); + return z; +} + +#define s3jni_jstring_to_mutf8(ARG) s3jni__jstring_to_mutf8(env, (ARG)) +#define s3jni_mutf8_release(ARG,VAR) if( VAR ) (*env)->ReleaseStringUTFChars(env, ARG, VAR) + +/* +** If jBA is not NULL then its GetByteArrayElements() value is +** returned. If jBA is not NULL and nBA is not NULL then *nBA is set +** to the GetArrayLength() of jBA. If GetByteArrayElements() requires +** an allocation and that allocation fails then this function either +** fails fatally or returns 0, depending on build-time options. + */ +static jbyte * s3jni__jbyteArray_bytes2(JNIEnv * const env, jbyteArray jBA, jsize * nBA ){ + jbyte * const rv = jBA ? (*env)->GetByteArrayElements(env, jBA, NULL) : 0; + s3jni_oom_check( jBA ? !!rv : 1 ); + if( jBA && nBA ) *nBA = (*env)->GetArrayLength(env, jBA); + return rv; +} + +#define s3jni_jbyteArray_bytes2(jByteArray,ptrToSz) \ + s3jni__jbyteArray_bytes2(env, (jByteArray), (ptrToSz)) +#define s3jni_jbyteArray_bytes(jByteArray) s3jni__jbyteArray_bytes2(env, (jByteArray), 0) +#define s3jni_jbyteArray_release(jByteArray,jBytes) \ + if( jBytes ) (*env)->ReleaseByteArrayElements(env, jByteArray, jBytes, JNI_ABORT) +#define s3jni_jbyteArray_commit(jByteArray,jBytes) \ + if( jBytes ) (*env)->ReleaseByteArrayElements(env, jByteArray, jBytes, JNI_COMMIT) + +/* +** If jbb is-a java.nio.Buffer object and the JNI environment supports +** it, *pBuf is set to the buffer's memory and *pN is set to its +** limit() (as opposed to its capacity()). If jbb is NULL, not a +** Buffer, or the JNI environment does not support that operation, +** *pBuf is set to 0 and *pN is set to 0. +** +** Note that the length of the buffer can be larger than SQLITE_LIMIT +** but this function does not know what byte range of the buffer is +** required so cannot check for that violation. The caller is required +** to ensure that any to-be-bind()ed range fits within SQLITE_LIMIT. +** +** Sidebar: it is unfortunate that we cannot get ByteBuffer.limit() +** via a JNI method like we can for ByteBuffer.capacity(). We instead +** have to call back into Java to get the limit(). Depending on how +** the ByteBuffer is used, the limit and capacity might be the same, +** but when reusing a buffer, the limit may well change whereas the +** capacity is fixed. The problem with, e.g., read()ing blob data to a +** ByteBuffer's memory based on its capacity is that Java-level code +** is restricted to accessing the range specified in +** ByteBuffer.limit(). If we were to honor only the capacity, we +** could end up writing to, or reading from, parts of a ByteBuffer +** which client code itself cannot access without explicitly modifying +** the limit. The penalty we pay for this correctness is that we must +** call into Java to get the limit() of every ByteBuffer we work with. +** +** An alternative to having to call into ByteBuffer.limit() from here +** would be to add private native impls of all ByteBuffer-using +** methods, each of which adds a jint parameter which _must_ be set to +** theBuffer.limit() by public Java APIs which use those private impls +** to do the real work. +*/ +static void s3jni__get_nio_buffer(JNIEnv * const env, jobject jbb, void **pBuf, jint * pN ){ + *pBuf = 0; + *pN = 0; + if( jbb ){ + *pBuf = (*env)->GetDirectBufferAddress(env, jbb); + if( *pBuf ){ + /* + ** Maintenance reminder: do not use + ** (*env)->GetDirectBufferCapacity(env,jbb), even though it + ** would be much faster, for reasons explained in this + ** function's comments. + */ + *pN = (*env)->CallIntMethod(env, jbb, SJG.g.byteBuffer.midLimit); + S3JniExceptionIsFatal("Error calling ByteBuffer.limit() method."); + } + } +} +#define s3jni_get_nio_buffer(JOBJ,vpOut,jpOut) \ + s3jni__get_nio_buffer(env,(JOBJ),(vpOut),(jpOut)) + +/* +** Returns the current JNIEnv object. Fails fatally if it cannot find +** the object. +*/ +static JNIEnv * s3jni_env(void){ + JNIEnv * env = 0; + if( (*SJG.jvm)->GetEnv(SJG.jvm, (void **)&env, + JNI_VERSION_1_8) ){ + fprintf(stderr, "Fatal error: cannot get current JNIEnv.\n"); + abort(); + } + return env; +} + +/* +** Fetches the S3JniGlobal.envCache row for the given env, allocing a +** row if needed. When a row is allocated, its state is initialized +** insofar as possible. Calls (*env)->FatalError() if allocation of an +** entry fails. That's hypothetically possible but "shouldn't happen." +*/ +static S3JniEnv * S3JniEnv__get(JNIEnv * const env){ + struct S3JniEnv * row; + S3JniEnv_mutex_enter; + row = SJG.envCache.aHead; + for( ; row; row = row->pNext ){ + if( row->env == env ){ + s3jni_incr( &SJG.metrics.nEnvHit ); + S3JniEnv_mutex_leave; + return row; + } + } + s3jni_incr( &SJG.metrics.nEnvMiss ); + row = SJG.envCache.aFree; + if( row ){ + SJG.envCache.aFree = row->pNext; + }else{ + row = s3jni_malloc_or_die(env, sizeof(*row)); + s3jni_incr( &SJG.metrics.nEnvAlloc ); + } + memset(row, 0, sizeof(*row)); + row->pNext = SJG.envCache.aHead; + SJG.envCache.aHead = row; + row->env = env; + + S3JniEnv_mutex_leave; + return row; +} + +#define S3JniEnv_get() S3JniEnv__get(env) + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own Java/JNI bindings. +** +** For purposes of certain hand-crafted JNI function bindings, we +** need a way of reporting errors which is consistent with the rest of +** the C API, as opposed to throwing Java exceptions. To that end, this +** internal-use-only function is a thin proxy around +** sqlite3ErrorWithMessage(). The intent is that it only be used from +** JNI bindings such as sqlite3_prepare_v2/v3(), and definitely not +** from client code. +** +** Returns err_code. +*/ +static int s3jni_db_error(sqlite3* const db, int err_code, + const char * const zMsg){ + if( db!=0 ){ + if( 0==zMsg ){ + sqlite3Error(db, err_code); + }else{ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + } + } + return err_code; +} + +/* +** Creates a new jByteArray of length nP, copies p's contents into it, +** and returns that byte array (NULL on OOM unless fail-fast alloc +** errors are enabled). p may be NULL, in which case the array is +** created but no bytes are filled. +*/ +static jbyteArray s3jni__new_jbyteArray(JNIEnv * const env, + const void * const p, int nP){ + jbyteArray jba = (*env)->NewByteArray(env, (jint)nP); + + s3jni_oom_check( jba ); + if( jba && p ){ + (*env)->SetByteArrayRegion(env, jba, 0, (jint)nP, (const jbyte*)p); + } + return jba; +} + +#define s3jni_new_jbyteArray(P,n) s3jni__new_jbyteArray(env, P, n) + + +/* +** Uses the java.lang.String(byte[],Charset) constructor to create a +** new String from UTF-8 string z. n is the number of bytes to +** copy. If n<0 then sqlite3Strlen30() is used to calculate it. +** +** Returns NULL if z is NULL or on OOM, else returns a new jstring +** owned by the caller. +** +** Sidebar: this is a painfully inefficient way to convert from +** standard UTF-8 to a Java string, but JNI offers only algorithms for +** working with MUTF-8, not UTF-8. +*/ +static jstring s3jni__utf8_to_jstring(JNIEnv * const env, + const char * const z, int n){ + jstring rv = NULL; + if( 0==n || (n<0 && z && !z[0]) ){ + /* Fast-track the empty-string case via the MUTF-8 API. We could + hypothetically do this for any strings where n<4 and z is + NUL-terminated and none of z[0..3] are NUL bytes. */ + rv = (*env)->NewStringUTF(env, ""); + s3jni_oom_check( rv ); + }else if( z ){ + jbyteArray jba; + if( n<0 ) n = sqlite3Strlen30(z); + jba = s3jni_new_jbyteArray((unsigned const char *)z, n); + if( jba ){ + rv = (*env)->NewObject(env, SJG.g.cString, SJG.g.ctorStringBA, + jba, SJG.g.oCharsetUtf8); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + } + S3JniUnrefLocal(jba); + } + s3jni_oom_check( rv ); + } + return rv; +} +#define s3jni_utf8_to_jstring(CStr,n) s3jni__utf8_to_jstring(env, CStr, n) + +/* +** Converts the given java.lang.String object into a NUL-terminated +** UTF-8 C-string by calling jstr.getBytes(StandardCharset.UTF_8). +** Returns NULL if jstr is NULL or on allocation error. If jstr is not +** NULL and nLen is not NULL then nLen is set to the length of the +** returned string, not including the terminating NUL. If jstr is not +** NULL and it returns NULL, this indicates an allocation error. In +** that case, if nLen is not NULL then it is either set to 0 (if +** fetching of jstr's bytes fails to allocate) or set to what would +** have been the length of the string had C-string allocation +** succeeded. +** +** The returned memory is allocated from sqlite3_malloc() and +** ownership is transferred to the caller. +*/ +static char * s3jni__jstring_to_utf8(JNIEnv * const env, + jstring jstr, int *nLen){ + jbyteArray jba; + jsize nBA; + char *rv; + + if( !jstr ) return 0; + jba = (*env)->CallObjectMethod(env, jstr, SJG.g.stringGetBytes, + SJG.g.oCharsetUtf8); + + if( (*env)->ExceptionCheck(env) || !jba + /* order of these checks is significant for -Xlint:jni */ ) { + S3JniExceptionReport; + s3jni_oom_check( jba ); + if( nLen ) *nLen = 0; + return 0; + } + nBA = (*env)->GetArrayLength(env, jba); + if( nLen ) *nLen = (int)nBA; + rv = s3jni_malloc( nBA + 1 ); + if( rv ){ + (*env)->GetByteArrayRegion(env, jba, 0, nBA, (jbyte*)rv); + rv[nBA] = 0; + } + S3JniUnrefLocal(jba); + return rv; +} +#define s3jni_jstring_to_utf8(JStr,n) s3jni__jstring_to_utf8(env, JStr, n) + +/* +** Expects to be passed a pointer from sqlite3_column_text16() or +** sqlite3_value_text16() and a byte-length value from +** sqlite3_column_bytes16() or sqlite3_value_bytes16(). It creates a +** Java String of exactly half that character length, returning NULL +** if !p or (*env)->NewString() fails. +*/ +static jstring s3jni_text16_to_jstring(JNIEnv * const env, const void * const p, int nP){ + jstring const rv = p + ? (*env)->NewString(env, (const jchar *)p, (jsize)(nP/2)) + : NULL; + s3jni_oom_check( p ? !!rv : 1 ); + return rv; +} + +/* +** Creates a new ByteBuffer instance with a capacity of n. assert()s +** that SJG.g.byteBuffer.klazz is not 0 and n>0. +*/ +static jobject s3jni__new_ByteBuffer(JNIEnv * const env, int n){ + jobject rv = 0; + assert( SJG.g.byteBuffer.klazz ); + assert( SJG.g.byteBuffer.midAlloc ); + assert( n > 0 ); + rv = (*env)->CallStaticObjectMethod(env, SJG.g.byteBuffer.klazz, + SJG.g.byteBuffer.midAlloc, (jint)n); + S3JniIfThrew { + S3JniExceptionReport; + S3JniExceptionClear; + } + s3jni_oom_check( rv ); + return rv; +} + +/* +** If n>0 and sqlite3_jni_supports_nio() is true then this creates a +** new ByteBuffer object and copies n bytes from p to it. Returns NULL +** if n is 0, sqlite3_jni_supports_nio() is false, or on allocation +** error (unless fatal alloc failures are enabled). +*/ +static jobject s3jni__blob_to_ByteBuffer(JNIEnv * const env, + const void * p, int n){ + jobject rv = NULL; + assert( n >= 0 ); + if( 0==n || !SJG.g.byteBuffer.klazz ){ + return NULL; + } + rv = s3jni__new_ByteBuffer(env, n); + if( rv ){ + void * tgt = (*env)->GetDirectBufferAddress(env, rv); + memcpy(tgt, p, (size_t)n); + } + return rv; +} + + +/* +** Requires jx to be a Throwable. Calls its toString() method and +** returns its value converted to a UTF-8 string. The caller owns the +** returned string and must eventually sqlite3_free() it. Returns 0 +** if there is a problem fetching the info or on OOM. +** +** Design note: we use toString() instead of getMessage() because the +** former includes the exception type's name: +** +** Exception e = new RuntimeException("Hi"); +** System.out.println(e.toString()); // java.lang.RuntimeException: Hi +** System.out.println(e.getMessage()); // Hi +*/ +static char * s3jni_exception_error_msg(JNIEnv * const env, jthrowable jx){ + jmethodID mid; + jstring msg; + char * zMsg; + jclass const klazz = (*env)->GetObjectClass(env, jx); + mid = (*env)->GetMethodID(env, klazz, "toString", "()Ljava/lang/String;"); + S3JniUnrefLocal(klazz); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + return 0; + } + msg = (*env)->CallObjectMethod(env, jx, mid); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + return 0; + } + zMsg = s3jni_jstring_to_utf8( msg, 0); + S3JniUnrefLocal(msg); + return zMsg; +} + +/* +** Extracts env's current exception, sets ps->pDb's error message to +** its message string, and clears the exception. If errCode is non-0, +** it is used as-is, else SQLITE_ERROR is assumed. If there's a +** problem extracting the exception's message, it's treated as +** non-fatal and zDfltMsg is used in its place. +** +** Locks the global S3JniDb mutex. +** +** This must only be called if a JNI exception is pending. +** +** Returns errCode unless it is 0, in which case SQLITE_ERROR is +** returned. +*/ +static int s3jni__db_exception(JNIEnv * const env, sqlite3 * const pDb, + int errCode, const char *zDfltMsg){ + jthrowable const ex = (*env)->ExceptionOccurred(env); + + if( 0==errCode ) errCode = SQLITE_ERROR; + if( ex ){ + char * zMsg; + S3JniExceptionClear; + zMsg = s3jni_exception_error_msg(env, ex); + s3jni_db_error(pDb, errCode, zMsg ? zMsg : zDfltMsg); + sqlite3_free(zMsg); + S3JniUnrefLocal(ex); + }else if( zDfltMsg ){ + s3jni_db_error(pDb, errCode, zDfltMsg); + } + return errCode; +} +#define s3jni_db_exception(pDb,ERRCODE,DFLTMSG) \ + s3jni__db_exception(env, (pDb), (ERRCODE), (DFLTMSG) ) + +/* +** Extracts the (void xDestroy()) method from jObj and applies it to +** jObj. If jObj is NULL, this is a no-op. The lack of an xDestroy() +** method is silently ignored. Any exceptions thrown by xDestroy() +** trigger a warning to stdout or stderr and then the exception is +** suppressed. +*/ +static void s3jni__call_xDestroy(JNIEnv * const env, jobject jObj){ + if( jObj ){ + jclass const klazz = (*env)->GetObjectClass(env, jObj); + jmethodID method = (*env)->GetMethodID(env, klazz, "xDestroy", "()V"); + + S3JniUnrefLocal(klazz); + if( method ){ + s3jni_incr( &SJG.metrics.nDestroy ); + (*env)->CallVoidMethod(env, jObj, method); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("xDestroy() callback"); + S3JniExceptionClear; + } + }else{ + /* Non-fatal. */ + S3JniExceptionClear; + } + } +} +#define s3jni_call_xDestroy(JOBJ) s3jni__call_xDestroy(env, (JOBJ)) + +/* +** Internal helper for many hook callback impls. Locks the S3JniDb +** mutex, makes a copy of src into dest, with a some differences: (1) +** if src->jObj or src->jExtra are not NULL then dest will be a new +** LOCAL ref to it instead of a copy of the prior GLOBAL ref. (2) +** dest->doXDestroy is always false. +** +** If dest->jObj is not NULL when this returns then the caller is +** obligated to eventually free the new ref by passing *dest to +** S3JniHook_localundup(). The dest pointer must NOT be passed to +** S3JniHook_unref(), as that routine assumes that dest->jObj/jExtra +** are GLOBAL refs (it's illegal to try to unref the wrong ref type). +** +** Background: when running a hook we need a call-local copy lest +** another thread modify the hook while we're running it. That copy +** has to have its own Java reference, but it need only be call-local. +*/ +static void S3JniHook__localdup( JNIEnv * const env, S3JniHook const * const src, + S3JniHook * const dest ){ + S3JniHook_mutex_enter; + *dest = *src; + if(src->jObj) dest->jObj = S3JniRefLocal(src->jObj); + if(src->jExtra) dest->jExtra = S3JniRefLocal(src->jExtra); + dest->doXDestroy = 0; + S3JniHook_mutex_leave; +} +#define S3JniHook_localdup(src,dest) S3JniHook__localdup(env,src,dest) + +static void S3JniHook__localundup( JNIEnv * const env, S3JniHook * const h ){ + S3JniUnrefLocal(h->jObj); + S3JniUnrefLocal(h->jExtra); + *h = S3JniHook_empty; +} +#define S3JniHook_localundup(HOOK) S3JniHook__localundup(env, &(HOOK)) + +/* +** Removes any Java references from s and clears its state. If +** doXDestroy is true and s->jObj is not NULL, s->jObj +** is passed to s3jni_call_xDestroy() before any references are +** cleared. It is legal to call this when the object has no Java +** references. s must not be NULL. +*/ +static void S3JniHook__unref(JNIEnv * const env, S3JniHook * const s){ + if( s->jObj ){ + if( s->doXDestroy ){ + s3jni_call_xDestroy(s->jObj); + } + S3JniUnrefGlobal(s->jObj); + S3JniUnrefGlobal(s->jExtra); + }else{ + assert( !s->jExtra ); + } + *s = S3JniHook_empty; +} +#define S3JniHook_unref(hook) S3JniHook__unref(env, (hook)) + +/* +** Allocates one blank S3JniHook object from the recycling bin, if +** available, else from the heap. Returns NULL or dies on OOM, +** depending on build options. Locks on SJG.hooks.mutex. +*/ +static S3JniHook *S3JniHook__alloc(JNIEnv * const env){ + S3JniHook * p = 0; + S3JniHook_mutex_enter; + if( SJG.hook.aFree ){ + p = SJG.hook.aFree; + SJG.hook.aFree = p->pNext; + p->pNext = 0; + s3jni_incr(&SJG.metrics.nHookRecycled); + } + S3JniHook_mutex_leave; + if( 0==p ){ + p = s3jni_malloc(sizeof(S3JniHook)); + if( p ){ + s3jni_incr(&SJG.metrics.nHookAlloc); + } + } + if( p ){ + *p = S3JniHook_empty; + } + return p; +} +#define S3JniHook_alloc() S3JniHook__alloc(env) + +/* +** The rightful fate of all results from S3JniHook_alloc(). Locks on +** SJG.hook.mutex. +*/ +static void S3JniHook__free(JNIEnv * const env, S3JniHook * const p){ + if(p){ + assert( !p->pNext ); + S3JniHook_unref(p); + S3JniHook_mutex_enter; + p->pNext = SJG.hook.aFree; + SJG.hook.aFree = p; + S3JniHook_mutex_leave; + } +} +#define S3JniHook_free(hook) S3JniHook__free(env, hook) + +#if 0 +/* S3JniHook__free() without the lock: caller must hold the global mutex */ +static void S3JniHook__free_unlocked(JNIEnv * const env, S3JniHook * const p){ + if(p){ + assert( !p->pNext ); + assert( p->pNext != SJG.hook.aFree ); + S3JniHook_unref(p); + p->pNext = SJG.hook.aFree; + SJG.hook.aFree = p; + } +} +#define S3JniHook_free_unlocked(hook) S3JniHook__free_unlocked(env, hook) +#endif + +/* +** Clears all of s's state. Requires that that the caller has locked +** S3JniGlobal.perDb.mutex. Make sure to do anything needed with +** s->pNext and s->pPrev before calling this, as this clears them. +*/ +static void S3JniDb_clear(JNIEnv * const env, S3JniDb * const s){ + S3JniDb_mutex_assertLocker; + sqlite3_free( s->zMainDbName ); +#define UNHOOK(MEMBER) \ + S3JniHook_unref(&s->hooks.MEMBER) + UNHOOK(auth); + UNHOOK(busyHandler); + UNHOOK(collationNeeded); + UNHOOK(commit); + UNHOOK(progress); + UNHOOK(rollback); + UNHOOK(trace); + UNHOOK(update); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + UNHOOK(preUpdate); +#endif +#undef UNHOOK + S3JniUnrefGlobal(s->jDb); + memset(s, 0, sizeof(S3JniDb)); +} + +/* +** Clears s's state and moves it to the free-list. Requires that +** S3JniGlobal.perDb.mutex is locked. +*/ +static void S3JniDb__set_aside_unlocked(JNIEnv * const env, S3JniDb * const s){ + assert( s ); + S3JniDb_mutex_assertLocker; + if( s ){ + S3JniDb_clear(env, s); + s->pNext = SJG.perDb.aFree; + SJG.perDb.aFree = s; + } +} +#define S3JniDb_set_aside_unlocked(JniDb) S3JniDb__set_aside_unlocked(env, JniDb) + +static void S3JniDb__set_aside(JNIEnv * const env, S3JniDb * const s){ + S3JniDb_mutex_enter; + S3JniDb_set_aside_unlocked(s); + S3JniDb_mutex_leave; +} +#define S3JniDb_set_aside(JNIDB) S3JniDb__set_aside(env, JNIDB) + +/* +** Uncache any state for the given JNIEnv, clearing all Java +** references the cache owns. Returns true if env was cached and false +** if it was not found in the cache. Ownership of the S3JniEnv object +** associated with the given argument is transferred to this function, +** which makes it free for re-use. +** +** Requires that the env mutex be locked. +*/ +static int S3JniEnv_uncache(JNIEnv * const env){ + struct S3JniEnv * row; + struct S3JniEnv * pPrev = 0; + + S3JniEnv_mutex_assertLocked; + row = SJG.envCache.aHead; + for( ; row; pPrev = row, row = row->pNext ){ + if( row->env == env ){ + break; + } + } + if( !row ){ + return 0; + } + if( pPrev) pPrev->pNext = row->pNext; + else{ + assert( SJG.envCache.aHead == row ); + SJG.envCache.aHead = row->pNext; + } + memset(row, 0, sizeof(S3JniEnv)); + row->pNext = SJG.envCache.aFree; + SJG.envCache.aFree = row; + return 1; +} + +/* +** Fetches the given nph-ref from cache the cache and returns the +** object with its klazz member set. This is an O(1) operation except +** on the first call for a given pRef, during which pRef->klazz and +** pRef->pRef are initialized thread-safely. In the latter case it's +** still effectively O(1), but with a much longer 1. +** +** It is up to the caller to populate the other members of the +** returned object if needed, taking care to lock the modification +** with S3JniNph_mutex_enter/leave. +** +** This simple cache catches >99% of searches in the current +** (2023-07-31) tests. +*/ +static S3JniNphOp * s3jni__nphop(JNIEnv * const env, S3JniNphOp const* pRef){ + S3JniNphOp * const pNC = &SJG.nph.list[pRef->index]; + + assert( (void*)pRef>=(void*)&S3JniNphOps && (void*)pRef<(void*)(&S3JniNphOps + 1) + && "pRef is out of range" ); + assert( pRef->index>=0 + && (pRef->index < (sizeof(S3JniNphOps) / sizeof(S3JniNphOp))) + && "pRef->index is out of range" ); + if( !pNC->klazz ){ + S3JniNph_mutex_enter; + if( !pNC->klazz ){ + jclass const klazz = (*env)->FindClass(env, pRef->zName); + //printf("FindClass %s\n", pRef->zName); + S3JniExceptionIsFatal("FindClass() unexpectedly threw"); + pNC->klazz = S3JniRefGlobal(klazz); + } + S3JniNph_mutex_leave; + } + assert( pNC->klazz ); + return pNC; +} + +#define s3jni_nphop(PRef) s3jni__nphop(env, PRef) + +/* +** Common code for accessor functions for NativePointerHolder and +** OutputPointer types. pRef must be a pointer from S3JniNphOps. jOut +** must be an instance of that class (Java's type safety takes care of +** that requirement). If necessary, this fetches the jfieldID for +** jOut's pRef->zMember, which must be of the type represented by the +** JNI type signature pRef->zTypeSig, and stores it in +** S3JniGlobal.nph.list[pRef->index]. Fails fatally if the pRef->zMember +** property is not found, as that presents a serious internal misuse. +** +** Property lookups are cached on a per-pRef basis. +*/ +static jfieldID s3jni_nphop_field(JNIEnv * const env, S3JniNphOp const* pRef){ + S3JniNphOp * const pNC = s3jni_nphop(pRef); + + if( !pNC->fidValue ){ + S3JniNph_mutex_enter; + if( !pNC->fidValue ){ + pNC->fidValue = (*env)->GetFieldID(env, pNC->klazz, + pRef->zMember, pRef->zTypeSig); + S3JniExceptionIsFatal("Code maintenance required: missing " + "required S3JniNphOp::fidValue."); + } + S3JniNph_mutex_leave; + } + assert( pNC->fidValue ); + return pNC->fidValue; +} + +/* +** Sets a native ptr value in NativePointerHolder object jNph, +** which must be of the native type described by pRef. jNph +** may not be NULL. +*/ +static void NativePointerHolder__set(JNIEnv * const env, S3JniNphOp const* pRef, + jobject jNph, const void * p){ + assert( jNph ); + (*env)->SetLongField(env, jNph, s3jni_nphop_field(env, pRef), + S3JniCast_P2L(p)); + S3JniExceptionIsFatal("Could not set NativePointerHolder.nativePointer."); +} + +#define NativePointerHolder_set(PREF,JNPH,P) \ + NativePointerHolder__set(env, PREF, JNPH, P) + +/* +** Fetches a native ptr value from NativePointerHolder object jNph, +** which must be of the native type described by pRef. This is a +** no-op if jNph is NULL. +*/ +static void * NativePointerHolder__get(JNIEnv * env, jobject jNph, + S3JniNphOp const* pRef){ + void * rv = 0; + if( jNph ){ + rv = S3JniCast_L2P( + (*env)->GetLongField(env, jNph, s3jni_nphop_field(env, pRef)) + ); + S3JniExceptionIsFatal("Cannot fetch NativePointerHolder.nativePointer."); + } + return rv; +} + +#define NativePointerHolder_get(JOBJ,NPHREF) \ + NativePointerHolder__get(env, (JOBJ), (NPHREF)) + +/* +** Helpers for extracting pointers from jobjects, noting that we rely +** on the corresponding Java interfaces having already done the +** type-checking. OBJ must be a jobject referring to a +** NativePointerHolder, where T matches PtrGet_T. Don't use these +** in contexts where that's not the case. Note that these aren't +** type-safe in the strictest sense: +** +** sqlite3 * s = PtrGet_sqlite3_stmt(...) +** +** will work, despite the incorrect macro name, so long as the +** argument is a Java sqlite3 object, as this operation only has void +** pointers to work with. +*/ +#define PtrGet_T(T,JOBJ) (T*)NativePointerHolder_get((JOBJ), S3JniNph(T)) +#define PtrGet_sqlite3(JOBJ) PtrGet_T(sqlite3, (JOBJ)) +#define PtrGet_sqlite3_backup(JOBJ) PtrGet_T(sqlite3_backup, (JOBJ)) +#define PtrGet_sqlite3_blob(JOBJ) PtrGet_T(sqlite3_blob, (JOBJ)) +#define PtrGet_sqlite3_context(JOBJ) PtrGet_T(sqlite3_context, (JOBJ)) +#define PtrGet_sqlite3_stmt(JOBJ) PtrGet_T(sqlite3_stmt, (JOBJ)) +#define PtrGet_sqlite3_value(JOBJ) PtrGet_T(sqlite3_value, (JOBJ)) +/* +** LongPtrGet_T(X,Y) expects X to be an unqualified sqlite3 struct +** type name and Y to be a native pointer to such an object in the +** form of a jlong value. The jlong is simply cast to (X*). This +** approach is, as of 2023-09-27, supplanting the former approach. We +** now do the native pointer extraction in the Java side, rather than +** the C side, because it's reportedly significantly faster. The +** intptr_t part here is necessary for compatibility with (at least) +** ARM32. +** +** 2023-11-09: testing has not revealed any measurable performance +** difference between the approach of passing type T to C compared to +** passing pointer-to-T to C, and adding support for the latter +** everywhere requires sigificantly more code. As of this writing, the +** older/simpler approach is being applied except for (A) where the +** newer approach has already been applied and (B) hot-spot APIs where +** a difference of microseconds (i.e. below our testing measurement +** threshold) might add up. +*/ +#define LongPtrGet_T(T,JLongAsPtr) (T*)((intptr_t)((JLongAsPtr))) +#define LongPtrGet_sqlite3(JLongAsPtr) LongPtrGet_T(sqlite3,(JLongAsPtr)) +#define LongPtrGet_sqlite3_backup(JLongAsPtr) LongPtrGet_T(sqlite3_backup,(JLongAsPtr)) +#define LongPtrGet_sqlite3_blob(JLongAsPtr) LongPtrGet_T(sqlite3_blob,(JLongAsPtr)) +#define LongPtrGet_sqlite3_stmt(JLongAsPtr) LongPtrGet_T(sqlite3_stmt,(JLongAsPtr)) +#define LongPtrGet_sqlite3_value(JLongAsPtr) LongPtrGet_T(sqlite3_value,(JLongAsPtr)) +/* +** Extracts the new S3JniDb instance from the free-list, or allocates +** one if needed, associates it with pDb, and returns. Returns NULL +** on OOM. The returned object MUST, on success of the calling +** operation, subsequently be associated with jDb via +** NativePointerHolder_set() or freed using S3JniDb_set_aside(). +*/ +static S3JniDb * S3JniDb_alloc(JNIEnv * const env, jobject jDb){ + S3JniDb * rv = 0; + S3JniDb_mutex_enter; + if( SJG.perDb.aFree ){ + rv = SJG.perDb.aFree; + SJG.perDb.aFree = rv->pNext; + rv->pNext = 0; + s3jni_incr( &SJG.metrics.nPdbRecycled ); + } + S3JniDb_mutex_leave; + if( 0==rv ){ + rv = s3jni_malloc(sizeof(S3JniDb)); + if( rv ){ + s3jni_incr( &SJG.metrics.nPdbAlloc ); + } + } + if( rv ){ + memset(rv, 0, sizeof(S3JniDb)); + rv->jDb = S3JniRefGlobal(jDb); + } + return rv; +} + +/* +** Returns the S3JniDb object for the given org.sqlite.jni.capi.sqlite3 +** object, or NULL if jDb is NULL, no pointer can be extracted +** from it, or no matching entry can be found. +*/ +static S3JniDb * S3JniDb__from_java(JNIEnv * const env, jobject jDb){ + sqlite3 * const pDb = jDb ? PtrGet_sqlite3(jDb) : 0; + return pDb ? S3JniDb_from_clientdata(pDb) : 0; +} +#define S3JniDb_from_java(jObject) S3JniDb__from_java(env,(jObject)) + +/* +** S3JniDb finalizer for use with sqlite3_set_clientdata(). +*/ +static void S3JniDb_xDestroy(void *p){ + S3JniDeclLocal_env; + S3JniDb * const ps = p; + assert( !ps->pNext && "Else ps is already in the free-list."); + S3JniDb_set_aside(ps); +} + +/* +** Evaluates to the S3JniDb object for the given sqlite3 object, or +** NULL if pDb is NULL or was not initialized via the JNI interfaces. +*/ +#define S3JniDb_from_c(sqlite3Ptr) \ + ((sqlite3Ptr) ? S3JniDb_from_clientdata(sqlite3Ptr) : 0) +#define S3JniDb_from_jlong(sqlite3PtrAsLong) \ + S3JniDb_from_c(LongPtrGet_T(sqlite3,sqlite3PtrAsLong)) + +/* +** Unref any Java-side state in (S3JniAutoExtension*) AX and zero out +** AX. +*/ +#define S3JniAutoExtension_clear(AX) S3JniHook_unref(AX); + +/* +** Initializes a pre-allocated S3JniAutoExtension object. Returns +** non-0 if there is an error collecting the required state from +** jAutoExt (which must be an AutoExtensionCallback object). On error, +** it passes ax to S3JniAutoExtension_clear(). +*/ +static int S3JniAutoExtension_init(JNIEnv *const env, + S3JniAutoExtension * const ax, + jobject const jAutoExt){ + jclass const klazz = (*env)->GetObjectClass(env, jAutoExt); + + S3JniAutoExt_mutex_assertLocker; + *ax = S3JniHook_empty; + ax->midCallback = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/capi/sqlite3;)I"); + S3JniUnrefLocal(klazz); + S3JniExceptionWarnIgnore; + if( !ax->midCallback ){ + S3JniAutoExtension_clear(ax); + return SQLITE_ERROR; + } + ax->jObj = S3JniRefGlobal(jAutoExt); + return 0; +} + +/* +** Sets the value property of the OutputPointer.Bool jOut object to +** v. +*/ +static void OutputPointer_set_Bool(JNIEnv * const env, jobject const jOut, + int v){ + (*env)->SetBooleanField(env, jOut, s3jni_nphop_field( + env, S3JniNph(OutputPointer_Bool) + ), v ? JNI_TRUE : JNI_FALSE ); + S3JniExceptionIsFatal("Cannot set OutputPointer.Bool.value"); +} + +/* +** Sets the value property of the OutputPointer.Int32 jOut object to +** v. +*/ +static void OutputPointer_set_Int32(JNIEnv * const env, jobject const jOut, + int v){ + (*env)->SetIntField(env, jOut, s3jni_nphop_field( + env, S3JniNph(OutputPointer_Int32) + ), (jint)v); + S3JniExceptionIsFatal("Cannot set OutputPointer.Int32.value"); +} + +/* +** Sets the value property of the OutputPointer.Int64 jOut object to +** v. +*/ +static void OutputPointer_set_Int64(JNIEnv * const env, jobject const jOut, + jlong v){ + (*env)->SetLongField(env, jOut, s3jni_nphop_field( + env, S3JniNph(OutputPointer_Int64) + ), v); + S3JniExceptionIsFatal("Cannot set OutputPointer.Int64.value"); +} + +/* +** Internal helper for OutputPointer_set_TYPE() where TYPE is an +** Object type. +*/ +static void OutputPointer_set_obj(JNIEnv * const env, + S3JniNphOp const * const pRef, + jobject const jOut, + jobject v){ + (*env)->SetObjectField(env, jOut, s3jni_nphop_field(env, pRef), v); + S3JniExceptionIsFatal("Cannot set OutputPointer.T.value"); +} + +#ifdef SQLITE_ENABLE_FTS5 +#if 0 +/* +** Sets the value property of the OutputPointer.ByteArray jOut object +** to v. +*/ +static void OutputPointer_set_ByteArray(JNIEnv * const env, jobject const jOut, + jbyteArray const v){ + OutputPointer_set_obj(env, S3JniNph(OutputPointer_ByteArray), jOut, v); +} +#endif +#endif /* SQLITE_ENABLE_FTS5 */ + +/* +** Sets the value property of the OutputPointer.String jOut object to +** v. +*/ +static void OutputPointer_set_String(JNIEnv * const env, jobject const jOut, + jstring const v){ + OutputPointer_set_obj(env, S3JniNph(OutputPointer_String), jOut, v); +} + +/* +** Returns true if eTextRep is a valid sqlite3 encoding constant, else +** returns false. +*/ +static int encodingTypeIsValid(int eTextRep){ + switch( eTextRep ){ + case SQLITE_UTF8: case SQLITE_UTF16: + case SQLITE_UTF16LE: case SQLITE_UTF16BE: + return 1; + default: + return 0; + } +} + +/* For use with sqlite3_result_pointer(), sqlite3_value_pointer(), + sqlite3_bind_java_object(), and sqlite3_column_java_object(). */ +static const char * const s3jni__value_jref_key = "org.sqlite.jni.capi.ResultJavaVal"; + +/* +** If v is not NULL, it must be a jobject global reference. Its +** reference is relinquished. +*/ +static void S3Jni_jobject_finalizer(void *v){ + if( v ){ + S3JniDeclLocal_env; + S3JniUnrefGlobal((jobject)v); + } +} + +/* +** Returns a new Java instance of the class referred to by pRef, which +** MUST be interface-compatible with NativePointerHolder and MUST have +** a no-arg constructor. The NativePointerHolder_set() method is +** passed the new Java object (which must not be NULL) and pNative +** (which may be NULL). Hypothetically returns NULL if Java fails to +** allocate, but the JNI docs are not entirely clear on that detail. +** +** Always use a static pointer from the S3JniNphOps struct for the +** 2nd argument. +*/ +static jobject NativePointerHolder_new(JNIEnv * const env, + S3JniNphOp const * pRef, + const void * pNative){ + jobject rv = 0; + S3JniNphOp * const pNC = s3jni_nphop(pRef); + if( !pNC->midCtor ){ + S3JniNph_mutex_enter; + if( !pNC->midCtor ){ + pNC->midCtor = (*env)->GetMethodID(env, pNC->klazz, "", "()V"); + S3JniExceptionIsFatal("Cannot find constructor for class."); + } + S3JniNph_mutex_leave; + } + rv = (*env)->NewObject(env, pNC->klazz, pNC->midCtor); + S3JniExceptionIsFatal("No-arg constructor threw."); + s3jni_oom_check(rv); + if( rv ) NativePointerHolder_set(pRef, rv, pNative); + return rv; +} + +static inline jobject new_java_sqlite3(JNIEnv * const env, sqlite3 *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3), sv); +} +static inline jobject new_java_sqlite3_backup(JNIEnv * const env, sqlite3_backup *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_backup), sv); +} +static inline jobject new_java_sqlite3_blob(JNIEnv * const env, sqlite3_blob *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_blob), sv); +} +static inline jobject new_java_sqlite3_context(JNIEnv * const env, sqlite3_context *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_context), sv); +} +static inline jobject new_java_sqlite3_stmt(JNIEnv * const env, sqlite3_stmt *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_stmt), sv); +} +static inline jobject new_java_sqlite3_value(JNIEnv * const env, sqlite3_value *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_value), sv); +} + +/* Helper typedefs for UDF callback types. */ +typedef void (*udf_xFunc_f)(sqlite3_context*,int,sqlite3_value**); +typedef void (*udf_xStep_f)(sqlite3_context*,int,sqlite3_value**); +typedef void (*udf_xFinal_f)(sqlite3_context*); +/*typedef void (*udf_xValue_f)(sqlite3_context*);*/ +/*typedef void (*udf_xInverse_f)(sqlite3_context*,int,sqlite3_value**);*/ + +/* +** Allocate a new S3JniUdf (User-defined Function) and associate it +** with the SQLFunction-type jObj. Returns NULL on OOM. If the +** returned object's type==UDF_UNKNOWN_TYPE then the type of UDF was +** not unambiguously detected based on which callback members it has, +** which falls into the category of user error. +** +** The caller must arrange for the returned object to eventually be +** passed to S3JniUdf_free(). +*/ +static S3JniUdf * S3JniUdf_alloc(JNIEnv * const env, jobject jObj){ + S3JniUdf * s = 0; + + S3JniGlobal_mutex_enter; + s3jni_incr(&SJG.metrics.nMutexUdf); + if( SJG.udf.aFree ){ + s = SJG.udf.aFree; + SJG.udf.aFree = s->pNext; + s->pNext = 0; + s3jni_incr(&SJG.metrics.nUdfRecycled); + } + S3JniGlobal_mutex_leave; + if( !s ){ + s = s3jni_malloc( sizeof(*s)); + s3jni_incr(&SJG.metrics.nUdfAlloc); + } + if( s ){ + const char * zFSI = /* signature for xFunc, xStep, xInverse */ + "(Lorg/sqlite/jni/capi/sqlite3_context;[Lorg/sqlite/jni/capi/sqlite3_value;)V"; + const char * zFV = /* signature for xFinal, xValue */ + "(Lorg/sqlite/jni/capi/sqlite3_context;)V"; + jclass const klazz = (*env)->GetObjectClass(env, jObj); + + memset(s, 0, sizeof(*s)); + s->jObj = S3JniRefGlobal(jObj); + +#define FGET(FuncName,FuncSig,Field) \ + s->Field = (*env)->GetMethodID(env, klazz, FuncName, FuncSig); \ + if( !s->Field ) (*env)->ExceptionClear(env) + + FGET("xFunc", zFSI, jmidxFunc); + FGET("xStep", zFSI, jmidxStep); + FGET("xFinal", zFV, jmidxFinal); + FGET("xValue", zFV, jmidxValue); + FGET("xInverse", zFSI, jmidxInverse); +#undef FGET + + S3JniUnrefLocal(klazz); + if( s->jmidxFunc ) s->type = UDF_SCALAR; + else if( s->jmidxStep && s->jmidxFinal ){ + s->type = (s->jmidxValue && s->jmidxInverse) + ? UDF_WINDOW : UDF_AGGREGATE; + }else{ + s->type = UDF_UNKNOWN_TYPE; + } + } + return s; +} + +/* +** Frees up all resources owned by s, clears its state, then either +** caches it for reuse (if cacheIt is true) or frees it. The former +** requires locking the global mutex, so it must not be held when this +** is called. +*/ +static void S3JniUdf_free(JNIEnv * const env, S3JniUdf * const s, + int cacheIt){ + assert( !s->pNext ); + if( s->jObj ){ + s3jni_call_xDestroy(s->jObj); + S3JniUnrefGlobal(s->jObj); + sqlite3_free(s->zFuncName); + assert( !s->pNext ); + memset(s, 0, sizeof(*s)); + } + if( cacheIt ){ + S3JniGlobal_mutex_enter; + s->pNext = S3JniGlobal.udf.aFree; + S3JniGlobal.udf.aFree = s; + S3JniGlobal_mutex_leave; + }else{ + sqlite3_free( s ); + } +} + +/* Finalizer for sqlite3_create_function() and friends. */ +static void S3JniUdf_finalizer(void * s){ + S3JniUdf_free(s3jni_env(), (S3JniUdf*)s, 1); +} + +/* +** Helper for processing args to UDF handlers with signature +** (sqlite3_context*,int,sqlite3_value**). +*/ +typedef struct { + jobject jcx /* sqlite3_context */; + jobjectArray jargv /* sqlite3_value[] */; +} udf_jargs; + +/* +** Converts the given (cx, argc, argv) into arguments for the given +** UDF, writing the result (Java wrappers for cx and argv) in the +** final 2 arguments. Returns 0 on success, SQLITE_NOMEM on allocation +** error. On error *jCx and *jArgv will be set to 0. The output +** objects are of type org.sqlite.jni.capi.sqlite3_context and +** array-of-org.sqlite.jni.capi.sqlite3_value, respectively. +*/ +static int udf_args(JNIEnv *env, + sqlite3_context * const cx, + int argc, sqlite3_value**argv, + jobject * jCx, jobjectArray *jArgv){ + jobjectArray ja = 0; + jobject jcx = new_java_sqlite3_context(env, cx); + jint i; + *jCx = 0; + *jArgv = 0; + if( !jcx ) goto error_oom; + ja = (*env)->NewObjectArray( + env, argc, s3jni_nphop(S3JniNph(sqlite3_value))->klazz, + NULL); + s3jni_oom_check( ja ); + if( !ja ) goto error_oom; + for(i = 0; i < argc; ++i){ + jobject jsv = new_java_sqlite3_value(env, argv[i]); + if( !jsv ) goto error_oom; + (*env)->SetObjectArrayElement(env, ja, i, jsv); + S3JniUnrefLocal(jsv)/*ja has a ref*/; + } + *jCx = jcx; + *jArgv = ja; + return 0; +error_oom: + S3JniUnrefLocal(jcx); + S3JniUnrefLocal(ja); + return SQLITE_NOMEM; +} + +/* +** Requires that jCx and jArgv are sqlite3_context +** resp. array-of-sqlite3_value values initialized by udf_args(). The +** latter will be 0-and-NULL for UDF types with no arguments. This +** function zeroes out the nativePointer member of jCx and each entry +** in jArgv. This is a safety-net precaution to avoid undefined +** behavior if a Java-side UDF holds a reference to its context or one +** of its arguments. This MUST be called from any function which +** successfully calls udf_args(), after calling the corresponding UDF +** and checking its exception status, or which Java-wraps a +** sqlite3_context for use with a UDF(ish) call. It MUST NOT be called +** in any other case. +*/ +static void udf_unargs(JNIEnv *env, jobject jCx, int argc, jobjectArray jArgv){ + int i = 0; + assert(jCx); + NativePointerHolder_set(S3JniNph(sqlite3_context), jCx, 0); + for( ; i < argc; ++i ){ + jobject jsv = (*env)->GetObjectArrayElement(env, jArgv, i); + /* + ** There is a potential Java-triggerable case of Undefined + ** Behavior here, but it would require intentional misuse of the + ** API: + ** + ** If a Java UDF grabs an sqlite3_value from its argv and then + ** assigns that element to null, it becomes unreachable to us so + ** we cannot clear out its pointer. That Java-side object's + ** getNativePointer() will then refer to a stale value, so passing + ** it into (e.g.) sqlite3_value_SOMETHING() would invoke UB. + ** + ** High-level wrappers can avoid that possibility if they do not + ** expose sqlite3_value directly to clients (as is the case in + ** org.sqlite.jni.wrapper1.SqlFunction). + ** + ** One potential (but expensive) workaround for this would be to + ** privately store a duplicate argv array in each sqlite3_context + ** wrapper object, and clear the native pointers from that copy. + */ + assert(jsv && "Someone illegally modified a UDF argument array."); + if( jsv ){ + NativePointerHolder_set(S3JniNph(sqlite3_value), jsv, 0); + } + } +} + + +/* +** Must be called immediately after a Java-side UDF callback throws. +** If translateToErr is true then it sets the exception's message in +** the result error using sqlite3_result_error(). If translateToErr is +** false then it emits a warning that the function threw but should +** not do so. In either case, it clears the exception state. +** +** Returns SQLITE_NOMEM if an allocation fails, else SQLITE_ERROR. In +** the former case it calls sqlite3_result_error_nomem(). +*/ +static int udf_report_exception(JNIEnv * const env, int translateToErr, + sqlite3_context * cx, + const char *zFuncName, const char *zFuncType ){ + jthrowable const ex = (*env)->ExceptionOccurred(env); + int rc = SQLITE_ERROR; + + assert(ex && "This must only be called when a Java exception is pending."); + if( translateToErr ){ + char * zMsg; + char * z; + + S3JniExceptionClear; + zMsg = s3jni_exception_error_msg(env, ex); + z = sqlite3_mprintf("Client-defined SQL function %s.%s() threw: %s", + zFuncName ? zFuncName : "", zFuncType, + zMsg ? zMsg : "Unknown exception" ); + sqlite3_free(zMsg); + if( z ){ + sqlite3_result_error(cx, z, -1); + sqlite3_free(z); + }else{ + sqlite3_result_error_nomem(cx); + rc = SQLITE_NOMEM; + } + }else{ + S3JniExceptionWarnCallbackThrew("client-defined SQL function"); + S3JniExceptionClear; + } + S3JniUnrefLocal(ex); + return rc; +} + +/* +** Sets up the state for calling a Java-side xFunc/xStep/xInverse() +** UDF, calls it, and returns 0 on success. +*/ +static int udf_xFSI(sqlite3_context* const pCx, int argc, + sqlite3_value** const argv, S3JniUdf * const s, + jmethodID xMethodID, const char * const zFuncType){ + S3JniDeclLocal_env; + udf_jargs args = {0,0}; + int rc = udf_args(env, pCx, argc, argv, &args.jcx, &args.jargv); + + if( 0 == rc ){ + (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); + S3JniIfThrew{ + rc = udf_report_exception(env, 'F'==zFuncType[1]/*xFunc*/, pCx, + s->zFuncName, zFuncType); + } + udf_unargs(env, args.jcx, argc, args.jargv); + } + S3JniUnrefLocal(args.jcx); + S3JniUnrefLocal(args.jargv); + return rc; +} + +/* +** Sets up the state for calling a Java-side xFinal/xValue() UDF, +** calls it, and returns 0 on success. +*/ +static int udf_xFV(sqlite3_context* cx, S3JniUdf * s, + jmethodID xMethodID, + const char *zFuncType){ + S3JniDeclLocal_env; + jobject jcx = new_java_sqlite3_context(env, cx); + int rc = 0; + int const isFinal = 'F'==zFuncType[1]/*xFinal*/; + + if( jcx ){ + (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx); + S3JniIfThrew{ + rc = udf_report_exception(env, isFinal, cx, s->zFuncName, + zFuncType); + } + udf_unargs(env, jcx, 0, 0); + S3JniUnrefLocal(jcx); + }else{ + if( isFinal ) sqlite3_result_error_nomem(cx); + rc = SQLITE_NOMEM; + } + return rc; +} + +/* Proxy for C-to-Java xFunc. */ +static void udf_xFunc(sqlite3_context* cx, int argc, + sqlite3_value** argv){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nFunc ); + udf_xFSI(cx, argc, argv, s, s->jmidxFunc, "xFunc"); +} +/* Proxy for C-to-Java xStep. */ +static void udf_xStep(sqlite3_context* cx, int argc, + sqlite3_value** argv){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nStep ); + udf_xFSI(cx, argc, argv, s, s->jmidxStep, "xStep"); +} +/* Proxy for C-to-Java xFinal. */ +static void udf_xFinal(sqlite3_context* cx){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nFinal ); + udf_xFV(cx, s, s->jmidxFinal, "xFinal"); +} +/* Proxy for C-to-Java xValue. */ +static void udf_xValue(sqlite3_context* cx){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nValue ); + udf_xFV(cx, s, s->jmidxValue, "xValue"); +} +/* Proxy for C-to-Java xInverse. */ +static void udf_xInverse(sqlite3_context* cx, int argc, + sqlite3_value** argv){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nInverse ); + udf_xFSI(cx, argc, argv, s, s->jmidxInverse, "xInverse"); +} + + +//////////////////////////////////////////////////////////////////////// +// What follows is the JNI/C bindings. They are in alphabetical order +// except for this macro-generated subset which are kept together +// (alphabetized) here at the front... +//////////////////////////////////////////////////////////////////////// + +/** Create a trivial JNI wrapper for (int CName(void)). */ +#define WRAP_INT_VOID(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass){ \ + return (jint)CName(); \ + } +/** Create a trivial JNI wrapper for (int CName(int)). */ +#define WRAP_INT_INT(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jint arg){ \ + return (jint)CName((int)arg); \ + } +/* +** Create a trivial JNI wrapper for (const mutf8_string * +** CName(void)). This is only valid for functions which are known to +** return ASCII or text which is equivalent in UTF-8 and MUTF-8. +*/ +#define WRAP_MUTF8_VOID(JniNameSuffix,CName) \ + JniDecl(jstring,JniNameSuffix)(JniArgsEnvClass){ \ + jstring const rv = (*env)->NewStringUTF( env, CName() ); \ + s3jni_oom_check(rv); \ + return rv; \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3_stmt*)). */ +#define WRAP_INT_STMT(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpStmt){ \ + return (jint)CName(LongPtrGet_sqlite3_stmt(jpStmt)); \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3_stmt*,int)). */ +#define WRAP_INT_STMT_INT(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpStmt, jint n){ \ + return (jint)CName(LongPtrGet_sqlite3_stmt(jpStmt), (int)n); \ + } +/** Create a trivial JNI wrapper for (boolean CName(sqlite3_stmt*)). */ +#define WRAP_BOOL_STMT(JniNameSuffix,CName) \ + JniDecl(jboolean,JniNameSuffix)(JniArgsEnvClass, jobject jStmt){ \ + return CName(PtrGet_sqlite3_stmt(jStmt)) ? JNI_TRUE : JNI_FALSE; \ + } +/** Create a trivial JNI wrapper for (jstring CName(sqlite3_stmt*,int)). */ +#define WRAP_STR_STMT_INT(JniNameSuffix,CName) \ + JniDecl(jstring,JniNameSuffix)(JniArgsEnvClass, jlong jpStmt, jint ndx){ \ + return s3jni_utf8_to_jstring( \ + CName(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx), \ + -1); \ + } +/** Create a trivial JNI wrapper for (boolean CName(sqlite3*)). */ +#define WRAP_BOOL_DB(JniNameSuffix,CName) \ + JniDecl(jboolean,JniNameSuffix)(JniArgsEnvClass, jlong jpDb){ \ + return CName(LongPtrGet_sqlite3(jpDb)) ? JNI_TRUE : JNI_FALSE; \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3*)). */ +#define WRAP_INT_DB(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpDb){ \ + return (jint)CName(LongPtrGet_sqlite3(jpDb)); \ + } +/** Create a trivial JNI wrapper for (int64 CName(sqlite3*)). */ +#define WRAP_INT64_DB(JniNameSuffix,CName) \ + JniDecl(jlong,JniNameSuffix)(JniArgsEnvClass, jlong jpDb){ \ + return (jlong)CName(LongPtrGet_sqlite3(jpDb)); \ + } +/** Create a trivial JNI wrapper for (jstring CName(sqlite3*,int)). */ +#define WRAP_STR_DB_INT(JniNameSuffix,CName) \ + JniDecl(jstring,JniNameSuffix)(JniArgsEnvClass, jlong jpDb, jint ndx){ \ + return s3jni_utf8_to_jstring( \ + CName(LongPtrGet_sqlite3(jpDb), (int)ndx), \ + -1); \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3_value*)). */ +#define WRAP_INT_SVALUE(JniNameSuffix,CName,DfltOnNull) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpSValue){ \ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSValue); \ + return (jint)(sv ? CName(sv): DfltOnNull); \ + } +/** Create a trivial JNI wrapper for (boolean CName(sqlite3_value*)). */ +#define WRAP_BOOL_SVALUE(JniNameSuffix,CName,DfltOnNull) \ + JniDecl(jboolean,JniNameSuffix)(JniArgsEnvClass, jlong jpSValue){ \ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSValue); \ + return (jint)(sv ? CName(sv) : DfltOnNull) \ + ? JNI_TRUE : JNI_FALSE; \ + } + +WRAP_INT_DB(1changes, sqlite3_changes) +WRAP_INT64_DB(1changes64, sqlite3_changes64) +WRAP_INT_STMT(1clear_1bindings, sqlite3_clear_bindings) +WRAP_INT_STMT_INT(1column_1bytes, sqlite3_column_bytes) +WRAP_INT_STMT_INT(1column_1bytes16, sqlite3_column_bytes16) +WRAP_INT_STMT(1column_1count, sqlite3_column_count) +WRAP_STR_STMT_INT(1column_1decltype, sqlite3_column_decltype) +WRAP_STR_STMT_INT(1column_1name, sqlite3_column_name) +#ifdef SQLITE_ENABLE_COLUMN_METADATA +WRAP_STR_STMT_INT(1column_1database_1name, sqlite3_column_database_name) +WRAP_STR_STMT_INT(1column_1origin_1name, sqlite3_column_origin_name) +WRAP_STR_STMT_INT(1column_1table_1name, sqlite3_column_table_name) +#endif +WRAP_INT_STMT_INT(1column_1type, sqlite3_column_type) +WRAP_INT_STMT(1data_1count, sqlite3_data_count) +WRAP_STR_DB_INT(1db_1name, sqlite3_db_name) +WRAP_INT_DB(1error_1offset, sqlite3_error_offset) +WRAP_INT_DB(1extended_1errcode, sqlite3_extended_errcode) +WRAP_BOOL_DB(1get_1autocommit, sqlite3_get_autocommit) +WRAP_MUTF8_VOID(1libversion, sqlite3_libversion) +WRAP_INT_VOID(1libversion_1number, sqlite3_libversion_number) +WRAP_INT_VOID(1keyword_1count, sqlite3_keyword_count) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +WRAP_INT_DB(1preupdate_1blobwrite, sqlite3_preupdate_blobwrite) +WRAP_INT_DB(1preupdate_1count, sqlite3_preupdate_count) +WRAP_INT_DB(1preupdate_1depth, sqlite3_preupdate_depth) +#endif +WRAP_INT_INT(1release_1memory, sqlite3_release_memory) +WRAP_INT_INT(1sleep, sqlite3_sleep) +WRAP_MUTF8_VOID(1sourceid, sqlite3_sourceid) +WRAP_BOOL_STMT(1stmt_1busy, sqlite3_stmt_busy) +WRAP_INT_STMT_INT(1stmt_1explain, sqlite3_stmt_explain) +WRAP_INT_STMT(1stmt_1isexplain, sqlite3_stmt_isexplain) +WRAP_BOOL_STMT(1stmt_1readonly, sqlite3_stmt_readonly) +WRAP_INT_DB(1system_1errno, sqlite3_system_errno) +WRAP_INT_VOID(1threadsafe, sqlite3_threadsafe) +WRAP_INT_DB(1total_1changes, sqlite3_total_changes) +WRAP_INT64_DB(1total_1changes64, sqlite3_total_changes64) +WRAP_INT_SVALUE(1value_1encoding, sqlite3_value_encoding,SQLITE_UTF8) +WRAP_BOOL_SVALUE(1value_1frombind, sqlite3_value_frombind,0) +WRAP_INT_SVALUE(1value_1nochange, sqlite3_value_nochange,0) +WRAP_INT_SVALUE(1value_1numeric_1type, sqlite3_value_numeric_type,SQLITE_NULL) +WRAP_INT_SVALUE(1value_1subtype, sqlite3_value_subtype,0) +WRAP_INT_SVALUE(1value_1type, sqlite3_value_type,SQLITE_NULL) + +#undef WRAP_BOOL_DB +#undef WRAP_BOOL_STMT +#undef WRAP_BOOL_SVALUE +#undef WRAP_INT64_DB +#undef WRAP_INT_DB +#undef WRAP_INT_INT +#undef WRAP_INT_STMT +#undef WRAP_INT_STMT_INT +#undef WRAP_INT_SVALUE +#undef WRAP_INT_VOID +#undef WRAP_MUTF8_VOID +#undef WRAP_STR_STMT_INT +#undef WRAP_STR_DB_INT + +S3JniApi(sqlite3_aggregate_context(),jlong,1aggregate_1context)( + JniArgsEnvClass, jobject jCx, jboolean initialize +){ + sqlite3_context * const pCx = PtrGet_sqlite3_context(jCx); + void * const p = pCx + ? sqlite3_aggregate_context(pCx, (int)(initialize + ? (int)sizeof(void*) + : 0)) + : 0; + return S3JniCast_P2L(p); +} + +/* +** Central auto-extension runner for auto-extensions created in Java. +*/ +static int s3jni_run_java_auto_extensions(sqlite3 *pDb, const char **pzErr, + const struct sqlite3_api_routines *ignored){ + int rc = 0; + unsigned i, go = 1; + JNIEnv * env = 0; + S3JniDb * ps; + S3JniEnv * jc; + + if( 0==SJG.autoExt.nExt ) return 0; + env = s3jni_env(); + jc = S3JniEnv_get(); + S3JniDb_mutex_enter; + ps = jc->pdbOpening ? jc->pdbOpening : S3JniDb_from_c(pDb); + if( !ps ){ + *pzErr = sqlite3_mprintf("Unexpected arrival of null S3JniDb in " + "auto-extension runner."); + S3JniDb_mutex_leave; + return SQLITE_ERROR; + } + assert( ps->jDb ); + if( !ps->pDb ){ + assert( jc->pdbOpening == ps ); + rc = sqlite3_set_clientdata(pDb, S3JniDb_clientdata_key, + ps, 0/* we'll re-set this after open() + completes. */); + if( rc ){ + S3JniDb_mutex_leave; + return rc; + } + } + else{ + assert( ps == jc->pdbOpening ); + jc->pdbOpening = 0; + } + S3JniDb_mutex_leave; + NativePointerHolder_set(S3JniNph(sqlite3), ps->jDb, pDb) + /* As of here, the Java/C connection is complete except for the + (temporary) lack of finalizer for the ps object. */; + ps->pDb = pDb; + for( i = 0; go && 0==rc; ++i ){ + S3JniAutoExtension ax = S3JniHook_empty + /* We need a copy of the auto-extension object, with our own + ** local reference to it, to avoid a race condition with another + ** thread manipulating the list during the call and invaliding + ** what ax references. */; + S3JniAutoExt_mutex_enter; + if( i >= SJG.autoExt.nExt ){ + go = 0; + }else{ + S3JniHook_localdup(&SJG.autoExt.aExt[i], &ax); + } + S3JniAutoExt_mutex_leave; + if( ax.jObj ){ + rc = (*env)->CallIntMethod(env, ax.jObj, ax.midCallback, ps->jDb); + S3JniHook_localundup(ax); + S3JniIfThrew { + jthrowable const ex = (*env)->ExceptionOccurred(env); + char * zMsg; + S3JniExceptionClear; + zMsg = s3jni_exception_error_msg(env, ex); + S3JniUnrefLocal(ex); + *pzErr = sqlite3_mprintf("auto-extension threw: %s", zMsg); + sqlite3_free(zMsg); + rc = SQLITE_ERROR; + } + } + } + return rc; +} + +S3JniApi(sqlite3_auto_extension(),jint,1auto_1extension)( + JniArgsEnvClass, jobject jAutoExt +){ + int i; + S3JniAutoExtension * ax = 0; + int rc = 0; + + if( !jAutoExt ) return SQLITE_MISUSE; + S3JniAutoExt_mutex_enter; + for( i = 0; i < SJG.autoExt.nExt; ++i ){ + /* Look for a match. */ + ax = &SJG.autoExt.aExt[i]; + if( ax->jObj && (*env)->IsSameObject(env, ax->jObj, jAutoExt) ){ + /* same object, so this is a no-op. */ + S3JniAutoExt_mutex_leave; + return 0; + } + } + if( i == SJG.autoExt.nExt ){ + assert( SJG.autoExt.nExt <= SJG.autoExt.nAlloc ); + if( SJG.autoExt.nExt == SJG.autoExt.nAlloc ){ + /* Allocate another slot. */ + unsigned n = 1 + SJG.autoExt.nAlloc; + S3JniAutoExtension * const aNew = + s3jni_realloc( SJG.autoExt.aExt, n * sizeof(*ax) ); + if( !aNew ){ + rc = SQLITE_NOMEM; + }else{ + SJG.autoExt.aExt = aNew; + ++SJG.autoExt.nAlloc; + } + } + if( 0==rc ){ + ax = &SJG.autoExt.aExt[SJG.autoExt.nExt]; + rc = S3JniAutoExtension_init(env, ax, jAutoExt); + assert( rc ? (0==ax->jObj && 0==ax->midCallback) + : (0!=ax->jObj && 0!=ax->midCallback) ); + } + } + if( 0==rc ){ + static int once = 0; + if( 0==once && ++once ){ + rc = sqlite3_auto_extension( + (void(*)(void))s3jni_run_java_auto_extensions + /* Reminder: the JNI binding of sqlite3_reset_auto_extension() + ** does not call the core-lib impl. It only clears Java-side + ** auto-extensions. */ + ); + if( rc ){ + assert( ax ); + S3JniAutoExtension_clear(ax); + } + } + if( 0==rc ){ + ++SJG.autoExt.nExt; + } + } + S3JniAutoExt_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_backup_finish(),jint,1backup_1finish)( + JniArgsEnvClass, jlong jpBack +){ + int rc = 0; + if( jpBack!=0 ){ + rc = sqlite3_backup_finish( LongPtrGet_sqlite3_backup(jpBack) ); + } + return rc; +} + +S3JniApi(sqlite3_backup_init(),jobject,1backup_1init)( + JniArgsEnvClass, jlong jpDbDest, jstring jTDest, + jlong jpDbSrc, jstring jTSrc +){ + sqlite3 * const pDest = LongPtrGet_sqlite3(jpDbDest); + sqlite3 * const pSrc = LongPtrGet_sqlite3(jpDbSrc); + char * const zDest = s3jni_jstring_to_utf8(jTDest, 0); + char * const zSrc = s3jni_jstring_to_utf8(jTSrc, 0); + jobject rv = 0; + + if( pDest && pSrc && zDest && zSrc ){ + sqlite3_backup * const pB = + sqlite3_backup_init(pDest, zDest, pSrc, zSrc); + if( pB ){ + rv = new_java_sqlite3_backup(env, pB); + if( !rv ){ + sqlite3_backup_finish( pB ); + } + } + } + sqlite3_free(zDest); + sqlite3_free(zSrc); + return rv; +} + +S3JniApi(sqlite3_backup_pagecount(),jint,1backup_1pagecount)( + JniArgsEnvClass, jlong jpBack +){ + return sqlite3_backup_pagecount(LongPtrGet_sqlite3_backup(jpBack)); +} + +S3JniApi(sqlite3_backup_remaining(),jint,1backup_1remaining)( + JniArgsEnvClass, jlong jpBack +){ + return sqlite3_backup_remaining(LongPtrGet_sqlite3_backup(jpBack)); +} + +S3JniApi(sqlite3_backup_step(),jint,1backup_1step)( + JniArgsEnvClass, jlong jpBack, jint nPage +){ + return sqlite3_backup_step(LongPtrGet_sqlite3_backup(jpBack), (int)nPage); +} + +S3JniApi(sqlite3_bind_blob(),jint,1bind_1blob)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jbyteArray baData, jint nMax +){ + jsize nBA = 0; + jbyte * const pBuf = baData ? s3jni_jbyteArray_bytes2(baData, &nBA) : 0; + int rc; + if( pBuf ){ + if( nMax>nBA ){ + nMax = nBA; + } + rc = sqlite3_bind_blob(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, + pBuf, (int)nMax, SQLITE_TRANSIENT); + s3jni_jbyteArray_release(baData, pBuf); + }else{ + rc = baData + ? SQLITE_NOMEM + : sqlite3_bind_null( LongPtrGet_sqlite3_stmt(jpStmt), ndx ); + } + return (jint)rc; +} + +/** + Helper for use with s3jni_setup_nio_args(). +*/ +struct S3JniNioArgs { + jobject jBuf; /* input - ByteBuffer */ + jint iOffset; /* input - byte offset */ + jint iHowMany; /* input - byte count to bind/read/write */ + jint nBuf; /* output - jBuf's buffer size */ + void * p; /* output - jBuf's buffer memory */ + void * pStart; /* output - offset of p to bind/read/write */ + int nOut; /* output - number of bytes from pStart to bind/read/write */ +}; +typedef struct S3JniNioArgs S3JniNioArgs; +static const S3JniNioArgs S3JniNioArgs_empty = { + 0,0,0,0,0,0,0 +}; + +/* +** Internal helper for sqlite3_bind_nio_buffer(), +** sqlite3_result_nio_buffer(), and similar methods which take a +** ByteBuffer object as either input or output. Populates pArgs and +** returns 0 on success, non-0 if the operation should fail. The +** caller is required to check for SJG.g.byteBuffer.klazz!=0 before calling +** this and reporting it in a way appropriate for that routine. This +** function may assert() that SJG.g.byteBuffer.klazz is not 0. +** +** The (jBuffer, iOffset, iHowMany) arguments are the (ByteBuffer, offset, +** length) arguments to the bind/result method. +** +** If iHowMany is negative then it's treated as "until the end" and +** the calculated slice is trimmed to fit if needed. If iHowMany is +** positive and extends past the end of jBuffer then SQLITE_ERROR is +** returned. +** +** Returns 0 if everything looks to be in order, else some SQLITE_... +** result code +*/ +static int s3jni_setup_nio_args( + JNIEnv *env, S3JniNioArgs * pArgs, + jobject jBuffer, jint iOffset, jint iHowMany +){ + jlong iEnd = 0; + const int bAllowTruncate = iHowMany<0; + *pArgs = S3JniNioArgs_empty; + pArgs->jBuf = jBuffer; + pArgs->iOffset = iOffset; + pArgs->iHowMany = iHowMany; + assert( SJG.g.byteBuffer.klazz ); + if( pArgs->iOffset<0 ){ + return SQLITE_ERROR + /* SQLITE_MISUSE or SQLITE_RANGE would fit better but we use + SQLITE_ERROR for consistency with the code documented for a + negative target blob offset in sqlite3_blob_read/write(). */; + } + s3jni_get_nio_buffer(pArgs->jBuf, &pArgs->p, &pArgs->nBuf); + if( !pArgs->p ){ + return SQLITE_MISUSE; + }else if( pArgs->iOffset>=pArgs->nBuf ){ + pArgs->pStart = 0; + pArgs->nOut = 0; + return 0; + } + assert( pArgs->nBuf > 0 ); + assert( pArgs->iOffset < pArgs->nBuf ); + iEnd = pArgs->iHowMany<0 + ? pArgs->nBuf - pArgs->iOffset + : pArgs->iOffset + pArgs->iHowMany; + if( iEnd>(jlong)pArgs->nBuf ){ + if( bAllowTruncate ){ + iEnd = pArgs->nBuf - pArgs->iOffset; + }else{ + return SQLITE_ERROR + /* again: for consistency with blob_read/write(), though + SQLITE_MISUSE or SQLITE_RANGE would be a better fit. */; + } + } + if( iEnd - pArgs->iOffset > (jlong)SQLITE_MAX_LENGTH ){ + return SQLITE_TOOBIG; + } + assert( pArgs->iOffset >= 0 ); + assert( iEnd > pArgs->iOffset ); + pArgs->pStart = pArgs->p + pArgs->iOffset; + pArgs->nOut = (int)(iEnd - pArgs->iOffset); + assert( pArgs->nOut > 0 ); + assert( (pArgs->pStart + pArgs->nOut) <= (pArgs->p + pArgs->nBuf) ); + return 0; +} + +S3JniApi(sqlite3_bind_nio_buffer(),jint,1bind_1nio_1buffer)( + JniArgsEnvClass, jobject jpStmt, jint ndx, jobject jBuffer, + jint iOffset, jint iN +){ + sqlite3_stmt * pStmt = PtrGet_sqlite3_stmt(jpStmt); + S3JniNioArgs args; + int rc; + if( !pStmt || !SJG.g.byteBuffer.klazz ) return SQLITE_MISUSE; + rc = s3jni_setup_nio_args(env, &args, jBuffer, iOffset, iN); + if(rc){ + return rc; + }else if( !args.pStart || !args.nOut ){ + return sqlite3_bind_null(pStmt, ndx); + } + return sqlite3_bind_blob( pStmt, (int)ndx, args.pStart, + args.nOut, SQLITE_TRANSIENT ); +} + +S3JniApi(sqlite3_bind_double(),jint,1bind_1double)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jdouble val +){ + return (jint)sqlite3_bind_double(LongPtrGet_sqlite3_stmt(jpStmt), + (int)ndx, (double)val); +} + +S3JniApi(sqlite3_bind_int(),jint,1bind_1int)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jint val +){ + return (jint)sqlite3_bind_int(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, (int)val); +} + +S3JniApi(sqlite3_bind_int64(),jint,1bind_1int64)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jlong val +){ + return (jint)sqlite3_bind_int64(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, (sqlite3_int64)val); +} + +/* +** Bind a new global ref to Object `val` using sqlite3_bind_pointer(). +*/ +S3JniApi(sqlite3_bind_java_object(),jint,1bind_1java_1object)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jobject val +){ + sqlite3_stmt * const pStmt = LongPtrGet_sqlite3_stmt(jpStmt); + int rc = SQLITE_MISUSE; + + if(pStmt){ + jobject const rv = S3JniRefGlobal(val); + if( rv ){ + rc = sqlite3_bind_pointer(pStmt, ndx, rv, s3jni__value_jref_key, + S3Jni_jobject_finalizer); + }else if(val){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_bind_null(pStmt, ndx); + } + } + return rc; +} + +S3JniApi(sqlite3_bind_null(),jint,1bind_1null)( + JniArgsEnvClass, jlong jpStmt, jint ndx +){ + return (jint)sqlite3_bind_null(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_bind_parameter_count(),jint,1bind_1parameter_1count)( + JniArgsEnvClass, jlong jpStmt +){ + return (jint)sqlite3_bind_parameter_count(LongPtrGet_sqlite3_stmt(jpStmt)); +} + +S3JniApi(sqlite3_bind_parameter_index(),jint,1bind_1parameter_1index)( + JniArgsEnvClass, jlong jpStmt, jbyteArray jName +){ + int rc = 0; + jbyte * const pBuf = s3jni_jbyteArray_bytes(jName); + if( pBuf ){ + rc = sqlite3_bind_parameter_index(LongPtrGet_sqlite3_stmt(jpStmt), + (const char *)pBuf); + s3jni_jbyteArray_release(jName, pBuf); + } + return rc; +} + +S3JniApi(sqlite3_bind_parameter_name(),jstring,1bind_1parameter_1name)( + JniArgsEnvClass, jlong jpStmt, jint ndx +){ + const char *z = + sqlite3_bind_parameter_name(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx); + return z ? s3jni_utf8_to_jstring(z, -1) : 0; +} + +/* +** Impl of sqlite3_bind_text/text16(). +*/ +static int s3jni__bind_text(int is16, JNIEnv *env, jlong jpStmt, jint ndx, + jbyteArray baData, jint nMax){ + jsize nBA = 0; + jbyte * const pBuf = + baData ? s3jni_jbyteArray_bytes2(baData, &nBA) : 0; + int rc; + if( pBuf ){ + if( nMax>nBA ){ + nMax = nBA; + } + /* Note that we rely on the Java layer having assured that baData + is NUL-terminated if nMax is negative. In order to avoid UB for + such cases, we do not expose the byte-limit arguments in the + public API. */ + rc = is16 + ? sqlite3_bind_text16(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, + pBuf, (int)nMax, SQLITE_TRANSIENT) + : sqlite3_bind_text(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, + (const char *)pBuf, + (int)nMax, SQLITE_TRANSIENT); + }else{ + rc = baData + ? sqlite3_bind_null(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx) + : SQLITE_NOMEM; + } + s3jni_jbyteArray_release(baData, pBuf); + return (jint)rc; + +} + +S3JniApi(sqlite3_bind_text(),jint,1bind_1text)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jbyteArray baData, jint nMax +){ + return s3jni__bind_text(0, env, jpStmt, ndx, baData, nMax); +} + +S3JniApi(sqlite3_bind_text16(),jint,1bind_1text16)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jbyteArray baData, jint nMax +){ + return s3jni__bind_text(1, env, jpStmt, ndx, baData, nMax); +} + +S3JniApi(sqlite3_bind_value(),jint,1bind_1value)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jlong jpValue +){ + int rc = 0; + sqlite3_stmt * pStmt = LongPtrGet_sqlite3_stmt(jpStmt); + if( pStmt ){ + sqlite3_value *v = LongPtrGet_sqlite3_value(jpValue); + if( v ){ + rc = sqlite3_bind_value(pStmt, (int)ndx, v); + }else{ + rc = sqlite3_bind_null(pStmt, (int)ndx); + } + }else{ + rc = SQLITE_MISUSE; + } + return (jint)rc; +} + +S3JniApi(sqlite3_bind_zeroblob(),jint,1bind_1zeroblob)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jint n +){ + return (jint)sqlite3_bind_zeroblob(LongPtrGet_sqlite3_stmt(jpStmt), + (int)ndx, (int)n); +} + +S3JniApi(sqlite3_bind_zeroblob64(),jint,1bind_1zeroblob64)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jlong n +){ + return (jint)sqlite3_bind_zeroblob64(LongPtrGet_sqlite3_stmt(jpStmt), + (int)ndx, (sqlite3_uint64)n); +} + +S3JniApi(sqlite3_blob_bytes(),jint,1blob_1bytes)( + JniArgsEnvClass, jlong jpBlob +){ + return sqlite3_blob_bytes(LongPtrGet_sqlite3_blob(jpBlob)); +} + +S3JniApi(sqlite3_blob_close(),jint,1blob_1close)( + JniArgsEnvClass, jlong jpBlob +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + return b ? (jint)sqlite3_blob_close(b) : SQLITE_MISUSE; +} + +S3JniApi(sqlite3_blob_open(),jint,1blob_1open)( + JniArgsEnvClass, jlong jpDb, jstring jDbName, jstring jTbl, jstring jCol, + jlong jRowId, jint flags, jobject jOut +){ + sqlite3 * const db = LongPtrGet_sqlite3(jpDb); + sqlite3_blob * pBlob = 0; + char * zDbName = 0, * zTableName = 0, * zColumnName = 0; + int rc; + + if( !db || !jDbName || !jTbl || !jCol ) return SQLITE_MISUSE; + zDbName = s3jni_jstring_to_utf8(jDbName,0); + zTableName = zDbName ? s3jni_jstring_to_utf8(jTbl,0) : 0; + zColumnName = zTableName ? s3jni_jstring_to_utf8(jCol,0) : 0; + rc = zColumnName + ? sqlite3_blob_open(db, zDbName, zTableName, zColumnName, + (sqlite3_int64)jRowId, (int)flags, &pBlob) + : SQLITE_NOMEM; + if( 0==rc ){ + jobject rv = new_java_sqlite3_blob(env, pBlob); + if( !rv ){ + sqlite3_blob_close(pBlob); + rc = SQLITE_NOMEM; + } + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3_blob), jOut, rv); + } + sqlite3_free(zDbName); + sqlite3_free(zTableName); + sqlite3_free(zColumnName); + return rc; +} + +S3JniApi(sqlite3_blob_read(),jint,1blob_1read)( + JniArgsEnvClass, jlong jpBlob, jbyteArray jTgt, jint iOffset +){ + jbyte * const pBa = s3jni_jbyteArray_bytes(jTgt); + int rc = jTgt ? (pBa ? SQLITE_MISUSE : SQLITE_NOMEM) : SQLITE_MISUSE; + if( pBa ){ + jsize const nTgt = (*env)->GetArrayLength(env, jTgt); + rc = sqlite3_blob_read(LongPtrGet_sqlite3_blob(jpBlob), pBa, + (int)nTgt, (int)iOffset); + if( 0==rc ){ + s3jni_jbyteArray_commit(jTgt, pBa); + }else{ + s3jni_jbyteArray_release(jTgt, pBa); + } + } + return rc; +} + +S3JniApi(sqlite3_blob_read_nio_buffer(),jint,1blob_1read_1nio_1buffer)( + JniArgsEnvClass, jlong jpBlob, jint iSrcOff, jobject jBB, jint iTgtOff, jint iHowMany +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + S3JniNioArgs args; + int rc; + if( !b || !SJG.g.byteBuffer.klazz || iHowMany<0 ){ + return SQLITE_MISUSE; + }else if( iTgtOff<0 || iSrcOff<0 ){ + return SQLITE_ERROR + /* for consistency with underlying sqlite3_blob_read() */; + }else if( 0==iHowMany ){ + return 0; + } + rc = s3jni_setup_nio_args(env, &args, jBB, iTgtOff, iHowMany); + if(rc){ + return rc; + }else if( !args.pStart || !args.nOut ){ + return 0; + } + assert( args.iHowMany>0 ); + return sqlite3_blob_read( b, args.pStart, (int)args.nOut, (int)iSrcOff ); +} + +S3JniApi(sqlite3_blob_reopen(),jint,1blob_1reopen)( + JniArgsEnvClass, jlong jpBlob, jlong iNewRowId +){ + return (jint)sqlite3_blob_reopen(LongPtrGet_sqlite3_blob(jpBlob), + (sqlite3_int64)iNewRowId); +} + +S3JniApi(sqlite3_blob_write(),jint,1blob_1write)( + JniArgsEnvClass, jlong jpBlob, jbyteArray jBa, jint iOffset +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + jbyte * const pBuf = b ? s3jni_jbyteArray_bytes(jBa) : 0; + const jsize nBA = pBuf ? (*env)->GetArrayLength(env, jBa) : 0; + int rc = SQLITE_MISUSE; + if(b && pBuf){ + rc = sqlite3_blob_write( b, pBuf, (int)nBA, (int)iOffset ); + } + s3jni_jbyteArray_release(jBa, pBuf); + return (jint)rc; +} + +S3JniApi(sqlite3_blob_write_nio_buffer(),jint,1blob_1write_1nio_1buffer)( + JniArgsEnvClass, jlong jpBlob, jint iTgtOff, jobject jBB, jint iSrcOff, jint iHowMany +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + S3JniNioArgs args; + int rc; + if( !b || !SJG.g.byteBuffer.klazz ){ + return SQLITE_MISUSE; + }else if( iTgtOff<0 || iSrcOff<0 ){ + return SQLITE_ERROR + /* for consistency with underlying sqlite3_blob_write() */; + }else if( 0==iHowMany ){ + return 0; + } + rc = s3jni_setup_nio_args(env, &args, jBB, iSrcOff, iHowMany); + if(rc){ + return rc; + }else if( !args.pStart || !args.nOut ){ + return 0; + } + return sqlite3_blob_write( b, args.pStart, (int)args.nOut, (int)iTgtOff ); +} + +/* Central C-to-Java busy handler proxy. */ +static int s3jni_busy_handler(void* pState, int n){ + S3JniDb * const ps = (S3JniDb *)pState; + int rc = 0; + S3JniDeclLocal_env; + S3JniHook hook; + + S3JniHook_localdup(&ps->hooks.busyHandler, &hook); + if( hook.jObj ){ + rc = (*env)->CallIntMethod(env, hook.jObj, + hook.midCallback, (jint)n); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("sqlite3_busy_handler() callback"); + rc = s3jni_db_exception(ps->pDb, SQLITE_ERROR, + "sqlite3_busy_handler() callback threw."); + } + S3JniHook_localundup(hook); + } + return rc; +} + +S3JniApi(sqlite3_busy_handler(),jint,1busy_1handler)( + JniArgsEnvClass, jlong jpDb, jobject jBusy +){ + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + S3JniHook * const pHook = ps ? &ps->hooks.busyHandler : 0; + S3JniHook hook = S3JniHook_empty; + int rc = 0; + + if( !ps ) return (jint)SQLITE_MISUSE; + S3JniDb_mutex_enter; + if( jBusy ){ + if( pHook->jObj && (*env)->IsSameObject(env, pHook->jObj, jBusy) ){ + /* Same object - this is a no-op. */ + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jBusy); + hook.jObj = S3JniRefGlobal(jBusy); + hook.midCallback = (*env)->GetMethodID(env, klazz, "call", "(I)I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + rc = SQLITE_ERROR; + } + } + } + if( 0==rc ){ + if( jBusy ){ + if( hook.jObj ){ /* Replace handler */ + rc = sqlite3_busy_handler(ps->pDb, s3jni_busy_handler, ps); + if( 0==rc ){ + S3JniHook_unref(pHook); + *pHook = hook /* transfer Java ref ownership */; + hook = S3JniHook_empty; + } + }/* else no-op */ + }else{ /* Clear handler */ + rc = sqlite3_busy_handler(ps->pDb, 0, 0); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + } + } + S3JniHook_unref(&hook); + S3JniDb_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_busy_timeout(),jint,1busy_1timeout)( + JniArgsEnvClass, jlong jpDb, jint ms +){ + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + int rc = SQLITE_MISUSE; + if( ps ){ + S3JniDb_mutex_enter; + S3JniHook_unref(&ps->hooks.busyHandler); + rc = sqlite3_busy_timeout(ps->pDb, (int)ms); + S3JniDb_mutex_leave; + } + return rc; +} + +S3JniApi(sqlite3_cancel_auto_extension(),jboolean,1cancel_1auto_1extension)( + JniArgsEnvClass, jobject jAutoExt +){ + S3JniAutoExtension * ax; + jboolean rc = JNI_FALSE; + int i; + + if( !jAutoExt ){ + return rc; + } + S3JniAutoExt_mutex_enter; + /* This algo corresponds to the one in the core. */ + for( i = SJG.autoExt.nExt-1; i >= 0; --i ){ + ax = &SJG.autoExt.aExt[i]; + if( ax->jObj && (*env)->IsSameObject(env, ax->jObj, jAutoExt) ){ + S3JniAutoExtension_clear(ax); + /* Move final entry into this slot. */ + --SJG.autoExt.nExt; + *ax = SJG.autoExt.aExt[SJG.autoExt.nExt]; + SJG.autoExt.aExt[SJG.autoExt.nExt] = S3JniHook_empty; + assert( !SJG.autoExt.aExt[SJG.autoExt.nExt].jObj ); + rc = JNI_TRUE; + break; + } + } + S3JniAutoExt_mutex_leave; + return rc; +} + +/* Wrapper for sqlite3_close(_v2)(). */ +static jint s3jni_close_db(JNIEnv * const env, jlong jpDb, int version){ + int rc = 0; + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + + assert(version == 1 || version == 2); + if( ps ){ + rc = 1==version + ? (jint)sqlite3_close(ps->pDb) + : (jint)sqlite3_close_v2(ps->pDb); + } + return (jint)rc; +} + +S3JniApi(sqlite3_close(),jint,1close)(JniArgsEnvClass, jlong pDb){ + return s3jni_close_db(env, pDb, 1); +} + +S3JniApi(sqlite3_close_v2(),jint,1close_1v2)(JniArgsEnvClass, jlong pDb){ + return s3jni_close_db(env, pDb, 2); +} + +/* +** Assumes z is an array of unsigned short and returns the index in +** that array of the first element with the value 0. +*/ +static unsigned int s3jni_utf16_strlen(void const * z){ + unsigned int i = 0; + const unsigned short * p = z; + while( p[i] ) ++i; + return i; +} + +/* Descriptive alias for use with sqlite3_collation_needed(). */ +typedef S3JniHook S3JniCollationNeeded; + +/* Central C-to-Java sqlite3_collation_needed16() hook impl. */ +static void s3jni_collation_needed_impl16(void *pState, sqlite3 *pDb, + int eTextRep, const void * z16Name){ + S3JniCollationNeeded * const pHook = pState; + S3JniDeclLocal_env; + S3JniHook hook; + + S3JniHook_localdup(pHook, &hook); + if( hook.jObj ){ + unsigned int const nName = s3jni_utf16_strlen(z16Name); + jstring jName = (*env)->NewString(env, (jchar const *)z16Name, nName); + + s3jni_oom_check( jName ); + assert( hook.jExtra ); + S3JniIfThrew{ + S3JniExceptionClear; + }else if( hook.jExtra ){ + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, + hook.jExtra, (jint)eTextRep, jName); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("sqlite3_collation_needed() callback"); + } + } + S3JniUnrefLocal(jName); + S3JniHook_localundup(hook); + } +} + +S3JniApi(sqlite3_collation_needed(),jint,1collation_1needed)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + S3JniDb * ps; + S3JniCollationNeeded * pHook; + int rc = 0; + + S3JniDb_mutex_enter; + ps = S3JniDb_from_jlong(jpDb); + if( !ps ){ + S3JniDb_mutex_leave; + return SQLITE_MISUSE; + } + pHook = &ps->hooks.collationNeeded; + if( pHook->jObj && jHook && + (*env)->IsSameObject(env, pHook->jObj, jHook) ){ + /* no-op */ + }else if( !jHook ){ + rc = sqlite3_collation_needed(ps->pDb, 0, 0); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jHook); + jmethodID const xCallback = (*env)->GetMethodID( + env, klazz, "call", "(Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)V" + ); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + rc = s3jni_db_exception(ps->pDb, SQLITE_MISUSE, + "Cannot not find matching call() in " + "CollationNeededCallback object."); + }else{ + rc = sqlite3_collation_needed16(ps->pDb, pHook, + s3jni_collation_needed_impl16); + if( 0==rc ){ + S3JniHook_unref(pHook); + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jHook); + pHook->jExtra = S3JniRefGlobal(ps->jDb); + } + } + } + S3JniDb_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_column_blob(),jbyteArray,1column_1blob)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + void const * const p = sqlite3_column_blob(pStmt, (int)ndx); + int const n = p ? sqlite3_column_bytes(pStmt, (int)ndx) : 0; + + return p ? s3jni_new_jbyteArray(p, n) : 0; +} + +S3JniApi(sqlite3_column_double(),jdouble,1column_1double)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + return (jdouble)sqlite3_column_double(PtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_column_int(),jint,1column_1int)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + return (jint)sqlite3_column_int(PtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_column_int64(),jlong,1column_1int64)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + return (jlong)sqlite3_column_int64(PtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_column_java_object(),jobject,1column_1java_1object)( + JniArgsEnvClass, jlong jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = LongPtrGet_sqlite3_stmt(jpStmt); + jobject rv = 0; + if( stmt ){ + sqlite3 * const db = sqlite3_db_handle(stmt); + sqlite3_value * sv; + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + sv = sqlite3_column_value(stmt, (int)ndx); + if( sv ){ + rv = S3JniRefLocal( + sqlite3_value_pointer(sv, s3jni__value_jref_key) + ); + } + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + } + return rv; +} + +S3JniApi(sqlite3_column_nio_buffer(),jobject,1column_1nio_1buffer)( + JniArgsEnvClass, jobject jStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jStmt); + jobject rv = 0; + if( stmt ){ + const void * const p = sqlite3_column_blob(stmt, (int)ndx); + if( p ){ + const int n = sqlite3_column_bytes(stmt, (int)ndx); + rv = s3jni__blob_to_ByteBuffer(env, p, n); + } + } + return rv; +} + +S3JniApi(sqlite3_column_text(),jbyteArray,1column_1text)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jpStmt); + const unsigned char * const p = stmt ? sqlite3_column_text(stmt, (int)ndx) : 0; + const int n = p ? sqlite3_column_bytes(stmt, (int)ndx) : 0; + return p ? s3jni_new_jbyteArray(p, n) : NULL; +} + +#if 0 +// this impl might prove useful. +S3JniApi(sqlite3_column_text(),jstring,1column_1text)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jpStmt); + const unsigned char * const p = stmt ? sqlite3_column_text(stmt, (int)ndx) : 0; + const int n = p ? sqlite3_column_bytes(stmt, (int)ndx) : 0; + return p ? s3jni_utf8_to_jstring( (const char *)p, n) : 0; +} +#endif + +S3JniApi(sqlite3_column_text16(),jstring,1column_1text16)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jpStmt); + const void * const p = stmt ? sqlite3_column_text16(stmt, (int)ndx) : 0; + const int n = p ? sqlite3_column_bytes16(stmt, (int)ndx) : 0; + return s3jni_text16_to_jstring(env, p, n); +} + +S3JniApi(sqlite3_column_value(),jobject,1column_1value)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_value * const sv = + sqlite3_column_value(PtrGet_sqlite3_stmt(jpStmt), (int)ndx) + /* reminder: returns an SQL NULL if jpStmt==NULL */; + return new_java_sqlite3_value(env, sv); +} + +/* +** Impl for commit hooks (if isCommit is true) or rollback hooks. +*/ +static int s3jni_commit_rollback_hook_impl(int isCommit, S3JniDb * const ps){ + S3JniDeclLocal_env; + int rc = 0; + S3JniHook hook; + + S3JniHook_localdup(isCommit + ? &ps->hooks.commit : &ps->hooks.rollback, + &hook); + if( hook.jObj ){ + rc = isCommit + ? (int)(*env)->CallIntMethod(env, hook.jObj, hook.midCallback) + : (int)((*env)->CallVoidMethod(env, hook.jObj, hook.midCallback), 0); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, SQLITE_ERROR, + isCommit + ? "Commit hook callback threw" + : "Rollback hook callback threw"); + } + S3JniHook_localundup(hook); + } + return rc; +} + +/* C-to-Java commit hook wrapper. */ +static int s3jni_commit_hook_impl(void *pP){ + return s3jni_commit_rollback_hook_impl(1, pP); +} + +/* C-to-Java rollback hook wrapper. */ +static void s3jni_rollback_hook_impl(void *pP){ + (void)s3jni_commit_rollback_hook_impl(0, pP); +} + +/* +** Proxy for sqlite3_commit_hook() (if isCommit is true) or +** sqlite3_rollback_hook(). +*/ +static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, + jlong jpDb, jobject jHook){ + S3JniDb * ps; + jobject pOld = 0; /* previous hoook */ + S3JniHook * pHook; /* ps->hooks.commit|rollback */ + + S3JniDb_mutex_enter; + ps = S3JniDb_from_jlong(jpDb); + if( !ps ){ + s3jni_db_error(ps->pDb, SQLITE_MISUSE, 0); + S3JniDb_mutex_leave; + return 0; + } + pHook = isCommit ? &ps->hooks.commit : &ps->hooks.rollback; + pOld = pHook->jObj; + if( pOld && jHook && + (*env)->IsSameObject(env, pOld, jHook) ){ + /* No-op. */ + }else if( !jHook ){ + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + *pHook = S3JniHook_empty; + if( isCommit ) sqlite3_commit_hook(ps->pDb, 0, 0); + else sqlite3_rollback_hook(ps->pDb, 0, 0); + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jHook); + jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", + isCommit ? "()I" : "()V"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionReport; + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching call() method in" + "hook object."); + }else{ + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jHook); + if( isCommit ) sqlite3_commit_hook(ps->pDb, s3jni_commit_hook_impl, ps); + else sqlite3_rollback_hook(ps->pDb, s3jni_rollback_hook_impl, ps); + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + } + } + S3JniDb_mutex_leave; + return pOld; +} + +S3JniApi(sqlite3_commit_hook(),jobject,1commit_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + return s3jni_commit_rollback_hook(1, env, jpDb, jHook); +} + +S3JniApi(sqlite3_compileoption_get(),jstring,1compileoption_1get)( + JniArgsEnvClass, jint n +){ + const char * z = sqlite3_compileoption_get(n); + jstring const rv = z ? (*env)->NewStringUTF( env, z ) : 0; + /* We know these to be ASCII, so MUTF-8 is fine. */; + s3jni_oom_check(z ? !!rv : 1); + return rv; +} + +S3JniApi(sqlite3_compileoption_used(),jboolean,1compileoption_1used)( + JniArgsEnvClass, jstring name +){ + const char *zUtf8 = s3jni_jstring_to_mutf8(name) + /* We know these to be ASCII, so MUTF-8 is fine (and + hypothetically faster to convert). */; + const jboolean rc = + 0==sqlite3_compileoption_used(zUtf8) ? JNI_FALSE : JNI_TRUE; + s3jni_mutf8_release(name, zUtf8); + return rc; +} + +S3JniApi(sqlite3_complete(),jint,1complete)( + JniArgsEnvClass, jbyteArray jSql +){ + jbyte * const pBuf = s3jni_jbyteArray_bytes(jSql); + const jsize nBA = pBuf ? (*env)->GetArrayLength(env, jSql) : 0; + int rc; + + assert( (nBA>0 ? 0==pBuf[nBA-1] : (pBuf ? 0==*pBuf : 1)) + && "Byte array is not NUL-terminated." ); + rc = (pBuf && 0==pBuf[(nBA ? nBA-1 : 0)]) + ? sqlite3_complete( (const char *)pBuf ) + : (jSql ? SQLITE_NOMEM : SQLITE_MISUSE); + s3jni_jbyteArray_release(jSql, pBuf); + return rc; +} + +S3JniApi(sqlite3_config() /*for a small subset of options.*/ + sqlite3_config__enable()/* internal name to avoid name-mangling issues*/, + jint,1config_1_1enable)(JniArgsEnvClass, jint n){ + switch( n ){ + case SQLITE_CONFIG_SINGLETHREAD: + case SQLITE_CONFIG_MULTITHREAD: + case SQLITE_CONFIG_SERIALIZED: + return sqlite3_config( n ); + default: + return SQLITE_MISUSE; + } +} +/* C-to-Java SQLITE_CONFIG_LOG wrapper. */ +static void s3jni_config_log(void *ignored, int errCode, const char *z){ + S3JniDeclLocal_env; + S3JniHook hook = S3JniHook_empty; + + S3JniHook_localdup(&SJG.hook.configlog, &hook); + if( hook.jObj ){ + jstring const jArg1 = z ? s3jni_utf8_to_jstring(z, -1) : 0; + if( z ? !!jArg1 : 1 ){ + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, errCode, jArg1); + } + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("SQLITE_CONFIG_LOG callback"); + S3JniExceptionClear; + } + S3JniHook_localundup(hook); + S3JniUnrefLocal(jArg1); + } +} + +S3JniApi(sqlite3_config() /* for SQLITE_CONFIG_LOG */ + sqlite3_config__config_log() /* internal name */, + jint, 1config_1_1CONFIG_1LOG +)(JniArgsEnvClass, jobject jLog){ + S3JniHook * const pHook = &SJG.hook.configlog; + int rc = 0; + + S3JniGlobal_mutex_enter; + if( !jLog ){ + rc = sqlite3_config( SQLITE_CONFIG_LOG, NULL, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + }else if( pHook->jObj && (*env)->IsSameObject(env, jLog, pHook->jObj) ){ + /* No-op */ + }else { + jclass const klazz = (*env)->GetObjectClass(env, jLog); + jmethodID const midCallback = (*env)->GetMethodID(env, klazz, "call", + "(ILjava/lang/String;)V"); + S3JniUnrefLocal(klazz); + if( midCallback ){ + rc = sqlite3_config( SQLITE_CONFIG_LOG, s3jni_config_log, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + pHook->midCallback = midCallback; + pHook->jObj = S3JniRefGlobal(jLog); + } + }else{ + S3JniExceptionWarnIgnore; + rc = SQLITE_ERROR; + } + } + S3JniGlobal_mutex_leave; + return rc; +} + +#ifdef SQLITE_ENABLE_SQLLOG +/* C-to-Java SQLITE_CONFIG_SQLLOG wrapper. */ +static void s3jni_config_sqllog(void *ignored, sqlite3 *pDb, const char *z, int op){ + jobject jArg0 = 0; + jstring jArg1 = 0; + S3JniDeclLocal_env; + S3JniDb * const ps = S3JniDb_from_c(pDb); + S3JniHook hook = S3JniHook_empty; + + if( ps ){ + S3JniHook_localdup(&SJG.hook.sqllog, &hook); + } + if( !hook.jObj ) return; + jArg0 = S3JniRefLocal(ps->jDb); + switch( op ){ + case 0: /* db opened */ + case 1: /* SQL executed */ + jArg1 = s3jni_utf8_to_jstring( z, -1); + break; + case 2: /* db closed */ + break; + default: + (*env)->FatalError(env, "Unhandled 4th arg to SQLITE_CONFIG_SQLLOG."); + break; + } + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, jArg0, jArg1, op); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("SQLITE_CONFIG_SQLLOG callback"); + S3JniExceptionClear; + } + S3JniHook_localundup(hook); + S3JniUnrefLocal(jArg0); + S3JniUnrefLocal(jArg1); +} +//! Requirement of SQLITE_CONFIG_SQLLOG. +void sqlite3_init_sqllog(void){ + sqlite3_config( SQLITE_CONFIG_SQLLOG, s3jni_config_sqllog, 0 ); +} +#endif + +S3JniApi(sqlite3_config() /* for SQLITE_CONFIG_SQLLOG */ + sqlite3_config__SQLLOG() /*internal name*/, + jint, 1config_1_1SQLLOG +)(JniArgsEnvClass, jobject jLog){ +#ifndef SQLITE_ENABLE_SQLLOG + return SQLITE_MISUSE; +#else + S3JniHook * const pHook = &SJG.hook.sqllog; + int rc = 0; + + S3JniGlobal_mutex_enter; + if( !jLog ){ + rc = sqlite3_config( SQLITE_CONFIG_SQLLOG, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + }else if( pHook->jObj && (*env)->IsSameObject(env, jLog, pHook->jObj) ){ + /* No-op */ + }else { + jclass const klazz = (*env)->GetObjectClass(env, jLog); + jmethodID const midCallback = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/capi/sqlite3;" + "Ljava/lang/String;" + "I)V"); + S3JniUnrefLocal(klazz); + if( midCallback ){ + rc = sqlite3_config( SQLITE_CONFIG_SQLLOG, s3jni_config_sqllog, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + pHook->midCallback = midCallback; + pHook->jObj = S3JniRefGlobal(jLog); + } + }else{ + S3JniExceptionWarnIgnore; + rc = SQLITE_ERROR; + } + } + S3JniGlobal_mutex_leave; + return rc; +#endif +} + +S3JniApi(sqlite3_context_db_handle(),jobject,1context_1db_1handle)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_context * const pCx = PtrGet_sqlite3_context(jpCx); + sqlite3 * const pDb = pCx ? sqlite3_context_db_handle(pCx) : 0; + S3JniDb * const ps = pDb ? S3JniDb_from_c(pDb) : 0; + return ps ? ps->jDb : 0; +} + +/* +** State for CollationCallbacks. This used to be its own separate +** type, but has since been consolidated with S3JniHook. It retains +** its own typedef for code legibility and searchability reasons. +*/ +typedef S3JniHook S3JniCollationCallback; + +/* +** Proxy for Java-side CollationCallback.xCompare() callbacks. +*/ +static int CollationCallback_xCompare(void *pArg, int nLhs, const void *lhs, + int nRhs, const void *rhs){ + S3JniCollationCallback * const pCC = pArg; + S3JniDeclLocal_env; + jint rc = 0; + if( pCC->jObj ){ + jbyteArray jbaLhs = s3jni_new_jbyteArray(lhs, (jint)nLhs); + jbyteArray jbaRhs = jbaLhs + ? s3jni_new_jbyteArray(rhs, (jint)nRhs) : 0; + if( !jbaRhs ){ + S3JniUnrefLocal(jbaLhs); + /* We have no recovery strategy here. */ + s3jni_oom_check( jbaRhs ); + return 0; + } + rc = (*env)->CallIntMethod(env, pCC->jObj, pCC->midCallback, + jbaLhs, jbaRhs); + S3JniExceptionIgnore; + S3JniUnrefLocal(jbaLhs); + S3JniUnrefLocal(jbaRhs); + } + return (int)rc; +} + +/* CollationCallback finalizer for use by the sqlite3 internals. */ +static void CollationCallback_xDestroy(void *pArg){ + S3JniCollationCallback * const pCC = pArg; + S3JniDeclLocal_env; + S3JniHook_free(pCC); +} + +S3JniApi(sqlite3_create_collation() sqlite3_create_collation_v2(), + jint,1create_1collation +)(JniArgsEnvClass, jobject jDb, jstring name, jint eTextRep, + jobject oCollation){ + int rc; + S3JniDb * ps; + + if( !jDb || !name || !encodingTypeIsValid(eTextRep) ){ + return (jint)SQLITE_MISUSE; + } + S3JniDb_mutex_enter; + ps = S3JniDb_from_java(jDb); + jclass const klazz = (*env)->GetObjectClass(env, oCollation); + jmethodID const midCallback = + (*env)->GetMethodID(env, klazz, "call", "([B[B)I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew{ + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Could not get call() method from " + "CollationCallback object."); + }else{ + char * const zName = s3jni_jstring_to_utf8(name, 0); + S3JniCollationCallback * const pCC = + zName ? S3JniHook_alloc() : 0; + if( pCC ){ + rc = sqlite3_create_collation_v2(ps->pDb, zName, (int)eTextRep, + pCC, CollationCallback_xCompare, + CollationCallback_xDestroy); + if( 0==rc ){ + pCC->midCallback = midCallback; + pCC->jObj = S3JniRefGlobal(oCollation); + pCC->doXDestroy = 1; + }else{ + CollationCallback_xDestroy(pCC); + } + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zName); + } + S3JniDb_mutex_leave; + return (jint)rc; +} + +S3JniApi(sqlite3_create_function() sqlite3_create_function_v2() + sqlite3_create_window_function(), + jint,1create_1function +)(JniArgsEnvClass, jobject jDb, jstring jFuncName, jint nArg, + jint eTextRep, jobject jFunctor){ + S3JniUdf * s = 0; + int rc; + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + char * zFuncName = 0; + + if( !pDb || !jFuncName ){ + return SQLITE_MISUSE; + }else if( !encodingTypeIsValid(eTextRep) ){ + return s3jni_db_error(pDb, SQLITE_FORMAT, + "Invalid function encoding option."); + } + s = S3JniUdf_alloc(env, jFunctor); + if( !s ) return SQLITE_NOMEM; + + if( UDF_UNKNOWN_TYPE==s->type ){ + rc = s3jni_db_error(pDb, SQLITE_MISUSE, + "Cannot unambiguously determine function type."); + S3JniUdf_free(env, s, 1); + goto error_cleanup; + } + zFuncName = s3jni_jstring_to_utf8(jFuncName,0); + if( !zFuncName ){ + rc = SQLITE_NOMEM; + S3JniUdf_free(env, s, 1); + goto error_cleanup; + } + s->zFuncName = zFuncName /* pass on ownership */; + if( UDF_WINDOW == s->type ){ + rc = sqlite3_create_window_function(pDb, zFuncName, nArg, eTextRep, s, + udf_xStep, udf_xFinal, udf_xValue, + udf_xInverse, S3JniUdf_finalizer); + }else{ + udf_xFunc_f xFunc = 0; + udf_xStep_f xStep = 0; + udf_xFinal_f xFinal = 0; + if( UDF_SCALAR == s->type ){ + xFunc = udf_xFunc; + }else{ + assert( UDF_AGGREGATE == s->type ); + xStep = udf_xStep; + xFinal = udf_xFinal; + } + rc = sqlite3_create_function_v2(pDb, zFuncName, nArg, eTextRep, s, + xFunc, xStep, xFinal, S3JniUdf_finalizer); + } +error_cleanup: + /* Reminder: on sqlite3_create_function() error, s will be + ** destroyed via create_function(). */ + return (jint)rc; +} + + +S3JniApi(sqlite3_db_config() /*for MAINDBNAME*/, + jint,1db_1config__Lorg_sqlite_jni_capi_sqlite3_2ILjava_lang_String_2 +)(JniArgsEnvClass, jobject jDb, jint op, jstring jStr){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc; + char *zStr; + + switch( (ps && jStr) ? op : 0 ){ + case SQLITE_DBCONFIG_MAINDBNAME: + S3JniDb_mutex_enter + /* Protect against a race in modifying/freeing + ps->zMainDbName. */; + zStr = s3jni_jstring_to_utf8( jStr, 0); + if( zStr ){ + rc = sqlite3_db_config(ps->pDb, (int)op, zStr); + if( rc ){ + sqlite3_free( zStr ); + }else{ + sqlite3_free( ps->zMainDbName ); + ps->zMainDbName = zStr; + } + }else{ + rc = SQLITE_NOMEM; + } + S3JniDb_mutex_leave; + break; + case 0: + default: + rc = SQLITE_MISUSE; + } + return rc; +} + +S3JniApi( + sqlite3_db_config(), + /* WARNING: openjdk v19 creates a different mangled name for this + ** function than openjdk v8 does. We account for that by exporting + ** both versions of the name. */ + jint,1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2 +)( + JniArgsEnvClass, jobject jDb, jint op, jint onOff, jobject jOut +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc; + switch( ps ? op : 0 ){ + case SQLITE_DBCONFIG_ENABLE_FKEY: + case SQLITE_DBCONFIG_ENABLE_TRIGGER: + case SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: + case SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: + case SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: + case SQLITE_DBCONFIG_ENABLE_QPSG: + case SQLITE_DBCONFIG_TRIGGER_EQP: + case SQLITE_DBCONFIG_RESET_DATABASE: + case SQLITE_DBCONFIG_DEFENSIVE: + case SQLITE_DBCONFIG_WRITABLE_SCHEMA: + case SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: + case SQLITE_DBCONFIG_DQS_DML: + case SQLITE_DBCONFIG_DQS_DDL: + case SQLITE_DBCONFIG_ENABLE_VIEW: + case SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: + case SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case SQLITE_DBCONFIG_STMT_SCANSTATUS: + case SQLITE_DBCONFIG_REVERSE_SCANORDER: { + int pOut = 0; + rc = sqlite3_db_config( ps->pDb, (int)op, onOff, &pOut ); + if( 0==rc && jOut ){ + OutputPointer_set_Int32(env, jOut, pOut); + } + break; + } + default: + rc = SQLITE_MISUSE; + } + return (jint)rc; +} + +/* +** This is a workaround for openjdk v19 (and possibly others) encoding +** this function's name differently than JDK v8 does. If we do not +** install both names for this function then Java will not be able to +** find the function in both environments. +*/ +JniDecl(jint,1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_00024Int32_2)( + JniArgsEnvClass, jobject jDb, jint op, jint onOff, jobject jOut +){ + return JniFuncName(1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2)( + env, jKlazz, jDb, op, onOff, jOut + ); +} + +S3JniApi(sqlite3_db_filename(),jstring,1db_1filename)( + JniArgsEnvClass, jobject jDb, jstring jDbName +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + char *zDbName; + jstring jRv = 0; + int nStr = 0; + + if( !ps || !jDbName ){ + return 0; + } + zDbName = s3jni_jstring_to_utf8( jDbName, &nStr); + if( zDbName ){ + char const * zRv = sqlite3_db_filename(ps->pDb, zDbName); + sqlite3_free(zDbName); + if( zRv ){ + jRv = s3jni_utf8_to_jstring( zRv, -1); + } + } + return jRv; +} + +S3JniApi(sqlite3_db_handle(),jobject,1db_1handle)( + JniArgsEnvClass, jobject jpStmt +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + sqlite3 * const pDb = pStmt ? sqlite3_db_handle(pStmt) : 0; + S3JniDb * const ps = pDb ? S3JniDb_from_c(pDb) : 0; + return ps ? ps->jDb : 0; +} + +S3JniApi(sqlite3_db_readonly(),jint,1db_1readonly)( + JniArgsEnvClass, jobject jDb, jstring jDbName +){ + int rc = 0; + S3JniDb * const ps = S3JniDb_from_java(jDb); + char *zDbName = jDbName ? s3jni_jstring_to_utf8( jDbName, 0 ) : 0; + rc = sqlite3_db_readonly(ps ? ps->pDb : 0, zDbName); + sqlite3_free(zDbName); + return (jint)rc; +} + +S3JniApi(sqlite3_db_release_memory(),jint,1db_1release_1memory)( + JniArgsEnvClass, jobject jDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + return pDb ? sqlite3_db_release_memory(pDb) : SQLITE_MISUSE; +} + +S3JniApi(sqlite3_db_status(),jint,1db_1status)( + JniArgsEnvClass, jobject jDb, jint op, jobject jOutCurrent, + jobject jOutHigh, jboolean reset +){ + int iCur = 0, iHigh = 0; + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + int rc = sqlite3_db_status( pDb, op, &iCur, &iHigh, reset ); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCurrent, iCur); + OutputPointer_set_Int32(env, jOutHigh, iHigh); + } + return (jint)rc; +} + +S3JniApi(sqlite3_errcode(),jint,1errcode)( + JniArgsEnvClass, jobject jpDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + return pDb ? sqlite3_errcode(pDb) : SQLITE_MISUSE; +} + +S3JniApi(sqlite3_errmsg(),jstring,1errmsg)( + JniArgsEnvClass, jobject jpDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + return pDb ? s3jni_utf8_to_jstring( sqlite3_errmsg(pDb), -1) : 0 + /* We don't use errmsg16() directly only because it would cause an + additional level of internal encoding in sqlite3. The end + effect should be identical to using errmsg16(), however. */; +} + +S3JniApi(sqlite3_errstr(),jstring,1errstr)( + JniArgsEnvClass, jint rcCode +){ + jstring rv; + const char * z = sqlite3_errstr((int)rcCode); + if( !z ){ + /* This hypothetically cannot happen, but we'll behave like the + low-level library would in such a case... */ + z = "unknown error"; + } + rv = (*env)->NewStringUTF(env, z) + /* We know these values to be plain ASCII, so pose no MUTF-8 + ** incompatibility */; + s3jni_oom_check( rv ); + return rv; +} + +#ifndef SQLITE_ENABLE_NORMALIZE +/* Dummy stub for sqlite3_normalized_sql(). Never called. */ +static const char * sqlite3_normalized_sql(sqlite3_stmt *s){ + S3JniDeclLocal_env; + (*env)->FatalError(env, "dummy sqlite3_normalized_sql() was " + "impossibly called.") /* does not return */; + return 0; +} +#endif + +/* +** Impl for sqlite3_expanded_sql() (if isExpanded is true) and +** sqlite3_normalized_sql(). +*/ +static jstring s3jni_xn_sql(int isExpanded, JNIEnv *env, jobject jpStmt){ + jstring rv = 0; + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + + if( pStmt ){ + char * zSql = isExpanded + ? sqlite3_expanded_sql(pStmt) + : (char*)sqlite3_normalized_sql(pStmt); + s3jni_oom_fatal(zSql); + if( zSql ){ + rv = s3jni_utf8_to_jstring(zSql, -1); + if( isExpanded ) sqlite3_free(zSql); + } + } + return rv; +} + +S3JniApi(sqlite3_expanded_sql(),jstring,1expanded_1sql)( + JniArgsEnvClass, jobject jpStmt +){ + return s3jni_xn_sql(1, env, jpStmt); +} + +S3JniApi(sqlite3_normalized_sql(),jstring,1normalized_1sql)( + JniArgsEnvClass, jobject jpStmt +){ +#ifdef SQLITE_ENABLE_NORMALIZE + return s3jni_xn_sql(0, env, jpStmt); +#else + return 0; +#endif +} + +S3JniApi(sqlite3_extended_result_codes(),jint,1extended_1result_1codes)( + JniArgsEnvClass, jobject jpDb, jboolean onoff +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + int const rc = pDb + ? sqlite3_extended_result_codes(pDb, onoff ? 1 : 0) + : SQLITE_MISUSE; + return rc; +} + +S3JniApi(sqlite3_finalize(),jint,1finalize)( + JniArgsEnvClass, jlong jpStmt +){ + return jpStmt + ? sqlite3_finalize(LongPtrGet_sqlite3_stmt(jpStmt)) + : 0; +} + +S3JniApi(sqlite3_get_auxdata(),jobject,1get_1auxdata)( + JniArgsEnvClass, jobject jCx, jint n +){ + return sqlite3_get_auxdata(PtrGet_sqlite3_context(jCx), (int)n); +} + +S3JniApi(sqlite3_initialize(),jint,1initialize)( + JniArgsEnvClass +){ + return sqlite3_initialize(); +} + +S3JniApi(sqlite3_interrupt(),void,1interrupt)( + JniArgsEnvClass, jobject jpDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + if( pDb ){ + sqlite3_interrupt(pDb); + } +} + +S3JniApi(sqlite3_is_interrupted(),jboolean,1is_1interrupted)( + JniArgsEnvClass, jobject jpDb +){ + int rc = 0; + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + if( pDb ){ + rc = sqlite3_is_interrupted(pDb); + } + return rc ? JNI_TRUE : JNI_FALSE; +} + +/* +** Uncaches the current JNIEnv from the S3JniGlobal state, clearing +** any resources owned by that cache entry and making that slot +** available for re-use. +*/ +S3JniApi(sqlite3_java_uncache_thread(), jboolean, 1java_1uncache_1thread)( + JniArgsEnvClass +){ + int rc; + S3JniEnv_mutex_enter; + rc = S3JniEnv_uncache(env); + S3JniEnv_mutex_leave; + return rc ? JNI_TRUE : JNI_FALSE; +} + +S3JniApi(sqlite3_jni_db_error(), jint, 1jni_1db_1error)( + JniArgsEnvClass, jobject jDb, jint jRc, jstring jStr +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc = SQLITE_MISUSE; + if( ps ){ + char *zStr; + zStr = jStr + ? s3jni_jstring_to_utf8( jStr, 0) + : NULL; + rc = s3jni_db_error( ps->pDb, (int)jRc, zStr ); + sqlite3_free(zStr); + } + return rc; +} + +S3JniApi(sqlite3_jni_supports_nio(), jboolean,1jni_1supports_1nio)( + JniArgsEnvClass +){ + return SJG.g.byteBuffer.klazz ? JNI_TRUE : JNI_FALSE; +} + + +S3JniApi(sqlite3_keyword_check(),jboolean,1keyword_1check)( + JniArgsEnvClass, jstring jWord +){ + int nWord = 0; + char * zWord = s3jni_jstring_to_utf8(jWord, &nWord); + int rc = 0; + + s3jni_oom_check(jWord ? !!zWord : 1); + if( zWord && nWord ){ + rc = sqlite3_keyword_check(zWord, nWord); + } + sqlite3_free(zWord); + return rc ? JNI_TRUE : JNI_FALSE; +} + +S3JniApi(sqlite3_keyword_name(),jstring,1keyword_1name)( + JniArgsEnvClass, jint ndx +){ + const char * zWord = 0; + int n = 0; + jstring rv = 0; + + if( 0==sqlite3_keyword_name(ndx, &zWord, &n) ){ + rv = s3jni_utf8_to_jstring(zWord, n); + } + return rv; +} + + +S3JniApi(sqlite3_last_insert_rowid(),jlong,1last_1insert_1rowid)( + JniArgsEnvClass, jobject jpDb +){ + return (jlong)sqlite3_last_insert_rowid(PtrGet_sqlite3(jpDb)); +} + +S3JniApi(sqlite3_limit(),jint,1limit)( + JniArgsEnvClass, jobject jpDb, jint id, jint newVal +){ + jint rc = 0; + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + if( pDb ){ + rc = sqlite3_limit( pDb, (int)id, (int)newVal ); + } + return rc; +} + +/* Pre-open() code common to sqlite3_open[_v2](). */ +static int s3jni_open_pre(JNIEnv * const env, S3JniEnv **jc, + jstring jDbName, char **zDbName, + S3JniDb ** ps){ + int rc = 0; + jobject jDb = 0; + + *jc = S3JniEnv_get(); + if( !*jc ){ + rc = SQLITE_NOMEM; + goto end; + } + *zDbName = jDbName ? s3jni_jstring_to_utf8( jDbName, 0) : 0; + if( jDbName && !*zDbName ){ + rc = SQLITE_NOMEM; + goto end; + } + jDb = new_java_sqlite3(env, 0); + if( !jDb ){ + sqlite3_free(*zDbName); + *zDbName = 0; + rc = SQLITE_NOMEM; + goto end; + } + *ps = S3JniDb_alloc(env, jDb); + if( *ps ){ + (*jc)->pdbOpening = *ps; + }else{ + S3JniUnrefLocal(jDb); + rc = SQLITE_NOMEM; + } +end: + return rc; +} + +/* +** Post-open() code common to both the sqlite3_open() and +** sqlite3_open_v2() bindings. ps->jDb must be the +** org.sqlite.jni.capi.sqlite3 object which will hold the db's native +** pointer. theRc must be the result code of the open() op. If +** *ppDb is NULL then ps is set aside and its state cleared, +** else ps is associated with *ppDb. If *ppDb is not NULL then +** ps->jDb is stored in jOut (an OutputPointer.sqlite3 instance). +** +** Must be called if s3jni_open_pre() succeeds and must not be called +** if it doesn't. +** +** Returns theRc. +*/ +static int s3jni_open_post(JNIEnv * const env, S3JniEnv * const jc, + S3JniDb * ps, sqlite3 **ppDb, + jobject jOut, int theRc){ + int rc = 0; + jc->pdbOpening = 0; + if( *ppDb ){ + assert(ps->jDb); + if( 0==ps->pDb ){ + ps->pDb = *ppDb; + NativePointerHolder_set(S3JniNph(sqlite3), ps->jDb, *ppDb); + }else{ + assert( ps->pDb==*ppDb + && "Set up via s3jni_run_java_auto_extensions()" ); + } + rc = sqlite3_set_clientdata(ps->pDb, S3JniDb_clientdata_key, + ps, S3JniDb_xDestroy) + /* As of here, the Java/C connection is complete */; + }else{ + S3JniDb_set_aside(ps); + ps = 0; + } + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3), + jOut, ps ? ps->jDb : 0); + return theRc ? theRc : rc; +} + +S3JniApi(sqlite3_open(),jint,1open)( + JniArgsEnvClass, jstring strName, jobject jOut +){ + sqlite3 * pOut = 0; + char *zName = 0; + S3JniDb * ps = 0; + S3JniEnv * jc = 0; + int rc; + + if( 0==jOut ) return SQLITE_MISUSE; + rc = s3jni_open_pre(env, &jc, strName, &zName, &ps); + if( 0==rc ){ + rc = s3jni_open_post(env, jc, ps, &pOut, jOut, + sqlite3_open(zName, &pOut)); + assert(rc==0 ? pOut!=0 : 1); + sqlite3_free(zName); + } + return (jint)rc; +} + +S3JniApi(sqlite3_open_v2(),jint,1open_1v2)( + JniArgsEnvClass, jstring strName, + jobject jOut, jint flags, jstring strVfs +){ + sqlite3 * pOut = 0; + char *zName = 0; + S3JniDb * ps = 0; + S3JniEnv * jc = 0; + char *zVfs = 0; + int rc; + + if( 0==jOut ) return SQLITE_MISUSE; + rc = s3jni_open_pre(env, &jc, strName, &zName, &ps); + if( 0==rc ){ + if( strVfs ){ + zVfs = s3jni_jstring_to_utf8( strVfs, 0); + if( !zVfs ){ + rc = SQLITE_NOMEM; + } + } + if( 0==rc ){ + rc = sqlite3_open_v2(zName, &pOut, (int)flags, zVfs); + } + rc = s3jni_open_post(env, jc, ps, &pOut, jOut, rc); + } + assert(rc==0 ? pOut!=0 : 1); + sqlite3_free(zName); + sqlite3_free(zVfs); + return (jint)rc; +} + +/* Proxy for the sqlite3_prepare[_v2/3]() family. */ +static jint sqlite3_jni_prepare_v123( int prepVersion, JNIEnv * const env, + jclass self, + jlong jpDb, jbyteArray baSql, + jint nMax, jint prepFlags, + jobject jOutStmt, jobject outTail){ + sqlite3_stmt * pStmt = 0; + jobject jStmt = 0; + const char * zTail = 0; + sqlite3 * const pDb = LongPtrGet_sqlite3(jpDb); + jbyte * const pBuf = pDb ? s3jni_jbyteArray_bytes(baSql) : 0; + int rc = SQLITE_ERROR; + + assert(prepVersion==1 || prepVersion==2 || prepVersion==3); + if( !pDb || !jOutStmt ){ + rc = SQLITE_MISUSE; + goto end; + }else if( !pBuf ){ + rc = baSql ? SQLITE_NOMEM : SQLITE_MISUSE; + goto end; + } + jStmt = new_java_sqlite3_stmt(env, 0); + if( !jStmt ){ + rc = SQLITE_NOMEM; + goto end; + } + switch( prepVersion ){ + case 1: rc = sqlite3_prepare(pDb, (const char *)pBuf, + (int)nMax, &pStmt, &zTail); + break; + case 2: rc = sqlite3_prepare_v2(pDb, (const char *)pBuf, + (int)nMax, &pStmt, &zTail); + break; + case 3: rc = sqlite3_prepare_v3(pDb, (const char *)pBuf, + (int)nMax, (unsigned int)prepFlags, + &pStmt, &zTail); + break; + default: + assert(!"Invalid prepare() version"); + } +end: + s3jni_jbyteArray_release(baSql,pBuf); + if( 0==rc ){ + if( 0!=outTail ){ + /* Noting that pBuf is deallocated now but its address is all we need for + ** what follows... */ + assert(zTail ? ((void*)zTail>=(void*)pBuf) : 1); + assert(zTail ? (((int)((void*)zTail - (void*)pBuf)) >= 0) : 1); + OutputPointer_set_Int32( + env, outTail, (int)(zTail ? (zTail - (const char *)pBuf) : 0) + ); + } + if( pStmt ){ + NativePointerHolder_set(S3JniNph(sqlite3_stmt), jStmt, pStmt); + }else{ + /* Happens for comments and whitespace. */ + S3JniUnrefLocal(jStmt); + jStmt = 0; + } + }else{ + S3JniUnrefLocal(jStmt); + jStmt = 0; + } + if( jOutStmt ){ + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3_stmt), + jOutStmt, jStmt); + } + return (jint)rc; +} +S3JniApi(sqlite3_prepare(),jint,1prepare)( + JNIEnv * const env, jclass self, jlong jpDb, jbyteArray baSql, + jint nMax, jobject jOutStmt, jobject outTail +){ + return sqlite3_jni_prepare_v123(1, env, self, jpDb, baSql, nMax, 0, + jOutStmt, outTail); +} +S3JniApi(sqlite3_prepare_v2(),jint,1prepare_1v2)( + JNIEnv * const env, jclass self, jlong jpDb, jbyteArray baSql, + jint nMax, jobject jOutStmt, jobject outTail +){ + return sqlite3_jni_prepare_v123(2, env, self, jpDb, baSql, nMax, 0, + jOutStmt, outTail); +} +S3JniApi(sqlite3_prepare_v3(),jint,1prepare_1v3)( + JNIEnv * const env, jclass self, jlong jpDb, jbyteArray baSql, + jint nMax, jint prepFlags, jobject jOutStmt, jobject outTail +){ + return sqlite3_jni_prepare_v123(3, env, self, jpDb, baSql, nMax, + prepFlags, jOutStmt, outTail); +} + +/* +** Impl for C-to-Java of the callbacks for both sqlite3_update_hook() +** and sqlite3_preupdate_hook(). The differences are that for +** update_hook(): +** +** - pDb is NULL +** - iKey1 is the row ID +** - iKey2 is unused +*/ +static void s3jni_updatepre_hook_impl(void * pState, sqlite3 *pDb, int opId, + const char *zDb, const char *zTable, + sqlite3_int64 iKey1, sqlite3_int64 iKey2){ + S3JniDb * const ps = pState; + S3JniDeclLocal_env; + jstring jDbName; + jstring jTable; + const int isPre = 0!=pDb; + S3JniHook hook; + + S3JniHook_localdup(isPre ? +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + &ps->hooks.preUpdate +#else + &S3JniHook_empty +#endif + : &ps->hooks.update, &hook); + if( !hook.jObj ){ + return; + } + jDbName = s3jni_utf8_to_jstring( zDb, -1); + jTable = jDbName ? s3jni_utf8_to_jstring( zTable, -1) : 0; + S3JniIfThrew { + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_NOMEM, 0); + }else{ + assert( hook.jObj ); + assert( hook.midCallback ); + assert( ps->jDb ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( isPre ) (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, + ps->jDb, (jint)opId, jDbName, jTable, + (jlong)iKey1, (jlong)iKey2); + else +#endif + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, + (jint)opId, jDbName, jTable, (jlong)iKey1); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("sqlite3_(pre)update_hook() callback"); + s3jni_db_exception(ps->pDb, 0, + "sqlite3_(pre)update_hook() callback threw"); + } + } + S3JniUnrefLocal(jDbName); + S3JniUnrefLocal(jTable); + S3JniHook_localundup(hook); +} + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +static void s3jni_preupdate_hook_impl(void * pState, sqlite3 *pDb, int opId, + const char *zDb, const char *zTable, + sqlite3_int64 iKey1, sqlite3_int64 iKey2){ + return s3jni_updatepre_hook_impl(pState, pDb, opId, zDb, zTable, + iKey1, iKey2); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +static void s3jni_update_hook_impl(void * pState, int opId, const char *zDb, + const char *zTable, sqlite3_int64 nRowid){ + return s3jni_updatepre_hook_impl(pState, NULL, opId, zDb, zTable, nRowid, 0); +} + +#if !defined(SQLITE_ENABLE_PREUPDATE_HOOK) +/* We need no-op impls for preupdate_{count,depth,blobwrite}() */ +S3JniApi(sqlite3_preupdate_blobwrite(),jint,1preupdate_1blobwrite)( + JniArgsEnvClass, jlong jDb){ return SQLITE_MISUSE; } +S3JniApi(sqlite3_preupdate_count(),jint,1preupdate_1count)( + JniArgsEnvClass, jlong jDb){ return SQLITE_MISUSE; } +S3JniApi(sqlite3_preupdate_depth(),jint,1preupdate_1depth)( + JniArgsEnvClass, jlong jDb){ return SQLITE_MISUSE; } +#endif /* !SQLITE_ENABLE_PREUPDATE_HOOK */ + +/* +** JNI wrapper for both sqlite3_update_hook() and +** sqlite3_preupdate_hook() (if isPre is true). +*/ +static jobject s3jni_updatepre_hook(JNIEnv * env, int isPre, jlong jpDb, jobject jHook){ + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + jclass klazz; + jobject pOld = 0; + jmethodID xCallback; + S3JniHook * pHook; + + if( !ps ) return 0; + S3JniDb_mutex_enter; + pHook = isPre ? +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + &ps->hooks.preUpdate +#else + 0 +#endif + : &ps->hooks.update; + if( !pHook ){ + goto end; + } + pOld = pHook->jObj; + if( pOld && jHook && (*env)->IsSameObject(env, pOld, jHook) ){ + goto end; + } + if( !jHook ){ + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + *pHook = S3JniHook_empty; +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( isPre ) sqlite3_preupdate_hook(ps->pDb, 0, 0); + else +#endif + sqlite3_update_hook(ps->pDb, 0, 0); + goto end; + } + klazz = (*env)->GetObjectClass(env, jHook); + xCallback = isPre + ? (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/capi/sqlite3;" + "I" + "Ljava/lang/String;" + "Ljava/lang/String;" + "JJ)V") + : (*env)->GetMethodID(env, klazz, "call", + "(ILjava/lang/String;Ljava/lang/String;J)V"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching callback on " + "(pre)update hook object."); + }else{ + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jHook); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( isPre ) sqlite3_preupdate_hook(ps->pDb, s3jni_preupdate_hook_impl, ps); + else +#endif + sqlite3_update_hook(ps->pDb, s3jni_update_hook_impl, ps); + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + } +end: + S3JniDb_mutex_leave; + return pOld; +} + + +S3JniApi(sqlite3_preupdate_hook(),jobject,1preupdate_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + return s3jni_updatepre_hook(env, 1, jpDb, jHook); +#else + return NULL; +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +} + +/* Impl for sqlite3_preupdate_{new,old}(). */ +static int s3jni_preupdate_newold(JNIEnv * const env, int isNew, jlong jpDb, + jint iCol, jobject jOut){ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3 * const pDb = LongPtrGet_sqlite3(jpDb); + int rc = SQLITE_MISUSE; + if( pDb ){ + sqlite3_value * pOut = 0; + int (*fOrig)(sqlite3*,int,sqlite3_value**) = + isNew ? sqlite3_preupdate_new : sqlite3_preupdate_old; + rc = fOrig(pDb, (int)iCol, &pOut); + if( 0==rc ){ + jobject pWrap = new_java_sqlite3_value(env, pOut); + if( !pWrap ){ + rc = SQLITE_NOMEM; + } + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3_value), + jOut, pWrap); + S3JniUnrefLocal(pWrap); + } + } + return rc; +#else + return SQLITE_MISUSE; +#endif +} + +S3JniApi(sqlite3_preupdate_new(),jint,1preupdate_1new)( + JniArgsEnvClass, jlong jpDb, jint iCol, jobject jOut +){ + return s3jni_preupdate_newold(env, 1, jpDb, iCol, jOut); +} + +S3JniApi(sqlite3_preupdate_old(),jint,1preupdate_1old)( + JniArgsEnvClass, jlong jpDb, jint iCol, jobject jOut +){ + return s3jni_preupdate_newold(env, 0, jpDb, iCol, jOut); +} + + +/* Central C-to-Java sqlite3_progress_handler() proxy. */ +static int s3jni_progress_handler_impl(void *pP){ + S3JniDb * const ps = (S3JniDb *)pP; + int rc = 0; + S3JniDeclLocal_env; + S3JniHook hook; + + S3JniHook_localdup(&ps->hooks.progress, &hook); + if( hook.jObj ){ + rc = (int)(*env)->CallIntMethod(env, hook.jObj, hook.midCallback); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, rc, + "sqlite3_progress_handler() callback threw"); + } + S3JniHook_localundup(hook); + } + return rc; +} + +S3JniApi(sqlite3_progress_handler(),void,1progress_1handler)( + JniArgsEnvClass,jobject jDb, jint n, jobject jProgress +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + S3JniHook * const pHook = ps ? &ps->hooks.progress : 0; + + if( !ps ) return; + S3JniDb_mutex_enter; + if( n<1 || !jProgress ){ + S3JniHook_unref(pHook); + sqlite3_progress_handler(ps->pDb, 0, 0, 0); + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jProgress); + jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", "()I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching xCallback() on " + "ProgressHandler object."); + }else{ + S3JniUnrefGlobal(pHook->jObj); + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jProgress); + sqlite3_progress_handler(ps->pDb, (int)n, s3jni_progress_handler_impl, ps); + } + } + S3JniDb_mutex_leave; +} + +S3JniApi(sqlite3_randomness(),void,1randomness)( + JniArgsEnvClass, jbyteArray jTgt +){ + jbyte * const jba = s3jni_jbyteArray_bytes(jTgt); + if( jba ){ + jsize const nTgt = (*env)->GetArrayLength(env, jTgt); + sqlite3_randomness( (int)nTgt, jba ); + s3jni_jbyteArray_commit(jTgt, jba); + } +} + + +S3JniApi(sqlite3_reset(),jint,1reset)( + JniArgsEnvClass, jobject jpStmt +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + return pStmt ? sqlite3_reset(pStmt) : SQLITE_MISUSE; +} + +/* Clears all entries from S3JniGlobal.autoExt. */ +static void s3jni_reset_auto_extension(JNIEnv *env){ + int i; + S3JniAutoExt_mutex_enter; + for( i = 0; i < SJG.autoExt.nExt; ++i ){ + S3JniAutoExtension_clear( &SJG.autoExt.aExt[i] ); + } + SJG.autoExt.nExt = 0; + S3JniAutoExt_mutex_leave; +} + +S3JniApi(sqlite3_reset_auto_extension(),void,1reset_1auto_1extension)( + JniArgsEnvClass +){ + s3jni_reset_auto_extension(env); +} + +/* Impl for sqlite3_result_text/blob() and friends. */ +static void result_blob_text(int as64 /* true for text64/blob64() mode */, + int eTextRep /* 0 for blobs, else SQLITE_UTF... */, + JNIEnv * const env, sqlite3_context *pCx, + jbyteArray jBa, jlong nMax){ + int const asBlob = 0==eTextRep; + if( !pCx ){ + /* We should arguably emit a warning here. But where to log it? */ + return; + }else if( jBa ){ + jbyte * const pBuf = s3jni_jbyteArray_bytes(jBa); + jsize nBA = (*env)->GetArrayLength(env, jBa); + if( nMax>=0 && nBA>(jsize)nMax ){ + nBA = (jsize)nMax; + /** + From the sqlite docs: + + > If the 3rd parameter to any of the sqlite3_result_text* + interfaces other than sqlite3_result_text64() is negative, + then SQLite computes the string length itself by searching + the 2nd parameter for the first zero character. + + Note that the text64() interfaces take an unsigned value for + the length, which Java does not support. This binding takes + the approach of passing on negative values to the C API, + which will in turn fail with SQLITE_TOOBIG at some later + point (recall that the sqlite3_result_xyz() family do not + have result values). + */ + } + if( as64 ){ /* 64-bit... */ + static const jsize nLimit64 = + SQLITE_MAX_ALLOCATION_SIZE/*only _kinda_ arbitrary*/; + if( nBA > nLimit64 ){ + sqlite3_result_error_toobig(pCx); + }else if( asBlob ){ + sqlite3_result_blob64(pCx, pBuf, (sqlite3_uint64)nBA, + SQLITE_TRANSIENT); + }else{ /* text64... */ + if( encodingTypeIsValid(eTextRep) ){ + sqlite3_result_text64(pCx, (const char *)pBuf, + (sqlite3_uint64)nBA, + SQLITE_TRANSIENT, eTextRep); + }else{ + sqlite3_result_error_code(pCx, SQLITE_FORMAT); + } + } + }else{ /* 32-bit... */ + static const jsize nLimit = SQLITE_MAX_ALLOCATION_SIZE; + if( nBA > nLimit ){ + sqlite3_result_error_toobig(pCx); + }else if( asBlob ){ + sqlite3_result_blob(pCx, pBuf, (int)nBA, + SQLITE_TRANSIENT); + }else{ + switch( eTextRep ){ + case SQLITE_UTF8: + sqlite3_result_text(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + case SQLITE_UTF16: + sqlite3_result_text16(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + } + } + s3jni_jbyteArray_release(jBa, pBuf); + } + }else{ + sqlite3_result_null(pCx); + } +} + +S3JniApi(sqlite3_result_blob(),void,1result_1blob)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jint nMax +){ + return result_blob_text(0, 0, env, PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_blob64(),void,1result_1blob64)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jlong nMax +){ + return result_blob_text(1, 0, env, PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_double(),void,1result_1double)( + JniArgsEnvClass, jobject jpCx, jdouble v +){ + sqlite3_result_double(PtrGet_sqlite3_context(jpCx), v); +} + +S3JniApi(sqlite3_result_error(),void,1result_1error)( + JniArgsEnvClass, jobject jpCx, jbyteArray baMsg, jint eTextRep +){ + const char * zUnspecified = "Unspecified error."; + jsize const baLen = (*env)->GetArrayLength(env, baMsg); + jbyte * const pjBuf = baMsg ? s3jni_jbyteArray_bytes(baMsg) : NULL; + switch( pjBuf ? eTextRep : SQLITE_UTF8 ){ + case SQLITE_UTF8: { + const char *zMsg = pjBuf ? (const char *)pjBuf : zUnspecified; + int const n = pjBuf ? (int)baLen : (int)sqlite3Strlen30(zMsg); + sqlite3_result_error(PtrGet_sqlite3_context(jpCx), zMsg, n); + break; + } + case SQLITE_UTF16: { + const void *zMsg = pjBuf; + sqlite3_result_error16(PtrGet_sqlite3_context(jpCx), zMsg, (int)baLen); + break; + } + default: + sqlite3_result_error(PtrGet_sqlite3_context(jpCx), + "Invalid encoding argument passed " + "to sqlite3_result_error().", -1); + break; + } + s3jni_jbyteArray_release(baMsg,pjBuf); +} + +S3JniApi(sqlite3_result_error_code(),void,1result_1error_1code)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_error_code(PtrGet_sqlite3_context(jpCx), (int)v); +} + +S3JniApi(sqlite3_result_error_nomem(),void,1result_1error_1nomem)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_result_error_nomem(PtrGet_sqlite3_context(jpCx)); +} + +S3JniApi(sqlite3_result_error_toobig(),void,1result_1error_1toobig)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_result_error_toobig(PtrGet_sqlite3_context(jpCx)); +} + +S3JniApi(sqlite3_result_int(),void,1result_1int)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_int(PtrGet_sqlite3_context(jpCx), (int)v); +} + +S3JniApi(sqlite3_result_int64(),void,1result_1int64)( + JniArgsEnvClass, jobject jpCx, jlong v +){ + sqlite3_result_int64(PtrGet_sqlite3_context(jpCx), (sqlite3_int64)v); +} + +S3JniApi(sqlite3_result_java_object(),void,1result_1java_1object)( + JniArgsEnvClass, jobject jpCx, jobject v +){ + sqlite3_context * pCx = PtrGet_sqlite3_context(jpCx); + if( !pCx ) return; + else if( v ){ + jobject const rjv = S3JniRefGlobal(v); + if( rjv ){ + sqlite3_result_pointer(pCx, rjv, + s3jni__value_jref_key, S3Jni_jobject_finalizer); + }else{ + sqlite3_result_error_nomem(PtrGet_sqlite3_context(jpCx)); + } + }else{ + sqlite3_result_null(PtrGet_sqlite3_context(jpCx)); + } +} + +S3JniApi(sqlite3_result_nio_buffer(),void,1result_1nio_1buffer)( + JniArgsEnvClass, jobject jpCtx, jobject jBuffer, + jint iOffset, jint iN +){ + sqlite3_context * pCx = PtrGet_sqlite3_context(jpCtx); + int rc; + S3JniNioArgs args; + if( !pCx ){ + return; + }else if( !SJG.g.byteBuffer.klazz ){ + sqlite3_result_error( + pCx, "This JVM does not support JNI access to ByteBuffers.", -1 + ); + return; + } + rc = s3jni_setup_nio_args(env, &args, jBuffer, iOffset, iN); + if(rc){ + if( iOffset<0 ){ + sqlite3_result_error(pCx, "Start index may not be negative.", -1); + }else if( SQLITE_TOOBIG==rc ){ + sqlite3_result_error_toobig(pCx); + }else{ + sqlite3_result_error( + pCx, "Invalid arguments to sqlite3_result_nio_buffer().", -1 + ); + } + }else if( !args.pStart || !args.nOut ){ + sqlite3_result_null(pCx); + }else{ + sqlite3_result_blob(pCx, args.pStart, args.nOut, SQLITE_TRANSIENT); + } +} + + +S3JniApi(sqlite3_result_null(),void,1result_1null)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_result_null(PtrGet_sqlite3_context(jpCx)); +} + +S3JniApi(sqlite3_result_subtype(),void,1result_1subtype)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_subtype(PtrGet_sqlite3_context(jpCx), (unsigned int)v); +} + + +S3JniApi(sqlite3_result_text(),void,1result_1text)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jint nMax +){ + return result_blob_text(0, SQLITE_UTF8, env, + PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_text64(),void,1result_1text64)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jlong nMax, + jint eTextRep +){ + return result_blob_text(1, eTextRep, env, + PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_value(),void,1result_1value)( + JniArgsEnvClass, jobject jpCx, jobject jpSVal +){ + sqlite3_result_value(PtrGet_sqlite3_context(jpCx), + PtrGet_sqlite3_value(jpSVal)); +} + +S3JniApi(sqlite3_result_zeroblob(),void,1result_1zeroblob)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_zeroblob(PtrGet_sqlite3_context(jpCx), (int)v); +} + +S3JniApi(sqlite3_result_zeroblob64(),jint,1result_1zeroblob64)( + JniArgsEnvClass, jobject jpCx, jlong v +){ + return (jint)sqlite3_result_zeroblob64(PtrGet_sqlite3_context(jpCx), + (sqlite3_int64)v); +} + +S3JniApi(sqlite3_rollback_hook(),jobject,1rollback_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + return s3jni_commit_rollback_hook(0, env, jpDb, jHook); +} + +/* Callback for sqlite3_set_authorizer(). */ +int s3jni_xAuth(void* pState, int op,const char*z0, const char*z1, + const char*z2,const char*z3){ + S3JniDb * const ps = pState; + S3JniDeclLocal_env; + S3JniHook hook; + int rc = 0; + + S3JniHook_localdup(&ps->hooks.auth, &hook ); + if( hook.jObj ){ + jstring const s0 = z0 ? s3jni_utf8_to_jstring( z0, -1) : 0; + jstring const s1 = z1 ? s3jni_utf8_to_jstring( z1, -1) : 0; + jstring const s2 = z2 ? s3jni_utf8_to_jstring( z2, -1) : 0; + jstring const s3 = z3 ? s3jni_utf8_to_jstring( z3, -1) : 0; + + rc = (*env)->CallIntMethod(env, hook.jObj, hook.midCallback, (jint)op, + s0, s1, s3, s3); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, rc, "sqlite3_set_authorizer() callback"); + } + S3JniUnrefLocal(s0); + S3JniUnrefLocal(s1); + S3JniUnrefLocal(s2); + S3JniUnrefLocal(s3); + S3JniHook_localundup(hook); + } + return rc; +} + +S3JniApi(sqlite3_set_authorizer(),jint,1set_1authorizer)( + JniArgsEnvClass,jobject jDb, jobject jHook +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + S3JniHook * const pHook = ps ? &ps->hooks.auth : 0; + int rc = 0; + + if( !ps ) return SQLITE_MISUSE; + S3JniDb_mutex_enter; + if( !jHook ){ + S3JniHook_unref(pHook); + rc = sqlite3_set_authorizer( ps->pDb, 0, 0 ); + }else{ + jclass klazz; + if( pHook->jObj ){ + if( (*env)->IsSameObject(env, pHook->jObj, jHook) ){ + /* Same object - this is a no-op. */ + S3JniDb_mutex_leave; + return 0; + } + S3JniHook_unref(pHook); + } + pHook->jObj = S3JniRefGlobal(jHook); + klazz = (*env)->GetObjectClass(env, jHook); + pHook->midCallback = (*env)->GetMethodID(env, klazz, + "call", + "(I" + "Ljava/lang/String;" + "Ljava/lang/String;" + "Ljava/lang/String;" + "Ljava/lang/String;" + ")I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Error setting up Java parts of authorizer hook."); + }else{ + rc = sqlite3_set_authorizer(ps->pDb, s3jni_xAuth, ps); + } + if( rc ) S3JniHook_unref(pHook); + } + S3JniDb_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_set_auxdata(),void,1set_1auxdata)( + JniArgsEnvClass, jobject jCx, jint n, jobject jAux +){ + sqlite3_set_auxdata(PtrGet_sqlite3_context(jCx), (int)n, + S3JniRefGlobal(jAux), S3Jni_jobject_finalizer); +} + +S3JniApi(sqlite3_set_last_insert_rowid(),void,1set_1last_1insert_1rowid)( + JniArgsEnvClass, jobject jpDb, jlong rowId +){ + sqlite3_set_last_insert_rowid(PtrGet_sqlite3(jpDb), + (sqlite3_int64)rowId); +} + +S3JniApi(sqlite3_shutdown(),jint,1shutdown)( + JniArgsEnvClass +){ + s3jni_reset_auto_extension(env); +#ifdef SQLITE_ENABLE_SQLLOG + S3JniHook_unref(&SJG.hook.sqllog); +#endif + S3JniHook_unref(&SJG.hook.configlog); + /* Free up S3JniDb recycling bin. */ + S3JniDb_mutex_enter; { + while( S3JniGlobal.perDb.aFree ){ + S3JniDb * const d = S3JniGlobal.perDb.aFree; + S3JniGlobal.perDb.aFree = d->pNext; + S3JniDb_clear(env, d); + sqlite3_free(d); + } + } S3JniDb_mutex_leave; + S3JniGlobal_mutex_enter; { + /* Free up S3JniUdf recycling bin. */ + while( S3JniGlobal.udf.aFree ){ + S3JniUdf * const u = S3JniGlobal.udf.aFree; + S3JniGlobal.udf.aFree = u->pNext; + u->pNext = 0; + S3JniUdf_free(env, u, 0); + } + } S3JniGlobal_mutex_leave; + S3JniHook_mutex_enter; { + /* Free up S3JniHook recycling bin. */ + while( S3JniGlobal.hook.aFree ){ + S3JniHook * const u = S3JniGlobal.hook.aFree; + S3JniGlobal.hook.aFree = u->pNext; + u->pNext = 0; + assert( !u->doXDestroy ); + assert( !u->jObj ); + assert( !u->jExtra ); + sqlite3_free( u ); + } + } S3JniHook_mutex_leave; + /* Free up env cache. */ + S3JniEnv_mutex_enter; { + while( SJG.envCache.aHead ){ + S3JniEnv_uncache( SJG.envCache.aHead->env ); + } + } S3JniEnv_mutex_leave; + /* Do not clear S3JniGlobal.jvm or S3JniGlobal.g: it's legal to + ** restart the lib. */ + return sqlite3_shutdown(); +} + +S3JniApi(sqlite3_status(),jint,1status)( + JniArgsEnvClass, jint op, jobject jOutCurrent, jobject jOutHigh, + jboolean reset +){ + int iCur = 0, iHigh = 0; + int rc = sqlite3_status( op, &iCur, &iHigh, reset ); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCurrent, iCur); + OutputPointer_set_Int32(env, jOutHigh, iHigh); + } + return (jint)rc; +} + +S3JniApi(sqlite3_status64(),jint,1status64)( + JniArgsEnvClass, jint op, jobject jOutCurrent, jobject jOutHigh, + jboolean reset +){ + sqlite3_int64 iCur = 0, iHigh = 0; + int rc = sqlite3_status64( op, &iCur, &iHigh, reset ); + if( 0==rc ){ + OutputPointer_set_Int64(env, jOutCurrent, iCur); + OutputPointer_set_Int64(env, jOutHigh, iHigh); + } + return (jint)rc; +} + +S3JniApi(sqlite3_stmt_status(),jint,1stmt_1status)( + JniArgsEnvClass, jobject jStmt, jint op, jboolean reset +){ + return sqlite3_stmt_status(PtrGet_sqlite3_stmt(jStmt), + (int)op, reset ? 1 : 0); +} + + +static int s3jni_strlike_glob(int isLike, JNIEnv *const env, + jbyteArray baG, jbyteArray baT, jint escLike){ + int rc = 0; + jbyte * const pG = s3jni_jbyteArray_bytes(baG); + jbyte * const pT = s3jni_jbyteArray_bytes(baT); + + /* Note that we're relying on the byte arrays having been + NUL-terminated on the Java side. */ + rc = isLike + ? sqlite3_strlike((const char *)pG, (const char *)pT, + (unsigned int)escLike) + : sqlite3_strglob((const char *)pG, (const char *)pT); + s3jni_jbyteArray_release(baG, pG); + s3jni_jbyteArray_release(baT, pT); + return rc; +} + +S3JniApi(sqlite3_strglob(),jint,1strglob)( + JniArgsEnvClass, jbyteArray baG, jbyteArray baT +){ + return s3jni_strlike_glob(0, env, baG, baT, 0); +} + +S3JniApi(sqlite3_strlike(),jint,1strlike)( + JniArgsEnvClass, jbyteArray baG, jbyteArray baT, jint escChar +){ + return s3jni_strlike_glob(1, env, baG, baT, escChar); +} + +S3JniApi(sqlite3_sql(),jstring,1sql)( + JniArgsEnvClass, jobject jpStmt +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + jstring rv = 0; + if( pStmt ){ + const char * zSql = 0; + zSql = sqlite3_sql(pStmt); + rv = s3jni_utf8_to_jstring( zSql, -1); + } + return rv; +} + +S3JniApi(sqlite3_step(),jint,1step)( + JniArgsEnvClass, jlong jpStmt +){ + sqlite3_stmt * const pStmt = LongPtrGet_sqlite3_stmt(jpStmt); + return pStmt ? (jint)sqlite3_step(pStmt) : (jint)SQLITE_MISUSE; +} + +S3JniApi(sqlite3_table_column_metadata(),jint,1table_1column_1metadata)( + JniArgsEnvClass, jobject jDb, jstring jDbName, jstring jTableName, + jstring jColumnName, jobject jDataType, jobject jCollSeq, jobject jNotNull, + jobject jPrimaryKey, jobject jAutoinc +){ + sqlite3 * const db = PtrGet_sqlite3(jDb); + char * zDbName = 0, * zTableName = 0, * zColumnName = 0; + const char * pzCollSeq = 0; + const char * pzDataType = 0; + int pNotNull = 0, pPrimaryKey = 0, pAutoinc = 0; + int rc; + + if( !db || !jDbName || !jTableName ) return SQLITE_MISUSE; + zDbName = s3jni_jstring_to_utf8(jDbName,0); + zTableName = zDbName ? s3jni_jstring_to_utf8(jTableName,0) : 0; + zColumnName = (zTableName && jColumnName) + ? s3jni_jstring_to_utf8(jColumnName,0) : 0; + rc = zTableName + ? sqlite3_table_column_metadata(db, zDbName, zTableName, + zColumnName, &pzDataType, &pzCollSeq, + &pNotNull, &pPrimaryKey, &pAutoinc) + : SQLITE_NOMEM; + if( 0==rc ){ + jstring jseq = jCollSeq + ? (pzCollSeq ? s3jni_utf8_to_jstring(pzCollSeq, -1) : 0) + : 0; + jstring jdtype = jDataType + ? (pzDataType ? s3jni_utf8_to_jstring(pzDataType, -1) : 0) + : 0; + if( (jCollSeq && pzCollSeq && !jseq) + || (jDataType && pzDataType && !jdtype) ){ + rc = SQLITE_NOMEM; + }else{ + if( jNotNull ) OutputPointer_set_Bool(env, jNotNull, pNotNull); + if( jPrimaryKey ) OutputPointer_set_Bool(env, jPrimaryKey, pPrimaryKey); + if( jAutoinc ) OutputPointer_set_Bool(env, jAutoinc, pAutoinc); + if( jCollSeq ) OutputPointer_set_String(env, jCollSeq, jseq); + if( jDataType ) OutputPointer_set_String(env, jDataType, jdtype); + } + S3JniUnrefLocal(jseq); + S3JniUnrefLocal(jdtype); + } + sqlite3_free(zDbName); + sqlite3_free(zTableName); + sqlite3_free(zColumnName); + return rc; +} + +static int s3jni_trace_impl(unsigned traceflag, void *pC, void *pP, void *pX){ + S3JniDb * const ps = (S3JniDb *)pC; + S3JniDeclLocal_env; + jobject jX = NULL /* the tracer's X arg */; + jobject jP = NULL /* the tracer's P arg */; + jobject jPUnref = NULL /* potentially a local ref to jP */; + int rc = 0; + S3JniHook hook; + + S3JniHook_localdup(&ps->hooks.trace, &hook ); + if( !hook.jObj ){ + return 0; + } + switch( traceflag ){ + case SQLITE_TRACE_STMT: + jX = s3jni_utf8_to_jstring( (const char *)pX, -1); + if( !jX ) rc = SQLITE_NOMEM; + break; + case SQLITE_TRACE_PROFILE: + jX = (*env)->NewObject(env, SJG.g.cLong, SJG.g.ctorLong1, + (jlong)*((sqlite3_int64*)pX)); + // hmm. ^^^ (*pX) really is zero. + // MARKER(("profile time = %llu\n", *((sqlite3_int64*)pX))); + s3jni_oom_check( jX ); + if( !jX ) rc = SQLITE_NOMEM; + break; + case SQLITE_TRACE_ROW: + break; + case SQLITE_TRACE_CLOSE: + jP = jPUnref = S3JniRefLocal(ps->jDb); + break; + default: + assert(!"cannot happen - unknown trace flag"); + rc = SQLITE_ERROR; + } + if( 0==rc ){ + if( !jP ){ + /* Create a new temporary sqlite3_stmt wrapper */ + jP = jPUnref = new_java_sqlite3_stmt(env, pP); + if( !jP ){ + rc = SQLITE_NOMEM; + } + } + if( 0==rc ){ + assert(jP); + rc = (int)(*env)->CallIntMethod(env, hook.jObj, hook.midCallback, + (jint)traceflag, jP, jX); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, SQLITE_ERROR, + "sqlite3_trace_v2() callback threw."); + } + } + } + S3JniUnrefLocal(jPUnref); + S3JniUnrefLocal(jX); + S3JniHook_localundup(hook); + return rc; +} + +S3JniApi(sqlite3_trace_v2(),jint,1trace_1v2)( + JniArgsEnvClass,jobject jDb, jint traceMask, jobject jTracer +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc; + + if( !ps ) return SQLITE_MISUSE; + if( !traceMask || !jTracer ){ + S3JniDb_mutex_enter; + rc = (jint)sqlite3_trace_v2(ps->pDb, 0, 0, 0); + S3JniHook_unref(&ps->hooks.trace); + S3JniDb_mutex_leave; + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jTracer); + S3JniHook hook = S3JniHook_empty; + hook.midCallback = (*env)->GetMethodID( + env, klazz, "call", "(ILjava/lang/Object;Ljava/lang/Object;)I" + ); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionClear; + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching call() on " + "TracerCallback object."); + }else{ + hook.jObj = S3JniRefGlobal(jTracer); + S3JniDb_mutex_enter; + rc = sqlite3_trace_v2(ps->pDb, (unsigned)traceMask, s3jni_trace_impl, ps); + if( 0==rc ){ + S3JniHook_unref(&ps->hooks.trace); + ps->hooks.trace = hook /* transfer ownership of reference */; + }else{ + S3JniHook_unref(&hook); + } + S3JniDb_mutex_leave; + } + } + return rc; +} + +S3JniApi(sqlite3_txn_state(),jint,1txn_1state)( + JniArgsEnvClass,jobject jDb, jstring jSchema +){ + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + int rc = SQLITE_MISUSE; + if( pDb ){ + char * zSchema = jSchema + ? s3jni_jstring_to_utf8(jSchema, 0) + : 0; + if( !jSchema || (zSchema && jSchema) ){ + rc = sqlite3_txn_state(pDb, zSchema); + sqlite3_free(zSchema); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +S3JniApi(sqlite3_update_hook(),jobject,1update_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + return s3jni_updatepre_hook(env, 0, jpDb, jHook); +} + + +S3JniApi(sqlite3_value_blob(),jbyteArray,1value_1blob)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const jbyte * pBytes = sv ? sqlite3_value_blob(sv) : 0; + int const nLen = pBytes ? sqlite3_value_bytes(sv) : 0; + + s3jni_oom_check( nLen ? !!pBytes : 1 ); + return pBytes + ? s3jni_new_jbyteArray(pBytes, nLen) + : NULL; +} + +S3JniApi(sqlite3_value_bytes(),jint,1value_1bytes)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return sv ? sqlite3_value_bytes(sv) : 0; +} + +S3JniApi(sqlite3_value_bytes16(),jint,1value_1bytes16)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return sv ? sqlite3_value_bytes16(sv) : 0; +} + + +S3JniApi(sqlite3_value_double(),jdouble,1value_1double)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return (jdouble) (sv ? sqlite3_value_double(sv) : 0.0); +} + + +S3JniApi(sqlite3_value_dup(),jobject,1value_1dup)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + sqlite3_value * const sd = sv ? sqlite3_value_dup(sv) : 0; + jobject rv = sd ? new_java_sqlite3_value(env, sd) : 0; + if( sd && !rv ) { + /* OOM */ + sqlite3_value_free(sd); + } + return rv; +} + +S3JniApi(sqlite3_value_free(),void,1value_1free)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + if( sv ){ + sqlite3_value_free(sv); + } +} + +S3JniApi(sqlite3_value_int(),jint,1value_1int)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return (jint) (sv ? sqlite3_value_int(sv) : 0); +} + +S3JniApi(sqlite3_value_int64(),jlong,1value_1int64)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return (jlong) (sv ? sqlite3_value_int64(sv) : 0LL); +} + +S3JniApi(sqlite3_value_java_object(),jobject,1value_1java_1object)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return sv + ? sqlite3_value_pointer(sv, s3jni__value_jref_key) + : 0; +} + +S3JniApi(sqlite3_value_nio_buffer(),jobject,1value_1nio_1buffer)( + JniArgsEnvClass, jobject jVal +){ + sqlite3_value * const sv = PtrGet_sqlite3_value(jVal); + jobject rv = 0; + if( sv ){ + const void * const p = sqlite3_value_blob(sv); + if( p ){ + const int n = sqlite3_value_bytes(sv); + rv = s3jni__blob_to_ByteBuffer(env, p, n); + } + } + return rv; +} + +S3JniApi(sqlite3_value_text(),jbyteArray,1value_1text)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const unsigned char * const p = sv ? sqlite3_value_text(sv) : 0; + int const n = p ? sqlite3_value_bytes(sv) : 0; + return p ? s3jni_new_jbyteArray(p, n) : 0; +} + +#if 0 +// this impl might prove useful. +S3JniApi(sqlite3_value_text(),jstring,1value_1text)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const unsigned char * const p = sv ? sqlite3_value_text(sv) : 0; + int const n = p ? sqlite3_value_bytes(sv) : 0; + return p ? s3jni_utf8_to_jstring( (const char *)p, n) : 0; +} +#endif + +S3JniApi(sqlite3_value_text16(),jstring,1value_1text16)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const int n = sv ? sqlite3_value_bytes16(sv) : 0; + const void * const p = sv ? sqlite3_value_text16(sv) : 0; + return p ? s3jni_text16_to_jstring(env, p, n) : 0; +} + +JniDecl(void,1jni_1internal_1details)(JniArgsEnvClass){ + MARKER(("\nVarious bits of internal info:\n")); + puts("FTS5 is " +#ifdef SQLITE_ENABLE_FTS5 + "available" +#else + "unavailable" +#endif + "." + ); + puts("sizeofs:"); +#define SO(T) printf("\tsizeof(" #T ") = %u\n", (unsigned)sizeof(T)) + SO(void*); + SO(jmethodID); + SO(jfieldID); + SO(S3JniEnv); + SO(S3JniHook); + SO(S3JniDb); + SO(S3JniNphOps); + printf("\t(^^^ %u NativePointerHolder/OutputPointer.T types)\n", + (unsigned)S3Jni_NphCache_size); + SO(S3JniGlobal); + SO(S3JniGlobal.nph); + SO(S3JniGlobal.metrics); + SO(S3JniAutoExtension); + SO(S3JniUdf); +#undef SO +#ifdef SQLITE_JNI_ENABLE_METRICS + printf("Cache info:\n"); + printf("\tJNIEnv cache: %u allocs, %u misses, %u hits\n", + SJG.metrics.nEnvAlloc, SJG.metrics.nEnvMiss, + SJG.metrics.nEnvHit); + printf("Mutex entry:" + "\n\tglobal = %u" + "\n\tenv = %u" + "\n\tnph = %u for S3JniNphOp init" + "\n\thook = %u" + "\n\tperDb = %u" + "\n\tautoExt list = %u" + "\n\tS3JniUdf = %u (free-list)" + "\n\tmetrics = %u\n", + SJG.metrics.nMutexGlobal, SJG.metrics.nMutexEnv, + SJG.metrics.nMutexNph, SJG.metrics.nMutexHook, + SJG.metrics.nMutexPerDb, SJG.metrics.nMutexAutoExt, + SJG.metrics.nMutexUdf, SJG.metrics.nMetrics); + puts("Allocs:"); + printf("\tS3JniDb: %u alloced (*%u = %u bytes), %u recycled\n", + SJG.metrics.nPdbAlloc, (unsigned) sizeof(S3JniDb), + (unsigned)(SJG.metrics.nPdbAlloc * sizeof(S3JniDb)), + SJG.metrics.nPdbRecycled); + printf("\tS3JniUdf: %u alloced (*%u = %u bytes), %u recycled\n", + SJG.metrics.nUdfAlloc, (unsigned) sizeof(S3JniUdf), + (unsigned)(SJG.metrics.nUdfAlloc * sizeof(S3JniUdf)), + SJG.metrics.nUdfRecycled); + printf("\tS3JniHook: %u alloced (*%u = %u bytes), %u recycled\n", + SJG.metrics.nHookAlloc, (unsigned) sizeof(S3JniHook), + (unsigned)(SJG.metrics.nHookAlloc * sizeof(S3JniHook)), + SJG.metrics.nHookRecycled); + printf("\tS3JniEnv: %u alloced (*%u = %u bytes)\n", + SJG.metrics.nEnvAlloc, (unsigned) sizeof(S3JniEnv), + (unsigned)(SJG.metrics.nEnvAlloc * sizeof(S3JniEnv))); + puts("Java-side UDF calls:"); +#define UDF(T) printf("\t%-8s = %u\n", "x" #T, SJG.metrics.udf.n##T) + UDF(Func); UDF(Step); UDF(Final); UDF(Value); UDF(Inverse); +#undef UDF + printf("xDestroy calls across all callback types: %u\n", + SJG.metrics.nDestroy); +#else + puts("Built without SQLITE_JNI_ENABLE_METRICS."); +#endif +} + +//////////////////////////////////////////////////////////////////////// +// End of the sqlite3_... API bindings. Next up, FTS5... +//////////////////////////////////////////////////////////////////////// +#ifdef SQLITE_ENABLE_FTS5 + +/* Creates a verbose JNI Fts5 function name. */ +#define JniFuncNameFtsXA(Suffix) \ + Java_org_sqlite_jni_fts5_Fts5ExtensionApi_ ## Suffix +#define JniFuncNameFtsApi(Suffix) \ + Java_org_sqlite_jni_fts5_fts5_1api_ ## Suffix +#define JniFuncNameFtsTok(Suffix) \ + Java_org_sqlite_jni_fts5_fts5_tokenizer_ ## Suffix + +#define JniDeclFtsXA(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JniFuncNameFtsXA(Suffix) +#define JniDeclFtsApi(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JniFuncNameFtsApi(Suffix) +#define JniDeclFtsTok(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JniFuncNameFtsTok(Suffix) + +#define PtrGet_fts5_api(OBJ) NativePointerHolder_get(OBJ,S3JniNph(fts5_api)) +#define PtrGet_fts5_tokenizer(OBJ) NativePointerHolder_get(OBJ,S3JniNph(fts5_tokenizer)) +#define PtrGet_Fts5Context(OBJ) NativePointerHolder_get(OBJ,S3JniNph(Fts5Context)) +#define PtrGet_Fts5Tokenizer(OBJ) NativePointerHolder_get(OBJ,S3JniNph(Fts5Tokenizer)) +#define s3jni_ftsext() &sFts5Api/*singleton from sqlite3.c*/ +#define Fts5ExtDecl Fts5ExtensionApi const * const ext = s3jni_ftsext() + +/** + State for binding Java-side FTS5 auxiliary functions. +*/ +typedef struct { + jobject jObj /* functor instance */; + jobject jUserData /* 2nd arg to JNI binding of + xCreateFunction(), ostensibly the 3rd arg + to the lib-level xCreateFunction(), except + that we necessarily use that slot for a + Fts5JniAux instance. */; + char * zFuncName /* Only for error reporting and debug logging */; + jmethodID jmid /* callback member's method ID */; +} Fts5JniAux; + +static void Fts5JniAux_free(Fts5JniAux * const s){ + S3JniDeclLocal_env; + if( env ){ + /*MARKER(("FTS5 aux function cleanup: %s\n", s->zFuncName));*/ + s3jni_call_xDestroy(s->jObj); + S3JniUnrefGlobal(s->jObj); + S3JniUnrefGlobal(s->jUserData); + } + sqlite3_free(s->zFuncName); + sqlite3_free(s); +} + +static void Fts5JniAux_xDestroy(void *p){ + if( p ) Fts5JniAux_free(p); +} + +static Fts5JniAux * Fts5JniAux_alloc(JNIEnv * const env, jobject jObj){ + Fts5JniAux * s = s3jni_malloc( sizeof(Fts5JniAux)); + + if( s ){ + jclass klazz; + memset(s, 0, sizeof(Fts5JniAux)); + s->jObj = S3JniRefGlobal(jObj); + klazz = (*env)->GetObjectClass(env, jObj); + s->jmid = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/fts5/Fts5ExtensionApi;" + "Lorg/sqlite/jni/fts5/Fts5Context;" + "Lorg/sqlite/jni/capi/sqlite3_context;" + "[Lorg/sqlite/jni/capi/sqlite3_value;)V"); + S3JniUnrefLocal(klazz); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + Fts5JniAux_free(s); + s = 0; + } + } + return s; +} + +static inline jobject new_java_Fts5Context(JNIEnv * const env, Fts5Context *sv){ + return NativePointerHolder_new(env, S3JniNph(Fts5Context), sv); +} +static inline jobject new_java_fts5_api(JNIEnv * const env, fts5_api *sv){ + return NativePointerHolder_new(env, S3JniNph(fts5_api), sv); +} + +/* +** Returns a per-JNIEnv global ref to the Fts5ExtensionApi singleton +** instance, or NULL on OOM. +*/ +static jobject s3jni_getFts5ExensionApi(JNIEnv * const env){ + if( !SJG.fts5.jExt ){ + S3JniGlobal_mutex_enter; + if( !SJG.fts5.jExt ){ + jobject const pNPH = NativePointerHolder_new( + env, S3JniNph(Fts5ExtensionApi), s3jni_ftsext() + ); + if( pNPH ){ + SJG.fts5.jExt = S3JniRefGlobal(pNPH); + S3JniUnrefLocal(pNPH); + } + } + S3JniGlobal_mutex_leave; + } + return SJG.fts5.jExt; +} + +/* +** Returns a pointer to the fts5_api instance for database connection +** db. If an error occurs, returns NULL and leaves an error in the +** database handle (accessible using sqlite3_errcode()/errmsg()). +*/ +static fts5_api *s3jni_fts5_api_from_db(sqlite3 *db){ + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){ + sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +JniDeclFtsApi(jobject,getInstanceForDb)(JniArgsEnvClass,jobject jDb){ + S3JniDb * const ps = S3JniDb_from_java(jDb); +#if 0 + jobject rv = 0; + if( !ps ) return 0; + else if( ps->fts.jApi ){ + rv = ps->fts.jApi; + }else{ + fts5_api * const pApi = s3jni_fts5_api_from_db(ps->pDb); + if( pApi ){ + rv = new_java_fts5_api(env, pApi); + ps->fts.jApi = rv ? S3JniRefGlobal(rv) : 0; + } + } + return rv; +#else + if( ps && !ps->fts.jApi ){ + S3JniDb_mutex_enter; + if( !ps->fts.jApi ){ + fts5_api * const pApi = s3jni_fts5_api_from_db(ps->pDb); + if( pApi ){ + jobject const rv = new_java_fts5_api(env, pApi); + ps->fts.jApi = rv ? S3JniRefGlobal(rv) : 0; + } + } + S3JniDb_mutex_leave; + } + return ps ? ps->fts.jApi : 0; +#endif +} + + +JniDeclFtsXA(jobject,getInstance)(JniArgsEnvClass){ + return s3jni_getFts5ExensionApi(env); +} + +JniDeclFtsXA(jint,xColumnCount)(JniArgsEnvObj,jobject jCtx){ + Fts5ExtDecl; + return (jint)ext->xColumnCount(PtrGet_Fts5Context(jCtx)); +} + +JniDeclFtsXA(jint,xColumnSize)(JniArgsEnvObj,jobject jCtx, jint iIdx, jobject jOut32){ + Fts5ExtDecl; + int n1 = 0; + int const rc = ext->xColumnSize(PtrGet_Fts5Context(jCtx), (int)iIdx, &n1); + if( 0==rc ) OutputPointer_set_Int32(env, jOut32, n1); + return rc; +} + +JniDeclFtsXA(jint,xColumnText)(JniArgsEnvObj,jobject jCtx, jint iCol, + jobject jOut){ + Fts5ExtDecl; + const char *pz = 0; + int pn = 0; + int rc = ext->xColumnText(PtrGet_Fts5Context(jCtx), (int)iCol, + &pz, &pn); + if( 0==rc ){ + jstring jstr = pz ? s3jni_utf8_to_jstring( pz, pn) : 0; + if( pz ){ + if( jstr ){ + OutputPointer_set_String(env, jOut, jstr); + S3JniUnrefLocal(jstr)/*jOut has a reference*/; + }else{ + rc = SQLITE_NOMEM; + } + } + } + return (jint)rc; +} + +JniDeclFtsXA(jint,xColumnTotalSize)(JniArgsEnvObj,jobject jCtx, jint iCol, jobject jOut64){ + Fts5ExtDecl; + sqlite3_int64 nOut = 0; + int const rc = ext->xColumnTotalSize(PtrGet_Fts5Context(jCtx), (int)iCol, &nOut); + if( 0==rc && jOut64 ) OutputPointer_set_Int64(env, jOut64, (jlong)nOut); + return (jint)rc; +} + +/* +** Proxy for fts5_extension_function instances plugged in via +** fts5_api::xCreateFunction(). +*/ +static void s3jni_fts5_extension_function(Fts5ExtensionApi const *pApi, + Fts5Context *pFts, + sqlite3_context *pCx, + int argc, + sqlite3_value **argv){ + Fts5JniAux * const pAux = pApi->xUserData(pFts); + jobject jpCx = 0; + jobjectArray jArgv = 0; + jobject jpFts = 0; + jobject jFXA; + int rc; + S3JniDeclLocal_env; + + assert(pAux); + jFXA = s3jni_getFts5ExensionApi(env); + if( !jFXA ) goto error_oom; + jpFts = new_java_Fts5Context(env, pFts); + if( !jpFts ) goto error_oom; + rc = udf_args(env, pCx, argc, argv, &jpCx, &jArgv); + if( rc ) goto error_oom; + (*env)->CallVoidMethod(env, pAux->jObj, pAux->jmid, + jFXA, jpFts, jpCx, jArgv); + S3JniIfThrew{ + udf_report_exception(env, 1, pCx, pAux->zFuncName, "call"); + } + udf_unargs(env, jpCx, argc, jArgv); + S3JniUnrefLocal(jpFts); + S3JniUnrefLocal(jpCx); + S3JniUnrefLocal(jArgv); + return; +error_oom: + s3jni_db_oom( sqlite3_context_db_handle(pCx) ); + assert( !jArgv ); + assert( !jpCx ); + S3JniUnrefLocal(jpFts); + sqlite3_result_error_nomem(pCx); + return; +} + +JniDeclFtsApi(jint,xCreateFunction)(JniArgsEnvObj, jstring jName, + jobject jUserData, jobject jFunc){ + fts5_api * const pApi = PtrGet_fts5_api(jSelf); + int rc; + char * zName; + Fts5JniAux * pAux; + + assert(pApi); + zName = s3jni_jstring_to_utf8( jName, 0); + if(!zName) return SQLITE_NOMEM; + pAux = Fts5JniAux_alloc(env, jFunc); + if( pAux ){ + rc = pApi->xCreateFunction(pApi, zName, pAux, + s3jni_fts5_extension_function, + Fts5JniAux_xDestroy); + }else{ + rc = SQLITE_NOMEM; + } + if( 0==rc ){ + pAux->jUserData = jUserData ? S3JniRefGlobal(jUserData) : 0; + pAux->zFuncName = zName; + }else{ + sqlite3_free(zName); + } + return (jint)rc; +} + + +typedef struct S3JniFts5AuxData S3JniFts5AuxData; +/* +** TODO: this middle-man struct is no longer necessary. Conider +** removing it and passing around jObj itself instead. +*/ +struct S3JniFts5AuxData { + jobject jObj; +}; + +static void S3JniFts5AuxData_xDestroy(void *x){ + if( x ){ + S3JniFts5AuxData * const p = x; + if( p->jObj ){ + S3JniDeclLocal_env; + s3jni_call_xDestroy(p->jObj); + S3JniUnrefGlobal(p->jObj); + } + sqlite3_free(x); + } +} + +JniDeclFtsXA(jobject,xGetAuxdata)(JniArgsEnvObj,jobject jCtx, jboolean bClear){ + Fts5ExtDecl; + jobject rv = 0; + S3JniFts5AuxData * const pAux = ext->xGetAuxdata(PtrGet_Fts5Context(jCtx), bClear); + if( pAux ){ + if( bClear ){ + if( pAux->jObj ){ + rv = S3JniRefLocal(pAux->jObj); + S3JniUnrefGlobal(pAux->jObj); + } + /* Note that we do not call xDestroy() in this case. */ + sqlite3_free(pAux); + }else{ + rv = pAux->jObj; + } + } + return rv; +} + +JniDeclFtsXA(jint,xInst)(JniArgsEnvObj,jobject jCtx, jint iIdx, jobject jOutPhrase, + jobject jOutCol, jobject jOutOff){ + Fts5ExtDecl; + int n1 = 0, n2 = 2, n3 = 0; + int const rc = ext->xInst(PtrGet_Fts5Context(jCtx), (int)iIdx, &n1, &n2, &n3); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutPhrase, n1); + OutputPointer_set_Int32(env, jOutCol, n2); + OutputPointer_set_Int32(env, jOutOff, n3); + } + return rc; +} + +JniDeclFtsXA(jint,xInstCount)(JniArgsEnvObj,jobject jCtx, jobject jOut32){ + Fts5ExtDecl; + int nOut = 0; + int const rc = ext->xInstCount(PtrGet_Fts5Context(jCtx), &nOut); + if( 0==rc && jOut32 ) OutputPointer_set_Int32(env, jOut32, nOut); + return (jint)rc; +} + +JniDeclFtsXA(jint,xPhraseCount)(JniArgsEnvObj,jobject jCtx){ + Fts5ExtDecl; + return (jint)ext->xPhraseCount(PtrGet_Fts5Context(jCtx)); +} + +/* Copy the 'a' and 'b' fields from pSrc to Fts5PhraseIter object jIter. */ +static void s3jni_phraseIter_NToJ(JNIEnv *const env, + Fts5PhraseIter const * const pSrc, + jobject jIter){ + S3JniGlobalType * const g = &S3JniGlobal; + assert(g->fts5.jPhraseIter.fidA); + (*env)->SetLongField(env, jIter, g->fts5.jPhraseIter.fidA, + S3JniCast_P2L(pSrc->a)); + S3JniExceptionIsFatal("Cannot set Fts5PhraseIter.a field."); + (*env)->SetLongField(env, jIter, g->fts5.jPhraseIter.fidB, + S3JniCast_P2L(pSrc->b)); + S3JniExceptionIsFatal("Cannot set Fts5PhraseIter.b field."); +} + +/* Copy the 'a' and 'b' fields from Fts5PhraseIter object jIter to pDest. */ +static void s3jni_phraseIter_JToN(JNIEnv *const env, jobject jIter, + Fts5PhraseIter * const pDest){ + S3JniGlobalType * const g = &S3JniGlobal; + assert(g->fts5.jPhraseIter.fidA); + pDest->a = S3JniCast_L2P( + (*env)->GetLongField(env, jIter, g->fts5.jPhraseIter.fidA) + ); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.a field."); + pDest->b = S3JniCast_L2P( + (*env)->GetLongField(env, jIter, g->fts5.jPhraseIter.fidB) + ); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.b field."); +} + +JniDeclFtsXA(jint,xPhraseFirst)(JniArgsEnvObj,jobject jCtx, jint iPhrase, + jobject jIter, jobject jOutCol, + jobject jOutOff){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int rc, iCol = 0, iOff = 0; + rc = ext->xPhraseFirst(PtrGet_Fts5Context(jCtx), (int)iPhrase, + &iter, &iCol, &iOff); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCol, iCol); + OutputPointer_set_Int32(env, jOutOff, iOff); + s3jni_phraseIter_NToJ(env, &iter, jIter); + } + return rc; +} + +JniDeclFtsXA(jint,xPhraseFirstColumn)(JniArgsEnvObj,jobject jCtx, jint iPhrase, + jobject jIter, jobject jOutCol){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int rc, iCol = 0; + rc = ext->xPhraseFirstColumn(PtrGet_Fts5Context(jCtx), (int)iPhrase, + &iter, &iCol); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCol, iCol); + s3jni_phraseIter_NToJ(env, &iter, jIter); + } + return rc; +} + +JniDeclFtsXA(void,xPhraseNext)(JniArgsEnvObj,jobject jCtx, jobject jIter, + jobject jOutCol, jobject jOutOff){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int iCol = 0, iOff = 0; + s3jni_phraseIter_JToN(env, jIter, &iter); + ext->xPhraseNext(PtrGet_Fts5Context(jCtx), &iter, &iCol, &iOff); + OutputPointer_set_Int32(env, jOutCol, iCol); + OutputPointer_set_Int32(env, jOutOff, iOff); + s3jni_phraseIter_NToJ(env, &iter, jIter); +} + +JniDeclFtsXA(void,xPhraseNextColumn)(JniArgsEnvObj,jobject jCtx, jobject jIter, + jobject jOutCol){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int iCol = 0; + s3jni_phraseIter_JToN(env, jIter, &iter); + ext->xPhraseNextColumn(PtrGet_Fts5Context(jCtx), &iter, &iCol); + OutputPointer_set_Int32(env, jOutCol, iCol); + s3jni_phraseIter_NToJ(env, &iter, jIter); +} + + +JniDeclFtsXA(jint,xPhraseSize)(JniArgsEnvObj,jobject jCtx, jint iPhrase){ + Fts5ExtDecl; + return (jint)ext->xPhraseSize(PtrGet_Fts5Context(jCtx), (int)iPhrase); +} + +/* State for use with xQueryPhrase() and xTokenize(). */ +struct s3jni_xQueryPhraseState { + Fts5ExtensionApi const * ext; + jmethodID midCallback; /* jCallback->call() method */ + jobject jCallback; /* Fts5ExtensionApi.XQueryPhraseCallback instance */ + jobject jFcx; /* (Fts5Context*) for xQueryPhrase() + callback. This is NOT the instance that is + passed to xQueryPhrase(), it's the one + created by xQueryPhrase() for use by its + callback. */ + /* State for xTokenize() */ + struct { + const char * zPrev; + int nPrev; + jbyteArray jba; + } tok; +}; + +static int s3jni_xQueryPhrase(const Fts5ExtensionApi *xapi, + Fts5Context * pFcx, void *pData){ + struct s3jni_xQueryPhraseState * const s = pData; + S3JniDeclLocal_env; + + if( !s->jFcx ){ + s->jFcx = new_java_Fts5Context(env, pFcx); + if( !s->jFcx ) return SQLITE_NOMEM; + } + int rc = (int)(*env)->CallIntMethod(env, s->jCallback, s->midCallback, + SJG.fts5.jExt, s->jFcx); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("xQueryPhrase() callback"); + S3JniExceptionClear; + rc = SQLITE_ERROR; + } + return rc; +} + +JniDeclFtsXA(jint,xQueryPhrase)(JniArgsEnvObj,jobject jFcx, jint iPhrase, + jobject jCallback){ + Fts5ExtDecl; + int rc; + struct s3jni_xQueryPhraseState s; + jclass klazz = jCallback ? (*env)->GetObjectClass(env, jCallback) : NULL; + + if( !klazz ) return SQLITE_MISUSE; + s.jCallback = jCallback; + s.jFcx = 0; + s.ext = ext; + s.midCallback = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/fts5/Fts5ExtensionApi;" + "Lorg/sqlite/jni/fts5/Fts5Context;)I"); + S3JniUnrefLocal(klazz); + S3JniExceptionIsFatal("Could not extract xQueryPhraseCallback.call() method."); + rc = ext->xQueryPhrase(PtrGet_Fts5Context(jFcx), iPhrase, &s, + s3jni_xQueryPhrase); + S3JniUnrefLocal(s.jFcx); + return (jint)rc; +} + + +JniDeclFtsXA(jint,xRowCount)(JniArgsEnvObj,jobject jCtx, jobject jOut64){ + Fts5ExtDecl; + sqlite3_int64 nOut = 0; + int const rc = ext->xRowCount(PtrGet_Fts5Context(jCtx), &nOut); + if( 0==rc && jOut64 ) OutputPointer_set_Int64(env, jOut64, (jlong)nOut); + return (jint)rc; +} + +JniDeclFtsXA(jlong,xRowid)(JniArgsEnvObj,jobject jCtx){ + Fts5ExtDecl; + return (jlong)ext->xRowid(PtrGet_Fts5Context(jCtx)); +} + +JniDeclFtsXA(jint,xSetAuxdata)(JniArgsEnvObj,jobject jCtx, jobject jAux){ + Fts5ExtDecl; + int rc; + S3JniFts5AuxData * pAux; + + pAux = s3jni_malloc( sizeof(*pAux)); + if( !pAux ){ + if( jAux ){ + /* Emulate how xSetAuxdata() behaves when it cannot alloc + ** its auxdata wrapper. */ + s3jni_call_xDestroy(jAux); + } + return SQLITE_NOMEM; + } + pAux->jObj = S3JniRefGlobal(jAux); + rc = ext->xSetAuxdata(PtrGet_Fts5Context(jCtx), pAux, + S3JniFts5AuxData_xDestroy); + return rc; +} + +/* xToken() impl for xTokenize(). */ +static int s3jni_xTokenize_xToken(void *p, int tFlags, const char* z, + int nZ, int iStart, int iEnd){ + int rc; + S3JniDeclLocal_env; + struct s3jni_xQueryPhraseState * const s = p; + jbyteArray jba; + + S3JniUnrefLocal(s->tok.jba); + s->tok.zPrev = z; + s->tok.nPrev = nZ; + s->tok.jba = s3jni_new_jbyteArray(z, nZ); + if( !s->tok.jba ) return SQLITE_NOMEM; + jba = s->tok.jba; + rc = (int)(*env)->CallIntMethod(env, s->jCallback, s->midCallback, + (jint)tFlags, jba, (jint)iStart, + (jint)iEnd); + S3JniIfThrew { + S3JniExceptionWarnCallbackThrew("xTokenize() callback"); + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Proxy for Fts5ExtensionApi.xTokenize() and +** fts5_tokenizer.xTokenize() +*/ +static jint s3jni_fts5_xTokenize(JniArgsEnvObj, S3JniNphOp const *pRef, + jint tokFlags, jobject jFcx, + jbyteArray jbaText, jobject jCallback){ + Fts5ExtDecl; + struct s3jni_xQueryPhraseState s; + int rc = 0; + jbyte * const pText = jCallback ? s3jni_jbyteArray_bytes(jbaText) : 0; + jsize nText = pText ? (*env)->GetArrayLength(env, jbaText) : 0; + jclass const klazz = jCallback ? (*env)->GetObjectClass(env, jCallback) : NULL; + + if( !klazz ) return SQLITE_MISUSE; + memset(&s, 0, sizeof(s)); + s.jCallback = jCallback; + s.jFcx = jFcx; + s.ext = ext; + s.midCallback = (*env)->GetMethodID(env, klazz, "call", "(I[BII)I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionReport; + S3JniExceptionClear; + s3jni_jbyteArray_release(jbaText, pText); + return SQLITE_ERROR; + } + s.tok.jba = S3JniRefLocal(jbaText); + s.tok.zPrev = (const char *)pText; + s.tok.nPrev = (int)nText; + if( pRef == S3JniNph(Fts5ExtensionApi) ){ + rc = ext->xTokenize(PtrGet_Fts5Context(jFcx), + (const char *)pText, (int)nText, + &s, s3jni_xTokenize_xToken); + }else if( pRef == S3JniNph(fts5_tokenizer) ){ + fts5_tokenizer * const pTok = PtrGet_fts5_tokenizer(jSelf); + rc = pTok->xTokenize(PtrGet_Fts5Tokenizer(jFcx), &s, tokFlags, + (const char *)pText, (int)nText, + s3jni_xTokenize_xToken); + }else{ + (*env)->FatalError(env, "This cannot happen. Maintenance required."); + } + if( s.tok.jba ){ + assert( s.tok.zPrev ); + S3JniUnrefLocal(s.tok.jba); + } + s3jni_jbyteArray_release(jbaText, pText); + return (jint)rc; +} + +JniDeclFtsXA(jint,xTokenize)(JniArgsEnvObj,jobject jFcx, jbyteArray jbaText, + jobject jCallback){ + return s3jni_fts5_xTokenize(env, jSelf, S3JniNph(Fts5ExtensionApi), + 0, jFcx, jbaText, jCallback); +} + +JniDeclFtsTok(jint,xTokenize)(JniArgsEnvObj,jobject jFcx, jint tokFlags, + jbyteArray jbaText, jobject jCallback){ + return s3jni_fts5_xTokenize(env, jSelf, S3JniNph(Fts5Tokenizer), + tokFlags, jFcx, jbaText, jCallback); +} + + +JniDeclFtsXA(jobject,xUserData)(JniArgsEnvObj,jobject jFcx){ + Fts5ExtDecl; + Fts5JniAux * const pAux = ext->xUserData(PtrGet_Fts5Context(jFcx)); + return pAux ? pAux->jUserData : 0; +} + +#endif /* SQLITE_ENABLE_FTS5 */ + +//////////////////////////////////////////////////////////////////////// +// End of the main API bindings. Start of SQLTester bits... +//////////////////////////////////////////////////////////////////////// + +#ifdef SQLITE_JNI_ENABLE_SQLTester +typedef struct SQLTesterJni SQLTesterJni; +struct SQLTesterJni { + sqlite3_int64 nDup; +}; +static SQLTesterJni SQLTester = { + 0 +}; + +static void SQLTester_dup_destructor(void*pToFree){ + u64 *p = (u64*)pToFree; + assert( p!=0 ); + p--; + assert( p[0]==0x2bbf4b7c ); + p[0] = 0; + p[1] = 0; + sqlite3_free(p); +} + +/* +** Implementation of +** +** dup(TEXT) +** +** This SQL function simply makes a copy of its text argument. But it +** returns the result using a custom destructor, in order to provide +** tests for the use of Mem.xDel() in the SQLite VDBE. +*/ +static void SQLTester_dup_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + u64 *pOut; + char *z; + int n = sqlite3_value_bytes(argv[0]); + SQLTesterJni * const p = (SQLTesterJni *)sqlite3_user_data(context); + S3JniDeclLocal_env; + + ++p->nDup; + if( n>0 && (pOut = s3jni_malloc( (n+16)&~7 ))!=0 ){ + pOut[0] = 0x2bbf4b7c; + z = (char*)&pOut[1]; + memcpy(z, sqlite3_value_text(argv[0]), n); + z[n] = 0; + sqlite3_result_text(context, z, n, SQLTester_dup_destructor); + } + return; +} + +/* +** Return the number of calls to the dup() SQL function since the +** SQLTester context was opened or since the last dup_count() call. +*/ +static void SQLTester_dup_count_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SQLTesterJni * const p = (SQLTesterJni *)sqlite3_user_data(context); + sqlite3_result_int64(context, p->nDup); + p->nDup = 0; +} + +/* +** Return non-zero if string z matches glob pattern zGlob and zero if the +** pattern does not match. +** +** To repeat: +** +** zero == no match +** non-zero == match +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front, or a hexadecimal +** literal of the form 0x... +*/ +static int SQLTester_strnotglob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && SQLTester_strnotglob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( SQLTester_strnotglob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( z[0]=='0' + && (z[1]=='x' || z[1]=='X') + && sqlite3Isxdigit(z[2]) + ){ + z += 3; + while( sqlite3Isxdigit(z[0]) ){ z++; } + }else{ + if( (z[0]=='-' || z[0]=='+') && sqlite3Isdigit(z[1]) ) z++; + if( !sqlite3Isdigit(z[0]) ) return 0; + z++; + while( sqlite3Isdigit(z[0]) ){ z++; } + } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + return *z==0; +} + +JNIEXPORT jint JNICALL +Java_org_sqlite_jni_capi_SQLTester_strglob( + JniArgsEnvClass, jbyteArray baG, jbyteArray baT +){ + int rc = 0; + jbyte * const pG = s3jni_jbyteArray_bytes(baG); + jbyte * const pT = pG ? s3jni_jbyteArray_bytes(baT) : 0; + + s3jni_oom_fatal(pT); + /* Note that we're relying on the byte arrays having been + NUL-terminated on the Java side. */ + rc = !SQLTester_strnotglob((const char *)pG, (const char *)pT); + s3jni_jbyteArray_release(baG, pG); + s3jni_jbyteArray_release(baT, pT); + return rc; +} + + +static int SQLTester_auto_extension(sqlite3 *pDb, const char **pzErr, + const struct sqlite3_api_routines *ignored){ + sqlite3_create_function(pDb, "dup", 1, SQLITE_UTF8, &SQLTester, + SQLTester_dup_func, 0, 0); + sqlite3_create_function(pDb, "dup_count", 0, SQLITE_UTF8, &SQLTester, + SQLTester_dup_count_func, 0, 0); + return 0; +} + +JNIEXPORT void JNICALL +Java_org_sqlite_jni_capi_SQLTester_installCustomExtensions(JniArgsEnvClass){ + sqlite3_auto_extension( (void(*)(void))SQLTester_auto_extension ); +} + +#endif /* SQLITE_JNI_ENABLE_SQLTester */ +//////////////////////////////////////////////////////////////////////// +// End of SQLTester bindings. Start of lower-level bits. +//////////////////////////////////////////////////////////////////////// + +/* +** Called during static init of the CApi class to set up global +** state. +*/ +JNIEXPORT void JNICALL +Java_org_sqlite_jni_capi_CApi_init(JniArgsEnvClass){ + jclass klazz; + + memset(&S3JniGlobal, 0, sizeof(S3JniGlobal)); + if( (*env)->GetJavaVM(env, &SJG.jvm) ){ + (*env)->FatalError(env, "GetJavaVM() failure shouldn't be possible."); + return; + } + + /* Grab references to various global classes and objects... */ + SJG.g.cLong = S3JniRefGlobal((*env)->FindClass(env,"java/lang/Long")); + S3JniExceptionIsFatal("Error getting reference to Long class."); + SJG.g.ctorLong1 = (*env)->GetMethodID(env, SJG.g.cLong, + "", "(J)V"); + S3JniExceptionIsFatal("Error getting reference to Long constructor."); + + SJG.g.cString = S3JniRefGlobal((*env)->FindClass(env,"java/lang/String")); + S3JniExceptionIsFatal("Error getting reference to String class."); + SJG.g.ctorStringBA = + (*env)->GetMethodID(env, SJG.g.cString, + "", "([BLjava/nio/charset/Charset;)V"); + S3JniExceptionIsFatal("Error getting reference to String(byte[],Charset) ctor."); + SJG.g.stringGetBytes = + (*env)->GetMethodID(env, SJG.g.cString, + "getBytes", "(Ljava/nio/charset/Charset;)[B"); + S3JniExceptionIsFatal("Error getting reference to String.getBytes(Charset)."); + + { /* java.nio.charset.StandardCharsets.UTF_8 */ + jfieldID fUtf8; + klazz = (*env)->FindClass(env,"java/nio/charset/StandardCharsets"); + S3JniExceptionIsFatal("Error getting reference to StandardCharsets class."); + fUtf8 = (*env)->GetStaticFieldID(env, klazz, "UTF_8", + "Ljava/nio/charset/Charset;"); + S3JniExceptionIsFatal("Error getting StandardCharsets.UTF_8 field."); + SJG.g.oCharsetUtf8 = + S3JniRefGlobal((*env)->GetStaticObjectField(env, klazz, fUtf8)); + S3JniExceptionIsFatal("Error getting reference to StandardCharsets.UTF_8."); + S3JniUnrefLocal(klazz); + } + +#ifdef SQLITE_ENABLE_FTS5 + klazz = (*env)->FindClass(env, "org/sqlite/jni/fts5/Fts5PhraseIter"); + S3JniExceptionIsFatal("Error getting reference to org.sqlite.jni.fts5.Fts5PhraseIter."); + SJG.fts5.jPhraseIter.fidA = (*env)->GetFieldID(env, klazz, "a", "J"); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.a field."); + SJG.fts5.jPhraseIter.fidB = (*env)->GetFieldID(env, klazz, "b", "J"); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.b field."); + S3JniUnrefLocal(klazz); +#endif + + SJG.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.mutex ); + SJG.hook.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.hook.mutex ); + SJG.nph.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.nph.mutex ); + SJG.envCache.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.envCache.mutex ); + SJG.perDb.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.perDb.mutex ); + SJG.autoExt.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.autoExt.mutex ); + +#if S3JNI_METRICS_MUTEX + SJG.metrics.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.metrics.mutex ); +#endif + + { + /* Test whether this JVM supports direct memory access via + ByteBuffer. */ + unsigned char buf[16] = {0}; + jobject bb = (*env)->NewDirectByteBuffer(env, buf, 16); + if( bb ){ + SJG.g.byteBuffer.klazz = S3JniRefGlobal((*env)->GetObjectClass(env, bb)); + SJG.g.byteBuffer.midAlloc = (*env)->GetStaticMethodID( + env, SJG.g.byteBuffer.klazz, "allocateDirect", "(I)Ljava/nio/ByteBuffer;" + ); + S3JniExceptionIsFatal("Error getting ByteBuffer.allocateDirect() method."); + SJG.g.byteBuffer.midLimit = (*env)->GetMethodID( + env, SJG.g.byteBuffer.klazz, "limit", "()I" + ); + S3JniExceptionIsFatal("Error getting ByteBuffer.limit() method."); + S3JniUnrefLocal(bb); + }else{ + SJG.g.byteBuffer.klazz = 0; + SJG.g.byteBuffer.midAlloc = 0; + } + } + + sqlite3_shutdown() + /* So that it becomes legal for Java-level code to call + ** sqlite3_config(). */; +} diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h new file mode 100644 index 0000000000000000000000000000000000000000..082a202122c89b1884b16149f37e1c5877efcea1 --- /dev/null +++ b/ext/jni/src/c/sqlite3-jni.h @@ -0,0 +1,2461 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_capi_CApi */ + +#ifndef _Included_org_sqlite_jni_capi_CApi +#define _Included_org_sqlite_jni_capi_CApi +#ifdef __cplusplus +extern "C" { +#endif +#undef org_sqlite_jni_capi_CApi_SQLITE_ACCESS_EXISTS +#define org_sqlite_jni_capi_CApi_SQLITE_ACCESS_EXISTS 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READWRITE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READ +#define org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READ 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_DENY +#define org_sqlite_jni_capi_CApi_SQLITE_DENY 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_IGNORE +#define org_sqlite_jni_capi_CApi_SQLITE_IGNORE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_INDEX 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TABLE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_INDEX 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TABLE 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TRIGGER 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_VIEW 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TRIGGER 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_VIEW 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_DELETE +#define org_sqlite_jni_capi_CApi_SQLITE_DELETE 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_INDEX 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TABLE 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_INDEX 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TABLE 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TRIGGER 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_VIEW 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TRIGGER 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_VIEW 17L +#undef org_sqlite_jni_capi_CApi_SQLITE_INSERT +#define org_sqlite_jni_capi_CApi_SQLITE_INSERT 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_PRAGMA +#define org_sqlite_jni_capi_CApi_SQLITE_PRAGMA 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_READ +#define org_sqlite_jni_capi_CApi_SQLITE_READ 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_SELECT +#define org_sqlite_jni_capi_CApi_SQLITE_SELECT 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRANSACTION +#define org_sqlite_jni_capi_CApi_SQLITE_TRANSACTION 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_UPDATE +#define org_sqlite_jni_capi_CApi_SQLITE_UPDATE 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_ATTACH +#define org_sqlite_jni_capi_CApi_SQLITE_ATTACH 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_DETACH +#define org_sqlite_jni_capi_CApi_SQLITE_DETACH 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_ALTER_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_ALTER_TABLE 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_REINDEX +#define org_sqlite_jni_capi_CApi_SQLITE_REINDEX 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_ANALYZE +#define org_sqlite_jni_capi_CApi_SQLITE_ANALYZE 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_VTABLE +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_VTABLE 29L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_VTABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_VTABLE 30L +#undef org_sqlite_jni_capi_CApi_SQLITE_FUNCTION +#define org_sqlite_jni_capi_CApi_SQLITE_FUNCTION 31L +#undef org_sqlite_jni_capi_CApi_SQLITE_SAVEPOINT +#define org_sqlite_jni_capi_CApi_SQLITE_SAVEPOINT 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_RECURSIVE +#define org_sqlite_jni_capi_CApi_SQLITE_RECURSIVE 33L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATIC +#define org_sqlite_jni_capi_CApi_SQLITE_STATIC 0LL +#undef org_sqlite_jni_capi_CApi_SQLITE_TRANSIENT +#define org_sqlite_jni_capi_CApi_SQLITE_TRANSIENT -1LL +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETSTART_INVERT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETSTART_INVERT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_NOSAVEPOINT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_NOSAVEPOINT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_INVERT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_INVERT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_IGNORENOOP +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_IGNORENOOP 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_DATA +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_DATA 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_NOTFOUND +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_NOTFOUND 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONFLICT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONFLICT 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONSTRAINT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONSTRAINT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_FOREIGN_KEY +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_FOREIGN_KEY 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_OMIT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_OMIT 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_REPLACE +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_REPLACE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_ABORT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_ABORT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SINGLETHREAD +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SINGLETHREAD 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MULTITHREAD +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MULTITHREAD 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SERIALIZED +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SERIALIZED 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MALLOC +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MALLOC 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMALLOC +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMALLOC 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SCRATCH +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SCRATCH 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PAGECACHE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PAGECACHE 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_HEAP +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_HEAP 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMSTATUS +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMSTATUS 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MUTEX 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMUTEX 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOOKASIDE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOOKASIDE 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOG +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOG 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_URI +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_URI 17L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE2 +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE2 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE2 +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE2 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_COVERING_INDEX_SCAN +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_COVERING_INDEX_SCAN 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SQLLOG +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SQLLOG 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MMAP_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MMAP_SIZE 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_WIN32_HEAPSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_WIN32_HEAPSIZE 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE_HDRSZ +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE_HDRSZ 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PMASZ +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PMASZ 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_STMTJRNL_SPILL +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_STMTJRNL_SPILL 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SMALL_MALLOC +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SMALL_MALLOC 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SORTERREF_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SORTERREF_SIZE 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMDB_MAXSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMDB_MAXSIZE 29L +#undef org_sqlite_jni_capi_CApi_SQLITE_INTEGER +#define org_sqlite_jni_capi_CApi_SQLITE_INTEGER 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_FLOAT +#define org_sqlite_jni_capi_CApi_SQLITE_FLOAT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_TEXT +#define org_sqlite_jni_capi_CApi_SQLITE_TEXT 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_BLOB +#define org_sqlite_jni_capi_CApi_SQLITE_BLOB 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_NULL +#define org_sqlite_jni_capi_CApi_SQLITE_NULL 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAINDBNAME +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAINDBNAME 1000L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LOOKASIDE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LOOKASIDE 1001L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FKEY +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FKEY 1002L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_TRIGGER 1003L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_QPSG +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_QPSG 1007L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRIGGER_EQP +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRIGGER_EQP 1008L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_RESET_DATABASE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_RESET_DATABASE 1009L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DEFENSIVE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DEFENSIVE 1010L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_WRITABLE_SCHEMA +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_ALTER_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DML +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DML 1013L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DDL +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DDL 1014L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_VIEW 1015L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRUSTED_SCHEMA +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_STMT_SCANSTATUS +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_STMT_SCANSTATUS 1018L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_REVERSE_SCANORDER +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_REVERSE_SCANORDER 1019L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAX +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAX 1019L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_USED 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_SCHEMA_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_SCHEMA_USED 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_STMT_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_STMT_USED 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_HIT +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_HIT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_HIT +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_HIT 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_MISS +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_MISS 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_WRITE 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_DEFERRED_FKS +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_DEFERRED_FKS 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF8 +#define org_sqlite_jni_capi_CApi_SQLITE_UTF8 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16LE +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16LE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16BE +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16BE 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16 +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16_ALIGNED +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16_ALIGNED 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCKSTATE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCKSTATE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_GET_LOCKPROXYFILE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_GET_LOCKPROXYFILE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SET_LOCKPROXYFILE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SET_LOCKPROXYFILE 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LAST_ERRNO +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LAST_ERRNO 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_HINT +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_HINT 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CHUNK_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CHUNK_SIZE 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_FILE_POINTER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_FILE_POINTER 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC_OMITTED +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC_OMITTED 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_AV_RETRY +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_AV_RETRY 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PERSIST_WAL +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PERSIST_WAL 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_OVERWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_OVERWRITE 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFSNAME +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFSNAME 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_POWERSAFE_OVERWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_POWERSAFE_OVERWRITE 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PRAGMA +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PRAGMA 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BUSYHANDLER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BUSYHANDLER 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TEMPFILENAME +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TEMPFILENAME 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_MMAP_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_MMAP_SIZE 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TRACE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TRACE 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_HAS_MOVED +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_HAS_MOVED 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_PHASETWO +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_PHASETWO 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_SET_HANDLE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_SET_HANDLE 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WAL_BLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WAL_BLOCK 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ZIPVFS +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ZIPVFS 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RBU +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RBU 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFS_POINTER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFS_POINTER 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_JOURNAL_POINTER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_JOURNAL_POINTER 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_GET_HANDLE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_GET_HANDLE 29L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PDB +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PDB 30L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BEGIN_ATOMIC_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_ATOMIC_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCK_TIMEOUT +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCK_TIMEOUT 34L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_DATA_VERSION +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_DATA_VERSION 35L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_LIMIT +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_LIMIT 36L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_DONE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_DONE 37L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESERVE_BYTES +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESERVE_BYTES 38L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_START +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_START 39L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_EXTERNAL_READER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_EXTERNAL_READER 40L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKSM_FILE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKSM_FILE 41L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESET_CACHE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESET_CACHE 42L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_NONE +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_NONE 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_SHARED +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_SHARED 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_RESERVED +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_RESERVED 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_PENDING +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_PENDING 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_EXCLUSIVE +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_EXCLUSIVE 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC512 +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC512 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC1K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC1K 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC2K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC2K 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC4K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC4K 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC8K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC8K 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC16K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC16K 64L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC32K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC32K 128L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC64K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC64K 256L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SAFE_APPEND +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SAFE_APPEND 512L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SEQUENTIAL +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SEQUENTIAL 1024L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 2048L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_POWERSAFE_OVERWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_POWERSAFE_OVERWRITE 4096L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_IMMUTABLE +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_IMMUTABLE 8192L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_BATCH_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_BATCH_ATOMIC 16384L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LENGTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LENGTH 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_SQL_LENGTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_SQL_LENGTH 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COLUMN +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COLUMN 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_EXPR_DEPTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_EXPR_DEPTH 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COMPOUND_SELECT +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COMPOUND_SELECT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VDBE_OP +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VDBE_OP 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_FUNCTION_ARG +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_FUNCTION_ARG 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_ATTACHED +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_ATTACHED 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LIKE_PATTERN_LENGTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VARIABLE_NUMBER +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VARIABLE_NUMBER 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_TRIGGER_DEPTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_TRIGGER_DEPTH 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_WORKER_THREADS +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_WORKER_THREADS 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_READONLY +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_READONLY 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_READWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_READWRITE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_CREATE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_CREATE 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_URI +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_URI 64L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_MEMORY +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_MEMORY 128L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOMUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOMUTEX 32768L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_FULLMUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_FULLMUTEX 65536L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_SHAREDCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_SHAREDCACHE 131072L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_PRIVATECACHE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_PRIVATECACHE 262144L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOFOLLOW +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOFOLLOW 16777216L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_EXRESCODE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_EXRESCODE 33554432L +#undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_PERSISTENT +#define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_PERSISTENT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NO_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NO_VTAB 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_OK +#define org_sqlite_jni_capi_CApi_SQLITE_OK 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_INTERNAL +#define org_sqlite_jni_capi_CApi_SQLITE_INTERNAL 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_PERM +#define org_sqlite_jni_capi_CApi_SQLITE_PERM 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_ABORT +#define org_sqlite_jni_capi_CApi_SQLITE_ABORT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCKED +#define org_sqlite_jni_capi_CApi_SQLITE_LOCKED 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOMEM +#define org_sqlite_jni_capi_CApi_SQLITE_NOMEM 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_INTERRUPT +#define org_sqlite_jni_capi_CApi_SQLITE_INTERRUPT 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTFOUND +#define org_sqlite_jni_capi_CApi_SQLITE_NOTFOUND 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_FULL +#define org_sqlite_jni_capi_CApi_SQLITE_FULL 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_PROTOCOL +#define org_sqlite_jni_capi_CApi_SQLITE_PROTOCOL 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_EMPTY +#define org_sqlite_jni_capi_CApi_SQLITE_EMPTY 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_SCHEMA +#define org_sqlite_jni_capi_CApi_SQLITE_SCHEMA 17L +#undef org_sqlite_jni_capi_CApi_SQLITE_TOOBIG +#define org_sqlite_jni_capi_CApi_SQLITE_TOOBIG 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_MISMATCH +#define org_sqlite_jni_capi_CApi_SQLITE_MISMATCH 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_MISUSE +#define org_sqlite_jni_capi_CApi_SQLITE_MISUSE 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOLFS +#define org_sqlite_jni_capi_CApi_SQLITE_NOLFS 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_AUTH +#define org_sqlite_jni_capi_CApi_SQLITE_AUTH 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_FORMAT +#define org_sqlite_jni_capi_CApi_SQLITE_FORMAT 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_RANGE +#define org_sqlite_jni_capi_CApi_SQLITE_RANGE 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTADB +#define org_sqlite_jni_capi_CApi_SQLITE_NOTADB 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTICE +#define org_sqlite_jni_capi_CApi_SQLITE_NOTICE 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_WARNING +#define org_sqlite_jni_capi_CApi_SQLITE_WARNING 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_ROW +#define org_sqlite_jni_capi_CApi_SQLITE_ROW 100L +#undef org_sqlite_jni_capi_CApi_SQLITE_DONE +#define org_sqlite_jni_capi_CApi_SQLITE_DONE 101L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR_MISSING_COLLSEQ +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR_MISSING_COLLSEQ 257L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR_RETRY +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR_RETRY 513L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR_SNAPSHOT +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR_SNAPSHOT 769L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_READ +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_READ 266L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHORT_READ +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHORT_READ 522L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_WRITE 778L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSYNC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSYNC 1034L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_FSYNC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_FSYNC 1290L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_TRUNCATE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_TRUNCATE 1546L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSTAT +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSTAT 1802L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_UNLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_UNLOCK 2058L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_RDLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_RDLOCK 2314L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE 2570L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_BLOCKED +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_BLOCKED 2826L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_NOMEM +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_NOMEM 3082L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_ACCESS +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_ACCESS 3338L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CHECKRESERVEDLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CHECKRESERVEDLOCK 3594L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_LOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_LOCK 3850L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CLOSE 4106L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_CLOSE 4362L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMOPEN +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMOPEN 4618L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMSIZE 4874L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMLOCK 5130L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMMAP +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMMAP 5386L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SEEK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SEEK 5642L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE_NOENT +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE_NOENT 5898L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_MMAP +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_MMAP 6154L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_GETTEMPPATH +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_GETTEMPPATH 6410L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CONVPATH +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CONVPATH 6666L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_VNODE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_VNODE 6922L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_AUTH +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_AUTH 7178L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_BEGIN_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_BEGIN_ATOMIC 7434L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_COMMIT_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_COMMIT_ATOMIC 7690L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_ROLLBACK_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_ROLLBACK_ATOMIC 7946L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DATA +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DATA 8202L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CORRUPTFS +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CORRUPTFS 8458L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCKED_SHAREDCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_LOCKED_SHAREDCACHE 262L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCKED_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_LOCKED_VTAB 518L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY_RECOVERY +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY_RECOVERY 261L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY_SNAPSHOT +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY_SNAPSHOT 517L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY_TIMEOUT +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY_TIMEOUT 773L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_NOTEMPDIR +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_NOTEMPDIR 270L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_ISDIR +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_ISDIR 526L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_FULLPATH +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_FULLPATH 782L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_CONVPATH +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_CONVPATH 1038L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_SYMLINK +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_SYMLINK 1550L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_VTAB 267L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_SEQUENCE +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_SEQUENCE 523L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_INDEX 779L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_RECOVERY +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_RECOVERY 264L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTLOCK 520L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_ROLLBACK 776L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_DBMOVED +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_DBMOVED 1032L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTINIT +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTINIT 1288L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_DIRECTORY +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_DIRECTORY 1544L +#undef org_sqlite_jni_capi_CApi_SQLITE_ABORT_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_ABORT_ROLLBACK 516L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_CHECK +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_CHECK 275L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_COMMITHOOK +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_COMMITHOOK 531L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FOREIGNKEY +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FOREIGNKEY 787L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FUNCTION +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FUNCTION 1043L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_NOTNULL +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_NOTNULL 1299L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PRIMARYKEY +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PRIMARYKEY 1555L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_TRIGGER 1811L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_UNIQUE +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_UNIQUE 2067L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_VTAB 2323L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_ROWID +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_ROWID 2579L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PINNED +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PINNED 2835L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_DATATYPE +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_DATATYPE 3091L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_WAL +#define org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_WAL 283L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_ROLLBACK 539L +#undef org_sqlite_jni_capi_CApi_SQLITE_WARNING_AUTOINDEX +#define org_sqlite_jni_capi_CApi_SQLITE_WARNING_AUTOINDEX 284L +#undef org_sqlite_jni_capi_CApi_SQLITE_AUTH_USER +#define org_sqlite_jni_capi_CApi_SQLITE_AUTH_USER 279L +#undef org_sqlite_jni_capi_CApi_SQLITE_OK_LOAD_PERMANENTLY +#define org_sqlite_jni_capi_CApi_SQLITE_OK_LOAD_PERMANENTLY 256L +#undef org_sqlite_jni_capi_CApi_SQLITE_SERIALIZE_NOCOPY +#define org_sqlite_jni_capi_CApi_SQLITE_SERIALIZE_NOCOPY 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_FREEONCLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_FREEONCLOSE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_READONLY +#define org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_READONLY 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_RESIZEABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_RESIZEABLE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_SESSION_CONFIG_STRMSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_SESSION_CONFIG_STRMSIZE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_SESSION_OBJCONFIG_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_SESSION_OBJCONFIG_SIZE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_MEMORY_USED +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_MEMORY_USED 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_USED +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_USED 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_OVERFLOW +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_OVERFLOW 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_SIZE 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PARSER_STACK +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PARSER_STACK 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_SIZE 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_COUNT +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_COUNT 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FULLSCAN_STEP +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FULLSCAN_STEP 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_SORT +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_SORT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_AUTOINDEX +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_AUTOINDEX 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_VM_STEP +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_VM_STEP 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_REPREPARE +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_REPREPARE 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_RUN +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_RUN 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_MISS +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_MISS 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_HIT +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_HIT 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_MEMUSED +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_MEMUSED 99L +#undef org_sqlite_jni_capi_CApi_SQLITE_SYNC_NORMAL +#define org_sqlite_jni_capi_CApi_SQLITE_SYNC_NORMAL 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_SYNC_FULL +#define org_sqlite_jni_capi_CApi_SQLITE_SYNC_FULL 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_SYNC_DATAONLY +#define org_sqlite_jni_capi_CApi_SQLITE_SYNC_DATAONLY 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_STMT +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_STMT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_PROFILE +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_PROFILE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_ROW +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_ROW 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_CLOSE 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_TXN_NONE +#define org_sqlite_jni_capi_CApi_SQLITE_TXN_NONE 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_TXN_READ +#define org_sqlite_jni_capi_CApi_SQLITE_TXN_READ 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_TXN_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_TXN_WRITE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_DETERMINISTIC +#define org_sqlite_jni_capi_CApi_SQLITE_DETERMINISTIC 2048L +#undef org_sqlite_jni_capi_CApi_SQLITE_DIRECTONLY +#define org_sqlite_jni_capi_CApi_SQLITE_DIRECTONLY 524288L +#undef org_sqlite_jni_capi_CApi_SQLITE_SUBTYPE +#define org_sqlite_jni_capi_CApi_SQLITE_SUBTYPE 1048576L +#undef org_sqlite_jni_capi_CApi_SQLITE_INNOCUOUS +#define org_sqlite_jni_capi_CApi_SQLITE_INNOCUOUS 2097152L +#undef org_sqlite_jni_capi_CApi_SQLITE_RESULT_SUBTYPE +#define org_sqlite_jni_capi_CApi_SQLITE_RESULT_SUBTYPE 16777216L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_SCAN_UNIQUE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_SCAN_UNIQUE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_EQ +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_EQ 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LE 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LT 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GE 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_MATCH +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_MATCH 64L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIKE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIKE 65L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GLOB +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GLOB 66L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_REGEXP +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_REGEXP 67L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_NE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_NE 68L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOT 69L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOTNULL +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNULL +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNULL 71L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_IS +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_IS 72L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIMIT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIMIT 73L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_OFFSET +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_OFFSET 74L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_FUNCTION +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_FUNCTION 150L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_CONSTRAINT_SUPPORT +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_CONSTRAINT_SUPPORT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_INNOCUOUS +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_INNOCUOUS 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_DIRECTONLY +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_DIRECTONLY 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_USES_ALL_SCHEMAS +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_USES_ALL_SCHEMAS 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_ROLLBACK 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_FAIL +#define org_sqlite_jni_capi_CApi_SQLITE_FAIL 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_REPLACE +#define org_sqlite_jni_capi_CApi_SQLITE_REPLACE 5L +/* + * Class: org_sqlite_jni_capi_CApi + * Method: init + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_init + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_java_uncache_thread + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1java_1uncache_1thread + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_jni_supports_nio + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1supports_1nio + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_jni_db_error + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1db_1error + (JNIEnv *, jclass, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_aggregate_context + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Z)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1aggregate_1context + (JNIEnv *, jclass, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_auto_extension + * Signature: (Lorg/sqlite/jni/capi/AutoExtensionCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1auto_1extension + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_finish + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1finish + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_init + * Signature: (JLjava/lang/String;JLjava/lang/String;)Lorg/sqlite/jni/capi/sqlite3_backup; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1init + (JNIEnv *, jclass, jlong, jstring, jlong, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_pagecount + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1pagecount + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_remaining + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1remaining + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_step + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1step + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_blob + * Signature: (JI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1blob + (JNIEnv *, jclass, jlong, jint, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_double + * Signature: (JID)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1double + (JNIEnv *, jclass, jlong, jint, jdouble); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_int + * Signature: (JII)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1int + (JNIEnv *, jclass, jlong, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_int64 + * Signature: (JIJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1int64 + (JNIEnv *, jclass, jlong, jint, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_java_object + * Signature: (JILjava/lang/Object;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1java_1object + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;ILjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1nio_1buffer + (JNIEnv *, jclass, jobject, jint, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_null + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1null + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_parameter_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1parameter_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_parameter_index + * Signature: (J[B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1parameter_1index + (JNIEnv *, jclass, jlong, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_parameter_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1parameter_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_text + * Signature: (JI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1text + (JNIEnv *, jclass, jlong, jint, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_text16 + * Signature: (JI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1text16 + (JNIEnv *, jclass, jlong, jint, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_value + * Signature: (JIJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1value + (JNIEnv *, jclass, jlong, jint, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_zeroblob + * Signature: (JII)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1zeroblob + (JNIEnv *, jclass, jlong, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_zeroblob64 + * Signature: (JIJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1zeroblob64 + (JNIEnv *, jclass, jlong, jint, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_bytes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1bytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_close + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1close + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_open + * Signature: (JLjava/lang/String;Ljava/lang/String;Ljava/lang/String;JILorg/sqlite/jni/capi/OutputPointer/sqlite3_blob;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1open + (JNIEnv *, jclass, jlong, jstring, jstring, jstring, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_read + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1read + (JNIEnv *, jclass, jlong, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_read_nio_buffer + * Signature: (JILjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1read_1nio_1buffer + (JNIEnv *, jclass, jlong, jint, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_reopen + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1reopen + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_write + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1write + (JNIEnv *, jclass, jlong, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_write_nio_buffer + * Signature: (JILjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1write_1nio_1buffer + (JNIEnv *, jclass, jlong, jint, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_busy_handler + * Signature: (JLorg/sqlite/jni/capi/BusyHandlerCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1busy_1handler + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_busy_timeout + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1busy_1timeout + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_cancel_auto_extension + * Signature: (Lorg/sqlite/jni/capi/AutoExtensionCallback;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1cancel_1auto_1extension + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_changes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1changes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_changes64 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1changes64 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_clear_bindings + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1clear_1bindings + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_close + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1close + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_close_v2 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1close_1v2 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_blob + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1blob + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_bytes + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1bytes + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_bytes16 + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1bytes16 + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_database_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1database_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_decltype + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1decltype + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_double + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)D + */ +JNIEXPORT jdouble JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1double + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_int + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1int + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_int64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1int64 + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_java_object + * Signature: (JI)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1java_1object + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Ljava/nio/ByteBuffer; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1nio_1buffer + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_origin_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1origin_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_table_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1table_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_text + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1text + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_text16 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1text16 + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_type + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1type + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_value + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Lorg/sqlite/jni/capi/sqlite3_value; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1value + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_collation_needed + * Signature: (JLorg/sqlite/jni/capi/CollationNeededCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1collation_1needed + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_commit_hook + * Signature: (JLorg/sqlite/jni/capi/CommitHookCallback;)Lorg/sqlite/jni/capi/CommitHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1commit_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_compileoption_get + * Signature: (I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1compileoption_1get + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_compileoption_used + * Signature: (Ljava/lang/String;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1compileoption_1used + (JNIEnv *, jclass, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_complete + * Signature: ([B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1complete + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_config__enable + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1enable + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_config__CONFIG_LOG + * Signature: (Lorg/sqlite/jni/capi/ConfigLogCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1CONFIG_1LOG + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_config__SQLLOG + * Signature: (Lorg/sqlite/jni/capi/ConfigSqlLogCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1SQLLOG + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_context_db_handle + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)Lorg/sqlite/jni/capi/sqlite3; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1context_1db_1handle + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_create_collation + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;ILorg/sqlite/jni/capi/CollationCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1create_1collation + (JNIEnv *, jclass, jobject, jstring, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_create_function + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;IILorg/sqlite/jni/capi/SQLFunction;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1create_1function + (JNIEnv *, jclass, jobject, jstring, jint, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_data_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1data_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_config + * Signature: (Lorg/sqlite/jni/capi/sqlite3;IILorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2 + (JNIEnv *, jclass, jobject, jint, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_config + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2ILjava_lang_String_2 + (JNIEnv *, jclass, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_filename + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1filename + (JNIEnv *, jclass, jobject, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_handle + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Lorg/sqlite/jni/capi/sqlite3; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1handle + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_readonly + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1readonly + (JNIEnv *, jclass, jobject, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_release_memory + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1release_1memory + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_status + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1status + (JNIEnv *, jclass, jobject, jint, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_errcode + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errcode + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_errmsg + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errmsg + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_error_offset + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1error_1offset + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_errstr + * Signature: (I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errstr + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_expanded_sql + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1expanded_1sql + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_extended_errcode + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1errcode + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_extended_result_codes + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1result_1codes + (JNIEnv *, jclass, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_get_autocommit + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1get_1autocommit + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_get_auxdata + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1get_1auxdata + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_finalize + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1finalize + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_initialize + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1initialize + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_interrupt + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1interrupt + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_is_interrupted + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1is_1interrupted + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_keyword_check + * Signature: (Ljava/lang/String;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1keyword_1check + (JNIEnv *, jclass, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_keyword_count + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1keyword_1count + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_keyword_name + * Signature: (I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1keyword_1name + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_last_insert_rowid + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1last_1insert_1rowid + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_libversion + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1libversion + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_libversion_number + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1libversion_1number + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_limit + * Signature: (Lorg/sqlite/jni/capi/sqlite3;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1limit + (JNIEnv *, jclass, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_normalized_sql + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1normalized_1sql + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_open + * Signature: (Ljava/lang/String;Lorg/sqlite/jni/capi/OutputPointer/sqlite3;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1open + (JNIEnv *, jclass, jstring, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_open_v2 + * Signature: (Ljava/lang/String;Lorg/sqlite/jni/capi/OutputPointer/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1open_1v2 + (JNIEnv *, jclass, jstring, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_prepare + * Signature: (J[BILorg/sqlite/jni/capi/OutputPointer/sqlite3_stmt;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1prepare + (JNIEnv *, jclass, jlong, jbyteArray, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_prepare_v2 + * Signature: (J[BILorg/sqlite/jni/capi/OutputPointer/sqlite3_stmt;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1prepare_1v2 + (JNIEnv *, jclass, jlong, jbyteArray, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_prepare_v3 + * Signature: (J[BIILorg/sqlite/jni/capi/OutputPointer/sqlite3_stmt;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1prepare_1v3 + (JNIEnv *, jclass, jlong, jbyteArray, jint, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_blobwrite + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1blobwrite + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_depth + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1depth + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_hook + * Signature: (JLorg/sqlite/jni/capi/PreupdateHookCallback;)Lorg/sqlite/jni/capi/PreupdateHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_new + * Signature: (JILorg/sqlite/jni/capi/OutputPointer/sqlite3_value;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1new + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_old + * Signature: (JILorg/sqlite/jni/capi/OutputPointer/sqlite3_value;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1old + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_progress_handler + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILorg/sqlite/jni/capi/ProgressHandlerCallback;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1progress_1handler + (JNIEnv *, jclass, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_randomness + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1randomness + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_release_memory + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1release_1memory + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_reset + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1reset + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_reset_auto_extension + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1reset_1auto_1extension + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_double + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;D)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1double + (JNIEnv *, jclass, jobject, jdouble); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error + (JNIEnv *, jclass, jobject, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error_toobig + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1toobig + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error_nomem + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1nomem + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error_code + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1code + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_int + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1int + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_int64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;J)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1int64 + (JNIEnv *, jclass, jobject, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_java_object + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1java_1object + (JNIEnv *, jclass, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Ljava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1nio_1buffer + (JNIEnv *, jclass, jobject, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_null + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1null + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_subtype + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1subtype + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_value + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Lorg/sqlite/jni/capi/sqlite3_value;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1value + (JNIEnv *, jclass, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_zeroblob + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1zeroblob + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_zeroblob64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1zeroblob64 + (JNIEnv *, jclass, jobject, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_blob + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1blob + (JNIEnv *, jclass, jobject, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_blob64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BJ)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1blob64 + (JNIEnv *, jclass, jobject, jbyteArray, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_text + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1text + (JNIEnv *, jclass, jobject, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_text64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BJI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1text64 + (JNIEnv *, jclass, jobject, jbyteArray, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_rollback_hook + * Signature: (JLorg/sqlite/jni/capi/RollbackHookCallback;)Lorg/sqlite/jni/capi/RollbackHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1rollback_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_authorizer + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Lorg/sqlite/jni/capi/AuthorizerCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1authorizer + (JNIEnv *, jclass, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_auxdata + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;ILjava/lang/Object;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1auxdata + (JNIEnv *, jclass, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_last_insert_rowid + * Signature: (Lorg/sqlite/jni/capi/sqlite3;J)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1last_1insert_1rowid + (JNIEnv *, jclass, jobject, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_shutdown + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1shutdown + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_sleep + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1sleep + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_sourceid + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1sourceid + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_sql + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1sql + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_status + * Signature: (ILorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1status + (JNIEnv *, jclass, jint, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_status64 + * Signature: (ILorg/sqlite/jni/capi/OutputPointer/Int64;Lorg/sqlite/jni/capi/OutputPointer/Int64;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1status64 + (JNIEnv *, jclass, jint, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_step + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1step + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_busy + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1busy + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_explain + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1explain + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_isexplain + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1isexplain + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_readonly + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1readonly + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_status + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;IZ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1status + (JNIEnv *, jclass, jobject, jint, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_strglob + * Signature: ([B[B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1strglob + (JNIEnv *, jclass, jbyteArray, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_strlike + * Signature: ([B[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1strlike + (JNIEnv *, jclass, jbyteArray, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_system_errno + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1system_1errno + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_table_column_metadata + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lorg/sqlite/jni/capi/OutputPointer/String;Lorg/sqlite/jni/capi/OutputPointer/String;Lorg/sqlite/jni/capi/OutputPointer/Bool;Lorg/sqlite/jni/capi/OutputPointer/Bool;Lorg/sqlite/jni/capi/OutputPointer/Bool;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1table_1column_1metadata + (JNIEnv *, jclass, jobject, jstring, jstring, jstring, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_threadsafe + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1threadsafe + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_total_changes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1total_1changes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_total_changes64 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1total_1changes64 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_trace_v2 + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILorg/sqlite/jni/capi/TraceV2Callback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1trace_1v2 + (JNIEnv *, jclass, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_txn_state + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1txn_1state + (JNIEnv *, jclass, jobject, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_update_hook + * Signature: (JLorg/sqlite/jni/capi/UpdateHookCallback;)Lorg/sqlite/jni/capi/UpdateHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1update_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_blob + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1blob + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_bytes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1bytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_bytes16 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1bytes16 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_double + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1double + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_dup + * Signature: (J)Lorg/sqlite/jni/capi/sqlite3_value; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1dup + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_encoding + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1encoding + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_free + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1free + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_frombind + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1frombind + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_int + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1int + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_int64 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1int64 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_java_object + * Signature: (J)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1java_1object + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_value;)Ljava/nio/ByteBuffer; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1nio_1buffer + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_nochange + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1nochange + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_numeric_type + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1numeric_1type + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_subtype + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1subtype + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_text + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1text + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_text16 + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1text16 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_type + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1type + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_jni_internal_details + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1internal_1details + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_capi_SQLTester */ + +#ifndef _Included_org_sqlite_jni_capi_SQLTester +#define _Included_org_sqlite_jni_capi_SQLTester +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_capi_SQLTester + * Method: strglob + * Signature: ([B[B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_SQLTester_strglob + (JNIEnv *, jclass, jbyteArray, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_SQLTester + * Method: installCustomExtensions + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_SQLTester_installCustomExtensions + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_Fts5ExtensionApi */ + +#ifndef _Included_org_sqlite_jni_fts5_Fts5ExtensionApi +#define _Included_org_sqlite_jni_fts5_Fts5ExtensionApi +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: getInstance + * Signature: ()Lorg/sqlite/jni/fts5/Fts5ExtensionApi; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_getInstance + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnCount + (JNIEnv *, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnSize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnSize + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnText + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnText + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnTotalSize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/Int64;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnTotalSize + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xGetAuxdata + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Z)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xGetAuxdata + (JNIEnv *, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xInst + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xInst + (JNIEnv *, jobject, jobject, jint, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xInstCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xInstCount + (JNIEnv *, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseCount + (JNIEnv *, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseFirst + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseFirst + (JNIEnv *, jobject, jobject, jint, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseFirstColumn + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseFirstColumn + (JNIEnv *, jobject, jobject, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseNext + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseNext + (JNIEnv *, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseNextColumn + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseNextColumn + (JNIEnv *, jobject, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseSize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseSize + (JNIEnv *, jobject, jobject, jint); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xQueryPhrase + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/fts5/Fts5ExtensionApi/XQueryPhraseCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xQueryPhrase + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xRowCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/capi/OutputPointer/Int64;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xRowCount + (JNIEnv *, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xRowid + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xRowid + (JNIEnv *, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xSetAuxdata + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Ljava/lang/Object;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xSetAuxdata + (JNIEnv *, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xTokenize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;[BLorg/sqlite/jni/fts5/XTokenizeCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xTokenize + (JNIEnv *, jobject, jobject, jbyteArray, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xUserData + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xUserData + (JNIEnv *, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_fts5_api */ + +#ifndef _Included_org_sqlite_jni_fts5_fts5_api +#define _Included_org_sqlite_jni_fts5_fts5_api +#ifdef __cplusplus +extern "C" { +#endif +#undef org_sqlite_jni_fts5_fts5_api_iVersion +#define org_sqlite_jni_fts5_fts5_api_iVersion 2L +/* + * Class: org_sqlite_jni_fts5_fts5_api + * Method: getInstanceForDb + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)Lorg/sqlite/jni/fts5/fts5_api; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_fts5_1api_getInstanceForDb + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_fts5_fts5_api + * Method: xCreateFunction + * Signature: (Ljava/lang/String;Ljava/lang/Object;Lorg/sqlite/jni/fts5/fts5_extension_function;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_fts5_1api_xCreateFunction + (JNIEnv *, jobject, jstring, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_fts5_tokenizer */ + +#ifndef _Included_org_sqlite_jni_fts5_fts5_tokenizer +#define _Included_org_sqlite_jni_fts5_fts5_tokenizer +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_fts5_fts5_tokenizer + * Method: xTokenize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Tokenizer;I[BLorg/sqlite/jni/fts5/XTokenizeCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_fts5_1tokenizer_xTokenize + (JNIEnv *, jobject, jobject, jint, jbyteArray, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/ext/jni/src/org/sqlite/jni/annotation/Experimental.java b/ext/jni/src/org/sqlite/jni/annotation/Experimental.java new file mode 100644 index 0000000000000000000000000000000000000000..190435c8582855ae880b4bb8c3ee450792747670 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/Experimental.java @@ -0,0 +1,30 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file houses the Experimental annotation for the sqlite3 C API. +*/ +package org.sqlite.jni.annotation; +import java.lang.annotation.*; + +/** + This annotation is for flagging methods, constructors, and types + which are expressly experimental and subject to any amount of + change or outright removal. Client code should not rely on such + features. +*/ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target({ + ElementType.METHOD, + ElementType.CONSTRUCTOR, + ElementType.TYPE +}) +public @interface Experimental{} diff --git a/ext/jni/src/org/sqlite/jni/annotation/NotNull.java b/ext/jni/src/org/sqlite/jni/annotation/NotNull.java new file mode 100644 index 0000000000000000000000000000000000000000..0c31782f236fa79963001d27aa3c844957a05ce9 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/NotNull.java @@ -0,0 +1,71 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file houses the NotNull annotation for the sqlite3 C API. +*/ +package org.sqlite.jni.annotation; +import java.lang.annotation.*; + +/** + This annotation is for flagging parameters which may not legally be + null or point to closed/finalized C-side resources. + +

In the case of Java types which map directly to C struct types + (e.g. {@link org.sqlite.jni.capi.sqlite3}, {@link + org.sqlite.jni.capi.sqlite3_stmt}, and {@link + org.sqlite.jni.capi.sqlite3_context}), a closed/finalized resource + is also considered to be null for purposes this annotation because + the C-side effect of passing such a handle is the same as if null + is passed.

+ +

When used in the context of Java interfaces which are called + from the C APIs, this annotation communicates that the C API will + never pass a null value to the callback for that parameter.

+ +

Passing a null, for this annotation's definition of null, for + any parameter marked with this annoation specifically invokes + undefined behavior (see below).

+ +

Passing 0 (i.e. C NULL) or a negative value for any long-type + parameter marked with this annoation specifically invokes undefined + behavior (see below). Such values are treated as C pointers in the + JNI layer.

+ +

Undefined behaviour: the JNI build uses the {@code + SQLITE_ENABLE_API_ARMOR} build flag, meaning that the C code + invoked with invalid NULL pointers and the like will not invoke + undefined behavior in the conventional C sense, but may, for + example, return result codes which are not documented for the + affected APIs or may otherwise behave unpredictably. In no known + cases will such arguments result in C-level code dereferencing a + NULL pointer or accessing out-of-bounds (or otherwise invalid) + memory. In other words, they may cause unexpected behavior but + should never cause an outright crash or security issue.

+ +

Note that the C-style API does not throw any exceptions on its + own because it has a no-throw policy in order to retain its C-style + semantics, but it may trigger NullPointerExceptions (or similar) if + passed a null for a parameter flagged with this annotation.

+ +

This annotation is informational only. No policy is in place to + programmatically ensure that NotNull is conformed to in client + code.

+ +

This annotation is solely for the use by the classes in the + org.sqlite.jni package and subpackages, but is made public so that + javadoc will link to it from the annotated functions. It is not + part of the public API and client-level code must not rely on + it.

+*/ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target(ElementType.PARAMETER) +public @interface NotNull{} diff --git a/ext/jni/src/org/sqlite/jni/annotation/Nullable.java b/ext/jni/src/org/sqlite/jni/annotation/Nullable.java new file mode 100644 index 0000000000000000000000000000000000000000..e3fa30efc9b536a1973e10b1f321ca72130a0d20 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/Nullable.java @@ -0,0 +1,33 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file houses the Nullable annotation for the sqlite3 C API. +*/ +package org.sqlite.jni.annotation; +import java.lang.annotation.*; + +/** + This annotation is for flagging parameters which may legally be + null, noting that they may behave differently if passed null but + are prepared to expect null as a value. When used in the context of + callback methods which are called into from the C APIs, this + annotation communicates that the C API may pass a null value to the + callback. + +

This annotation is solely for the use by the classes in this + package but is made public so that javadoc will link to it from the + annotated functions. It is not part of the public API and + client-level code must not rely on it. +*/ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target(ElementType.PARAMETER) +public @interface Nullable{} diff --git a/ext/jni/src/org/sqlite/jni/annotation/package-info.java b/ext/jni/src/org/sqlite/jni/annotation/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..20ac7a3017e4bd12e8d3536e9c61e233d74d2ff3 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/package-info.java @@ -0,0 +1,17 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +/** + This package houses annotations specific to the JNI bindings of the + SQLite3 C API. +*/ +package org.sqlite.jni.annotation; diff --git a/ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java b/ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..925536636ebff95349442752f4f05c4b8c90b544 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java @@ -0,0 +1,34 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.NotNull; + +/** + An implementation of {@link CollationCallback} which provides a + no-op xDestroy() method. +*/ +public abstract class AbstractCollationCallback + implements CollationCallback, XDestroyCallback { + /** + Must compare the given byte arrays and return the result using + {@code memcmp()} semantics. + */ + public abstract int call(@NotNull byte[] lhs, @NotNull byte[] rhs); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. This implementation does nothing. + */ + public void xDestroy(){} +} diff --git a/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java b/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..1fa6c6b8051d15b863db0fb751bb7bed65c856e0 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java @@ -0,0 +1,138 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + + +/** + A SQLFunction implementation for aggregate functions. Its T is the + data type of its "accumulator" state, an instance of which is + intended to be be managed using the getAggregateState() and + takeAggregateState() methods. +*/ +public abstract class AggregateFunction implements SQLFunction { + + /** + As for the xStep() argument of the C API's + sqlite3_create_function(). If this function throws, the + exception is not propagated and a warning might be emitted to a + debugging channel. + */ + public abstract void xStep(sqlite3_context cx, sqlite3_value[] args); + + /** + As for the xFinal() argument of the C API's sqlite3_create_function(). + If this function throws, it is translated into an sqlite3_result_error(). + */ + public abstract void xFinal(sqlite3_context cx); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. + */ + public void xDestroy() {} + + /** + PerContextState assists aggregate and window functions in + managing their accumulator state across calls to the UDF's + callbacks. + +

T must be of a type which can be legally stored as a value in + java.util.HashMap. + +

If a given aggregate or window function is called multiple times + in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)..., + then the clients need some way of knowing which call is which so + that they can map their state between their various UDF callbacks + and reset it via xFinal(). This class takes care of such + mappings. + +

This class works by mapping + sqlite3_context.getAggregateContext() to a single piece of + state, of a client-defined type (the T part of this class), which + persists across a "matching set" of the UDF's callbacks. + +

This class is a helper providing commonly-needed functionality + - it is not required for use with aggregate or window functions. + Client UDFs are free to perform such mappings using custom + approaches. The provided {@link AggregateFunction} and {@link + WindowFunction} classes use this. + */ + public static final class PerContextState { + private final java.util.Map> map + = new java.util.HashMap<>(); + + /** + Should be called from a UDF's xStep(), xValue(), and xInverse() + methods, passing it that method's first argument and an initial + value for the persistent state. If there is currently no + mapping for the given context within the map, one is created + using the given initial value, else the existing one is used + and the 2nd argument is ignored. It returns a ValueHolder + which can be used to modify that state directly without + requiring that the client update the underlying map's entry. + +

The caller is obligated to eventually call + takeAggregateState() to clear the mapping. + */ + public ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ + final Long key = cx.getAggregateContext(true); + ValueHolder rc = null==key ? null : map.get(key); + if( null==rc ){ + map.put(key, rc = new ValueHolder<>(initialValue)); + } + return rc; + } + + /** + Should be called from a UDF's xFinal() method and passed that + method's first argument. This function removes the value + associated with cx.getAggregateContext() from the map and + returns it, returning null if no other UDF method has been + called to set up such a mapping. The latter condition will be + the case if a UDF is used in a statement which has no result + rows. + */ + public T takeAggregateState(sqlite3_context cx){ + final ValueHolder h = map.remove(cx.getAggregateContext(false)); + return null==h ? null : h.value; + } + } + + /** Per-invocation state for the UDF. */ + private final PerContextState map = new PerContextState<>(); + + /** + To be called from the implementation's xStep() method, as well + as the xValue() and xInverse() methods of the {@link WindowFunction} + subclass, to fetch the current per-call UDF state. On the + first call to this method for any given sqlite3_context + argument, the context is set to the given initial value. On all other + calls, the 2nd argument is ignored. + + @see SQLFunction.PerContextState#getAggregateState + */ + protected final ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ + return map.getAggregateState(cx, initialValue); + } + + /** + To be called from the implementation's xFinal() method to fetch + the final state of the UDF and remove its mapping. + + see SQLFunction.PerContextState#takeAggregateState + */ + protected final T takeAggregateState(sqlite3_context cx){ + return map.takeAggregateState(cx); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java b/ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..298e3a59002fadd47e485c623697410758e72b6f --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java @@ -0,0 +1,29 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.*; + +/** + Callback for use with {@link CApi#sqlite3_set_authorizer}. +*/ +public interface AuthorizerCallback extends CallbackProxy { + /** + Must function as described for the C-level + sqlite3_set_authorizer() callback. If it throws, the error is + converted to a db-level error and the exception is suppressed. + */ + int call(int opId, @Nullable String s1, @Nullable String s2, + @Nullable String s3, @Nullable String s4); + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java b/ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..7a54132d2941ba2d47f98018e2f4774f89b6d061 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java @@ -0,0 +1,40 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with the {@link CApi#sqlite3_auto_extension} + family of APIs. +*/ +public interface AutoExtensionCallback extends CallbackProxy { + /** + Must function as described for a C-level + sqlite3_auto_extension() callback. + +

This callback may throw and the exception's error message will + be set as the db's error string. + +

Tips for implementations: + +

- Opening a database from an auto-extension handler will lead to + an endless recursion of the auto-handler triggering itself + indirectly for each newly-opened database. + +

- If this routine is stateful, it may be useful to make the + overridden method synchronized. + +

- Results are undefined if the given db is closed by an auto-extension. + */ + int call(sqlite3 db); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java b/ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..00223f0b66895c2bd6cee8d65676b4e6b01b01d2 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_busy_handler}. +*/ +public interface BusyHandlerCallback extends CallbackProxy { + /** + Must function as documented for the C-level + sqlite3_busy_handler() callback argument, minus the (void*) + argument the C-level function requires. + */ + int call(int n); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java new file mode 100644 index 0000000000000000000000000000000000000000..b5d08306e269bd47a0a380a59faf3b0e82560877 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -0,0 +1,2897 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file declares the main JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import java.nio.charset.StandardCharsets; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import org.sqlite.jni.annotation.*; +import java.util.Arrays; + +/** + This class contains the entire C-style sqlite3 JNI API binding, + minus a few bits and pieces declared in other files. For client-side + use, a static import is recommended: + +

{@code
+  import static org.sqlite.jni.capi.CApi.*;
+  }
+ +

The C-side part can be found in sqlite3-jni.c. + +

Only functions which materially differ from their C counterparts + are documented here, and only those material differences are + documented. The C documentation is otherwise applicable for these + APIs: + +

https://sqlite.org/c3ref/intro.html + +

A handful of Java-specific APIs have been added which are + documented here. A number of convenience overloads are provided + which are not documented but whose semantics map 1-to-1 in an + intuitive manner. e.g. {@link + #sqlite3_result_set(sqlite3_context,int)} is equivalent to {@link + #sqlite3_result_int}, and sqlite3_result_set() has many + type-specific overloads. + +

Notes regarding Java's Modified UTF-8 vs standard UTF-8: + +

SQLite internally uses UTF-8 encoding, whereas Java natively uses + UTF-16. Java JNI has routines for converting to and from UTF-8, + but JNI uses what its docs call modified UTF-8 (see links below) + Care must be taken when converting Java strings to or from standard + UTF-8 to ensure that the proper conversion is performed. In short, + Java's {@code String.getBytes(StandardCharsets.UTF_8)} performs the proper + conversion in Java, and there are no JNI C APIs for that conversion + (JNI's {@code NewStringUTF()} requires its input to be in MUTF-8). + +

The known consequences and limitations this discrepancy places on + the SQLite3 JNI binding include: + +

    + +
  • C functions which take C-style strings without a length argument + require special care when taking input from Java. In particular, + Java strings converted to byte arrays for encoding purposes are not + NUL-terminated, and conversion to a Java byte array must sometimes + be careful to add one. Functions which take a length do not require + this so long as the length is provided. Search the CApi class + for "\0" for examples. + +
+ +

Further reading: + +

https://stackoverflow.com/questions/57419723 +

https://stackoverflow.com/questions/7921016 +

https://itecnote.com/tecnote/java-getting-true-utf-8-characters-in-java-jni/ +

https://docs.oracle.com/javase/8/docs/api/java/lang/Character.html#unicode +

https://docs.oracle.com/javase/8/docs/api/java/io/DataInput.html#modified-utf-8 + +*/ +public final class CApi { + static { + System.loadLibrary("sqlite3-jni"); + } + //! Not used + private CApi(){} + //! Called from static init code. + private static native void init(); + + /** + Returns a nul-terminated copy of s as a UTF-8-encoded byte array, + or null if s is null. + */ + private static byte[] nulTerminateUtf8(String s){ + return null==s ? null : (s+"\0").getBytes(StandardCharsets.UTF_8); + } + + /** + Each thread which uses the SQLite3 JNI APIs should call + sqlite3_jni_uncache_thread() when it is done with the library - + either right before it terminates or when it finishes using the + SQLite API. This will clean up any cached per-thread info. + +

This process does not close any databases or finalize + any prepared statements because their ownership does not depend on + a given thread. For proper library behavior, and to + avoid C-side leaks, be sure to finalize all statements and close + all databases before calling this function. + +

Calling this from the main application thread is not strictly + required. Additional threads must call this before ending or they + will leak cache entries in the C heap, which in turn may keep + numerous Java-side global references active. + +

This routine returns false without side effects if the current + JNIEnv is not cached, else returns true, but this information is + primarily for testing of the JNI bindings and is not information + which client-level code can use to make any informed + decisions. Its return type and semantics are not considered + stable and may change at any time. + */ + public static native boolean sqlite3_java_uncache_thread(); + + /** + Returns true if this JVM has JNI-level support for C-level direct + memory access using java.nio.ByteBuffer, else returns false. + */ + @Experimental + public static native boolean sqlite3_jni_supports_nio(); + + /** + For internal use only. Sets the given db's error code and + (optionally) string. If rc is 0, it defaults to SQLITE_ERROR. + + On success it returns rc. On error it may return a more serious + code, such as SQLITE_NOMEM. Returns SQLITE_MISUSE if db is null. + */ + static native int sqlite3_jni_db_error(@NotNull sqlite3 db, + int rc, @Nullable String msg); + + /** + Convenience overload which uses e.toString() as the error + message. + */ + static int sqlite3_jni_db_error(@NotNull sqlite3 db, + int rc, @NotNull Exception e){ + return sqlite3_jni_db_error(db, rc, e.toString()); + } + + ////////////////////////////////////////////////////////////////////// + // Maintenance reminder: please keep the sqlite3_.... functions + // alphabetized. The SQLITE_... values. on the other hand, are + // grouped by category. + + /** + Functions exactly like the native form except that (A) the 2nd + argument is a boolean instead of an int and (B) the returned + value is not a pointer address and is only intended for use as a + per-UDF-call lookup key in a higher-level data structure. + +

Passing a true second argument is analogous to passing some + unspecified small, non-0 positive value to the C API and passing + false is equivalent to passing 0 to the C API. + +

Like the C API, it returns 0 if allocation fails or if + initialize is false and no prior aggregate context was allocated + for cx. If initialize is true then it returns 0 only on + allocation error. In all casses, 0 is considered the sentinel + "not a key" value. + */ + public static native long sqlite3_aggregate_context(sqlite3_context cx, boolean initialize); + + /** + Functions almost as documented for the C API, with these + exceptions: + +

- The callback interface is shorter because of + cross-language differences. Specifically, 3rd argument to the C + auto-extension callback interface is unnecessary here. + +

The C API docs do not specifically say so, but if the list of + auto-extensions is manipulated from an auto-extension, it is + undefined which, if any, auto-extensions will subsequently + execute for the current database. That is, doing so will result + in unpredictable, but not undefined, behavior. + +

See the AutoExtension class docs for more information. + */ + public static native int sqlite3_auto_extension(@NotNull AutoExtensionCallback callback); + + private static native int sqlite3_backup_finish(@NotNull long ptrToBackup); + + public static int sqlite3_backup_finish(@NotNull sqlite3_backup b){ + return null==b ? 0 : sqlite3_backup_finish(b.clearNativePointer()); + } + + private static native sqlite3_backup sqlite3_backup_init( + @NotNull long ptrToDbDest, @NotNull String destTableName, + @NotNull long ptrToDbSrc, @NotNull String srcTableName + ); + + public static sqlite3_backup sqlite3_backup_init( + @NotNull sqlite3 dbDest, @NotNull String destTableName, + @NotNull sqlite3 dbSrc, @NotNull String srcTableName + ){ + return sqlite3_backup_init( dbDest.getNativePointer(), destTableName, + dbSrc.getNativePointer(), srcTableName ); + } + + private static native int sqlite3_backup_pagecount(@NotNull long ptrToBackup); + + public static int sqlite3_backup_pagecount(@NotNull sqlite3_backup b){ + return sqlite3_backup_pagecount(b.getNativePointer()); + } + + private static native int sqlite3_backup_remaining(@NotNull long ptrToBackup); + + public static int sqlite3_backup_remaining(@NotNull sqlite3_backup b){ + return sqlite3_backup_remaining(b.getNativePointer()); + } + + private static native int sqlite3_backup_step(@NotNull long ptrToBackup, int nPage); + + public static int sqlite3_backup_step(@NotNull sqlite3_backup b, int nPage){ + return sqlite3_backup_step(b.getNativePointer(), nPage); + } + + private static native int sqlite3_bind_blob( + @NotNull long ptrToStmt, int ndx, @Nullable byte[] data, int n + ); + + /** + If n is negative, SQLITE_MISUSE is returned. If n>data.length + then n is silently truncated to data.length. + */ + public static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data, int n + ){ + return sqlite3_bind_blob(stmt.getNativePointer(), ndx, data, n); + } + + public static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data + ){ + return (null==data) + ? sqlite3_bind_null(stmt.getNativePointer(), ndx) + : sqlite3_bind_blob(stmt.getNativePointer(), ndx, data, data.length); + } + + /** + Convenience overload which is a simple proxy for + sqlite3_bind_nio_buffer(). + */ + @Experimental + /*public*/ static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data, + int begin, int n + ){ + return sqlite3_bind_nio_buffer(stmt, ndx, data, begin, n); + } + + /** + Convenience overload which is equivalant to passing its arguments + to sqlite3_bind_nio_buffer() with the values 0 and -1 for the + final two arguments. + */ + @Experimental + /*public*/ static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data + ){ + return sqlite3_bind_nio_buffer(stmt, ndx, data, 0, -1); + } + + private static native int sqlite3_bind_double( + @NotNull long ptrToStmt, int ndx, double v + ); + + public static int sqlite3_bind_double( + @NotNull sqlite3_stmt stmt, int ndx, double v + ){ + return sqlite3_bind_double(stmt.getNativePointer(), ndx, v); + } + + private static native int sqlite3_bind_int( + @NotNull long ptrToStmt, int ndx, int v + ); + + public static int sqlite3_bind_int( + @NotNull sqlite3_stmt stmt, int ndx, int v + ){ + return sqlite3_bind_int(stmt.getNativePointer(), ndx, v); + } + + private static native int sqlite3_bind_int64( + @NotNull long ptrToStmt, int ndx, long v + ); + + public static int sqlite3_bind_int64(@NotNull sqlite3_stmt stmt, int ndx, long v){ + return sqlite3_bind_int64( stmt.getNativePointer(), ndx, v ); + } + + private static native int sqlite3_bind_java_object( + @NotNull long ptrToStmt, int ndx, @Nullable Object o + ); + + /** + Binds the contents of the given buffer object as a blob. + + The byte range of the buffer may be restricted by providing a + start index and a number of bytes. beginPos may not be negative. + Negative howMany is interpretated as the remainder of the buffer + past the given start position, up to the buffer's limit() (as + opposed its capacity()). + + If beginPos+howMany would extend past the limit() of the buffer + then SQLITE_ERROR is returned. + + If any of the following are true, this function behaves like + sqlite3_bind_null(): the buffer is null, beginPos is at or past + the end of the buffer, howMany is 0, or the calculated slice of + the blob has a length of 0. + + If ndx is out of range, it returns SQLITE_RANGE, as documented + for sqlite3_bind_blob(). If beginPos is negative or if + sqlite3_jni_supports_nio() returns false then SQLITE_MISUSE is + returned. Note that this function is bound (as it were) by the + SQLITE_LIMIT_LENGTH constraint and SQLITE_TOOBIG is returned if + the resulting slice of the buffer exceeds that limit. + + This function does not modify the buffer's streaming-related + cursors. + + If the buffer is modified in a separate thread while this + operation is running, results are undefined and will likely + result in corruption of the bound data or a segmentation fault. + + Design note: this function should arguably take a java.nio.Buffer + instead of ByteBuffer, but it can only operate on "direct" + buffers and the only such class offered by Java is (apparently) + ByteBuffer. + + @see https://docs.oracle.com/javase/8/docs/api/java/nio/Buffer.html + */ + @Experimental + /*public*/ static native int sqlite3_bind_nio_buffer( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data, + int beginPos, int howMany + ); + + /** + Convenience overload which binds the given buffer's entire + contents, up to its limit() (as opposed to its capacity()). + */ + @Experimental + /*public*/ static int sqlite3_bind_nio_buffer( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data + ){ + return sqlite3_bind_nio_buffer(stmt, ndx, data, 0, -1); + } + + /** + Binds the given object at the given index. If o is null then this behaves like + sqlite3_bind_null(). + + @see #sqlite3_result_java_object + */ + public static int sqlite3_bind_java_object( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable Object o + ){ + return sqlite3_bind_java_object(stmt.getNativePointer(), ndx, o); + } + + private static native int sqlite3_bind_null(@NotNull long ptrToStmt, int ndx); + + public static int sqlite3_bind_null(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_bind_null(stmt.getNativePointer(), ndx); + } + + private static native int sqlite3_bind_parameter_count(@NotNull long ptrToStmt); + + public static int sqlite3_bind_parameter_count(@NotNull sqlite3_stmt stmt){ + return sqlite3_bind_parameter_count(stmt.getNativePointer()); + } + + /** + Requires that paramName be a NUL-terminated UTF-8 string. + + This overload is private because: (A) to keep users from + inadvertently passing non-NUL-terminated byte arrays (an easy + thing to do). (B) it is cheaper to NUL-terminate the + String-to-byte-array conversion in the public-facing Java-side + overload than to do that in C, so that signature is the + public-facing one. + */ + private static native int sqlite3_bind_parameter_index( + @NotNull long ptrToStmt, @NotNull byte[] paramName + ); + + public static int sqlite3_bind_parameter_index( + @NotNull sqlite3_stmt stmt, @NotNull String paramName + ){ + final byte[] utf8 = nulTerminateUtf8(paramName); + return null==utf8 ? 0 : sqlite3_bind_parameter_index(stmt.getNativePointer(), utf8); + } + + private static native String sqlite3_bind_parameter_name( + @NotNull long ptrToStmt, int index + ); + + public static String sqlite3_bind_parameter_name(@NotNull sqlite3_stmt stmt, int index){ + return sqlite3_bind_parameter_name(stmt.getNativePointer(), index); + } + + private static native int sqlite3_bind_text( + @NotNull long ptrToStmt, int ndx, @Nullable byte[] utf8, int maxBytes + ); + + /** + Works like the C-level sqlite3_bind_text() but assumes + SQLITE_TRANSIENT for the final C API parameter. The byte array is + assumed to be in UTF-8 encoding. + +

If data is not null and maxBytes>utf8.length then maxBytes is + silently truncated to utf8.length. If maxBytes is negative then + results are undefined if data is not null and does not contain a + NUL byte. + */ + static int sqlite3_bind_text( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] utf8, int maxBytes + ){ + return sqlite3_bind_text(stmt.getNativePointer(), ndx, utf8, maxBytes); + } + + /** + Converts data, if not null, to a UTF-8-encoded byte array and + binds it as such, returning the result of the C-level + sqlite3_bind_null() or sqlite3_bind_text(). + */ + public static int sqlite3_bind_text( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable String data + ){ + if( null==data ) return sqlite3_bind_null(stmt.getNativePointer(), ndx); + final byte[] utf8 = data.getBytes(StandardCharsets.UTF_8); + return sqlite3_bind_text(stmt.getNativePointer(), ndx, utf8, utf8.length); + } + + /** + Requires that utf8 be null or in UTF-8 encoding. + */ + public static int sqlite3_bind_text( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] utf8 + ){ + return ( null==utf8 ) + ? sqlite3_bind_null(stmt.getNativePointer(), ndx) + : sqlite3_bind_text(stmt.getNativePointer(), ndx, utf8, utf8.length); + } + + private static native int sqlite3_bind_text16( + @NotNull long ptrToStmt, int ndx, @Nullable byte[] data, int maxBytes + ); + + /** + Identical to the sqlite3_bind_text() overload with the same + signature but requires that its input be encoded in UTF-16 in + platform byte order. + */ + static int sqlite3_bind_text16( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data, int maxBytes + ){ + return sqlite3_bind_text16(stmt.getNativePointer(), ndx, data, maxBytes); + } + + /** + Converts its string argument to UTF-16 and binds it as such, returning + the result of the C-side function of the same name. The 3rd argument + may be null. + */ + public static int sqlite3_bind_text16( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable String data + ){ + if(null == data) return sqlite3_bind_null(stmt, ndx); + final byte[] bytes = data.getBytes(StandardCharsets.UTF_16); + return sqlite3_bind_text16(stmt.getNativePointer(), ndx, bytes, bytes.length); + } + + /** + Requires that data be null or in UTF-16 encoding in platform byte + order. Returns the result of the C-level sqlite3_bind_null() or + sqlite3_bind_text16(). + */ + public static int sqlite3_bind_text16( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data + ){ + return (null == data) + ? sqlite3_bind_null(stmt.getNativePointer(), ndx) + : sqlite3_bind_text16(stmt.getNativePointer(), ndx, data, data.length); + } + + private static native int sqlite3_bind_value(@NotNull long ptrToStmt, int ndx, long ptrToValue); + + /** + Functions like the C-level sqlite3_bind_value(), or + sqlite3_bind_null() if val is null. + */ + public static int sqlite3_bind_value(@NotNull sqlite3_stmt stmt, int ndx, sqlite3_value val){ + return sqlite3_bind_value(stmt.getNativePointer(), ndx, + null==val ? 0L : val.getNativePointer()); + } + + private static native int sqlite3_bind_zeroblob(@NotNull long ptrToStmt, int ndx, int n); + + public static int sqlite3_bind_zeroblob(@NotNull sqlite3_stmt stmt, int ndx, int n){ + return sqlite3_bind_zeroblob(stmt.getNativePointer(), ndx, n); + } + + private static native int sqlite3_bind_zeroblob64( + @NotNull long ptrToStmt, int ndx, long n + ); + + public static int sqlite3_bind_zeroblob64(@NotNull sqlite3_stmt stmt, int ndx, long n){ + return sqlite3_bind_zeroblob64(stmt.getNativePointer(), ndx, n); + } + + private static native int sqlite3_blob_bytes(@NotNull long ptrToBlob); + + public static int sqlite3_blob_bytes(@NotNull sqlite3_blob blob){ + return sqlite3_blob_bytes(blob.getNativePointer()); + } + + private static native int sqlite3_blob_close(@Nullable long ptrToBlob); + + public static int sqlite3_blob_close(@Nullable sqlite3_blob blob){ + return null==blob ? 0 : sqlite3_blob_close(blob.clearNativePointer()); + } + + private static native int sqlite3_blob_open( + @NotNull long ptrToDb, @NotNull String dbName, + @NotNull String tableName, @NotNull String columnName, + long iRow, int flags, @NotNull OutputPointer.sqlite3_blob out + ); + + public static int sqlite3_blob_open( + @NotNull sqlite3 db, @NotNull String dbName, + @NotNull String tableName, @NotNull String columnName, + long iRow, int flags, @NotNull OutputPointer.sqlite3_blob out + ){ + return sqlite3_blob_open(db.getNativePointer(), dbName, tableName, + columnName, iRow, flags, out); + } + + /** + Convenience overload. + */ + public static sqlite3_blob sqlite3_blob_open( + @NotNull sqlite3 db, @NotNull String dbName, + @NotNull String tableName, @NotNull String columnName, + long iRow, int flags ){ + final OutputPointer.sqlite3_blob out = new OutputPointer.sqlite3_blob(); + sqlite3_blob_open(db.getNativePointer(), dbName, tableName, columnName, + iRow, flags, out); + return out.take(); + }; + + private static native int sqlite3_blob_read( + @NotNull long ptrToBlob, @NotNull byte[] target, int srcOffset + ); + + /** + As per C's sqlite3_blob_read(), but writes its output to the + given byte array. Note that the final argument is the offset of + the source buffer, not the target array. + */ + public static int sqlite3_blob_read( + @NotNull sqlite3_blob src, @NotNull byte[] target, int srcOffset + ){ + return sqlite3_blob_read(src.getNativePointer(), target, srcOffset); + } + + /** + An internal level of indirection. + */ + @Experimental + private static native int sqlite3_blob_read_nio_buffer( + @NotNull long ptrToBlob, int srcOffset, + @NotNull java.nio.ByteBuffer tgt, int tgtOffset, int howMany + ); + + /** + Reads howMany bytes from offset srcOffset of src into position + tgtOffset of tgt. + + Returns SQLITE_MISUSE if src is null, tgt is null, or + sqlite3_jni_supports_nio() returns false. Returns SQLITE_ERROR if + howMany or either offset are negative. If argument validation + succeeds, it returns the result of the underlying call to + sqlite3_blob_read() (0 on success). + */ + @Experimental + /*public*/ static int sqlite3_blob_read_nio_buffer( + @NotNull sqlite3_blob src, int srcOffset, + @NotNull java.nio.ByteBuffer tgt, int tgtOffset, int howMany + ){ + return (JNI_SUPPORTS_NIO && src!=null && tgt!=null) + ? sqlite3_blob_read_nio_buffer( + src.getNativePointer(), srcOffset, tgt, tgtOffset, howMany + ) + : SQLITE_MISUSE; + } + + /** + Convenience overload which reads howMany bytes from position + srcOffset of src and returns the result as a new ByteBuffer. + + srcOffset may not be negative. If howMany is negative, it is + treated as all bytes following srcOffset. + + Returns null if sqlite3_jni_supports_nio(), any arguments are + invalid, if the number of bytes to read is 0 or is larger than + the src blob, or the underlying call to sqlite3_blob_read() fails + for any reason. + */ + @Experimental + /*public*/ static java.nio.ByteBuffer sqlite3_blob_read_nio_buffer( + @NotNull sqlite3_blob src, int srcOffset, int howMany + ){ + if( !JNI_SUPPORTS_NIO || src==null ) return null; + else if( srcOffset<0 ) return null; + final int nB = sqlite3_blob_bytes(src); + if( srcOffset>=nB ) return null; + else if( howMany<0 ) howMany = nB - srcOffset; + if( srcOffset + howMany > nB ) return null; + final java.nio.ByteBuffer tgt = + java.nio.ByteBuffer.allocateDirect(howMany); + final int rc = sqlite3_blob_read_nio_buffer( + src.getNativePointer(), srcOffset, tgt, 0, howMany + ); + return 0==rc ? tgt : null; + } + + /** + Overload alias for sqlite3_blob_read_nio_buffer(). + */ + @Experimental + /*public*/ static int sqlite3_blob_read( + @NotNull sqlite3_blob src, int srcOffset, + @NotNull java.nio.ByteBuffer tgt, + int tgtOffset, int howMany + ){ + return sqlite3_blob_read_nio_buffer( + src, srcOffset, tgt, tgtOffset, howMany + ); + } + + /** + Convenience overload which uses 0 for both src and tgt offsets + and reads a number of bytes equal to the smaller of + sqlite3_blob_bytes(src) and tgt.limit(). + + On success it sets tgt.limit() to the number of bytes read. On + error, tgt.limit() is not modified. + + Returns 0 on success. Returns SQLITE_MISUSE is either argument is + null or sqlite3_jni_supports_nio() returns false. Else it returns + the result of the underlying call to sqlite3_blob_read(). + */ + @Experimental + /*public*/ static int sqlite3_blob_read( + @NotNull sqlite3_blob src, + @NotNull java.nio.ByteBuffer tgt + ){ + if(!JNI_SUPPORTS_NIO || src==null || tgt==null) return SQLITE_MISUSE; + final int nSrc = sqlite3_blob_bytes(src); + final int nTgt = tgt.limit(); + final int nRead = nTgt T sqlite3_column_java_object( + @NotNull sqlite3_stmt stmt, int ndx, @NotNull Class type + ){ + final Object o = sqlite3_column_java_object(stmt, ndx); + return type.isInstance(o) ? (T)o : null; + } + + private static native String sqlite3_column_name(@NotNull long ptrToStmt, int ndx); + + public static String sqlite3_column_name(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_name(stmt.getNativePointer(), ndx); + } + + /** + A variant of sqlite3_column_blob() which returns the blob as a + ByteBuffer object. Returns null if its argument is null, if + sqlite3_jni_supports_nio() is false, or if sqlite3_column_blob() + would return null for the same inputs. + */ + @Experimental + /*public*/ static native java.nio.ByteBuffer sqlite3_column_nio_buffer( + @NotNull sqlite3_stmt stmt, int ndx + ); + + private static native String sqlite3_column_origin_name(@NotNull long ptrToStmt, int ndx); + + /** + Only available if built with SQLITE_ENABLE_COLUMN_METADATA. + */ + public static String sqlite3_column_origin_name(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_origin_name(stmt.getNativePointer(), ndx); + } + + private static native String sqlite3_column_table_name(@NotNull long ptrToStmt, int ndx); + + /** + Only available if built with SQLITE_ENABLE_COLUMN_METADATA. + */ + public static String sqlite3_column_table_name(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_table_name(stmt.getNativePointer(), ndx); + } + + /** + Functions identially to the C API, and this note is just to + stress that the returned bytes are encoded as UTF-8. It returns + null if the underlying C-level sqlite3_column_text() returns NULL + or on allocation error. + + @see #sqlite3_column_text16(sqlite3_stmt,int) + */ + public static native byte[] sqlite3_column_text( + @NotNull sqlite3_stmt stmt, int ndx + ); + + public static native String sqlite3_column_text16( + @NotNull sqlite3_stmt stmt, int ndx + ); + + // The real utility of this function is questionable. + // /** + // Returns a Java value representation based on the value of + // sqlite_value_type(). For integer types it returns either Integer + // or Long, depending on whether the value will fit in an + // Integer. For floating-point values it always returns type Double. + + // If the column was bound using sqlite3_result_java_object() then + // that value, as an Object, is returned. + // */ + // public static Object sqlite3_column_to_java(@NotNull sqlite3_stmt stmt, + // int ndx){ + // sqlite3_value v = sqlite3_column_value(stmt, ndx); + // Object rv = null; + // if(null == v) return v; + // v = sqlite3_value_dup(v)/*need a protected value*/; + // if(null == v) return v /* OOM error in C */; + // if(112/* 'p' */ == sqlite3_value_subtype(v)){ + // rv = sqlite3_value_java_object(v); + // }else{ + // switch(sqlite3_value_type(v)){ + // case SQLITE_INTEGER: { + // final long i = sqlite3_value_int64(v); + // rv = (i<=0x7fffffff && i>=-0x7fffffff-1) + // ? new Integer((int)i) : new Long(i); + // break; + // } + // case SQLITE_FLOAT: rv = new Double(sqlite3_value_double(v)); break; + // case SQLITE_BLOB: rv = sqlite3_value_blob(v); break; + // case SQLITE_TEXT: rv = sqlite3_value_text16(v); break; + // default: break; + // } + // } + // sqlite3_value_free(v); + // return rv; + // } + + private static native int sqlite3_column_type(@NotNull long ptrToStmt, int ndx); + + public static int sqlite3_column_type(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_type(stmt.getNativePointer(), ndx); + } + + public static native sqlite3_value sqlite3_column_value( + @NotNull sqlite3_stmt stmt, int ndx + ); + + private static native int sqlite3_collation_needed( + @NotNull long ptrToDb, @Nullable CollationNeededCallback callback + ); + + /** + This functions like C's sqlite3_collation_needed16() because + Java's string type is inherently compatible with that interface. + */ + public static int sqlite3_collation_needed( + @NotNull sqlite3 db, @Nullable CollationNeededCallback callback + ){ + return sqlite3_collation_needed(db.getNativePointer(), callback); + } + + private static native CommitHookCallback sqlite3_commit_hook( + @NotNull long ptrToDb, @Nullable CommitHookCallback hook + ); + + public static CommitHookCallback sqlite3_commit_hook( + @NotNull sqlite3 db, @Nullable CommitHookCallback hook + ){ + return sqlite3_commit_hook(db.getNativePointer(), hook); + } + + public static native String sqlite3_compileoption_get(int n); + + public static native boolean sqlite3_compileoption_used(String optName); + + /** + This implementation is private because it's too easy to pass it + non-NUL-terminated byte arrays from client code. + */ + private static native int sqlite3_complete( + @NotNull byte[] nulTerminatedUtf8Sql + ); + + /** + Unlike the C API, this returns SQLITE_MISUSE if its argument is + null (as opposed to invoking UB). + */ + public static int sqlite3_complete(@NotNull String sql){ + return sqlite3_complete( nulTerminateUtf8(sql) ); + } + + /** + Internal level of indirection for sqlite3_config(int). + */ + private static native int sqlite3_config__enable(int op); + + /** + Internal level of indirection for sqlite3_config(ConfigLogCallback). + */ + private static native int sqlite3_config__CONFIG_LOG( + @Nullable ConfigLogCallback logger + ); + + /** + Internal level of indirection for sqlite3_config(ConfigSqlLogCallback). + */ + private static native int sqlite3_config__SQLLOG( + @Nullable ConfigSqlLogCallback logger + ); + + /** +

Works like in the C API with the exception that it only supports + the following subset of configution flags: + +

SQLITE_CONFIG_SINGLETHREAD + SQLITE_CONFIG_MULTITHREAD + SQLITE_CONFIG_SERIALIZED + +

Others may be added in the future. It returns SQLITE_MISUSE if + given an argument it does not handle. + +

Note that sqlite3_config() is not threadsafe with regards to + the rest of the library. This must not be called when any other + library APIs are being called. + */ + public static int sqlite3_config(int op){ + return sqlite3_config__enable(op); + } + + /** + If the native library was built with SQLITE_ENABLE_SQLLOG defined + then this acts as a proxy for C's + sqlite3_config(SQLITE_CONFIG_SQLLOG,...). This sets or clears the + logger. If installation of a logger fails, any previous logger is + retained. + +

If not built with SQLITE_ENABLE_SQLLOG defined, this returns + SQLITE_MISUSE. + +

Note that sqlite3_config() is not threadsafe with regards to + the rest of the library. This must not be called when any other + library APIs are being called. + */ + public static int sqlite3_config( @Nullable ConfigSqlLogCallback logger ){ + return sqlite3_config__SQLLOG(logger); + } + + /** + The sqlite3_config() overload for handling the SQLITE_CONFIG_LOG + option. + */ + public static int sqlite3_config( @Nullable ConfigLogCallback logger ){ + return sqlite3_config__CONFIG_LOG(logger); + } + + /** + Unlike the C API, this returns null if its argument is + null (as opposed to invoking UB). + */ + public static native sqlite3 sqlite3_context_db_handle( + @NotNull sqlite3_context cx + ); + + public static native int sqlite3_create_collation( + @NotNull sqlite3 db, @NotNull String name, int eTextRep, + @NotNull CollationCallback col + ); + + /** + The Java counterpart to the C-native sqlite3_create_function(), + sqlite3_create_function_v2(), and + sqlite3_create_window_function(). Which one it behaves like + depends on which methods the final argument implements. See + SQLFunction's subclasses (ScalarFunction, AggregateFunction, + and WindowFunction) for details. + +

Unlike the C API, this returns SQLITE_MISUSE null if its db or + functionName arguments are null (as opposed to invoking UB). + */ + public static native int sqlite3_create_function( + @NotNull sqlite3 db, @NotNull String functionName, + int nArg, int eTextRep, @NotNull SQLFunction func + ); + + private static native int sqlite3_data_count(@NotNull long ptrToStmt); + + public static int sqlite3_data_count(@NotNull sqlite3_stmt stmt){ + return sqlite3_data_count(stmt.getNativePointer()); + } + + /** + Overload for sqlite3_db_config() calls which take (int,int*) + variadic arguments. Returns SQLITE_MISUSE if op is not one of the + SQLITE_DBCONFIG_... options which uses this call form. + +

Unlike the C API, this returns SQLITE_MISUSE if its db argument + is null (as opposed to invoking UB). + */ + public static native int sqlite3_db_config( + @NotNull sqlite3 db, int op, int onOff, @Nullable OutputPointer.Int32 out + ); + + /** + Overload for sqlite3_db_config() calls which take a (const char*) + variadic argument. As of SQLite3 v3.43 the only such option is + SQLITE_DBCONFIG_MAINDBNAME. Returns SQLITE_MISUSE if op is not + SQLITE_DBCONFIG_MAINDBNAME, but that set of options may be + extended in future versions. + */ + public static native int sqlite3_db_config( + @NotNull sqlite3 db, int op, @NotNull String val + ); + + private static native String sqlite3_db_name(@NotNull long ptrToDb, int ndx); + + public static String sqlite3_db_name(@NotNull sqlite3 db, int ndx){ + return null==db ? null : sqlite3_db_name(db.getNativePointer(), ndx); + } + + public static native String sqlite3_db_filename( + @NotNull sqlite3 db, @NotNull String dbName + ); + + public static native sqlite3 sqlite3_db_handle(@NotNull sqlite3_stmt stmt); + + public static native int sqlite3_db_readonly(@NotNull sqlite3 db, String dbName); + + public static native int sqlite3_db_release_memory(sqlite3 db); + + public static native int sqlite3_db_status( + @NotNull sqlite3 db, int op, @NotNull OutputPointer.Int32 pCurrent, + @NotNull OutputPointer.Int32 pHighwater, boolean reset + ); + + public static native int sqlite3_errcode(@NotNull sqlite3 db); + + public static native String sqlite3_errmsg(@NotNull sqlite3 db); + + private static native int sqlite3_error_offset(@NotNull long ptrToDb); + + /** + Note that the returned byte offset values assume UTF-8-encoded + inputs, so won't always match character offsets in Java Strings. + */ + public static int sqlite3_error_offset(@NotNull sqlite3 db){ + return sqlite3_error_offset(db.getNativePointer()); + } + + public static native String sqlite3_errstr(int resultCode); + + public static native String sqlite3_expanded_sql(@NotNull sqlite3_stmt stmt); + + private static native int sqlite3_extended_errcode(@NotNull long ptrToDb); + + public static int sqlite3_extended_errcode(@NotNull sqlite3 db){ + return sqlite3_extended_errcode(db.getNativePointer()); + } + + public static native int sqlite3_extended_result_codes( + @NotNull sqlite3 db, boolean on + ); + + private static native boolean sqlite3_get_autocommit(@NotNull long ptrToDb); + + public static boolean sqlite3_get_autocommit(@NotNull sqlite3 db){ + return sqlite3_get_autocommit(db.getNativePointer()); + } + + public static native Object sqlite3_get_auxdata( + @NotNull sqlite3_context cx, int n + ); + + private static native int sqlite3_finalize(long ptrToStmt); + + public static int sqlite3_finalize(@NotNull sqlite3_stmt stmt){ + return null==stmt ? 0 : sqlite3_finalize(stmt.clearNativePointer()); + } + + public static native int sqlite3_initialize(); + + public static native void sqlite3_interrupt(@NotNull sqlite3 db); + + public static native boolean sqlite3_is_interrupted(@NotNull sqlite3 db); + + public static native boolean sqlite3_keyword_check(@NotNull String word); + + public static native int sqlite3_keyword_count(); + + public static native String sqlite3_keyword_name(int index); + + + public static native long sqlite3_last_insert_rowid(@NotNull sqlite3 db); + + public static native String sqlite3_libversion(); + + public static native int sqlite3_libversion_number(); + + public static native int sqlite3_limit(@NotNull sqlite3 db, int id, int newVal); + + /** + Only available if built with SQLITE_ENABLE_NORMALIZE. If not, it always + returns null. + */ + public static native String sqlite3_normalized_sql(@NotNull sqlite3_stmt stmt); + + /** + Works like its C counterpart and makes the native pointer of the + underling (sqlite3*) object available via + ppDb.getNativePointer(). That pointer is necessary for looking up + the JNI-side native, but clients need not pay it any + heed. Passing the object to sqlite3_close() or sqlite3_close_v2() + will clear that pointer mapping. + +

Recall that even if opening fails, the output pointer might be + non-null. Any error message about the failure will be in that + object and it is up to the caller to sqlite3_close() that + db handle. + */ + public static native int sqlite3_open( + @Nullable String filename, @NotNull OutputPointer.sqlite3 ppDb + ); + + /** + Convenience overload which returns its db handle directly. The returned + object might not have been successfully opened: use sqlite3_errcode() to + check whether it is in an error state. + +

Ownership of the returned value is passed to the caller, who must eventually + pass it to sqlite3_close() or sqlite3_close_v2(). + */ + public static sqlite3 sqlite3_open(@Nullable String filename){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + sqlite3_open(filename, out); + return out.take(); + }; + + public static native int sqlite3_open_v2( + @Nullable String filename, @NotNull OutputPointer.sqlite3 ppDb, + int flags, @Nullable String zVfs + ); + + /** + Has the same semantics as the sqlite3-returning sqlite3_open() + but uses sqlite3_open_v2() instead of sqlite3_open(). + */ + public static sqlite3 sqlite3_open_v2(@Nullable String filename, int flags, + @Nullable String zVfs){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + sqlite3_open_v2(filename, out, flags, zVfs); + return out.take(); + }; + + /** + The sqlite3_prepare() family of functions require slightly + different signatures than their native counterparts, but (A) they + retain functionally equivalent semantics and (B) overloading + allows us to install several convenience forms. + +

All of them which take their SQL in the form of a byte[] require + that it be in UTF-8 encoding unless explicitly noted otherwise. + +

The forms which take a "tail" output pointer return (via that + output object) the index into their SQL byte array at which the + end of the first SQL statement processed by the call was + found. That's fundamentally how the C APIs work but making use of + that value requires more copying of the input SQL into + consecutively smaller arrays in order to consume all of + it. (There is an example of doing that in this project's Tester1 + class.) For that vast majority of uses, that capability is not + necessary, however, and overloads are provided which gloss over + that. + +

Results are undefined if maxBytes>sqlUtf8.length. + +

This routine is private because its maxBytes value is not + strictly necessary in the Java interface, as sqlUtf8.length tells + us the information we need. Making this public would give clients + more ways to shoot themselves in the foot without providing any + real utility. + */ + private static native int sqlite3_prepare( + @NotNull long ptrToDb, @NotNull byte[] sqlUtf8, int maxBytes, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ); + + /** + Works like the canonical sqlite3_prepare() but its "tail" output + argument is returned as the index offset into the given + UTF-8-encoded byte array at which SQL parsing stopped. The + semantics are otherwise identical to the C API counterpart. + +

Several overloads provided simplified call signatures. + */ + public static int sqlite3_prepare( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ){ + return sqlite3_prepare(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, pTailOffset); + } + + public static int sqlite3_prepare( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + return sqlite3_prepare(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, null); + } + + public static int sqlite3_prepare( + @NotNull sqlite3 db, @NotNull String sql, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + final byte[] utf8 = sql.getBytes(StandardCharsets.UTF_8); + return sqlite3_prepare(db.getNativePointer(), utf8, utf8.length, + outStmt, null); + } + + /** + Convenience overload which returns its statement handle directly, + or null on error or when reading only whitespace or + comments. sqlite3_errcode() can be used to determine whether + there was an error or the input was empty. Ownership of the + returned object is passed to the caller, who must eventually pass + it to sqlite3_finalize(). + */ + public static sqlite3_stmt sqlite3_prepare( + @NotNull sqlite3 db, @NotNull String sql + ){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + sqlite3_prepare(db, sql, out); + return out.take(); + } + /** + @see #sqlite3_prepare + */ + private static native int sqlite3_prepare_v2( + @NotNull long ptrToDb, @NotNull byte[] sqlUtf8, int maxBytes, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ); + + /** + Works like the canonical sqlite3_prepare_v2() but its "tail" + output paramter is returned as the index offset into the given + byte array at which SQL parsing stopped. + */ + public static int sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ){ + return sqlite3_prepare_v2(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, pTailOffset); + } + + public static int sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + return sqlite3_prepare_v2(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, null); + } + + public static int sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull String sql, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + final byte[] utf8 = sql.getBytes(StandardCharsets.UTF_8); + return sqlite3_prepare_v2(db.getNativePointer(), utf8, utf8.length, + outStmt, null); + } + + /** + Works identically to the sqlite3_stmt-returning sqlite3_prepare() + but uses sqlite3_prepare_v2(). + */ + public static sqlite3_stmt sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull String sql + ){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + sqlite3_prepare_v2(db, sql, out); + return out.take(); + } + + /** + @see #sqlite3_prepare + */ + private static native int sqlite3_prepare_v3( + @NotNull long ptrToDb, @NotNull byte[] sqlUtf8, int maxBytes, + int prepFlags, @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ); + + /** + Works like the canonical sqlite3_prepare_v2() but its "tail" + output paramter is returned as the index offset into the given + byte array at which SQL parsing stopped. + */ + public static int sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, int prepFlags, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ){ + return sqlite3_prepare_v3(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + prepFlags, outStmt, pTailOffset); + } + + /** + Convenience overload which elides the seldom-used pTailOffset + parameter. + */ + public static int sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, int prepFlags, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + return sqlite3_prepare_v3(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + prepFlags, outStmt, null); + } + + /** + Convenience overload which elides the seldom-used pTailOffset + parameter and converts the given string to UTF-8 before passing + it on. + */ + public static int sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull String sql, int prepFlags, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + final byte[] utf8 = sql.getBytes(StandardCharsets.UTF_8); + return sqlite3_prepare_v3(db.getNativePointer(), utf8, utf8.length, + prepFlags, outStmt, null); + } + + /** + Works identically to the sqlite3_stmt-returning sqlite3_prepare() + but uses sqlite3_prepare_v3(). + */ + public static sqlite3_stmt sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull String sql, int prepFlags + ){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + sqlite3_prepare_v3(db, sql, prepFlags, out); + return out.take(); + } + + /** + A convenience wrapper around sqlite3_prepare_v3() which accepts + an arbitrary amount of input provided as a UTF-8-encoded byte + array. It loops over the input bytes looking for + statements. Each one it finds is passed to p.call(), passing + ownership of it to that function. If p.call() returns 0, looping + continues, else the loop stops and p.call()'s result code is + returned. If preparation of any given segment fails, looping + stops and that result code is returned. + +

If p.call() throws, the exception is converted to a db-level + error and a non-0 code is returned, in order to retain the + C-style error semantics of the API. + +

How each statement is handled, including whether it is finalized + or not, is up to the callback object. e.g. the callback might + collect them for later use. If it does not collect them then it + must finalize them. See PrepareMultiCallback.Finalize for a + simple proxy which does that. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + int prepFlags, + @NotNull PrepareMultiCallback p){ + final OutputPointer.Int32 oTail = new OutputPointer.Int32(); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + int rc = 0; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + while( 0==rc && pos0 ){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, + sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v3(db, sqlChunk, prepFlags, outStmt, oTail); + if( 0!=rc ) break; + pos = oTail.value; + stmt = outStmt.take(); + if( null==stmt ){ + // empty statement (whitespace/comments) + continue; + } + try{ + rc = p.call(stmt); + }catch(Exception e){ + rc = sqlite3_jni_db_error( db, SQLITE_ERROR, e ); + } + } + return rc; + } + + /** + Convenience overload which accepts its SQL as a String and uses + no statement-preparation flags. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, sqlUtf8, 0, p); + } + + /** + Convenience overload which accepts its SQL as a String. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String sql, int prepFlags, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi( + db, sql.getBytes(StandardCharsets.UTF_8), prepFlags, p + ); + } + + /** + Convenience overload which accepts its SQL as a String and uses + no statement-preparation flags. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String sql, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, sql, 0, p); + } + + /** + Convenience overload which accepts its SQL as a String + array. They will be concatenated together as-is, with no + separator, and passed on to one of the other overloads. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String[] sql, int prepFlags, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, String.join("",sql), prepFlags, p); + } + + /** + Convenience overload which uses no statement-preparation flags. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String[] sql, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, sql, 0, p); + } + + private static native int sqlite3_preupdate_blobwrite(@NotNull long ptrToDb); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_blobwrite(), else it returns + SQLITE_MISUSE with no side effects. + */ + public static int sqlite3_preupdate_blobwrite(@NotNull sqlite3 db){ + return sqlite3_preupdate_blobwrite(db.getNativePointer()); + } + + private static native int sqlite3_preupdate_count(@NotNull long ptrToDb); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_count(), else it returns + SQLITE_MISUSE with no side effects. + */ + public static int sqlite3_preupdate_count(@NotNull sqlite3 db){ + return sqlite3_preupdate_count(db.getNativePointer()); + } + + private static native int sqlite3_preupdate_depth(@NotNull long ptrToDb); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_depth(), else it returns + SQLITE_MISUSE with no side effects. + */ + public static int sqlite3_preupdate_depth(@NotNull sqlite3 db){ + return sqlite3_preupdate_depth(db.getNativePointer()); + } + + private static native PreupdateHookCallback sqlite3_preupdate_hook( + @NotNull long ptrToDb, @Nullable PreupdateHookCallback hook + ); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_hook(), else it returns null + with no side effects. + */ + public static PreupdateHookCallback sqlite3_preupdate_hook( + @NotNull sqlite3 db, @Nullable PreupdateHookCallback hook + ){ + return sqlite3_preupdate_hook(db.getNativePointer(), hook); + } + + private static native int sqlite3_preupdate_new(@NotNull long ptrToDb, int col, + @NotNull OutputPointer.sqlite3_value out); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, + this acts as a proxy for C's sqlite3_preupdate_new(), else it + returns SQLITE_MISUSE with no side effects. + + WARNING: client code _must not_ hold a reference to the returned + sqlite3_value object beyond the scope of the preupdate hook in + which this function is called. Doing so will leave the client + holding a stale pointer, the address of which could point to + anything at all after the pre-update hook is complete. This API + has no way to record such objects and clear/invalidate them at + the end of a pre-update hook. We "could" add infrastructure to do + so, but would require significant levels of bookkeeping. + */ + public static int sqlite3_preupdate_new(@NotNull sqlite3 db, int col, + @NotNull OutputPointer.sqlite3_value out){ + return sqlite3_preupdate_new(db.getNativePointer(), col, out); + } + + /** + Convenience wrapper for the 3-arg sqlite3_preupdate_new() which returns + null on error. + */ + public static sqlite3_value sqlite3_preupdate_new(@NotNull sqlite3 db, int col){ + final OutputPointer.sqlite3_value out = new OutputPointer.sqlite3_value(); + sqlite3_preupdate_new(db.getNativePointer(), col, out); + return out.take(); + } + + private static native int sqlite3_preupdate_old(@NotNull long ptrToDb, int col, + @NotNull OutputPointer.sqlite3_value out); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, + this acts as a proxy for C's sqlite3_preupdate_old(), else it + returns SQLITE_MISUSE with no side effects. + + WARNING: see warning in sqlite3_preupdate_new() regarding the + potential for stale sqlite3_value handles. + */ + public static int sqlite3_preupdate_old(@NotNull sqlite3 db, int col, + @NotNull OutputPointer.sqlite3_value out){ + return sqlite3_preupdate_old(db.getNativePointer(), col, out); + } + + /** + Convenience wrapper for the 3-arg sqlite3_preupdate_old() which returns + null on error. + */ + public static sqlite3_value sqlite3_preupdate_old(@NotNull sqlite3 db, int col){ + final OutputPointer.sqlite3_value out = new OutputPointer.sqlite3_value(); + sqlite3_preupdate_old(db.getNativePointer(), col, out); + return out.take(); + } + + public static native void sqlite3_progress_handler( + @NotNull sqlite3 db, int n, @Nullable ProgressHandlerCallback h + ); + + public static native void sqlite3_randomness(byte[] target); + + public static native int sqlite3_release_memory(int n); + + public static native int sqlite3_reset(@NotNull sqlite3_stmt stmt); + + /** + Works like the C API except that it has no side effects if auto + extensions are currently running. (The JNI-level list of + extensions cannot be manipulated while it is being traversed.) + */ + public static native void sqlite3_reset_auto_extension(); + + public static native void sqlite3_result_double( + @NotNull sqlite3_context cx, double v + ); + + /** + The main sqlite3_result_error() impl of which all others are + proxies. eTextRep must be one of SQLITE_UTF8 or SQLITE_UTF16 and + msg must be encoded correspondingly. Any other eTextRep value + results in the C-level sqlite3_result_error() being called with a + complaint about the invalid argument. + */ + private static native void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull byte[] msg, int eTextRep + ); + + public static void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull byte[] utf8 + ){ + sqlite3_result_error(cx, utf8, SQLITE_UTF8); + } + + public static void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull String msg + ){ + final byte[] utf8 = msg.getBytes(StandardCharsets.UTF_8); + sqlite3_result_error(cx, utf8, SQLITE_UTF8); + } + + public static void sqlite3_result_error16( + @NotNull sqlite3_context cx, @NotNull byte[] utf16 + ){ + sqlite3_result_error(cx, utf16, SQLITE_UTF16); + } + + public static void sqlite3_result_error16( + @NotNull sqlite3_context cx, @NotNull String msg + ){ + final byte[] utf16 = msg.getBytes(StandardCharsets.UTF_16); + sqlite3_result_error(cx, utf16, SQLITE_UTF16); + } + + /** + Equivalent to passing e.toString() to {@link + #sqlite3_result_error(sqlite3_context,String)}. Note that + toString() is used instead of getMessage() because the former + prepends the exception type name to the message. + */ + public static void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull Exception e + ){ + sqlite3_result_error(cx, e.toString()); + } + + public static native void sqlite3_result_error_toobig( + @NotNull sqlite3_context cx + ); + + public static native void sqlite3_result_error_nomem( + @NotNull sqlite3_context cx + ); + + public static native void sqlite3_result_error_code( + @NotNull sqlite3_context cx, int c + ); + + public static native void sqlite3_result_int( + @NotNull sqlite3_context cx, int v + ); + + public static native void sqlite3_result_int64( + @NotNull sqlite3_context cx, long v + ); + + /** + Binds the SQL result to the given object, or {@link + #sqlite3_result_null} if {@code o} is null. Use {@link + #sqlite3_value_java_object} to fetch it. + +

This is implemented in terms of C's sqlite3_result_pointer(), + but that function is not exposed to JNI because (A) + cross-language semantic mismatch and (B) Java doesn't need that + argument for its intended purpose (type safety). + + @see #sqlite3_value_java_object + @see #sqlite3_bind_java_object + */ + public static native void sqlite3_result_java_object( + @NotNull sqlite3_context cx, @NotNull Object o + ); + + /** + Similar to sqlite3_bind_nio_buffer(), this works like + sqlite3_result_blob() but accepts a java.nio.ByteBuffer as its + input source. See sqlite3_bind_nio_buffer() for the semantics of + the second and subsequent arguments. + + If cx is null then this function will silently fail. If + sqlite3_jni_supports_nio() returns false or iBegin is negative, + an error result is set. If (begin+n) extends beyond the end of + the buffer, it is silently truncated to fit. + + If any of the following apply, this function behaves like + sqlite3_result_null(): the blob is null, the resulting slice of + the blob is empty. + + If the resulting slice of the buffer exceeds SQLITE_LIMIT_LENGTH + then this function behaves like sqlite3_result_error_toobig(). + */ + @Experimental + /*public*/ static native void sqlite3_result_nio_buffer( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob, + int begin, int n + ); + + /** + Convenience overload which uses the whole input object + as the result blob content. + */ + @Experimental + /*public*/ static void sqlite3_result_nio_buffer( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob + ){ + sqlite3_result_nio_buffer(cx, blob, 0, -1); + } + + public static native void sqlite3_result_null( + @NotNull sqlite3_context cx + ); + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Boolean v + ){ + sqlite3_result_int(cx, v ? 1 : 0); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, boolean v + ){ + sqlite3_result_int(cx, v ? 1 : 0); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Double v + ){ + sqlite3_result_double(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, double v + ){ + sqlite3_result_double(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Integer v + ){ + sqlite3_result_int(cx, v); + } + + public static void sqlite3_result_set(@NotNull sqlite3_context cx, int v){ + sqlite3_result_int(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Long v + ){ + sqlite3_result_int64(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, long v + ){ + sqlite3_result_int64(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @Nullable String v + ){ + if( null==v ) sqlite3_result_null(cx); + else sqlite3_result_text(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @Nullable byte[] blob + ){ + if( null==blob ) sqlite3_result_null(cx); + else sqlite3_result_blob(cx, blob, blob.length); + } + + public static native void sqlite3_result_subtype( + @NotNull sqlite3_context cx, int val + ); + + public static native void sqlite3_result_value( + @NotNull sqlite3_context cx, @NotNull sqlite3_value v + ); + + public static native void sqlite3_result_zeroblob( + @NotNull sqlite3_context cx, int n + ); + + public static native int sqlite3_result_zeroblob64( + @NotNull sqlite3_context cx, long n + ); + + /** + This overload is private because its final parameter is arguably + unnecessary in Java. + */ + private static native void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable byte[] blob, int maxLen + ); + + public static void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable byte[] blob + ){ + sqlite3_result_blob(cx, blob, (int)(null==blob ? 0 : blob.length)); + } + + /** + Convenience overload which behaves like + sqlite3_result_nio_buffer(). + */ + @Experimental + /*public*/ static void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob, + int begin, int n + ){ + sqlite3_result_nio_buffer(cx, blob, begin, n); + } + + /** + Convenience overload which behaves like the two-argument overload of + sqlite3_result_nio_buffer(). + */ + @Experimental + /*public*/ static void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob + ){ + sqlite3_result_nio_buffer(cx, blob); + } + + /** + Binds the given text using C's sqlite3_result_blob64() unless: + +

    + +
  • @param blob is null: translates to sqlite3_result_null()
  • + +
  • @param blob is too large: translates to + sqlite3_result_error_toobig()
  • + +
+ +

If @param maxLen is larger than blob.length, it is truncated + to that value. If it is negative, results are undefined.

+ +

This overload is private because its final parameter is + arguably unnecessary in Java.

+ */ + private static native void sqlite3_result_blob64( + @NotNull sqlite3_context cx, @Nullable byte[] blob, long maxLen + ); + + public static void sqlite3_result_blob64( + @NotNull sqlite3_context cx, @Nullable byte[] blob + ){ + sqlite3_result_blob64(cx, blob, (long)(null==blob ? 0 : blob.length)); + } + + /** + This overload is private because its final parameter is + arguably unnecessary in Java. + */ + private static native void sqlite3_result_text( + @NotNull sqlite3_context cx, @Nullable byte[] utf8, int maxLen + ); + + public static void sqlite3_result_text( + @NotNull sqlite3_context cx, @Nullable byte[] utf8 + ){ + sqlite3_result_text(cx, utf8, null==utf8 ? 0 : utf8.length); + } + + public static void sqlite3_result_text( + @NotNull sqlite3_context cx, @Nullable String text + ){ + if(null == text) sqlite3_result_null(cx); + else{ + final byte[] utf8 = text.getBytes(StandardCharsets.UTF_8); + sqlite3_result_text(cx, utf8, utf8.length); + } + } + + /** + Binds the given text using C's sqlite3_result_text64() unless: + +
    + +
  • text is null: translates to a call to {@link + #sqlite3_result_null}
  • + +
  • text is too large: translates to a call to + {@link #sqlite3_result_error_toobig}
  • + +
  • The @param encoding argument has an invalid value: translates to + {@link sqlite3_result_error_code} with code SQLITE_FORMAT.
  • + +
+ + If maxLength (in bytes, not characters) is larger than + text.length, it is silently truncated to text.length. If it is + negative, results are undefined. If text is null, the subsequent + arguments are ignored. + + This overload is private because its maxLength parameter is + arguably unnecessary in Java. + */ + private static native void sqlite3_result_text64( + @NotNull sqlite3_context cx, @Nullable byte[] text, + long maxLength, int encoding + ); + + /** + Sets the current UDF result to the given bytes, which are assumed + be encoded in UTF-16 using the platform's byte order. + */ + public static void sqlite3_result_text16( + @NotNull sqlite3_context cx, @Nullable byte[] utf16 + ){ + if(null == utf16) sqlite3_result_null(cx); + else sqlite3_result_text64(cx, utf16, utf16.length, SQLITE_UTF16); + } + + public static void sqlite3_result_text16( + @NotNull sqlite3_context cx, @Nullable String text + ){ + if(null == text) sqlite3_result_null(cx); + else{ + final byte[] b = text.getBytes(StandardCharsets.UTF_16); + sqlite3_result_text64(cx, b, b.length, SQLITE_UTF16); + } + } + + private static native RollbackHookCallback sqlite3_rollback_hook( + @NotNull long ptrToDb, @Nullable RollbackHookCallback hook + ); + + public static RollbackHookCallback sqlite3_rollback_hook( + @NotNull sqlite3 db, @Nullable RollbackHookCallback hook + ){ + return sqlite3_rollback_hook(db.getNativePointer(), hook); + } + + public static native int sqlite3_set_authorizer( + @NotNull sqlite3 db, @Nullable AuthorizerCallback auth + ); + + public static native void sqlite3_set_auxdata( + @NotNull sqlite3_context cx, int n, @Nullable Object data + ); + + public static native void sqlite3_set_last_insert_rowid( + @NotNull sqlite3 db, long rowid + ); + + + /** + In addition to calling the C-level sqlite3_shutdown(), the JNI + binding also cleans up all stale per-thread state managed by the + library, as well as any registered auto-extensions, and frees up + various bits of memory. Calling this while database handles or + prepared statements are still active will leak resources. Trying + to use those objects after this routine is called invoked + undefined behavior. + */ + public static synchronized native int sqlite3_shutdown(); + + public static native int sqlite3_sleep(int ms); + + public static native String sqlite3_sourceid(); + + public static native String sqlite3_sql(@NotNull sqlite3_stmt stmt); + + //! Consider removing this. We can use sqlite3_status64() instead, + // or use that one's impl with this one's name. + public static native int sqlite3_status( + int op, @NotNull OutputPointer.Int32 pCurrent, + @NotNull OutputPointer.Int32 pHighwater, boolean reset + ); + + public static native int sqlite3_status64( + int op, @NotNull OutputPointer.Int64 pCurrent, + @NotNull OutputPointer.Int64 pHighwater, boolean reset + ); + + private static native int sqlite3_step(@NotNull long ptrToStmt); + + public static int sqlite3_step(@NotNull sqlite3_stmt stmt){ + return null==stmt ? SQLITE_MISUSE : sqlite3_step(stmt.getNativePointer()); + } + + public static native boolean sqlite3_stmt_busy(@NotNull sqlite3_stmt stmt); + + private static native int sqlite3_stmt_explain(@NotNull long ptrToStmt, int op); + + public static int sqlite3_stmt_explain(@NotNull sqlite3_stmt stmt, int op){ + return null==stmt ? SQLITE_MISUSE : sqlite3_stmt_explain(stmt.getNativePointer(), op); + } + + private static native int sqlite3_stmt_isexplain(@NotNull long ptrToStmt); + + public static int sqlite3_stmt_isexplain(@NotNull sqlite3_stmt stmt){ + return null==stmt ? 0 : sqlite3_stmt_isexplain(stmt.getNativePointer()); + } + + public static native boolean sqlite3_stmt_readonly(@NotNull sqlite3_stmt stmt); + + public static native int sqlite3_stmt_status( + @NotNull sqlite3_stmt stmt, int op, boolean reset + ); + + /** + Internal impl of the public sqlite3_strglob() method. Neither + argument may be null and both must be NUL-terminated UTF-8. + + This overload is private because: (A) to keep users from + inadvertently passing non-NUL-terminated byte arrays (an easy + thing to do). (B) it is cheaper to NUL-terminate the + String-to-byte-array conversion in the Java implementation + (sqlite3_strglob(String,String)) than to do that in C, so that + signature is the public-facing one. + */ + private static native int sqlite3_strglob( + @NotNull byte[] glob, @NotNull byte[] nulTerminatedUtf8 + ); + + public static int sqlite3_strglob( + @NotNull String glob, @NotNull String txt + ){ + return sqlite3_strglob(nulTerminateUtf8(glob), + nulTerminateUtf8(txt)); + } + + /** + The LIKE counterpart of the private sqlite3_strglob() method. + */ + private static native int sqlite3_strlike( + @NotNull byte[] glob, @NotNull byte[] nulTerminatedUtf8, + int escChar + ); + + public static int sqlite3_strlike( + @NotNull String glob, @NotNull String txt, char escChar + ){ + return sqlite3_strlike(nulTerminateUtf8(glob), + nulTerminateUtf8(txt), + (int)escChar); + } + + private static native int sqlite3_system_errno(@NotNull long ptrToDb); + + public static int sqlite3_system_errno(@NotNull sqlite3 db){ + return sqlite3_system_errno(db.getNativePointer()); + } + + public static native int sqlite3_table_column_metadata( + @NotNull sqlite3 db, @NotNull String zDbName, + @NotNull String zTableName, @NotNull String zColumnName, + @Nullable OutputPointer.String pzDataType, + @Nullable OutputPointer.String pzCollSeq, + @Nullable OutputPointer.Bool pNotNull, + @Nullable OutputPointer.Bool pPrimaryKey, + @Nullable OutputPointer.Bool pAutoinc + ); + + /** + Convenience overload which returns its results via a single + output object. If this function returns non-0 (error), the the + contents of the output object are not modified. + */ + public static int sqlite3_table_column_metadata( + @NotNull sqlite3 db, @NotNull String zDbName, + @NotNull String zTableName, @NotNull String zColumnName, + @NotNull TableColumnMetadata out){ + return sqlite3_table_column_metadata( + db, zDbName, zTableName, zColumnName, + out.pzDataType, out.pzCollSeq, out.pNotNull, + out.pPrimaryKey, out.pAutoinc); + } + + /** + Convenience overload which returns the column metadata object on + success and null on error. + */ + public static TableColumnMetadata sqlite3_table_column_metadata( + @NotNull sqlite3 db, @NotNull String zDbName, + @NotNull String zTableName, @NotNull String zColumnName){ + final TableColumnMetadata out = new TableColumnMetadata(); + return 0==sqlite3_table_column_metadata( + db, zDbName, zTableName, zColumnName, out + ) ? out : null; + } + + public static native int sqlite3_threadsafe(); + + private static native int sqlite3_total_changes(@NotNull long ptrToDb); + + public static int sqlite3_total_changes(@NotNull sqlite3 db){ + return sqlite3_total_changes(db.getNativePointer()); + } + + private static native long sqlite3_total_changes64(@NotNull long ptrToDb); + + public static long sqlite3_total_changes64(@NotNull sqlite3 db){ + return sqlite3_total_changes64(db.getNativePointer()); + } + + /** + Works like C's sqlite3_trace_v2() except that the 3rd argument to that + function is elided here because the roles of that functions' 3rd and 4th + arguments are encapsulated in the final argument to this function. + +

Unlike the C API, which is documented as always returning 0, + this implementation returns non-0 if initialization of the tracer + mapping state fails (e.g. on OOM). + */ + public static native int sqlite3_trace_v2( + @NotNull sqlite3 db, int traceMask, @Nullable TraceV2Callback tracer + ); + + public static native int sqlite3_txn_state( + @NotNull sqlite3 db, @Nullable String zSchema + ); + + private static native UpdateHookCallback sqlite3_update_hook( + @NotNull long ptrToDb, @Nullable UpdateHookCallback hook + ); + + public static UpdateHookCallback sqlite3_update_hook( + @NotNull sqlite3 db, @Nullable UpdateHookCallback hook + ){ + return sqlite3_update_hook(db.getNativePointer(), hook); + } + + /* + Note that: + + void * sqlite3_user_data(sqlite3_context*) + + Is not relevant in the JNI binding, as its feature is replaced by + the ability to pass an object, including any relevant state, to + sqlite3_create_function(). + */ + + private static native byte[] sqlite3_value_blob(@NotNull long ptrToValue); + + public static byte[] sqlite3_value_blob(@NotNull sqlite3_value v){ + return sqlite3_value_blob(v.getNativePointer()); + } + + private static native int sqlite3_value_bytes(@NotNull long ptrToValue); + + public static int sqlite3_value_bytes(@NotNull sqlite3_value v){ + return sqlite3_value_bytes(v.getNativePointer()); + } + + private static native int sqlite3_value_bytes16(@NotNull long ptrToValue); + + public static int sqlite3_value_bytes16(@NotNull sqlite3_value v){ + return sqlite3_value_bytes16(v.getNativePointer()); + } + + private static native double sqlite3_value_double(@NotNull long ptrToValue); + + public static double sqlite3_value_double(@NotNull sqlite3_value v){ + return sqlite3_value_double(v.getNativePointer()); + } + + private static native sqlite3_value sqlite3_value_dup(@NotNull long ptrToValue); + + public static sqlite3_value sqlite3_value_dup(@NotNull sqlite3_value v){ + return sqlite3_value_dup(v.getNativePointer()); + } + + private static native int sqlite3_value_encoding(@NotNull long ptrToValue); + + public static int sqlite3_value_encoding(@NotNull sqlite3_value v){ + return sqlite3_value_encoding(v.getNativePointer()); + } + + private static native void sqlite3_value_free(@Nullable long ptrToValue); + + public static void sqlite3_value_free(@Nullable sqlite3_value v){ + if( null!=v ) sqlite3_value_free(v.clearNativePointer()); + } + + private static native boolean sqlite3_value_frombind(@NotNull long ptrToValue); + + public static boolean sqlite3_value_frombind(@NotNull sqlite3_value v){ + return sqlite3_value_frombind(v.getNativePointer()); + } + + private static native int sqlite3_value_int(@NotNull long ptrToValue); + + public static int sqlite3_value_int(@NotNull sqlite3_value v){ + return sqlite3_value_int(v.getNativePointer()); + } + + private static native long sqlite3_value_int64(@NotNull long ptrToValue); + + public static long sqlite3_value_int64(@NotNull sqlite3_value v){ + return sqlite3_value_int64(v.getNativePointer()); + } + + private static native Object sqlite3_value_java_object(@NotNull long ptrToValue); + + /** + If the given value was set using {@link + #sqlite3_result_java_object} then this function returns that + object, else it returns null. + +

It is up to the caller to inspect the object to determine its + type, and cast it if necessary. + */ + public static Object sqlite3_value_java_object(@NotNull sqlite3_value v){ + return sqlite3_value_java_object(v.getNativePointer()); + } + + /** + A variant of sqlite3_value_java_object() which returns the + fetched object cast to T if the object is an instance of the + given Class, else it returns null. + */ + @SuppressWarnings("unchecked") + public static T sqlite3_value_java_object(@NotNull sqlite3_value v, + @NotNull Class type){ + final Object o = sqlite3_value_java_object(v); + return type.isInstance(o) ? (T)o : null; + } + + /** + A variant of sqlite3_column_blob() which returns the blob as a + ByteBuffer object. Returns null if its argument is null, if + sqlite3_jni_supports_nio() is false, or if sqlite3_value_blob() + would return null for the same input. + */ + @Experimental + /*public*/ static native java.nio.ByteBuffer sqlite3_value_nio_buffer( + @NotNull sqlite3_value v + ); + + private static native int sqlite3_value_nochange(@NotNull long ptrToValue); + + public static int sqlite3_value_nochange(@NotNull sqlite3_value v){ + return sqlite3_value_nochange(v.getNativePointer()); + } + + private static native int sqlite3_value_numeric_type(@NotNull long ptrToValue); + + public static int sqlite3_value_numeric_type(@NotNull sqlite3_value v){ + return sqlite3_value_numeric_type(v.getNativePointer()); + } + + private static native int sqlite3_value_subtype(@NotNull long ptrToValue); + + public static int sqlite3_value_subtype(@NotNull sqlite3_value v){ + return sqlite3_value_subtype(v.getNativePointer()); + } + + private static native byte[] sqlite3_value_text(@NotNull long ptrToValue); + + /** + Functions identially to the C API, and this note is just to + stress that the returned bytes are encoded as UTF-8. It returns + null if the underlying C-level sqlite3_value_text() returns NULL + or on allocation error. + */ + public static byte[] sqlite3_value_text(@NotNull sqlite3_value v){ + return sqlite3_value_text(v.getNativePointer()); + } + + private static native String sqlite3_value_text16(@NotNull long ptrToValue); + + public static String sqlite3_value_text16(@NotNull sqlite3_value v){ + return sqlite3_value_text16(v.getNativePointer()); + } + + private static native int sqlite3_value_type(@NotNull long ptrToValue); + + public static int sqlite3_value_type(@NotNull sqlite3_value v){ + return sqlite3_value_type(v.getNativePointer()); + } + + /** + This is NOT part of the public API. It exists solely as a place + for this code's developers to collect internal metrics and such. + It has no stable interface. It may go way or change behavior at + any time. + */ + public static native void sqlite3_jni_internal_details(); + + ////////////////////////////////////////////////////////////////////// + // SQLITE_... constants follow... + + // version info + public static final int SQLITE_VERSION_NUMBER = sqlite3_libversion_number(); + public static final String SQLITE_VERSION = sqlite3_libversion(); + public static final String SQLITE_SOURCE_ID = sqlite3_sourceid(); + + // access + public static final int SQLITE_ACCESS_EXISTS = 0; + public static final int SQLITE_ACCESS_READWRITE = 1; + public static final int SQLITE_ACCESS_READ = 2; + + // authorizer + public static final int SQLITE_DENY = 1; + public static final int SQLITE_IGNORE = 2; + public static final int SQLITE_CREATE_INDEX = 1; + public static final int SQLITE_CREATE_TABLE = 2; + public static final int SQLITE_CREATE_TEMP_INDEX = 3; + public static final int SQLITE_CREATE_TEMP_TABLE = 4; + public static final int SQLITE_CREATE_TEMP_TRIGGER = 5; + public static final int SQLITE_CREATE_TEMP_VIEW = 6; + public static final int SQLITE_CREATE_TRIGGER = 7; + public static final int SQLITE_CREATE_VIEW = 8; + public static final int SQLITE_DELETE = 9; + public static final int SQLITE_DROP_INDEX = 10; + public static final int SQLITE_DROP_TABLE = 11; + public static final int SQLITE_DROP_TEMP_INDEX = 12; + public static final int SQLITE_DROP_TEMP_TABLE = 13; + public static final int SQLITE_DROP_TEMP_TRIGGER = 14; + public static final int SQLITE_DROP_TEMP_VIEW = 15; + public static final int SQLITE_DROP_TRIGGER = 16; + public static final int SQLITE_DROP_VIEW = 17; + public static final int SQLITE_INSERT = 18; + public static final int SQLITE_PRAGMA = 19; + public static final int SQLITE_READ = 20; + public static final int SQLITE_SELECT = 21; + public static final int SQLITE_TRANSACTION = 22; + public static final int SQLITE_UPDATE = 23; + public static final int SQLITE_ATTACH = 24; + public static final int SQLITE_DETACH = 25; + public static final int SQLITE_ALTER_TABLE = 26; + public static final int SQLITE_REINDEX = 27; + public static final int SQLITE_ANALYZE = 28; + public static final int SQLITE_CREATE_VTABLE = 29; + public static final int SQLITE_DROP_VTABLE = 30; + public static final int SQLITE_FUNCTION = 31; + public static final int SQLITE_SAVEPOINT = 32; + public static final int SQLITE_RECURSIVE = 33; + + // blob finalizers: these should, because they are treated as + // special pointer values in C, ideally have the same sizeof() as + // the platform's (void*), but we can't know that size from here. + public static final long SQLITE_STATIC = 0; + public static final long SQLITE_TRANSIENT = -1; + + // changeset + public static final int SQLITE_CHANGESETSTART_INVERT = 2; + public static final int SQLITE_CHANGESETAPPLY_NOSAVEPOINT = 1; + public static final int SQLITE_CHANGESETAPPLY_INVERT = 2; + public static final int SQLITE_CHANGESETAPPLY_IGNORENOOP = 4; + public static final int SQLITE_CHANGESET_DATA = 1; + public static final int SQLITE_CHANGESET_NOTFOUND = 2; + public static final int SQLITE_CHANGESET_CONFLICT = 3; + public static final int SQLITE_CHANGESET_CONSTRAINT = 4; + public static final int SQLITE_CHANGESET_FOREIGN_KEY = 5; + public static final int SQLITE_CHANGESET_OMIT = 0; + public static final int SQLITE_CHANGESET_REPLACE = 1; + public static final int SQLITE_CHANGESET_ABORT = 2; + + // config + public static final int SQLITE_CONFIG_SINGLETHREAD = 1; + public static final int SQLITE_CONFIG_MULTITHREAD = 2; + public static final int SQLITE_CONFIG_SERIALIZED = 3; + public static final int SQLITE_CONFIG_MALLOC = 4; + public static final int SQLITE_CONFIG_GETMALLOC = 5; + public static final int SQLITE_CONFIG_SCRATCH = 6; + public static final int SQLITE_CONFIG_PAGECACHE = 7; + public static final int SQLITE_CONFIG_HEAP = 8; + public static final int SQLITE_CONFIG_MEMSTATUS = 9; + public static final int SQLITE_CONFIG_MUTEX = 10; + public static final int SQLITE_CONFIG_GETMUTEX = 11; + public static final int SQLITE_CONFIG_LOOKASIDE = 13; + public static final int SQLITE_CONFIG_PCACHE = 14; + public static final int SQLITE_CONFIG_GETPCACHE = 15; + public static final int SQLITE_CONFIG_LOG = 16; + public static final int SQLITE_CONFIG_URI = 17; + public static final int SQLITE_CONFIG_PCACHE2 = 18; + public static final int SQLITE_CONFIG_GETPCACHE2 = 19; + public static final int SQLITE_CONFIG_COVERING_INDEX_SCAN = 20; + public static final int SQLITE_CONFIG_SQLLOG = 21; + public static final int SQLITE_CONFIG_MMAP_SIZE = 22; + public static final int SQLITE_CONFIG_WIN32_HEAPSIZE = 23; + public static final int SQLITE_CONFIG_PCACHE_HDRSZ = 24; + public static final int SQLITE_CONFIG_PMASZ = 25; + public static final int SQLITE_CONFIG_STMTJRNL_SPILL = 26; + public static final int SQLITE_CONFIG_SMALL_MALLOC = 27; + public static final int SQLITE_CONFIG_SORTERREF_SIZE = 28; + public static final int SQLITE_CONFIG_MEMDB_MAXSIZE = 29; + + // data types + public static final int SQLITE_INTEGER = 1; + public static final int SQLITE_FLOAT = 2; + public static final int SQLITE_TEXT = 3; + public static final int SQLITE_BLOB = 4; + public static final int SQLITE_NULL = 5; + + // db config + public static final int SQLITE_DBCONFIG_MAINDBNAME = 1000; + public static final int SQLITE_DBCONFIG_LOOKASIDE = 1001; + public static final int SQLITE_DBCONFIG_ENABLE_FKEY = 1002; + public static final int SQLITE_DBCONFIG_ENABLE_TRIGGER = 1003; + public static final int SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER = 1004; + public static final int SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION = 1005; + public static final int SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE = 1006; + public static final int SQLITE_DBCONFIG_ENABLE_QPSG = 1007; + public static final int SQLITE_DBCONFIG_TRIGGER_EQP = 1008; + public static final int SQLITE_DBCONFIG_RESET_DATABASE = 1009; + public static final int SQLITE_DBCONFIG_DEFENSIVE = 1010; + public static final int SQLITE_DBCONFIG_WRITABLE_SCHEMA = 1011; + public static final int SQLITE_DBCONFIG_LEGACY_ALTER_TABLE = 1012; + public static final int SQLITE_DBCONFIG_DQS_DML = 1013; + public static final int SQLITE_DBCONFIG_DQS_DDL = 1014; + public static final int SQLITE_DBCONFIG_ENABLE_VIEW = 1015; + public static final int SQLITE_DBCONFIG_LEGACY_FILE_FORMAT = 1016; + public static final int SQLITE_DBCONFIG_TRUSTED_SCHEMA = 1017; + public static final int SQLITE_DBCONFIG_STMT_SCANSTATUS = 1018; + public static final int SQLITE_DBCONFIG_REVERSE_SCANORDER = 1019; + public static final int SQLITE_DBCONFIG_MAX = 1019; + + // db status + public static final int SQLITE_DBSTATUS_LOOKASIDE_USED = 0; + public static final int SQLITE_DBSTATUS_CACHE_USED = 1; + public static final int SQLITE_DBSTATUS_SCHEMA_USED = 2; + public static final int SQLITE_DBSTATUS_STMT_USED = 3; + public static final int SQLITE_DBSTATUS_LOOKASIDE_HIT = 4; + public static final int SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE = 5; + public static final int SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL = 6; + public static final int SQLITE_DBSTATUS_CACHE_HIT = 7; + public static final int SQLITE_DBSTATUS_CACHE_MISS = 8; + public static final int SQLITE_DBSTATUS_CACHE_WRITE = 9; + public static final int SQLITE_DBSTATUS_DEFERRED_FKS = 10; + public static final int SQLITE_DBSTATUS_CACHE_USED_SHARED = 11; + public static final int SQLITE_DBSTATUS_CACHE_SPILL = 12; + public static final int SQLITE_DBSTATUS_MAX = 12; + + // encodings + public static final int SQLITE_UTF8 = 1; + public static final int SQLITE_UTF16LE = 2; + public static final int SQLITE_UTF16BE = 3; + public static final int SQLITE_UTF16 = 4; + public static final int SQLITE_UTF16_ALIGNED = 8; + + // fcntl + public static final int SQLITE_FCNTL_LOCKSTATE = 1; + public static final int SQLITE_FCNTL_GET_LOCKPROXYFILE = 2; + public static final int SQLITE_FCNTL_SET_LOCKPROXYFILE = 3; + public static final int SQLITE_FCNTL_LAST_ERRNO = 4; + public static final int SQLITE_FCNTL_SIZE_HINT = 5; + public static final int SQLITE_FCNTL_CHUNK_SIZE = 6; + public static final int SQLITE_FCNTL_FILE_POINTER = 7; + public static final int SQLITE_FCNTL_SYNC_OMITTED = 8; + public static final int SQLITE_FCNTL_WIN32_AV_RETRY = 9; + public static final int SQLITE_FCNTL_PERSIST_WAL = 10; + public static final int SQLITE_FCNTL_OVERWRITE = 11; + public static final int SQLITE_FCNTL_VFSNAME = 12; + public static final int SQLITE_FCNTL_POWERSAFE_OVERWRITE = 13; + public static final int SQLITE_FCNTL_PRAGMA = 14; + public static final int SQLITE_FCNTL_BUSYHANDLER = 15; + public static final int SQLITE_FCNTL_TEMPFILENAME = 16; + public static final int SQLITE_FCNTL_MMAP_SIZE = 18; + public static final int SQLITE_FCNTL_TRACE = 19; + public static final int SQLITE_FCNTL_HAS_MOVED = 20; + public static final int SQLITE_FCNTL_SYNC = 21; + public static final int SQLITE_FCNTL_COMMIT_PHASETWO = 22; + public static final int SQLITE_FCNTL_WIN32_SET_HANDLE = 23; + public static final int SQLITE_FCNTL_WAL_BLOCK = 24; + public static final int SQLITE_FCNTL_ZIPVFS = 25; + public static final int SQLITE_FCNTL_RBU = 26; + public static final int SQLITE_FCNTL_VFS_POINTER = 27; + public static final int SQLITE_FCNTL_JOURNAL_POINTER = 28; + public static final int SQLITE_FCNTL_WIN32_GET_HANDLE = 29; + public static final int SQLITE_FCNTL_PDB = 30; + public static final int SQLITE_FCNTL_BEGIN_ATOMIC_WRITE = 31; + public static final int SQLITE_FCNTL_COMMIT_ATOMIC_WRITE = 32; + public static final int SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE = 33; + public static final int SQLITE_FCNTL_LOCK_TIMEOUT = 34; + public static final int SQLITE_FCNTL_DATA_VERSION = 35; + public static final int SQLITE_FCNTL_SIZE_LIMIT = 36; + public static final int SQLITE_FCNTL_CKPT_DONE = 37; + public static final int SQLITE_FCNTL_RESERVE_BYTES = 38; + public static final int SQLITE_FCNTL_CKPT_START = 39; + public static final int SQLITE_FCNTL_EXTERNAL_READER = 40; + public static final int SQLITE_FCNTL_CKSM_FILE = 41; + public static final int SQLITE_FCNTL_RESET_CACHE = 42; + + // flock + public static final int SQLITE_LOCK_NONE = 0; + public static final int SQLITE_LOCK_SHARED = 1; + public static final int SQLITE_LOCK_RESERVED = 2; + public static final int SQLITE_LOCK_PENDING = 3; + public static final int SQLITE_LOCK_EXCLUSIVE = 4; + + // iocap + public static final int SQLITE_IOCAP_ATOMIC = 1; + public static final int SQLITE_IOCAP_ATOMIC512 = 2; + public static final int SQLITE_IOCAP_ATOMIC1K = 4; + public static final int SQLITE_IOCAP_ATOMIC2K = 8; + public static final int SQLITE_IOCAP_ATOMIC4K = 16; + public static final int SQLITE_IOCAP_ATOMIC8K = 32; + public static final int SQLITE_IOCAP_ATOMIC16K = 64; + public static final int SQLITE_IOCAP_ATOMIC32K = 128; + public static final int SQLITE_IOCAP_ATOMIC64K = 256; + public static final int SQLITE_IOCAP_SAFE_APPEND = 512; + public static final int SQLITE_IOCAP_SEQUENTIAL = 1024; + public static final int SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN = 2048; + public static final int SQLITE_IOCAP_POWERSAFE_OVERWRITE = 4096; + public static final int SQLITE_IOCAP_IMMUTABLE = 8192; + public static final int SQLITE_IOCAP_BATCH_ATOMIC = 16384; + + // limits + public static final int SQLITE_LIMIT_LENGTH = 0; + public static final int SQLITE_LIMIT_SQL_LENGTH = 1; + public static final int SQLITE_LIMIT_COLUMN = 2; + public static final int SQLITE_LIMIT_EXPR_DEPTH = 3; + public static final int SQLITE_LIMIT_COMPOUND_SELECT = 4; + public static final int SQLITE_LIMIT_VDBE_OP = 5; + public static final int SQLITE_LIMIT_FUNCTION_ARG = 6; + public static final int SQLITE_LIMIT_ATTACHED = 7; + public static final int SQLITE_LIMIT_LIKE_PATTERN_LENGTH = 8; + public static final int SQLITE_LIMIT_VARIABLE_NUMBER = 9; + public static final int SQLITE_LIMIT_TRIGGER_DEPTH = 10; + public static final int SQLITE_LIMIT_WORKER_THREADS = 11; + + // open flags + + public static final int SQLITE_OPEN_READONLY = 0x00000001 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_READWRITE = 0x00000002 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_CREATE = 0x00000004 /* Ok for sqlite3_open_v2() */; + //public static final int SQLITE_OPEN_DELETEONCLOSE = 0x00000008 /* VFS only */; + //public static final int SQLITE_OPEN_EXCLUSIVE = 0x00000010 /* VFS only */; + //public static final int SQLITE_OPEN_AUTOPROXY = 0x00000020 /* VFS only */; + public static final int SQLITE_OPEN_URI = 0x00000040 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_MEMORY = 0x00000080 /* Ok for sqlite3_open_v2() */; + //public static final int SQLITE_OPEN_MAIN_DB = 0x00000100 /* VFS only */; + //public static final int SQLITE_OPEN_TEMP_DB = 0x00000200 /* VFS only */; + //public static final int SQLITE_OPEN_TRANSIENT_DB = 0x00000400 /* VFS only */; + //public static final int SQLITE_OPEN_MAIN_JOURNAL = 0x00000800 /* VFS only */; + //public static final int SQLITE_OPEN_TEMP_JOURNAL = 0x00001000 /* VFS only */; + //public static final int SQLITE_OPEN_SUBJOURNAL = 0x00002000 /* VFS only */; + //public static final int SQLITE_OPEN_SUPER_JOURNAL = 0x00004000 /* VFS only */; + public static final int SQLITE_OPEN_NOMUTEX = 0x00008000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_FULLMUTEX = 0x00010000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_SHAREDCACHE = 0x00020000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_PRIVATECACHE = 0x00040000 /* Ok for sqlite3_open_v2() */; + //public static final int SQLITE_OPEN_WAL = 0x00080000 /* VFS only */; + public static final int SQLITE_OPEN_NOFOLLOW = 0x01000000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_EXRESCODE = 0x02000000 /* Extended result codes */; + + // prepare flags + public static final int SQLITE_PREPARE_PERSISTENT = 1; + public static final int SQLITE_PREPARE_NO_VTAB = 4; + + // result codes + public static final int SQLITE_OK = 0; + public static final int SQLITE_ERROR = 1; + public static final int SQLITE_INTERNAL = 2; + public static final int SQLITE_PERM = 3; + public static final int SQLITE_ABORT = 4; + public static final int SQLITE_BUSY = 5; + public static final int SQLITE_LOCKED = 6; + public static final int SQLITE_NOMEM = 7; + public static final int SQLITE_READONLY = 8; + public static final int SQLITE_INTERRUPT = 9; + public static final int SQLITE_IOERR = 10; + public static final int SQLITE_CORRUPT = 11; + public static final int SQLITE_NOTFOUND = 12; + public static final int SQLITE_FULL = 13; + public static final int SQLITE_CANTOPEN = 14; + public static final int SQLITE_PROTOCOL = 15; + public static final int SQLITE_EMPTY = 16; + public static final int SQLITE_SCHEMA = 17; + public static final int SQLITE_TOOBIG = 18; + public static final int SQLITE_CONSTRAINT = 19; + public static final int SQLITE_MISMATCH = 20; + public static final int SQLITE_MISUSE = 21; + public static final int SQLITE_NOLFS = 22; + public static final int SQLITE_AUTH = 23; + public static final int SQLITE_FORMAT = 24; + public static final int SQLITE_RANGE = 25; + public static final int SQLITE_NOTADB = 26; + public static final int SQLITE_NOTICE = 27; + public static final int SQLITE_WARNING = 28; + public static final int SQLITE_ROW = 100; + public static final int SQLITE_DONE = 101; + public static final int SQLITE_ERROR_MISSING_COLLSEQ = 257; + public static final int SQLITE_ERROR_RETRY = 513; + public static final int SQLITE_ERROR_SNAPSHOT = 769; + public static final int SQLITE_IOERR_READ = 266; + public static final int SQLITE_IOERR_SHORT_READ = 522; + public static final int SQLITE_IOERR_WRITE = 778; + public static final int SQLITE_IOERR_FSYNC = 1034; + public static final int SQLITE_IOERR_DIR_FSYNC = 1290; + public static final int SQLITE_IOERR_TRUNCATE = 1546; + public static final int SQLITE_IOERR_FSTAT = 1802; + public static final int SQLITE_IOERR_UNLOCK = 2058; + public static final int SQLITE_IOERR_RDLOCK = 2314; + public static final int SQLITE_IOERR_DELETE = 2570; + public static final int SQLITE_IOERR_BLOCKED = 2826; + public static final int SQLITE_IOERR_NOMEM = 3082; + public static final int SQLITE_IOERR_ACCESS = 3338; + public static final int SQLITE_IOERR_CHECKRESERVEDLOCK = 3594; + public static final int SQLITE_IOERR_LOCK = 3850; + public static final int SQLITE_IOERR_CLOSE = 4106; + public static final int SQLITE_IOERR_DIR_CLOSE = 4362; + public static final int SQLITE_IOERR_SHMOPEN = 4618; + public static final int SQLITE_IOERR_SHMSIZE = 4874; + public static final int SQLITE_IOERR_SHMLOCK = 5130; + public static final int SQLITE_IOERR_SHMMAP = 5386; + public static final int SQLITE_IOERR_SEEK = 5642; + public static final int SQLITE_IOERR_DELETE_NOENT = 5898; + public static final int SQLITE_IOERR_MMAP = 6154; + public static final int SQLITE_IOERR_GETTEMPPATH = 6410; + public static final int SQLITE_IOERR_CONVPATH = 6666; + public static final int SQLITE_IOERR_VNODE = 6922; + public static final int SQLITE_IOERR_AUTH = 7178; + public static final int SQLITE_IOERR_BEGIN_ATOMIC = 7434; + public static final int SQLITE_IOERR_COMMIT_ATOMIC = 7690; + public static final int SQLITE_IOERR_ROLLBACK_ATOMIC = 7946; + public static final int SQLITE_IOERR_DATA = 8202; + public static final int SQLITE_IOERR_CORRUPTFS = 8458; + public static final int SQLITE_LOCKED_SHAREDCACHE = 262; + public static final int SQLITE_LOCKED_VTAB = 518; + public static final int SQLITE_BUSY_RECOVERY = 261; + public static final int SQLITE_BUSY_SNAPSHOT = 517; + public static final int SQLITE_BUSY_TIMEOUT = 773; + public static final int SQLITE_CANTOPEN_NOTEMPDIR = 270; + public static final int SQLITE_CANTOPEN_ISDIR = 526; + public static final int SQLITE_CANTOPEN_FULLPATH = 782; + public static final int SQLITE_CANTOPEN_CONVPATH = 1038; + public static final int SQLITE_CANTOPEN_SYMLINK = 1550; + public static final int SQLITE_CORRUPT_VTAB = 267; + public static final int SQLITE_CORRUPT_SEQUENCE = 523; + public static final int SQLITE_CORRUPT_INDEX = 779; + public static final int SQLITE_READONLY_RECOVERY = 264; + public static final int SQLITE_READONLY_CANTLOCK = 520; + public static final int SQLITE_READONLY_ROLLBACK = 776; + public static final int SQLITE_READONLY_DBMOVED = 1032; + public static final int SQLITE_READONLY_CANTINIT = 1288; + public static final int SQLITE_READONLY_DIRECTORY = 1544; + public static final int SQLITE_ABORT_ROLLBACK = 516; + public static final int SQLITE_CONSTRAINT_CHECK = 275; + public static final int SQLITE_CONSTRAINT_COMMITHOOK = 531; + public static final int SQLITE_CONSTRAINT_FOREIGNKEY = 787; + public static final int SQLITE_CONSTRAINT_FUNCTION = 1043; + public static final int SQLITE_CONSTRAINT_NOTNULL = 1299; + public static final int SQLITE_CONSTRAINT_PRIMARYKEY = 1555; + public static final int SQLITE_CONSTRAINT_TRIGGER = 1811; + public static final int SQLITE_CONSTRAINT_UNIQUE = 2067; + public static final int SQLITE_CONSTRAINT_VTAB = 2323; + public static final int SQLITE_CONSTRAINT_ROWID = 2579; + public static final int SQLITE_CONSTRAINT_PINNED = 2835; + public static final int SQLITE_CONSTRAINT_DATATYPE = 3091; + public static final int SQLITE_NOTICE_RECOVER_WAL = 283; + public static final int SQLITE_NOTICE_RECOVER_ROLLBACK = 539; + public static final int SQLITE_WARNING_AUTOINDEX = 284; + public static final int SQLITE_AUTH_USER = 279; + public static final int SQLITE_OK_LOAD_PERMANENTLY = 256; + + // serialize + public static final int SQLITE_SERIALIZE_NOCOPY = 1; + public static final int SQLITE_DESERIALIZE_FREEONCLOSE = 1; + public static final int SQLITE_DESERIALIZE_READONLY = 4; + public static final int SQLITE_DESERIALIZE_RESIZEABLE = 2; + + // session + public static final int SQLITE_SESSION_CONFIG_STRMSIZE = 1; + public static final int SQLITE_SESSION_OBJCONFIG_SIZE = 1; + + // sqlite3 status + public static final int SQLITE_STATUS_MEMORY_USED = 0; + public static final int SQLITE_STATUS_PAGECACHE_USED = 1; + public static final int SQLITE_STATUS_PAGECACHE_OVERFLOW = 2; + public static final int SQLITE_STATUS_MALLOC_SIZE = 5; + public static final int SQLITE_STATUS_PARSER_STACK = 6; + public static final int SQLITE_STATUS_PAGECACHE_SIZE = 7; + public static final int SQLITE_STATUS_MALLOC_COUNT = 9; + + // stmt status + public static final int SQLITE_STMTSTATUS_FULLSCAN_STEP = 1; + public static final int SQLITE_STMTSTATUS_SORT = 2; + public static final int SQLITE_STMTSTATUS_AUTOINDEX = 3; + public static final int SQLITE_STMTSTATUS_VM_STEP = 4; + public static final int SQLITE_STMTSTATUS_REPREPARE = 5; + public static final int SQLITE_STMTSTATUS_RUN = 6; + public static final int SQLITE_STMTSTATUS_FILTER_MISS = 7; + public static final int SQLITE_STMTSTATUS_FILTER_HIT = 8; + public static final int SQLITE_STMTSTATUS_MEMUSED = 99; + + // sync flags + public static final int SQLITE_SYNC_NORMAL = 2; + public static final int SQLITE_SYNC_FULL = 3; + public static final int SQLITE_SYNC_DATAONLY = 16; + + // tracing flags + public static final int SQLITE_TRACE_STMT = 1; + public static final int SQLITE_TRACE_PROFILE = 2; + public static final int SQLITE_TRACE_ROW = 4; + public static final int SQLITE_TRACE_CLOSE = 8; + + // transaction state + public static final int SQLITE_TXN_NONE = 0; + public static final int SQLITE_TXN_READ = 1; + public static final int SQLITE_TXN_WRITE = 2; + + // udf flags + public static final int SQLITE_DETERMINISTIC = 0x000000800; + public static final int SQLITE_DIRECTONLY = 0x000080000; + public static final int SQLITE_SUBTYPE = 0x000100000; + public static final int SQLITE_INNOCUOUS = 0x000200000; + public static final int SQLITE_RESULT_SUBTYPE = 0x001000000; + + // virtual tables + public static final int SQLITE_INDEX_SCAN_UNIQUE = 1; + public static final int SQLITE_INDEX_CONSTRAINT_EQ = 2; + public static final int SQLITE_INDEX_CONSTRAINT_GT = 4; + public static final int SQLITE_INDEX_CONSTRAINT_LE = 8; + public static final int SQLITE_INDEX_CONSTRAINT_LT = 16; + public static final int SQLITE_INDEX_CONSTRAINT_GE = 32; + public static final int SQLITE_INDEX_CONSTRAINT_MATCH = 64; + public static final int SQLITE_INDEX_CONSTRAINT_LIKE = 65; + public static final int SQLITE_INDEX_CONSTRAINT_GLOB = 66; + public static final int SQLITE_INDEX_CONSTRAINT_REGEXP = 67; + public static final int SQLITE_INDEX_CONSTRAINT_NE = 68; + public static final int SQLITE_INDEX_CONSTRAINT_ISNOT = 69; + public static final int SQLITE_INDEX_CONSTRAINT_ISNOTNULL = 70; + public static final int SQLITE_INDEX_CONSTRAINT_ISNULL = 71; + public static final int SQLITE_INDEX_CONSTRAINT_IS = 72; + public static final int SQLITE_INDEX_CONSTRAINT_LIMIT = 73; + public static final int SQLITE_INDEX_CONSTRAINT_OFFSET = 74; + public static final int SQLITE_INDEX_CONSTRAINT_FUNCTION = 150; + public static final int SQLITE_VTAB_CONSTRAINT_SUPPORT = 1; + public static final int SQLITE_VTAB_INNOCUOUS = 2; + public static final int SQLITE_VTAB_DIRECTONLY = 3; + public static final int SQLITE_VTAB_USES_ALL_SCHEMAS = 4; + public static final int SQLITE_ROLLBACK = 1; + public static final int SQLITE_FAIL = 3; + public static final int SQLITE_REPLACE = 5; + static { + init(); + } + /* Must come after static init(). */ + private static final boolean JNI_SUPPORTS_NIO = sqlite3_jni_supports_nio(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java b/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java new file mode 100644 index 0000000000000000000000000000000000000000..7df748e8d8381868f45ca14481c6c0a135b8ebd8 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java @@ -0,0 +1,45 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +/** + This marker interface exists soley for use as a documentation and + class-grouping tool. It should be applied to interfaces or + classes which have a call() method implementing some specific + callback interface on behalf of the C library. + +

Unless very explicitely documented otherwise, callbacks must + never throw. Any which do throw but should not might trigger debug + output regarding the error, but the exception will not be + propagated. For callback interfaces which support returning error + info to the core, the JNI binding will convert any exceptions to + C-level error information. For callback interfaces which do not + support returning error information, all exceptions will + necessarily be suppressed in order to retain the C-style no-throw + semantics and avoid invoking undefined behavior in the C layer. + +

Callbacks of this style follow a common naming convention: + +

1) They use the UpperCamelCase form of the C function they're + proxying for, minus the {@code sqlite3_} prefix, plus a {@code + Callback} suffix. e.g. {@code sqlite3_busy_handler()}'s callback is + named {@code BusyHandlerCallback}. Exceptions are made where that + would potentially be ambiguous, e.g. {@link ConfigSqllogCallback} + instead of {@code ConfigCallback} because the {@code + sqlite3_config()} interface may need to support more callback types + in the future. + +

2) They all have a {@code call()} method but its signature is + callback-specific. +*/ +public interface CallbackProxy {} diff --git a/ext/jni/src/org/sqlite/jni/capi/CollationCallback.java b/ext/jni/src/org/sqlite/jni/capi/CollationCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..ed8bd094753d2dc1cbf83c8e80657253cc5fe473 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CollationCallback.java @@ -0,0 +1,35 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.NotNull; + +/** + Callback for use with {@link CApi#sqlite3_create_collation}. + + @see AbstractCollationCallback +*/ +public interface CollationCallback + extends CallbackProxy, XDestroyCallback { + /** + Must compare the given byte arrays and return the result using + {@code memcmp()} semantics. + */ + int call(@NotNull byte[] lhs, @NotNull byte[] rhs); + + /** + Called by SQLite when the collation is destroyed. If a collation + requires custom cleanup, override this method. + */ + void xDestroy(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java b/ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..ffd7fa94aba1069af4f63e95482e13d4cb945c4f --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java @@ -0,0 +1,29 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_collation_needed}. +*/ +public interface CollationNeededCallback extends CallbackProxy { + /** + Has the same semantics as the C-level sqlite3_create_collation() + callback. + +

Because the C API has no mechanism for reporting errors + from this callbacks, any exceptions thrown by this callback + are suppressed. + */ + void call(sqlite3 db, int eTextRep, String collationName); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..e1e55c78d291703e86407c70354ecc9b925c945d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_commit_hook}. +*/ +public interface CommitHookCallback extends CallbackProxy { + /** + Works as documented for the C-level sqlite3_commit_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + int call(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b/ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..6513b0730de145b646f7396cc2f3862dd2a4e233 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java @@ -0,0 +1,25 @@ +/* +** 2023-08-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A callback for use with sqlite3_config(). +*/ +public interface ConfigLogCallback { + /** + Must function as described for a C-level callback for + {@link CApi#sqlite3_config(ConfigLogCallback)}, with the slight signature change. + */ + void call(int errCode, String msg); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java b/ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..a5530b49a460decfab5da8c416efb4bb2931778d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java @@ -0,0 +1,25 @@ +/* +** 2023-08-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A callback for use with sqlite3_config(). +*/ +public interface ConfigSqlLogCallback { + /** + Must function as described for a C-level callback for + {@link CApi#sqlite3_config(ConfigSqlLogCallback)}, with the slight signature change. + */ + void call(sqlite3 db, String msg, int msgType ); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b/ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..e82909e4246d98788c287447d6bb7a228ac3c076 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java @@ -0,0 +1,46 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A helper for passing pointers between JNI C code and Java, in + particular for output pointers of high-level object types in the + sqlite3 C API, e.g. (sqlite3**) and (sqlite3_stmt**). This is + intended to be subclassed and the ContextType is intended to be the + class which is doing the subclassing. The intent of the ContextType + is strictly to provide some level of type safety by avoiding that + NativePointerHolder is not inadvertently passed to an incompatible + function signature. + + These objects do not own the pointer they refer to. They are + intended simply to communicate that pointer between C and Java. +*/ +public class NativePointerHolder { + //! Only set from JNI, where access permissions don't matter. + private volatile long nativePointer = 0; + /** + For use ONLY by package-level APIs which act as proxies for + close/finalize operations. Such ops must call this to zero out + the pointer so that this object is not carrying a stale + pointer. This function returns the prior value of the pointer and + sets it to 0. + */ + final long clearNativePointer() { + final long rv = nativePointer; + nativePointer= 0; + return rv; + } + + public final long getNativePointer(){ return nativePointer; } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java b/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java new file mode 100644 index 0000000000000000000000000000000000000000..7bf7529da13954b63386bd52fd620a154d70827d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java @@ -0,0 +1,253 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Helper classes for handling JNI output pointers. + +

We do not use a generic OutputPointer because working with those + from the native JNI code is unduly quirky due to a lack of + autoboxing at that level. + +

The usage is similar for all of thes types: + +

{@code
+   OutputPointer.sqlite3 out = new OutputPointer.sqlite3();
+   assert( null==out.get() );
+   int rc = sqlite3_open(":memory:", out);
+   if( 0!=rc ) ... error;
+   assert( null!=out.get() );
+   sqlite3 db = out.take();
+   assert( null==out.get() );
+   }
+ +

With the minor exception that the primitive types permit direct + access to the object's value via the `value` property, whereas the + JNI-level opaque types do not permit client-level code to set that + property. + +

Warning: do not share instances of these classes across + threads. Doing so may lead to corrupting sqlite3-internal state. +*/ +public final class OutputPointer { + + /** + Output pointer for use with routines, such as sqlite3_open(), + which return a database handle via an output pointer. These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3 { + private org.sqlite.jni.capi.sqlite3 value; + /** Initializes with a null value. */ + public sqlite3(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public final org.sqlite.jni.capi.sqlite3 get(){return value;} + /** Equivalent to calling get() then clear(). */ + public final org.sqlite.jni.capi.sqlite3 take(){ + final org.sqlite.jni.capi.sqlite3 v = value; + value = null; + return v; + } + } + + /** + Output pointer for sqlite3_blob_open(). These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3_blob { + private org.sqlite.jni.capi.sqlite3_blob value; + /** Initializes with a null value. */ + public sqlite3_blob(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public final org.sqlite.jni.capi.sqlite3_blob get(){return value;} + /** Equivalent to calling get() then clear(). */ + public final org.sqlite.jni.capi.sqlite3_blob take(){ + final org.sqlite.jni.capi.sqlite3_blob v = value; + value = null; + return v; + } + } + + /** + Output pointer for use with routines, such as sqlite3_prepare(), + which return a statement handle via an output pointer. These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3_stmt { + private org.sqlite.jni.capi.sqlite3_stmt value; + /** Initializes with a null value. */ + public sqlite3_stmt(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public final org.sqlite.jni.capi.sqlite3_stmt get(){return value;} + /** Equivalent to calling get() then clear(). */ + public final org.sqlite.jni.capi.sqlite3_stmt take(){ + final org.sqlite.jni.capi.sqlite3_stmt v = value; + value = null; + return v; + } + } + + /** + Output pointer for use with routines, such as sqlite3_prepupdate_new(), + which return a sqlite3_value handle via an output pointer. These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3_value { + private org.sqlite.jni.capi.sqlite3_value value; + /** Initializes with a null value. */ + public sqlite3_value(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public final org.sqlite.jni.capi.sqlite3_value get(){return value;} + /** Equivalent to calling get() then clear(). */ + public final org.sqlite.jni.capi.sqlite3_value take(){ + final org.sqlite.jni.capi.sqlite3_value v = value; + value = null; + return v; + } + } + + /** + Output pointer for use with native routines which return booleans + via integer output pointers. + */ + public static final class Bool { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public boolean value; + /** Initializes with the value 0. */ + public Bool(){this(false);} + /** Initializes with the value v. */ + public Bool(boolean v){value = v;} + /** Returns the current value. */ + public final boolean get(){return value;} + /** Sets the current value to v. */ + public final void set(boolean v){value = v;} + } + + /** + Output pointer for use with native routines which return integers via + output pointers. + */ + public static final class Int32 { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public int value; + /** Initializes with the value 0. */ + public Int32(){this(0);} + /** Initializes with the value v. */ + public Int32(int v){value = v;} + /** Returns the current value. */ + public final int get(){return value;} + /** Sets the current value to v. */ + public final void set(int v){value = v;} + } + + /** + Output pointer for use with native routines which return 64-bit integers + via output pointers. + */ + public static final class Int64 { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public long value; + /** Initializes with the value 0. */ + public Int64(){this(0);} + /** Initializes with the value v. */ + public Int64(long v){value = v;} + /** Returns the current value. */ + public final long get(){return value;} + /** Sets the current value. */ + public final void set(long v){value = v;} + } + + /** + Output pointer for use with native routines which return strings via + output pointers. + */ + public static final class String { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public java.lang.String value; + /** Initializes with a null value. */ + public String(){this(null);} + /** Initializes with the value v. */ + public String(java.lang.String v){value = v;} + /** Returns the current value. */ + public final java.lang.String get(){return value;} + /** Sets the current value. */ + public final void set(java.lang.String v){value = v;} + } + + /** + Output pointer for use with native routines which return byte + arrays via output pointers. + */ + public static final class ByteArray { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public byte[] value; + /** Initializes with the value null. */ + public ByteArray(){this(null);} + /** Initializes with the value v. */ + public ByteArray(byte[] v){value = v;} + /** Returns the current value. */ + public final byte[] get(){return value;} + /** Sets the current value. */ + public final void set(byte[] v){value = v;} + } + + /** + Output pointer for use with native routines which return + blobs via java.nio.ByteBuffer. + + See {@link org.sqlite.jni.capi.CApi#sqlite3_jni_supports_nio} + */ + public static final class ByteBuffer { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public java.nio.ByteBuffer value; + /** Initializes with the value null. */ + public ByteBuffer(){this(null);} + /** Initializes with the value v. */ + public ByteBuffer(java.nio.ByteBuffer v){value = v;} + /** Returns the current value. */ + public final java.nio.ByteBuffer get(){return value;} + /** Sets the current value. */ + public final void set(java.nio.ByteBuffer v){value = v;} + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java b/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..9f6dd478ce41d1d42e917c7849a815611c42c904 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java @@ -0,0 +1,81 @@ +/* +** 2023-09-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_prepare_multi}. +*/ +public interface PrepareMultiCallback extends CallbackProxy { + + /** + Gets passed a sqlite3_stmt which it may handle in arbitrary ways, + transfering ownership of it to this function. + + sqlite3_prepare_multi() will _not_ finalize st - it is up + to the call() implementation how st is handled. + + Must return 0 on success or an SQLITE_... code on error. If it + throws, sqlite3_prepare_multi() will transform the exception into + a db-level error in order to retain the C-style error semantics + of the API. + + See the {@link Finalize} class for a wrapper which finalizes the + statement after calling a proxy PrepareMultiCallback. + */ + int call(sqlite3_stmt st); + + /** + A PrepareMultiCallback impl which wraps a separate impl and finalizes + any sqlite3_stmt passed to its callback. + */ + public static final class Finalize implements PrepareMultiCallback { + private final PrepareMultiCallback p; + /** + p is the proxy to call() when this.call() is called. + */ + public Finalize( PrepareMultiCallback p ){ + this.p = p; + } + /** + Calls the call() method of the proxied callback and either returns its + result or propagates an exception. Either way, it passes its argument to + sqlite3_finalize() before returning. + */ + @Override public int call(sqlite3_stmt st){ + try { + return this.p.call(st); + }finally{ + CApi.sqlite3_finalize(st); + } + } + } + + /** + A PrepareMultiCallback impl which steps entirely through a result set, + ignoring all non-error results. + */ + public static final class StepAll implements PrepareMultiCallback { + public StepAll(){} + /** + Calls sqlite3_step() on st until it returns something other than + SQLITE_ROW. If the final result is SQLITE_DONE then 0 is returned, + else the result of the final step is returned. + */ + @Override public int call(sqlite3_stmt st){ + int rc = CApi.SQLITE_DONE; + while( CApi.SQLITE_ROW == (rc = CApi.sqlite3_step(st)) ){} + return CApi.SQLITE_DONE==rc ? 0 : rc; + } + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..38f7c5613e573fee22959eb62e15bc9761091ed1 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java @@ -0,0 +1,27 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_preupdate_hook}. +*/ +public interface PreupdateHookCallback extends CallbackProxy { + /** + Must function as described for the C-level sqlite3_preupdate_hook() + callback. If it throws, the exception is translated to a + db-level error and the exception is suppressed. + */ + void call(sqlite3 db, int op, String dbName, String dbTable, + long iKey1, long iKey2 ); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java b/ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..464baa2e3d3e32acc5b3c8b1d144eef2beee9a6d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java @@ -0,0 +1,27 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_progress_handler}. +*/ +public interface ProgressHandlerCallback extends CallbackProxy { + /** + Works as documented for the C-level sqlite3_progress_handler() callback. + +

If it throws, the exception message is passed on to the db and + the exception is suppressed. + */ + int call(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ResultCode.java b/ext/jni/src/org/sqlite/jni/capi/ResultCode.java new file mode 100644 index 0000000000000000000000000000000000000000..5a8b2e6a189325c07f4375dc1936d555a4b7b8f0 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ResultCode.java @@ -0,0 +1,155 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + This enum contains all of the core and "extended" result codes used + by the sqlite3 library. It is provided not for use with the C-style + API (with which it won't work) but for higher-level code which may + find it useful to map SQLite result codes to human-readable names. +*/ +public enum ResultCode { + SQLITE_OK(CApi.SQLITE_OK), + SQLITE_ERROR(CApi.SQLITE_ERROR), + SQLITE_INTERNAL(CApi.SQLITE_INTERNAL), + SQLITE_PERM(CApi.SQLITE_PERM), + SQLITE_ABORT(CApi.SQLITE_ABORT), + SQLITE_BUSY(CApi.SQLITE_BUSY), + SQLITE_LOCKED(CApi.SQLITE_LOCKED), + SQLITE_NOMEM(CApi.SQLITE_NOMEM), + SQLITE_READONLY(CApi.SQLITE_READONLY), + SQLITE_INTERRUPT(CApi.SQLITE_INTERRUPT), + SQLITE_IOERR(CApi.SQLITE_IOERR), + SQLITE_CORRUPT(CApi.SQLITE_CORRUPT), + SQLITE_NOTFOUND(CApi.SQLITE_NOTFOUND), + SQLITE_FULL(CApi.SQLITE_FULL), + SQLITE_CANTOPEN(CApi.SQLITE_CANTOPEN), + SQLITE_PROTOCOL(CApi.SQLITE_PROTOCOL), + SQLITE_EMPTY(CApi.SQLITE_EMPTY), + SQLITE_SCHEMA(CApi.SQLITE_SCHEMA), + SQLITE_TOOBIG(CApi.SQLITE_TOOBIG), + SQLITE_CONSTRAINT(CApi.SQLITE_CONSTRAINT), + SQLITE_MISMATCH(CApi.SQLITE_MISMATCH), + SQLITE_MISUSE(CApi.SQLITE_MISUSE), + SQLITE_NOLFS(CApi.SQLITE_NOLFS), + SQLITE_AUTH(CApi.SQLITE_AUTH), + SQLITE_FORMAT(CApi.SQLITE_FORMAT), + SQLITE_RANGE(CApi.SQLITE_RANGE), + SQLITE_NOTADB(CApi.SQLITE_NOTADB), + SQLITE_NOTICE(CApi.SQLITE_NOTICE), + SQLITE_WARNING(CApi.SQLITE_WARNING), + SQLITE_ROW(CApi.SQLITE_ROW), + SQLITE_DONE(CApi.SQLITE_DONE), + SQLITE_ERROR_MISSING_COLLSEQ(CApi.SQLITE_ERROR_MISSING_COLLSEQ), + SQLITE_ERROR_RETRY(CApi.SQLITE_ERROR_RETRY), + SQLITE_ERROR_SNAPSHOT(CApi.SQLITE_ERROR_SNAPSHOT), + SQLITE_IOERR_READ(CApi.SQLITE_IOERR_READ), + SQLITE_IOERR_SHORT_READ(CApi.SQLITE_IOERR_SHORT_READ), + SQLITE_IOERR_WRITE(CApi.SQLITE_IOERR_WRITE), + SQLITE_IOERR_FSYNC(CApi.SQLITE_IOERR_FSYNC), + SQLITE_IOERR_DIR_FSYNC(CApi.SQLITE_IOERR_DIR_FSYNC), + SQLITE_IOERR_TRUNCATE(CApi.SQLITE_IOERR_TRUNCATE), + SQLITE_IOERR_FSTAT(CApi.SQLITE_IOERR_FSTAT), + SQLITE_IOERR_UNLOCK(CApi.SQLITE_IOERR_UNLOCK), + SQLITE_IOERR_RDLOCK(CApi.SQLITE_IOERR_RDLOCK), + SQLITE_IOERR_DELETE(CApi.SQLITE_IOERR_DELETE), + SQLITE_IOERR_BLOCKED(CApi.SQLITE_IOERR_BLOCKED), + SQLITE_IOERR_NOMEM(CApi.SQLITE_IOERR_NOMEM), + SQLITE_IOERR_ACCESS(CApi.SQLITE_IOERR_ACCESS), + SQLITE_IOERR_CHECKRESERVEDLOCK(CApi.SQLITE_IOERR_CHECKRESERVEDLOCK), + SQLITE_IOERR_LOCK(CApi.SQLITE_IOERR_LOCK), + SQLITE_IOERR_CLOSE(CApi.SQLITE_IOERR_CLOSE), + SQLITE_IOERR_DIR_CLOSE(CApi.SQLITE_IOERR_DIR_CLOSE), + SQLITE_IOERR_SHMOPEN(CApi.SQLITE_IOERR_SHMOPEN), + SQLITE_IOERR_SHMSIZE(CApi.SQLITE_IOERR_SHMSIZE), + SQLITE_IOERR_SHMLOCK(CApi.SQLITE_IOERR_SHMLOCK), + SQLITE_IOERR_SHMMAP(CApi.SQLITE_IOERR_SHMMAP), + SQLITE_IOERR_SEEK(CApi.SQLITE_IOERR_SEEK), + SQLITE_IOERR_DELETE_NOENT(CApi.SQLITE_IOERR_DELETE_NOENT), + SQLITE_IOERR_MMAP(CApi.SQLITE_IOERR_MMAP), + SQLITE_IOERR_GETTEMPPATH(CApi.SQLITE_IOERR_GETTEMPPATH), + SQLITE_IOERR_CONVPATH(CApi.SQLITE_IOERR_CONVPATH), + SQLITE_IOERR_VNODE(CApi.SQLITE_IOERR_VNODE), + SQLITE_IOERR_AUTH(CApi.SQLITE_IOERR_AUTH), + SQLITE_IOERR_BEGIN_ATOMIC(CApi.SQLITE_IOERR_BEGIN_ATOMIC), + SQLITE_IOERR_COMMIT_ATOMIC(CApi.SQLITE_IOERR_COMMIT_ATOMIC), + SQLITE_IOERR_ROLLBACK_ATOMIC(CApi.SQLITE_IOERR_ROLLBACK_ATOMIC), + SQLITE_IOERR_DATA(CApi.SQLITE_IOERR_DATA), + SQLITE_IOERR_CORRUPTFS(CApi.SQLITE_IOERR_CORRUPTFS), + SQLITE_LOCKED_SHAREDCACHE(CApi.SQLITE_LOCKED_SHAREDCACHE), + SQLITE_LOCKED_VTAB(CApi.SQLITE_LOCKED_VTAB), + SQLITE_BUSY_RECOVERY(CApi.SQLITE_BUSY_RECOVERY), + SQLITE_BUSY_SNAPSHOT(CApi.SQLITE_BUSY_SNAPSHOT), + SQLITE_BUSY_TIMEOUT(CApi.SQLITE_BUSY_TIMEOUT), + SQLITE_CANTOPEN_NOTEMPDIR(CApi.SQLITE_CANTOPEN_NOTEMPDIR), + SQLITE_CANTOPEN_ISDIR(CApi.SQLITE_CANTOPEN_ISDIR), + SQLITE_CANTOPEN_FULLPATH(CApi.SQLITE_CANTOPEN_FULLPATH), + SQLITE_CANTOPEN_CONVPATH(CApi.SQLITE_CANTOPEN_CONVPATH), + SQLITE_CANTOPEN_SYMLINK(CApi.SQLITE_CANTOPEN_SYMLINK), + SQLITE_CORRUPT_VTAB(CApi.SQLITE_CORRUPT_VTAB), + SQLITE_CORRUPT_SEQUENCE(CApi.SQLITE_CORRUPT_SEQUENCE), + SQLITE_CORRUPT_INDEX(CApi.SQLITE_CORRUPT_INDEX), + SQLITE_READONLY_RECOVERY(CApi.SQLITE_READONLY_RECOVERY), + SQLITE_READONLY_CANTLOCK(CApi.SQLITE_READONLY_CANTLOCK), + SQLITE_READONLY_ROLLBACK(CApi.SQLITE_READONLY_ROLLBACK), + SQLITE_READONLY_DBMOVED(CApi.SQLITE_READONLY_DBMOVED), + SQLITE_READONLY_CANTINIT(CApi.SQLITE_READONLY_CANTINIT), + SQLITE_READONLY_DIRECTORY(CApi.SQLITE_READONLY_DIRECTORY), + SQLITE_ABORT_ROLLBACK(CApi.SQLITE_ABORT_ROLLBACK), + SQLITE_CONSTRAINT_CHECK(CApi.SQLITE_CONSTRAINT_CHECK), + SQLITE_CONSTRAINT_COMMITHOOK(CApi.SQLITE_CONSTRAINT_COMMITHOOK), + SQLITE_CONSTRAINT_FOREIGNKEY(CApi.SQLITE_CONSTRAINT_FOREIGNKEY), + SQLITE_CONSTRAINT_FUNCTION(CApi.SQLITE_CONSTRAINT_FUNCTION), + SQLITE_CONSTRAINT_NOTNULL(CApi.SQLITE_CONSTRAINT_NOTNULL), + SQLITE_CONSTRAINT_PRIMARYKEY(CApi.SQLITE_CONSTRAINT_PRIMARYKEY), + SQLITE_CONSTRAINT_TRIGGER(CApi.SQLITE_CONSTRAINT_TRIGGER), + SQLITE_CONSTRAINT_UNIQUE(CApi.SQLITE_CONSTRAINT_UNIQUE), + SQLITE_CONSTRAINT_VTAB(CApi.SQLITE_CONSTRAINT_VTAB), + SQLITE_CONSTRAINT_ROWID(CApi.SQLITE_CONSTRAINT_ROWID), + SQLITE_CONSTRAINT_PINNED(CApi.SQLITE_CONSTRAINT_PINNED), + SQLITE_CONSTRAINT_DATATYPE(CApi.SQLITE_CONSTRAINT_DATATYPE), + SQLITE_NOTICE_RECOVER_WAL(CApi.SQLITE_NOTICE_RECOVER_WAL), + SQLITE_NOTICE_RECOVER_ROLLBACK(CApi.SQLITE_NOTICE_RECOVER_ROLLBACK), + SQLITE_WARNING_AUTOINDEX(CApi.SQLITE_WARNING_AUTOINDEX), + SQLITE_AUTH_USER(CApi.SQLITE_AUTH_USER), + SQLITE_OK_LOAD_PERMANENTLY(CApi.SQLITE_OK_LOAD_PERMANENTLY); + + public final int value; + + ResultCode(int rc){ + value = rc; + ResultCodeMap.set(rc, this); + } + + /** + Returns the entry from this enum for the given result code, or + null if no match is found. + */ + public static ResultCode getEntryForInt(int rc){ + return ResultCodeMap.get(rc); + } + + /** + Internal level of indirection required because we cannot initialize + static enum members in an enum before the enum constructor is + invoked. + */ + private static final class ResultCodeMap { + private static final java.util.Map i2e + = new java.util.HashMap<>(); + private static void set(int rc, ResultCode e){ i2e.put(rc, e); } + private static ResultCode get(int rc){ return i2e.get(rc); } + } + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..cf9c4b6e7a9f5da1267f677c7c8a5caa6ba02de7 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_rollback_hook}. +*/ +public interface RollbackHookCallback extends CallbackProxy { + /** + Must function as documented for the C-level sqlite3_rollback_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + void call(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/SQLFunction.java b/ext/jni/src/org/sqlite/jni/capi/SQLFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..7ad1381a7ada40b6d85e7dfa2b695412fdfc6ca8 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/SQLFunction.java @@ -0,0 +1,36 @@ +/* +** 2023-07-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + SQLFunction is used in conjunction with the + sqlite3_create_function() JNI-bound API to give that native code + access to the callback functions needed in order to implement SQL + functions in Java. + +

+ + This class is not used by itself, but is a marker base class. The + three UDF types are modelled by the inner classes Scalar, + Aggregate, and Window. Most simply, clients may subclass + those, or create anonymous classes from them, to implement + UDFs. Clients are free to create their own classes for use with + UDFs, so long as they conform to the public interfaces defined by + those three classes. The JNI layer only actively relies on the + SQLFunction base class and the method names and signatures used by + the UDF callback interfaces. +*/ +public interface SQLFunction { + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/SQLTester.java b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java new file mode 100644 index 0000000000000000000000000000000000000000..c68785e2c388df131d81b238f9834c3f792122c7 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java @@ -0,0 +1,1450 @@ +/* +** 2023-08-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the main application entry pointer for the +** SQLTester framework. +*/ +package org.sqlite.jni.capi; +import java.util.List; +import java.util.ArrayList; +import java.util.Arrays; +import java.nio.charset.StandardCharsets; +import java.util.regex.*; +import static org.sqlite.jni.capi.CApi.*; + +/** + Modes for how to escape (or not) column values and names from + SQLTester.execSql() to the result buffer output. +*/ +enum ResultBufferMode { + //! Do not append to result buffer + NONE, + //! Append output escaped. + ESCAPED, + //! Append output as-is + ASIS +}; + +/** + Modes to specify how to emit multi-row output from + SQLTester.execSql() to the result buffer. +*/ +enum ResultRowMode { + //! Keep all result rows on one line, space-separated. + ONELINE, + //! Add a newline between each result row. + NEWLINE +}; + +/** + Base exception type for test-related failures. +*/ +class SQLTesterException extends RuntimeException { + private boolean bFatal = false; + + SQLTesterException(String msg){ + super(msg); + } + + protected SQLTesterException(String msg, boolean fatal){ + super(msg); + bFatal = fatal; + } + + /** + Indicates whether the framework should consider this exception + type as immediately fatal to the test run or not. + */ + final boolean isFatal(){ return bFatal; } +} + +class DbException extends SQLTesterException { + DbException(sqlite3 db, int rc, boolean closeDb){ + super("DB error #"+rc+": "+sqlite3_errmsg(db),true); + if( closeDb ) sqlite3_close_v2(db); + } + DbException(sqlite3 db, int rc){ + this(db, rc, false); + } +} + +/** + Generic test-failed exception. + */ +class TestScriptFailed extends SQLTesterException { + public TestScriptFailed(TestScript ts, String msg){ + super(ts.getOutputPrefix()+": "+msg, true); + } +} + +/** + Thrown when an unknown test command is encountered in a script. +*/ +class UnknownCommand extends SQLTesterException { + public UnknownCommand(TestScript ts, String cmd){ + super(ts.getOutputPrefix()+": unknown command: "+cmd, false); + } +} + +/** + Thrown when an "incompatible directive" is found in a script. This + can be the presence of a C-preprocessor construct, specific + metadata tags within a test script's header, or specific test + constructs which are incompatible with this particular + implementation. +*/ +class IncompatibleDirective extends SQLTesterException { + public IncompatibleDirective(TestScript ts, String line){ + super(ts.getOutputPrefix()+": incompatible directive: "+line, false); + } +} + +/** + Console output utility class. +*/ +class Outer { + private int verbosity = 0; + + static void out(Object val){ + System.out.print(val); + } + + Outer out(Object... vals){ + for(Object v : vals) out(v); + return this; + } + + Outer outln(Object... vals){ + out(vals).out("\n"); + return this; + } + + Outer verbose(Object... vals){ + if(verbosity>0){ + out("VERBOSE",(verbosity>1 ? "+: " : ": ")).outln(vals); + } + return this; + } + + void setVerbosity(int level){ + verbosity = level; + } + + int getVerbosity(){ + return verbosity; + } + + public boolean isVerbose(){return verbosity > 0;} + +} + +/** +

This class provides an application which aims to implement the + rudimentary SQL-driven test tool described in the accompanying + {@code test-script-interpreter.md}. + +

This class is an internal testing tool, not part of the public + interface but is (A) in the same package as the library because + access permissions require it to be so and (B) the JDK8 javadoc + offers no way to filter individual classes out of the doc + generation process (it can only exclude packages, but see (A)). + +

An instance of this application provides a core set of services + which TestScript instances use for processing testing logic. + TestScripts, in turn, delegate the concrete test work to Command + objects, which the TestScript parses on their behalf. +*/ +public class SQLTester { + //! List of input script files. + private final java.util.List listInFiles = new ArrayList<>(); + //! Console output utility. + private final Outer outer = new Outer(); + //! Test input buffer. + private final StringBuilder inputBuffer = new StringBuilder(); + //! Test result buffer. + private final StringBuilder resultBuffer = new StringBuilder(); + //! Buffer for REQUIRED_PROPERTIES pragmas. + private final StringBuilder dbInitSql = new StringBuilder(); + //! Output representation of SQL NULL. + private String nullView = "nil"; + //! Total tests run. + private int nTotalTest = 0; + //! Total test script files run. + private int nTestFile = 0; + //! Number of scripts which were aborted. + private int nAbortedScript = 0; + //! Incremented by test case handlers + private int nTest = 0; + //! True to enable column name output from execSql() + private boolean emitColNames; + //! True to keep going regardless of how a test fails. + private boolean keepGoing = false; + //! The list of available db handles. + private final sqlite3[] aDb = new sqlite3[7]; + //! Index into aDb of the current db. + private int iCurrentDb = 0; + //! Name of the default db, re-created for each script. + private final String initialDbName = "test.db"; + + + public SQLTester(){ + reset(); + } + + void setVerbosity(int level){ + this.outer.setVerbosity( level ); + } + int getVerbosity(){ + return this.outer.getVerbosity(); + } + boolean isVerbose(){ + return this.outer.isVerbose(); + } + + void outputColumnNames(boolean b){ emitColNames = b; } + + void verbose(Object... vals){ + outer.verbose(vals); + } + + void outln(Object... vals){ + outer.outln(vals); + } + + void out(Object... vals){ + outer.out(vals); + } + + //! Adds the given test script to the to-test list. + public void addTestScript(String filename){ + listInFiles.add(filename); + //verbose("Added file ",filename); + } + + private void setupInitialDb() throws DbException { + if( null==aDb[0] ){ + Util.unlink(initialDbName); + openDb(0, initialDbName, true); + }else{ + outln("WARNING: setupInitialDb() unexpectedly ", + "triggered while it is opened."); + } + } + + static final String[] startEmoji = { + "🚴", "🏄", "🏇", "🤸", "⛹", "🏊", "⛷", "🧗", "🏋" + }; + static final int nStartEmoji = startEmoji.length; + static int iStartEmoji = 0; + + private static String nextStartEmoji(){ + return startEmoji[iStartEmoji++ % nStartEmoji]; + } + + public void runTests() throws Exception { + final long tStart = System.currentTimeMillis(); + for(String f : listInFiles){ + reset(); + ++nTestFile; + final TestScript ts = new TestScript(f); + outln(nextStartEmoji(), " starting [",f,"]"); + boolean threw = false; + final long timeStart = System.currentTimeMillis(); + try{ + ts.run(this); + }catch(SQLTesterException e){ + threw = true; + outln("🔥EXCEPTION: ",e.getClass().getSimpleName(),": ",e.getMessage()); + ++nAbortedScript; + if( keepGoing ) outln("Continuing anyway becaure of the keep-going option."); + else if( e.isFatal() ) throw e; + }finally{ + final long timeEnd = System.currentTimeMillis(); + outln("🏁",(threw ? "❌" : "✅")," ",nTest," test(s) in ", + (timeEnd-timeStart),"ms."); + } + } + final long tEnd = System.currentTimeMillis(); + outln("Total run-time: ",(tEnd-tStart),"ms"); + Util.unlink(initialDbName); + } + + private StringBuilder clearBuffer(StringBuilder b){ + b.setLength(0);; + return b; + } + + StringBuilder clearInputBuffer(){ + return clearBuffer(inputBuffer); + } + + StringBuilder clearResultBuffer(){ + return clearBuffer(resultBuffer); + } + + StringBuilder getInputBuffer(){ return inputBuffer; } + + void appendInput(String n, boolean addNL){ + inputBuffer.append(n); + if(addNL) inputBuffer.append('\n'); + } + + void appendResult(String n, boolean addNL){ + resultBuffer.append(n); + if(addNL) resultBuffer.append('\n'); + } + + void appendDbInitSql(String n) throws DbException { + dbInitSql.append(n).append('\n'); + if( null!=getCurrentDb() ){ + //outln("RUNNING DB INIT CODE: ",n); + execSql(null, true, ResultBufferMode.NONE, null, n); + } + } + String getDbInitSql(){ return dbInitSql.toString(); } + + String getInputText(){ return inputBuffer.toString(); } + + String getResultText(){ return resultBuffer.toString(); } + + private String takeBuffer(StringBuilder b){ + final String rc = b.toString(); + clearBuffer(b); + return rc; + } + + String takeInputBuffer(){ return takeBuffer(inputBuffer); } + + String takeResultBuffer(){ return takeBuffer(resultBuffer); } + + int getCurrentDbId(){ return iCurrentDb; } + + SQLTester affirmDbId(int n) throws IndexOutOfBoundsException { + if(n<0 || n>=aDb.length){ + throw new IndexOutOfBoundsException("illegal db number: "+n); + } + return this; + } + + sqlite3 setCurrentDb(int n) throws Exception{ + affirmDbId(n); + iCurrentDb = n; + return this.aDb[n]; + } + + sqlite3 getCurrentDb(){ return aDb[iCurrentDb]; } + + sqlite3 getDbById(int id) throws Exception{ + return affirmDbId(id).aDb[id]; + } + + void closeDb(int id) { + final sqlite3 db = affirmDbId(id).aDb[id]; + if( null != db ){ + sqlite3_close_v2(db); + aDb[id] = null; + } + } + + void closeDb() { closeDb(iCurrentDb); } + + void closeAllDbs(){ + for(int i = 0; i 0){ + //outln("RUNNING DB INIT CODE: ",dbInitSql.toString()); + rc = execSql(db, false, ResultBufferMode.NONE, + null, dbInitSql.toString()); + } + if( 0!=rc ){ + throw new DbException(db, rc, true); + } + return aDb[iCurrentDb] = db; + } + + sqlite3 openDb(int slot, String name, boolean createIfNeeded) throws DbException { + affirmDbId(slot); + iCurrentDb = slot; + return openDb(name, createIfNeeded); + } + + /** + Resets all tester context state except for that related to + tracking running totals. + */ + void reset(){ + clearInputBuffer(); + clearResultBuffer(); + clearBuffer(dbInitSql); + closeAllDbs(); + nTest = 0; + nullView = "nil"; + emitColNames = false; + iCurrentDb = 0; + //dbInitSql.append("SELECT 1;"); + } + + void setNullValue(String v){nullView = v;} + + /** + If true, encountering an unknown command in a script causes the + remainder of the script to be skipped, rather than aborting the + whole script run. + */ + boolean skipUnknownCommands(){ + // Currently hard-coded. Potentially a flag someday. + return true; + } + + void incrementTestCounter(){ ++nTest; ++nTotalTest; } + + //! "Special" characters - we have to escape output if it contains any. + static final Pattern patternSpecial = Pattern.compile( + "[\\x00-\\x20\\x22\\x5c\\x7b\\x7d]" + ); + //! Either of '{' or '}'. + static final Pattern patternSquiggly = Pattern.compile("[{}]"); + + /** + Returns v or some escaped form of v, as defined in the tester's + spec doc. + */ + String escapeSqlValue(String v){ + if( "".equals(v) ) return "{}"; + Matcher m = patternSpecial.matcher(v); + if( !m.find() ){ + return v /* no escaping needed */; + } + m = patternSquiggly.matcher(v); + if( !m.find() ){ + return "{"+v+"}"; + } + final StringBuilder sb = new StringBuilder("\""); + final int n = v.length(); + for(int i = 0; i < n; ++i){ + final char ch = v.charAt(i); + switch(ch){ + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + default: + //verbose("CHAR ",(int)ch," ",ch," octal=",String.format("\\%03o", (int)ch)); + if( (int)ch < 32 ) sb.append(String.format("\\%03o", (int)ch)); + else sb.append(ch); + break; + } + } + sb.append("\""); + return sb.toString(); + } + + private void appendDbErr(sqlite3 db, StringBuilder sb, int rc){ + sb.append(org.sqlite.jni.capi.ResultCode.getEntryForInt(rc)).append(' '); + final String msg = escapeSqlValue(sqlite3_errmsg(db)); + if( '{' == msg.charAt(0) ){ + sb.append(msg); + }else{ + sb.append('{').append(msg).append('}'); + } + } + + /** + Runs SQL on behalf of test commands and outputs the results following + the very specific rules of the test framework. + + If db is null, getCurrentDb() is assumed. If throwOnError is true then + any db-side error will result in an exception, else they result in + the db's result code. + + appendMode specifies how/whether to append results to the result + buffer. rowMode specifies whether to output all results in a + single line or one line per row. If appendMode is + ResultBufferMode.NONE then rowMode is ignored and may be null. + */ + public int execSql(sqlite3 db, boolean throwOnError, + ResultBufferMode appendMode, ResultRowMode rowMode, + String sql) throws SQLTesterException { + if( null==db && null==aDb[0] ){ + // Delay opening of the initial db to enable tests to change its + // name and inject on-connect code via, e.g., the MEMDB + // directive. this setup as the potential to misinteract with + // auto-extension timing and must be done carefully. + setupInitialDb(); + } + final OutputPointer.Int32 oTail = new OutputPointer.Int32(); + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + final byte[] sqlUtf8 = sql.getBytes(StandardCharsets.UTF_8); + if( null==db ) db = getCurrentDb(); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + int rc = 0; + sqlite3_stmt stmt = null; + int spacing = 0 /* emit a space for --result if>0 */ ; + final StringBuilder sb = (ResultBufferMode.NONE==appendMode) + ? null : resultBuffer; + //outln("sqlChunk len= = ",sqlChunk.length); + try{ + while(pos < sqlChunk.length){ + if(pos > 0){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, + sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v2(db, sqlChunk, outStmt, oTail); + /*outln("PREPARE rc ",rc," oTail=",oTail.get(),": ", + new String(sqlChunk,StandardCharsets.UTF_8),"\n");*/ + if( 0!=rc ){ + if(throwOnError){ + throw new DbException(db, rc); + }else if( null!=sb ){ + appendDbErr(db, sb, rc); + } + break; + } + pos = oTail.value; + stmt = outStmt.take(); + if( null == stmt ){ + // empty statement was parsed. + continue; + } + if( null!=sb ){ + // Add the output to the result buffer... + final int nCol = sqlite3_column_count(stmt); + String colName = null, val = null; + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){ + for(int i = 0; i < nCol; ++i){ + if( spacing++ > 0 ) sb.append(' '); + if( emitColNames ){ + colName = sqlite3_column_name(stmt, i); + switch(appendMode){ + case ASIS: + sb.append( colName ); + break; + case ESCAPED: + sb.append( escapeSqlValue(colName) ); + break; + default: + throw new SQLTesterException("Unhandled ResultBufferMode: "+appendMode); + } + sb.append(' '); + } + val = sqlite3_column_text16(stmt, i); + if( null==val ){ + sb.append( nullView ); + continue; + } + switch(appendMode){ + case ASIS: + sb.append( val ); + break; + case ESCAPED: + sb.append( escapeSqlValue(val) ); + break; + default: + throw new SQLTesterException("Unhandled ResultBufferMode: "+appendMode); + } + } + if( ResultRowMode.NEWLINE == rowMode ){ + spacing = 0; + sb.append('\n'); + } + } + }else{ + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){} + } + sqlite3_finalize(stmt); + stmt = null; + if(SQLITE_ROW==rc || SQLITE_DONE==rc) rc = 0; + else if( rc!=0 ){ + if( null!=sb ){ + appendDbErr(db, sb, rc); + } + break; + } + } + }finally{ + sqlite3_reset(stmt + /* In order to trigger an exception in the + INSERT...RETURNING locking scenario: + https://sqlite.org/forum/forumpost/36f7a2e7494897df */); + sqlite3_finalize(stmt); + } + if( 0!=rc && throwOnError ){ + throw new DbException(db, rc); + } + return rc; + } + + public static void main(String[] argv) throws Exception{ + installCustomExtensions(); + boolean dumpInternals = false; + final SQLTester t = new SQLTester(); + for(String a : argv){ + if(a.startsWith("-")){ + final String flag = a.replaceFirst("-+",""); + if( flag.equals("verbose") ){ + // Use --verbose up to 3 times + t.setVerbosity(t.getVerbosity() + 1); + }else if( flag.equals("keep-going") ){ + t.keepGoing = true; + }else if( flag.equals("internals") ){ + dumpInternals = true; + }else{ + throw new IllegalArgumentException("Unhandled flag: "+flag); + } + continue; + } + t.addTestScript(a); + } + final AutoExtensionCallback ax = new AutoExtensionCallback() { + private final SQLTester tester = t; + @Override public int call(sqlite3 db){ + final String init = tester.getDbInitSql(); + if( !init.isEmpty() ){ + tester.execSql(db, true, ResultBufferMode.NONE, null, init); + } + return 0; + } + }; + sqlite3_auto_extension(ax); + try { + t.runTests(); + }finally{ + sqlite3_cancel_auto_extension(ax); + t.outln("Processed ",t.nTotalTest," test(s) in ",t.nTestFile," file(s)."); + if( t.nAbortedScript > 0 ){ + t.outln("Aborted ",t.nAbortedScript," script(s)."); + } + if( dumpInternals ){ + sqlite3_jni_internal_details(); + } + } + } + + /** + Internal impl of the public strglob() method. Neither argument + may be NULL and both _MUST_ be NUL-terminated. + */ + private static native int strglob(byte[] glob, byte[] txt); + + /** + Works essentially the same as sqlite3_strglob() except that the + glob character '#' matches a sequence of one or more digits. It + does not match when it appears at the start or middle of a series + of digits, e.g. "#23" or "1#3", but will match at the end, + e.g. "12#". + */ + static int strglob(String glob, String txt){ + return strglob( + (glob+"\0").getBytes(StandardCharsets.UTF_8), + (txt+"\0").getBytes(StandardCharsets.UTF_8) + ); + } + + /** + Sets up C-side components needed by the test framework. This must + not be called until main() is triggered so that it does not + interfere with library clients who don't use this class. + */ + static native void installCustomExtensions(); + static { + System.loadLibrary("sqlite3-jni") + /* Interestingly, when SQLTester is the main app, we have to + load that lib from here. The same load from CApi does + not happen early enough. Without this, + installCustomExtensions() is an unresolved symbol. */; + } + +} + +/** + General utilities for the SQLTester bits. +*/ +final class Util { + + //! Throws a new T, appending all msg args into a string for the message. + static void toss(Class errorType, Object... msg) throws Exception { + StringBuilder sb = new StringBuilder(); + for(Object s : msg) sb.append(s); + final java.lang.reflect.Constructor ctor = + errorType.getConstructor(String.class); + throw ctor.newInstance(sb.toString()); + } + + static void toss(Object... msg) throws Exception{ + toss(RuntimeException.class, msg); + } + + //! Tries to delete the given file, silently ignoring failure. + static void unlink(String filename){ + try{ + final java.io.File f = new java.io.File(filename); + f.delete(); + }catch(Exception e){ + /* ignore */ + } + } + + /** + Appends all entries in argv[1..end] into a space-separated + string, argv[0] is not included because it's expected to be a + command name. + */ + static String argvToString(String[] argv){ + StringBuilder sb = new StringBuilder(); + for(int i = 1; i < argv.length; ++i ){ + if( i>1 ) sb.append(" "); + sb.append( argv[i] ); + } + return sb.toString(); + } + +} + +/** + Base class for test script commands. It provides a set of utility + APIs for concrete command implementations. + + Each subclass must have a public no-arg ctor and must implement + the process() method which is abstract in this class. + + Commands are intended to be stateless, except perhaps for counters + and similar internals. Specifically, no state which changes the + behavior between any two invocations of process() should be + retained. +*/ +abstract class Command { + protected Command(){} + + /** + Must process one command-unit of work and either return + (on success) or throw (on error). + + The first two arguments specify the context of the test. The TestScript + provides the content of the test and the SQLTester providers the sandbox + in which that script is being evaluated. + + argv is a list with the command name followed by any arguments to + that command. The argcCheck() method from this class provides + very basic argc validation. + */ + public abstract void process( + SQLTester st, TestScript ts, String[] argv + ) throws Exception; + + /** + If argv.length-1 (-1 because the command's name is in argv[0]) does not + fall in the inclusive range (min,max) then this function throws. Use + a max value of -1 to mean unlimited. + */ + protected final void argcCheck(TestScript ts, String[] argv, int min, int max) throws Exception{ + int argc = argv.length-1; + if(argc=0 && argc>max)){ + if( min==max ){ + ts.toss(argv[0]," requires exactly ",min," argument(s)"); + }else if(max>0){ + ts.toss(argv[0]," requires ",min,"-",max," arguments."); + }else{ + ts.toss(argv[0]," requires at least ",min," arguments."); + } + } + } + + /** + Equivalent to argcCheck(argv,argc,argc). + */ + protected final void argcCheck(TestScript ts, String[] argv, int argc) throws Exception{ + argcCheck(ts, argv, argc, argc); + } +} + +//! --close command +class CloseDbCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,0,1); + Integer id; + if(argv.length>1){ + String arg = argv[1]; + if("all".equals(arg)){ + t.closeAllDbs(); + return; + } + else{ + id = Integer.parseInt(arg); + } + }else{ + id = t.getCurrentDbId(); + } + t.closeDb(id); + } +} + +//! --column-names command +class ColumnNamesCommand extends Command { + public void process( + SQLTester st, TestScript ts, String[] argv + ) throws Exception{ + argcCheck(ts,argv,1); + st.outputColumnNames( Integer.parseInt(argv[1])!=0 ); + } +} + +//! --db command +class DbCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,1); + t.setCurrentDb( Integer.parseInt(argv[1]) ); + } +} + +//! --glob command +class GlobCommand extends Command { + private boolean negate = false; + public GlobCommand(){} + protected GlobCommand(boolean negate){ this.negate = negate; } + + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,1,-1); + t.incrementTestCounter(); + final String sql = t.takeInputBuffer(); + int rc = t.execSql(null, true, ResultBufferMode.ESCAPED, + ResultRowMode.ONELINE, sql); + final String result = t.getResultText(); + final String sArgs = Util.argvToString(argv); + //t2.verbose2(argv[0]," rc = ",rc," result buffer:\n", result,"\nargs:\n",sArgs); + final String glob = Util.argvToString(argv); + rc = SQLTester.strglob(glob, result); + if( (negate && 0==rc) || (!negate && 0!=rc) ){ + ts.toss(argv[0], " mismatch: ", glob," vs input: ",result); + } + } +} + +//! --json command +class JsonCommand extends ResultCommand { + public JsonCommand(){ super(ResultBufferMode.ASIS); } +} + +//! --json-block command +class JsonBlockCommand extends TableResultCommand { + public JsonBlockCommand(){ super(true); } +} + +//! --new command +class NewDbCommand extends OpenDbCommand { + public NewDbCommand(){ super(true); } + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + if(argv.length>1){ + Util.unlink(argv[1]); + } + super.process(t, ts, argv); + } + +} + +//! Placeholder dummy/no-op/unimplemented commands +class NoopCommand extends Command { + private boolean verbose = false; + public NoopCommand(boolean verbose){ + this.verbose = verbose; + } + public NoopCommand(){} + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + if( this.verbose ){ + t.outln("Skipping unhandled command: "+argv[0]); + } + } +} + +//! --notglob command +class NotGlobCommand extends GlobCommand { + public NotGlobCommand(){ + super(true); + } +} + +//! --null command +class NullCommand extends Command { + public void process( + SQLTester st, TestScript ts, String[] argv + ) throws Exception{ + argcCheck(ts,argv,1); + st.setNullValue( argv[1] ); + } +} + +//! --open command +class OpenDbCommand extends Command { + private boolean createIfNeeded = false; + public OpenDbCommand(){} + protected OpenDbCommand(boolean c){createIfNeeded = c;} + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,1); + t.openDb(argv[1], createIfNeeded); + } +} + +//! --print command +class PrintCommand extends Command { + public void process( + SQLTester st, TestScript ts, String[] argv + ) throws Exception{ + st.out(ts.getOutputPrefix(),": "); + if( 1==argv.length ){ + st.out( st.getInputText() ); + }else{ + st.outln( Util.argvToString(argv) ); + } + } +} + +//! --result command +class ResultCommand extends Command { + private final ResultBufferMode bufferMode; + protected ResultCommand(ResultBufferMode bm){ bufferMode = bm; } + public ResultCommand(){ this(ResultBufferMode.ESCAPED); } + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,0,-1); + t.incrementTestCounter(); + final String sql = t.takeInputBuffer(); + //ts.verbose2(argv[0]," SQL =\n",sql); + int rc = t.execSql(null, false, bufferMode, ResultRowMode.ONELINE, sql); + final String result = t.getResultText().trim(); + final String sArgs = argv.length>1 ? Util.argvToString(argv) : ""; + if( !result.equals(sArgs) ){ + t.outln(argv[0]," FAILED comparison. Result buffer:\n", + result,"\nExpected result:\n",sArgs); + ts.toss(argv[0]+" comparison failed."); + } + } +} + +//! --run command +class RunCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,0,1); + final sqlite3 db = (1==argv.length) + ? t.getCurrentDb() : t.getDbById( Integer.parseInt(argv[1]) ); + final String sql = t.takeInputBuffer(); + final int rc = t.execSql(db, false, ResultBufferMode.NONE, + ResultRowMode.ONELINE, sql); + if( 0!=rc && t.isVerbose() ){ + String msg = sqlite3_errmsg(db); + ts.verbose1(argv[0]," non-fatal command error #",rc,": ", + msg,"\nfor SQL:\n",sql); + } + } +} + +//! --tableresult command +class TableResultCommand extends Command { + private final boolean jsonMode; + protected TableResultCommand(boolean jsonMode){ this.jsonMode = jsonMode; } + public TableResultCommand(){ this(false); } + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,0); + t.incrementTestCounter(); + String body = ts.fetchCommandBody(t); + if( null==body ) ts.toss("Missing ",argv[0]," body."); + body = body.trim(); + if( !body.endsWith("\n--end") ){ + ts.toss(argv[0], " must be terminated with --end."); + }else{ + body = body.substring(0, body.length()-6); + } + final String[] globs = body.split("\\s*\\n\\s*"); + if( globs.length < 1 ){ + ts.toss(argv[0], " requires 1 or more ", + (jsonMode ? "json snippets" : "globs"),"."); + } + final String sql = t.takeInputBuffer(); + t.execSql(null, true, + jsonMode ? ResultBufferMode.ASIS : ResultBufferMode.ESCAPED, + ResultRowMode.NEWLINE, sql); + final String rbuf = t.getResultText(); + final String[] res = rbuf.split("\n"); + if( res.length != globs.length ){ + ts.toss(argv[0], " failure: input has ", res.length, + " row(s) but expecting ",globs.length); + } + for(int i = 0; i < res.length; ++i){ + final String glob = globs[i].replaceAll("\\s+"," ").trim(); + //ts.verbose2(argv[0]," <<",glob,">> vs <<",res[i],">>"); + if( jsonMode ){ + if( !glob.equals(res[i]) ){ + ts.toss(argv[0], " json <<",glob, ">> does not match: <<", + res[i],">>"); + } + }else if( 0 != SQLTester.strglob(glob, res[i]) ){ + ts.toss(argv[0], " glob <<",glob,">> does not match: <<",res[i],">>"); + } + } + } +} + +//! --testcase command +class TestCaseCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,1); + ts.setTestCaseName(argv[1]); + t.clearResultBuffer(); + t.clearInputBuffer(); + } +} + +//! --verbosity command +class VerbosityCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + argcCheck(ts,argv,1); + ts.setVerbosity( Integer.parseInt(argv[1]) ); + } +} + +class CommandDispatcher { + + private static java.util.Map commandMap = + new java.util.HashMap<>(); + + /** + Returns a (cached) instance mapped to name, or null if no match + is found. + */ + static Command getCommandByName(String name){ + Command rv = commandMap.get(name); + if( null!=rv ) return rv; + switch(name){ + case "close": rv = new CloseDbCommand(); break; + case "column-names": rv = new ColumnNamesCommand(); break; + case "db": rv = new DbCommand(); break; + case "glob": rv = new GlobCommand(); break; + case "json": rv = new JsonCommand(); break; + case "jsonglob": rv = new NoopCommand(true); break; + case "json-block": rv = new JsonBlockCommand(); break; + case "new": rv = new NewDbCommand(); break; + case "notglob": rv = new NotGlobCommand(); break; + case "null": rv = new NullCommand(); break; + case "oom": rv = new NoopCommand(); break; + case "open": rv = new OpenDbCommand(); break; + case "print": rv = new PrintCommand(); break; + case "result": rv = new ResultCommand(); break; + case "run": rv = new RunCommand(); break; + case "stmt-cache": rv = new NoopCommand(); break; + case "tableresult": rv = new TableResultCommand(); break; + case "testcase": rv = new TestCaseCommand(); break; + case "verbosity": rv = new VerbosityCommand(); break; + default: rv = null; break; + } + if( null!=rv ) commandMap.put(name, rv); + return rv; + } + + /** + Treats argv[0] as a command name, looks it up with + getCommandByName(), and calls process() on that instance, passing + it arguments given to this function. + */ + static void dispatch(SQLTester tester, TestScript ts, String[] argv) throws Exception{ + final Command cmd = getCommandByName(argv[0]); + if(null == cmd){ + throw new UnknownCommand(ts, argv[0]); + } + cmd.process(tester, ts, argv); + } +} + + +/** + This class represents a single test script. It handles (or + delegates) its the reading-in and parsing, but the details of + evaluation are delegated elsewhere. +*/ +class TestScript { + //! input file + private String filename = null; + //! Name pulled from the SCRIPT_MODULE_NAME directive of the file + private String moduleName = null; + //! Current test case name. + private String testCaseName = null; + //! Content buffer state. + private final Cursor cur = new Cursor(); + //! Utility for console output. + private final Outer outer = new Outer(); + + //! File content and parse state. + private static final class Cursor { + private final StringBuilder sb = new StringBuilder(); + byte[] src = null; + //! Current position in this.src. + int pos = 0; + //! Current line number. Starts at 0 for internal reasons and will + // line up with 1-based reality once parsing starts. + int lineNo = 0 /* yes, zero */; + //! Putback value for this.pos. + int putbackPos = 0; + //! Putback line number + int putbackLineNo = 0; + //! Peeked-to pos, used by peekLine() and consumePeeked(). + int peekedPos = 0; + //! Peeked-to line number. + int peekedLineNo = 0; + + //! Restore parsing state to the start of the stream. + void rewind(){ + sb.setLength(0); + pos = lineNo = putbackPos = putbackLineNo = peekedPos = peekedLineNo = 0 + /* kinda missing memset() about now. */; + } + } + + private byte[] readFile(String filename) throws Exception { + return java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(filename)); + } + + /** + Initializes the script with the content of the given file. + Throws if it cannot read the file. + */ + public TestScript(String filename) throws Exception{ + this.filename = filename; + setVerbosity(2); + cur.src = readFile(filename); + } + + public String getFilename(){ + return filename; + } + + public String getModuleName(){ + return moduleName; + } + + /** + Verbosity level 0 produces no debug/verbose output. Level 1 produces + some and level 2 produces more. + */ + public void setVerbosity(int level){ + outer.setVerbosity(level); + } + + public String getOutputPrefix(){ + String rc = "["+(moduleName==null ? "" : moduleName)+"]"; + if( null!=testCaseName ) rc += "["+testCaseName+"]"; + if( null!=filename ) rc += "["+filename+"]"; + return rc + " line "+ cur.lineNo; + } + + static final String[] verboseLabel = {"🔈",/*"🔉",*/"🔊","📢"}; + //! Output vals only if level<=current verbosity level. + private TestScript verboseN(int level, Object... vals){ + final int verbosity = outer.getVerbosity(); + if(verbosity>=level){ + outer.out( verboseLabel[level-1], getOutputPrefix(), " ",level,": " + ).outln(vals); + } + return this; + } + + TestScript verbose1(Object... vals){return verboseN(1,vals);} + TestScript verbose2(Object... vals){return verboseN(2,vals);} + TestScript verbose3(Object... vals){return verboseN(3,vals);} + + private void reset(){ + testCaseName = null; + cur.rewind(); + } + + void setTestCaseName(String n){ testCaseName = n; } + + /** + Returns the next line from the buffer, minus the trailing EOL. + + Returns null when all input is consumed. Throws if it reads + illegally-encoded input, e.g. (non-)characters in the range + 128-256. + */ + String getLine(){ + if( cur.pos==cur.src.length ){ + return null /* EOF */; + } + cur.putbackPos = cur.pos; + cur.putbackLineNo = cur.lineNo; + cur.sb.setLength(0); + final boolean skipLeadingWs = false; + byte b = 0, prevB = 0; + int i = cur.pos; + if(skipLeadingWs) { + /* Skip any leading spaces, including newlines. This will eliminate + blank lines. */ + for(; i < cur.src.length; ++i, prevB=b){ + b = cur.src[i]; + switch((int)b){ + case 32/*space*/: case 9/*tab*/: case 13/*CR*/: continue; + case 10/*NL*/: ++cur.lineNo; continue; + default: break; + } + break; + } + if( i==cur.src.length ){ + return null /* EOF */; + } + } + boolean doBreak = false; + final byte[] aChar = {0,0,0,0} /* multi-byte char buffer */; + int nChar = 0 /* number of bytes in the char */; + for(; i < cur.src.length && !doBreak; ++i){ + b = cur.src[i]; + switch( (int)b ){ + case 13/*CR*/: continue; + case 10/*NL*/: + ++cur.lineNo; + if(cur.sb.length()>0) doBreak = true; + // Else it's an empty string + break; + default: + /* Multi-byte chars need to be gathered up and appended at + one time. Appending individual bytes to the StringBuffer + appends their integer value. */ + nChar = 1; + switch( b & 0xF0 ){ + case 0xC0: nChar = 2; break; + case 0xE0: nChar = 3; break; + case 0xF0: nChar = 4; break; + default: + if( b > 127 ) this.toss("Invalid character (#"+(int)b+")."); + break; + } + if( 1==nChar ){ + cur.sb.append((char)b); + }else{ + for(int x = 0; x < nChar; ++x) aChar[x] = cur.src[i+x]; + cur.sb.append(new String(Arrays.copyOf(aChar, nChar), + StandardCharsets.UTF_8)); + i += nChar-1; + } + break; + } + } + cur.pos = i; + final String rv = cur.sb.toString(); + if( i==cur.src.length && 0==rv.length() ){ + return null /* EOF */; + } + return rv; + }/*getLine()*/ + + /** + Fetches the next line then resets the cursor to its pre-call + state. consumePeeked() can be used to consume this peeked line + without having to re-parse it. + */ + String peekLine(){ + final int oldPos = cur.pos; + final int oldPB = cur.putbackPos; + final int oldPBL = cur.putbackLineNo; + final int oldLine = cur.lineNo; + try{ return getLine(); } + finally{ + cur.peekedPos = cur.pos; + cur.peekedLineNo = cur.lineNo; + cur.pos = oldPos; + cur.lineNo = oldLine; + cur.putbackPos = oldPB; + cur.putbackLineNo = oldPBL; + } + } + + /** + Only valid after calling peekLine() and before calling getLine(). + This places the cursor to the position it would have been at had + the peekLine() had been fetched with getLine(). + */ + void consumePeeked(){ + cur.pos = cur.peekedPos; + cur.lineNo = cur.peekedLineNo; + } + + /** + Restores the cursor to the position it had before the previous + call to getLine(). + */ + void putbackLine(){ + cur.pos = cur.putbackPos; + cur.lineNo = cur.putbackLineNo; + } + + private boolean checkRequiredProperties(SQLTester t, String[] props) throws SQLTesterException{ + if( true ) return false; + int nOk = 0; + for(String rp : props){ + verbose1("REQUIRED_PROPERTIES: ",rp); + switch(rp){ + case "RECURSIVE_TRIGGERS": + t.appendDbInitSql("pragma recursive_triggers=on;"); + ++nOk; + break; + case "TEMPSTORE_FILE": + /* This _assumes_ that the lib is built with SQLITE_TEMP_STORE=1 or 2, + which we just happen to know is the case */ + t.appendDbInitSql("pragma temp_store=1;"); + ++nOk; + break; + case "TEMPSTORE_MEM": + /* This _assumes_ that the lib is built with SQLITE_TEMP_STORE=1 or 2, + which we just happen to know is the case */ + t.appendDbInitSql("pragma temp_store=0;"); + ++nOk; + break; + case "AUTOVACUUM": + t.appendDbInitSql("pragma auto_vacuum=full;"); + ++nOk; + case "INCRVACUUM": + t.appendDbInitSql("pragma auto_vacuum=incremental;"); + ++nOk; + default: + break; + } + } + return props.length == nOk; + } + + private static final Pattern patternRequiredProperties = + Pattern.compile(" REQUIRED_PROPERTIES:[ \\t]*(\\S.*)\\s*$"); + private static final Pattern patternScriptModuleName = + Pattern.compile(" SCRIPT_MODULE_NAME:[ \\t]*(\\S+)\\s*$"); + private static final Pattern patternMixedModuleName = + Pattern.compile(" ((MIXED_)?MODULE_NAME):[ \\t]*(\\S+)\\s*$"); + private static final Pattern patternCommand = + Pattern.compile("^--(([a-z-]+)( .*)?)$"); + + /** + Looks for "directives." If a compatible one is found, it is + processed and this function returns. If an incompatible one is found, + a description of it is returned and processing of the test must + end immediately. + */ + private void checkForDirective( + SQLTester tester, String line + ) throws IncompatibleDirective { + if(line.startsWith("#")){ + throw new IncompatibleDirective(this, "C-preprocessor input: "+line); + }else if(line.startsWith("---")){ + new IncompatibleDirective(this, "triple-dash: "+line); + } + Matcher m = patternScriptModuleName.matcher(line); + if( m.find() ){ + moduleName = m.group(1); + return; + } + m = patternRequiredProperties.matcher(line); + if( m.find() ){ + final String rp = m.group(1); + if( ! checkRequiredProperties( tester, rp.split("\\s+") ) ){ + throw new IncompatibleDirective(this, "REQUIRED_PROPERTIES: "+rp); + } + } + m = patternMixedModuleName.matcher(line); + if( m.find() ){ + throw new IncompatibleDirective(this, m.group(1)+": "+m.group(3)); + } + if( line.indexOf("\n|")>=0 ){ + throw new IncompatibleDirective(this, "newline-pipe combination."); + } + return; + } + + boolean isCommandLine(String line, boolean checkForImpl){ + final Matcher m = patternCommand.matcher(line); + boolean rc = m.find(); + if( rc && checkForImpl ){ + rc = null!=CommandDispatcher.getCommandByName(m.group(2)); + } + return rc; + } + + /** + If line looks like a command, returns an argv for that command + invocation, else returns null. + */ + String[] getCommandArgv(String line){ + final Matcher m = patternCommand.matcher(line); + return m.find() ? m.group(1).trim().split("\\s+") : null; + } + + /** + Fetches lines until the next recognized command. Throws if + checkForDirective() does. Returns null if there is no input or + it's only whitespace. The returned string retains all whitespace. + + Note that "subcommands", --command-like constructs in the body + which do not match a known command name are considered to be + content, not commands. + */ + String fetchCommandBody(SQLTester tester){ + final StringBuilder sb = new StringBuilder(); + String line; + while( (null != (line = peekLine())) ){ + checkForDirective(tester, line); + if( isCommandLine(line, true) ) break; + else { + sb.append(line).append("\n"); + consumePeeked(); + } + } + line = sb.toString(); + return line.trim().isEmpty() ? null : line; + } + + private void processCommand(SQLTester t, String[] argv) throws Exception{ + verbose1("running command: ",argv[0], " ", Util.argvToString(argv)); + if(outer.getVerbosity()>1){ + final String input = t.getInputText(); + if( !input.isEmpty() ) verbose3("Input buffer = ",input); + } + CommandDispatcher.dispatch(t, this, argv); + } + + void toss(Object... msg) throws TestScriptFailed { + StringBuilder sb = new StringBuilder(); + for(Object s : msg) sb.append(s); + throw new TestScriptFailed(this, sb.toString()); + } + + /** + Runs this test script in the context of the given tester object. + */ + public boolean run(SQLTester tester) throws Exception { + reset(); + setVerbosity(tester.getVerbosity()); + String line, directive; + String[] argv; + while( null != (line = getLine()) ){ + verbose3("input line: ",line); + checkForDirective(tester, line); + argv = getCommandArgv(line); + if( null!=argv ){ + processCommand(tester, argv); + continue; + } + tester.appendInput(line,true); + } + return true; + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java b/ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..95541bdcbaf86b79ac34089a8840b5b5164dce65 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java @@ -0,0 +1,33 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + + +/** + A SQLFunction implementation for scalar functions. +*/ +public abstract class ScalarFunction implements SQLFunction { + /** + As for the xFunc() argument of the C API's + sqlite3_create_function(). If this function throws, it is + translated into an sqlite3_result_error(). + */ + public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. This default implementation does nothing. + */ + public void xDestroy() {} +} diff --git a/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java b/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java new file mode 100644 index 0000000000000000000000000000000000000000..d8b6226ac9540406aa822e0973e14f32215b727d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java @@ -0,0 +1,35 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper object for use with sqlite3_table_column_metadata(). + They are populated only via that interface. +*/ +public final class TableColumnMetadata { + OutputPointer.Bool pNotNull = new OutputPointer.Bool(); + OutputPointer.Bool pPrimaryKey = new OutputPointer.Bool(); + OutputPointer.Bool pAutoinc = new OutputPointer.Bool(); + OutputPointer.String pzCollSeq = new OutputPointer.String(); + OutputPointer.String pzDataType = new OutputPointer.String(); + + public TableColumnMetadata(){ + } + + public String getDataType(){ return pzDataType.value; } + public String getCollation(){ return pzCollSeq.value; } + public boolean isNotNull(){ return pNotNull.value; } + public boolean isPrimaryKey(){ return pPrimaryKey.value; } + public boolean isAutoincrement(){ return pAutoinc.value; } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java new file mode 100644 index 0000000000000000000000000000000000000000..05b1cfeaedf2728db74ab16d31d44ce15320fed0 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java @@ -0,0 +1,2194 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a set of tests for the sqlite3 JNI bindings. +*/ +package org.sqlite.jni.capi; +import static org.sqlite.jni.capi.CApi.*; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + An annotation for Tester1 tests which we do not want to run in + reflection-driven test mode because either they are not suitable + for multi-threaded threaded mode or we have to control their execution + order. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface ManualTest{} +/** + Annotation for Tester1 tests which mark those which must be skipped + in multi-threaded mode. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface SingleThreadOnly{} + +/** + Annotation for Tester1 tests which must only be run if + sqlite3_jni_supports_nio() is true. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface RequiresJniNio{} + +public class Tester1 implements Runnable { + //! True when running in multi-threaded mode. + private static boolean mtMode = false; + //! True to sleep briefly between tests. + private static boolean takeNaps = false; + //! True to shuffle the order of the tests. + private static boolean shuffle = false; + //! True to dump the list of to-run tests to stdout. + private static int listRunTests = 0; + //! True to squelch all out() and outln() output. + private static boolean quietMode = false; + //! Total number of runTests() calls. + private static int nTestRuns = 0; + //! List of test*() methods to run. + private static List testMethods = null; + //! List of exceptions collected by run() + private static List listErrors = new ArrayList<>(); + private static final class Metrics { + //! Number of times createNewDb() (or equivalent) is invoked. + volatile int dbOpen = 0; + } + + private Integer tId; + + Tester1(Integer id){ + tId = id; + } + + static final Metrics metrics = new Metrics(); + + public static synchronized void outln(){ + if( !quietMode ){ + System.out.println(""); + } + } + + public static synchronized void outPrefix(){ + if( !quietMode ){ + System.out.print(Thread.currentThread().getName()+": "); + } + } + + public static synchronized void outln(Object val){ + if( !quietMode ){ + outPrefix(); + System.out.println(val); + } + } + + public static synchronized void out(Object val){ + if( !quietMode ){ + System.out.print(val); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void out(Object... vals){ + if( !quietMode ){ + outPrefix(); + for(Object v : vals) out(v); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void outln(Object... vals){ + if( !quietMode ){ + out(vals); out("\n"); + } + } + + static volatile int affirmCount = 0; + public static synchronized int affirm(Boolean v, String comment){ + ++affirmCount; + if( false ) assert( v /* prefer assert over exception if it's enabled because + the JNI layer sometimes has to suppress exceptions, + so they might be squelched on their way back to the + top. */); + if( !v ) throw new RuntimeException(comment); + return affirmCount; + } + + public static void affirm(Boolean v){ + affirm(v, "Affirmation failed."); + } + + @SingleThreadOnly /* because it's thread-agnostic */ + private void test1(){ + affirm(sqlite3_libversion_number() == SQLITE_VERSION_NUMBER); + } + + public static sqlite3 createNewDb(){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + int rc = sqlite3_open(":memory:", out); + ++metrics.dbOpen; + sqlite3 db = out.take(); + if( 0!=rc ){ + final String msg = + null==db ? sqlite3_errstr(rc) : sqlite3_errmsg(db); + sqlite3_close(db); + throw new RuntimeException("Opening db failed: "+msg); + } + affirm( null == out.get() ); + affirm( 0 != db.getNativePointer() ); + rc = sqlite3_busy_timeout(db, 2000); + affirm( 0 == rc ); + return db; + } + + public static void execSql(sqlite3 db, String[] sql){ + execSql(db, String.join("", sql)); + } + + public static int execSql(sqlite3 db, boolean throwOnError, String sql){ + OutputPointer.Int32 oTail = new OutputPointer.Int32(); + final byte[] sqlUtf8 = sql.getBytes(StandardCharsets.UTF_8); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + int rc = 0; + sqlite3_stmt stmt = null; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + while(pos < sqlChunk.length){ + if(pos > 0){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, + sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v2(db, sqlChunk, outStmt, oTail); + if(throwOnError) affirm(0 == rc); + else if( 0!=rc ) break; + pos = oTail.value; + stmt = outStmt.take(); + if( null == stmt ){ + // empty statement was parsed. + continue; + } + affirm(0 != stmt.getNativePointer()); + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){ + } + sqlite3_finalize(stmt); + affirm(0 == stmt.getNativePointer()); + if(0!=rc && SQLITE_ROW!=rc && SQLITE_DONE!=rc){ + break; + } + } + sqlite3_finalize(stmt); + if(SQLITE_ROW==rc || SQLITE_DONE==rc) rc = 0; + if( 0!=rc && throwOnError){ + throw new RuntimeException("db op failed with rc=" + +rc+": "+sqlite3_errmsg(db)); + } + return rc; + } + + public static void execSql(sqlite3 db, String sql){ + execSql(db, true, sql); + } + + public static sqlite3_stmt prepare(sqlite3 db, boolean throwOnError, String sql){ + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + int rc = sqlite3_prepare_v2(db, sql, outStmt); + if( throwOnError ){ + affirm( 0 == rc ); + } + final sqlite3_stmt rv = outStmt.take(); + affirm( null == outStmt.get() ); + if( throwOnError ){ + affirm( 0 != rv.getNativePointer() ); + } + return rv; + } + + public static sqlite3_stmt prepare(sqlite3 db, String sql){ + return prepare(db, true, sql); + } + + private void showCompileOption(){ + int i = 0; + String optName; + outln("compile options:"); + for( ; null != (optName = sqlite3_compileoption_get(i)); ++i){ + outln("\t"+optName+"\t (used="+ + sqlite3_compileoption_used(optName)+")"); + } + } + + private void testCompileOption(){ + int i = 0; + String optName; + for( ; null != (optName = sqlite3_compileoption_get(i)); ++i){ + } + affirm( i > 10 ); + affirm( null==sqlite3_compileoption_get(-1) ); + } + + private void testOpenDb1(){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + int rc = sqlite3_open(":memory:", out); + ++metrics.dbOpen; + sqlite3 db = out.get(); + affirm(0 == rc); + affirm(db.getNativePointer()!=0); + sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, null) + /* This function has different mangled names in jdk8 vs jdk19, + and this call is here to ensure that the build fails + if it cannot find both names. */; + + affirm( 0==sqlite3_db_readonly(db,"main") ); + affirm( 0==sqlite3_db_readonly(db,null) ); + affirm( 0>sqlite3_db_readonly(db,"nope") ); + affirm( 0>sqlite3_db_readonly(null,null) ); + affirm( 0==sqlite3_last_insert_rowid(null) ); + + // These interrupt checks are only to make sure that the JNI binding + // has the proper exported symbol names. They don't actually test + // anything useful. + affirm( !sqlite3_is_interrupted(db) ); + sqlite3_interrupt(db); + affirm( sqlite3_is_interrupted(db) ); + sqlite3_close_v2(db); + affirm(0 == db.getNativePointer()); + } + + private void testOpenDb2(){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + int rc = sqlite3_open_v2(":memory:", out, + SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE, null); + ++metrics.dbOpen; + affirm(0 == rc); + sqlite3 db = out.get(); + affirm(0 != db.getNativePointer()); + sqlite3_close_v2(db); + affirm(0 == db.getNativePointer()); + } + + private void testPrepare123(){ + sqlite3 db = createNewDb(); + int rc; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + rc = sqlite3_prepare(db, "CREATE TABLE t1(a);", outStmt); + affirm(0 == rc); + sqlite3_stmt stmt = outStmt.take(); + affirm(0 != stmt.getNativePointer()); + affirm( !sqlite3_stmt_readonly(stmt) ); + affirm( db == sqlite3_db_handle(stmt) ); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE == rc); + sqlite3_finalize(stmt); + affirm( null == sqlite3_db_handle(stmt) ); + affirm(0 == stmt.getNativePointer()); + + { /* Demonstrate how to use the "zTail" option of + sqlite3_prepare() family of functions. */ + OutputPointer.Int32 oTail = new OutputPointer.Int32(); + final byte[] sqlUtf8 = + "CREATE TABLE t2(a); INSERT INTO t2(a) VALUES(1),(2),(3)" + .getBytes(StandardCharsets.UTF_8); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + while(pos < sqlChunk.length){ + if(pos > 0){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, sqlChunk.length); + } + //outln("SQL chunk #"+n+" length = "+sqlChunk.length+", pos = "+pos); + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v2(db, sqlChunk, outStmt, oTail); + affirm(0 == rc); + stmt = outStmt.get(); + pos = oTail.value; + /*outln("SQL tail pos = "+pos+". Chunk = "+ + (new String(Arrays.copyOfRange(sqlChunk,0,pos), + StandardCharsets.UTF_8)));*/ + switch(n){ + case 1: affirm(19 == pos); break; + case 2: affirm(36 == pos); break; + default: affirm( false /* can't happen */ ); + + } + ++n; + affirm(0 != stmt.getNativePointer()); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE == rc); + sqlite3_finalize(stmt); + affirm(0 == stmt.getNativePointer()); + } + } + + + rc = sqlite3_prepare_v3(db, "INSERT INTO t2(a) VALUES(1),(2),(3)", + 0, outStmt); + affirm(0 == rc); + stmt = outStmt.get(); + affirm(0 != stmt.getNativePointer()); + sqlite3_finalize(stmt); + affirm(0 == stmt.getNativePointer() ); + + affirm( 0==sqlite3_errcode(db) ); + stmt = sqlite3_prepare(db, "intentional error"); + affirm( null==stmt ); + affirm( 0!=sqlite3_errcode(db) ); + affirm( 0==sqlite3_errmsg(db).indexOf("near \"intentional\"") ); + sqlite3_finalize(stmt); + stmt = sqlite3_prepare(db, "/* empty input*/\n-- comments only"); + affirm( null==stmt ); + affirm( 0==sqlite3_errcode(db) ); + sqlite3_close_v2(db); + } + + private void testBindFetchInt(){ + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(:a);"); + affirm(1 == sqlite3_bind_parameter_count(stmt)); + final int paramNdx = sqlite3_bind_parameter_index(stmt, ":a"); + affirm(1 == paramNdx); + affirm( ":a".equals(sqlite3_bind_parameter_name(stmt, paramNdx))); + int total1 = 0; + long rowid = -1; + int changes = sqlite3_changes(db); + int changesT = sqlite3_total_changes(db); + long changes64 = sqlite3_changes64(db); + long changesT64 = sqlite3_total_changes64(db); + int rc; + for(int i = 99; i < 102; ++i ){ + total1 += i; + rc = sqlite3_bind_int(stmt, paramNdx, i); + affirm(0 == rc); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + affirm(SQLITE_DONE == rc); + long x = sqlite3_last_insert_rowid(db); + affirm(x > rowid); + rowid = x; + } + sqlite3_finalize(stmt); + affirm(300 == total1); + affirm(sqlite3_changes(db) > changes); + affirm(sqlite3_total_changes(db) > changesT); + affirm(sqlite3_changes64(db) > changes64); + affirm(sqlite3_total_changes64(db) > changesT64); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + affirm( sqlite3_stmt_readonly(stmt) ); + affirm( !sqlite3_stmt_busy(stmt) ); + if( sqlite3_compileoption_used("ENABLE_COLUMN_METADATA") ){ + /* Unlike in native C code, JNI won't trigger an + UnsatisfiedLinkError until these are called (on Linux, at + least). */ + affirm("t".equals(sqlite3_column_table_name(stmt,0))); + affirm("main".equals(sqlite3_column_database_name(stmt,0))); + affirm("a".equals(sqlite3_column_origin_name(stmt,0))); + } + + int total2 = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + affirm( sqlite3_stmt_busy(stmt) ); + total2 += sqlite3_column_int(stmt, 0); + sqlite3_value sv = sqlite3_column_value(stmt, 0); + affirm( null != sv ); + affirm( 0 != sv.getNativePointer() ); + affirm( SQLITE_INTEGER == sqlite3_value_type(sv) ); + } + affirm( !sqlite3_stmt_busy(stmt) ); + sqlite3_finalize(stmt); + affirm(total1 == total2); + + // sqlite3_value_frombind() checks... + stmt = prepare(db, "SELECT 1, ?"); + sqlite3_bind_int(stmt, 1, 2); + rc = sqlite3_step(stmt); + affirm( SQLITE_ROW==rc ); + affirm( !sqlite3_value_frombind(sqlite3_column_value(stmt, 0)) ); + affirm( sqlite3_value_frombind(sqlite3_column_value(stmt, 1)) ); + sqlite3_finalize(stmt); + + sqlite3_close_v2(db); + affirm(0 == db.getNativePointer()); + } + + private void testBindFetchInt64(){ + try (sqlite3 db = createNewDb()){ + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + long total1 = 0; + for(long i = 0xffffffff; i < 0xffffffff + 3; ++i ){ + total1 += i; + sqlite3_bind_int64(stmt, 1, i); + sqlite3_step(stmt); + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + long total2 = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + total2 += sqlite3_column_int64(stmt, 0); + } + sqlite3_finalize(stmt); + affirm(total1 == total2); + //sqlite3_close_v2(db); + } + } + + private void testBindFetchDouble(){ + try (sqlite3 db = createNewDb()){ + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + double total1 = 0; + for(double i = 1.5; i < 5.0; i = i + 1.0 ){ + total1 += i; + sqlite3_bind_double(stmt, 1, i); + sqlite3_step(stmt); + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + double total2 = 0; + int counter = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + ++counter; + total2 += sqlite3_column_double(stmt, 0); + } + affirm(4 == counter); + sqlite3_finalize(stmt); + affirm(total2<=total1+0.01 && total2>=total1-0.01); + //sqlite3_close_v2(db); + } + } + + private void testBindFetchText(){ + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + String[] list1 = { "hell🤩", "w😃rld", "!🤩" }; + int rc; + int n = 0; + for( String e : list1 ){ + rc = (0==n) + ? sqlite3_bind_text(stmt, 1, e) + : sqlite3_bind_text16(stmt, 1, e); + affirm(0 == rc); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE==rc); + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + StringBuilder sbuf = new StringBuilder(); + n = 0; + final boolean tryNio = sqlite3_jni_supports_nio(); + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final sqlite3_value sv = sqlite3_value_dup(sqlite3_column_value(stmt,0)); + final String txt = sqlite3_column_text16(stmt, 0); + sbuf.append( txt ); + affirm( txt.equals(new String( + sqlite3_column_text(stmt, 0), + StandardCharsets.UTF_8 + )) ); + affirm( txt.length() < sqlite3_value_bytes(sv) ); + affirm( txt.equals(new String( + sqlite3_value_text(sv), + StandardCharsets.UTF_8)) ); + affirm( txt.length() == sqlite3_value_bytes16(sv)/2 ); + affirm( txt.equals(sqlite3_value_text16(sv)) ); + if( tryNio ){ + java.nio.ByteBuffer bu = sqlite3_value_nio_buffer(sv); + byte ba[] = sqlite3_value_blob(sv); + affirm( ba.length == bu.capacity() ); + int i = 0; + for( byte b : ba ){ + affirm( b == bu.get(i++) ); + } + } + sqlite3_value_free(sv); + ++n; + } + sqlite3_finalize(stmt); + affirm(3 == n); + affirm("w😃rldhell🤩!🤩".equals(sbuf.toString())); + + try( sqlite3_stmt stmt2 = prepare(db, "SELECT ?, ?") ){ + rc = sqlite3_bind_text(stmt2, 1, ""); + affirm( 0==rc ); + rc = sqlite3_bind_text(stmt2, 2, (String)null); + affirm( 0==rc ); + rc = sqlite3_step(stmt2); + affirm( SQLITE_ROW==rc ); + byte[] colBa = sqlite3_column_text(stmt2, 0); + affirm( 0==colBa.length ); + colBa = sqlite3_column_text(stmt2, 1); + affirm( null==colBa ); + //sqlite3_finalize(stmt); + } + + if(true){ + sqlite3_close_v2(db); + }else{ + // Let the Object.finalize() override deal with it. + } + } + + private void testBindFetchBlob(){ + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + byte[] list1 = { 0x32, 0x33, 0x34 }; + int rc = sqlite3_bind_blob(stmt, 1, list1); + affirm( 0==rc ); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE == rc); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + int n = 0; + int total = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + byte[] blob = sqlite3_column_blob(stmt, 0); + affirm(3 == blob.length); + int i = 0; + for(byte b : blob){ + affirm(b == list1[i++]); + total += b; + } + ++n; + } + sqlite3_finalize(stmt); + affirm(1 == n); + affirm(total == 0x32 + 0x33 + 0x34); + sqlite3_close_v2(db); + } + + @RequiresJniNio + private void testBindByteBuffer(){ + /* TODO: these tests need to be much more extensive to check the + begin/end range handling. */ + + java.nio.ByteBuffer zeroCheck = + java.nio.ByteBuffer.allocateDirect(0); + affirm( null != zeroCheck ); + zeroCheck = null; + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + + final java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocateDirect(10); + buf.put((byte)0x31)/*note that we'll skip this one*/ + .put((byte)0x32) + .put((byte)0x33) + .put((byte)0x34) + .put((byte)0x35)/*we'll skip this one too*/; + + final int expectTotal = buf.get(1) + buf.get(2) + buf.get(3); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + affirm( SQLITE_ERROR == sqlite3_bind_blob(stmt, 1, buf, -1, 0), + "Buffer offset may not be negative." ); + affirm( 0 == sqlite3_bind_blob(stmt, 1, buf, 1, 3) ); + affirm( SQLITE_DONE == sqlite3_step(stmt) ); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t;"); + int total = 0; + affirm( SQLITE_ROW == sqlite3_step(stmt) ); + byte blob[] = sqlite3_column_blob(stmt, 0); + java.nio.ByteBuffer nioBlob = + sqlite3_column_nio_buffer(stmt, 0); + affirm(3 == blob.length); + affirm(blob.length == nioBlob.capacity()); + affirm(blob.length == nioBlob.limit()); + int i = 0; + for(byte b : blob){ + affirm( i<=3 ); + affirm(b == buf.get(1 + i)); + affirm(b == nioBlob.get(i)); + ++i; + total += b; + } + affirm( SQLITE_DONE == sqlite3_step(stmt) ); + sqlite3_finalize(stmt); + affirm(total == expectTotal); + + SQLFunction func = + new ScalarFunction(){ + public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + sqlite3_result_blob(cx, buf, 1, 3); + } + }; + + affirm( 0 == sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func) ); + stmt = prepare(db, "SELECT myfunc()"); + affirm( SQLITE_ROW == sqlite3_step(stmt) ); + blob = sqlite3_column_blob(stmt, 0); + affirm(3 == blob.length); + i = 0; + total = 0; + for(byte b : blob){ + affirm( i<=3 ); + affirm(b == buf.get(1 + i++)); + total += b; + } + affirm( SQLITE_DONE == sqlite3_step(stmt) ); + sqlite3_finalize(stmt); + affirm(total == expectTotal); + + sqlite3_close_v2(db); + } + + private void testSql(){ + sqlite3 db = createNewDb(); + sqlite3_stmt stmt = prepare(db, "SELECT 1"); + affirm( "SELECT 1".equals(sqlite3_sql(stmt)) ); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT ?"); + sqlite3_bind_text(stmt, 1, "hell😃"); + final String expect = "SELECT 'hell😃'"; + affirm( expect.equals(sqlite3_expanded_sql(stmt)) ); + String n = sqlite3_normalized_sql(stmt); + affirm( null==n || "SELECT?;".equals(n) ); + sqlite3_finalize(stmt); + sqlite3_close(db); + } + + private void testCollation(){ + final sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + final ValueHolder xDestroyCalled = new ValueHolder<>(0); + final CollationCallback myCollation = new CollationCallback() { + private String myState = + "this is local state. There is much like it, but this is mine."; + @Override + // Reverse-sorts its inputs... + public int call(byte[] lhs, byte[] rhs){ + int len = lhs.length > rhs.length ? rhs.length : lhs.length; + int c = 0, i = 0; + for(i = 0; i < len; ++i){ + c = lhs[i] - rhs[i]; + if(0 != c) break; + } + if(0==c){ + if(i < lhs.length) c = 1; + else if(i < rhs.length) c = -1; + } + return -c; + } + @Override + public void xDestroy() { + // Just demonstrates that xDestroy is called. + ++xDestroyCalled.value; + } + }; + final CollationNeededCallback collLoader = new CollationNeededCallback(){ + @Override + public void call(sqlite3 dbArg, int eTextRep, String collationName){ + affirm(dbArg == db/* as opposed to a temporary object*/); + sqlite3_create_collation(dbArg, "reversi", eTextRep, myCollation); + } + }; + int rc = sqlite3_collation_needed(db, collLoader); + affirm( 0 == rc ); + rc = sqlite3_collation_needed(db, collLoader); + affirm( 0 == rc /* Installing the same object again is a no-op */); + sqlite3_stmt stmt = prepare(db, "SELECT a FROM t ORDER BY a COLLATE reversi"); + int counter = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final String val = sqlite3_column_text16(stmt, 0); + ++counter; + //outln("REVERSI'd row#"+counter+": "+val); + switch(counter){ + case 1: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 3: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a"); + counter = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final String val = sqlite3_column_text16(stmt, 0); + ++counter; + //outln("Non-REVERSI'd row#"+counter+": "+val); + switch(counter){ + case 3: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 1: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + sqlite3_finalize(stmt); + affirm( 0 == xDestroyCalled.value ); + rc = sqlite3_collation_needed(db, null); + affirm( 0 == rc ); + sqlite3_close_v2(db); + affirm( 0 == db.getNativePointer() ); + affirm( 1 == xDestroyCalled.value ); + } + + @SingleThreadOnly /* because it's thread-agnostic */ + private void testToUtf8(){ + /** + https://docs.oracle.com/javase/8/docs/api/java/nio/charset/Charset.html + + Let's ensure that we can convert to standard UTF-8 in Java code + (noting that the JNI native API has no way to do this). + */ + final byte[] ba = "a \0 b".getBytes(StandardCharsets.UTF_8); + affirm( 5 == ba.length /* as opposed to 6 in modified utf-8 */); + } + + private void testStatus(){ + final OutputPointer.Int64 cur64 = new OutputPointer.Int64(); + final OutputPointer.Int64 high64 = new OutputPointer.Int64(); + final OutputPointer.Int32 cur32 = new OutputPointer.Int32(); + final OutputPointer.Int32 high32 = new OutputPointer.Int32(); + final sqlite3 db = createNewDb(); + execSql(db, "create table t(a); insert into t values(1),(2),(3)"); + + int rc = sqlite3_status(SQLITE_STATUS_MEMORY_USED, cur32, high32, false); + affirm( 0 == rc ); + affirm( cur32.value > 0 ); + affirm( high32.value >= cur32.value ); + + rc = sqlite3_status64(SQLITE_STATUS_MEMORY_USED, cur64, high64, false); + affirm( 0 == rc ); + affirm( cur64.value > 0 ); + affirm( high64.value >= cur64.value ); + + cur32.value = 0; + high32.value = 1; + rc = sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, cur32, high32, false); + affirm( 0 == rc ); + affirm( cur32.value > 0 ); + affirm( high32.value == 0 /* always 0 for SCHEMA_USED */ ); + + sqlite3_close_v2(db); + } + + private void testUdf1(){ + final sqlite3 db = createNewDb(); + // These ValueHolders are just to confirm that the func did what we want... + final ValueHolder xDestroyCalled = new ValueHolder<>(false); + final ValueHolder xFuncAccum = new ValueHolder<>(0); + final ValueHolder neverEverDoThisInClientCode = new ValueHolder<>(null); + final ValueHolder neverEverDoThisInClientCode2 = new ValueHolder<>(null); + + // Create an SQLFunction instance using one of its 3 subclasses: + // Scalar, Aggregate, or Window: + SQLFunction func = + // Each of the 3 subclasses requires a different set of + // functions, all of which must be implemented. Anonymous + // classes are a convenient way to implement these. + new ScalarFunction(){ + public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + affirm(db == sqlite3_context_db_handle(cx)); + if( null==neverEverDoThisInClientCode.value ){ + /* !!!NEVER!!! hold a reference to an sqlite3_value or + sqlite3_context object like this in client code! They + are ONLY legal for the duration of their single + call. We do it here ONLY to test that the defenses + against clients doing this are working. */ + neverEverDoThisInClientCode2.value = cx; + neverEverDoThisInClientCode.value = args; + } + int result = 0; + for( sqlite3_value v : args ) result += sqlite3_value_int(v); + xFuncAccum.value += result;// just for post-run testing + sqlite3_result_int(cx, result); + } + /* OPTIONALLY override xDestroy... */ + public void xDestroy(){ + xDestroyCalled.value = true; + } + }; + + // Register and use the function... + int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); + affirm(0 == rc); + affirm(0 == xFuncAccum.value); + final sqlite3_stmt stmt = prepare(db, "SELECT myfunc(1,2,3)"); + int n = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + affirm( 6 == sqlite3_column_int(stmt, 0) ); + ++n; + } + sqlite3_finalize(stmt); + affirm(1 == n); + affirm(6 == xFuncAccum.value); + affirm( !xDestroyCalled.value ); + affirm( null!=neverEverDoThisInClientCode.value ); + affirm( null!=neverEverDoThisInClientCode2.value ); + affirm( 0 xFuncAccum = new ValueHolder<>(0); + + SQLFunction funcAgg = new AggregateFunction(){ + @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){ + /** Throwing from here should emit loud noise on stdout or stderr + but the exception is supressed because we have no way to inform + sqlite about it from these callbacks. */ + //throw new RuntimeException("Throwing from an xStep"); + } + @Override public void xFinal(sqlite3_context cx){ + throw new RuntimeException("Throwing from an xFinal"); + } + }; + int rc = sqlite3_create_function(db, "myagg", 1, SQLITE_UTF8, funcAgg); + affirm(0 == rc); + affirm(0 == xFuncAccum.value); + sqlite3_stmt stmt = prepare(db, "SELECT myagg(1)"); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + affirm( 0 != rc ); + affirm( sqlite3_errmsg(db).indexOf("an xFinal") > 0 ); + + SQLFunction funcSc = new ScalarFunction(){ + @Override public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + throw new RuntimeException("Throwing from an xFunc"); + } + }; + rc = sqlite3_create_function(db, "mysca", 0, SQLITE_UTF8, funcSc); + affirm(0 == rc); + affirm(0 == xFuncAccum.value); + stmt = prepare(db, "SELECT mysca()"); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + affirm( 0 != rc ); + affirm( sqlite3_errmsg(db).indexOf("an xFunc") > 0 ); + rc = sqlite3_create_function(db, "mysca", 1, -1, funcSc); + affirm( SQLITE_FORMAT==rc, "invalid encoding value." ); + sqlite3_close_v2(db); + } + + @SingleThreadOnly + private void testUdfJavaObject(){ + affirm( !mtMode ); + final sqlite3 db = createNewDb(); + final ValueHolder testResult = new ValueHolder<>(db); + final ValueHolder boundObj = new ValueHolder<>(42); + final SQLFunction func = new ScalarFunction(){ + public void xFunc(sqlite3_context cx, sqlite3_value args[]){ + sqlite3_result_java_object(cx, testResult.value); + affirm( sqlite3_value_java_object(args[0]) == boundObj ); + } + }; + int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); + affirm(0 == rc); + sqlite3_stmt stmt = prepare(db, "select myfunc(?)"); + affirm( 0 != stmt.getNativePointer() ); + affirm( testResult.value == db ); + rc = sqlite3_bind_java_object(stmt, 1, boundObj); + affirm( 0==rc ); + int n = 0; + if( SQLITE_ROW == sqlite3_step(stmt) ){ + affirm( testResult.value == sqlite3_column_java_object(stmt, 0) ); + affirm( testResult.value == sqlite3_column_java_object(stmt, 0, sqlite3.class) ); + affirm( null == sqlite3_column_java_object(stmt, 0, sqlite3_stmt.class) ); + affirm( null == sqlite3_column_java_object(stmt,1) ); + final sqlite3_value v = sqlite3_column_value(stmt, 0); + affirm( testResult.value == sqlite3_value_java_object(v) ); + affirm( testResult.value == sqlite3_value_java_object(v, sqlite3.class) ); + affirm( testResult.value == + sqlite3_value_java_object(v, testResult.value.getClass()) ); + affirm( testResult.value == sqlite3_value_java_object(v, Object.class) ); + affirm( null == sqlite3_value_java_object(v, String.class) ); + ++n; + } + sqlite3_finalize(stmt); + affirm( 1 == n ); + affirm( 0==sqlite3_db_release_memory(db) ); + sqlite3_close_v2(db); + } + + private void testUdfAggregate(){ + final sqlite3 db = createNewDb(); + final ValueHolder xFinalNull = + // To confirm that xFinal() is called with no aggregate state + // when the corresponding result set is empty. + new ValueHolder<>(false); + final ValueHolder neverEverDoThisInClientCode = new ValueHolder<>(null); + final ValueHolder neverEverDoThisInClientCode2 = new ValueHolder<>(null); + SQLFunction func = new AggregateFunction(){ + @Override + public void xStep(sqlite3_context cx, sqlite3_value[] args){ + if( null==neverEverDoThisInClientCode.value ){ + /* !!!NEVER!!! hold a reference to an sqlite3_value or + sqlite3_context object like this in client code! They + are ONLY legal for the duration of their single + call. We do it here ONLY to test that the defenses + against clients doing this are working. */ + neverEverDoThisInClientCode.value = args; + } + final ValueHolder agg = this.getAggregateState(cx, 0); + agg.value += sqlite3_value_int(args[0]); + affirm( agg == this.getAggregateState(cx, 0) ); + } + @Override + public void xFinal(sqlite3_context cx){ + if( null==neverEverDoThisInClientCode2.value ){ + neverEverDoThisInClientCode2.value = cx; + } + final Integer v = this.takeAggregateState(cx); + if(null == v){ + xFinalNull.value = true; + sqlite3_result_null(cx); + }else{ + sqlite3_result_int(cx, v); + } + } + }; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES(1),(2),(3)"); + int rc = sqlite3_create_function(db, "myfunc", 1, SQLITE_UTF8, func); + affirm(0 == rc); + sqlite3_stmt stmt = prepare(db, "select myfunc(a), myfunc(a+10) from t"); + affirm( 0==sqlite3_stmt_status(stmt, SQLITE_STMTSTATUS_RUN, false) ); + int n = 0; + if( SQLITE_ROW == sqlite3_step(stmt) ){ + int v = sqlite3_column_int(stmt, 0); + affirm( 6 == v ); + int v2 = sqlite3_column_int(stmt, 1); + affirm( 30+v == v2 ); + ++n; + } + affirm( 1==n ); + affirm(!xFinalNull.value); + affirm( null!=neverEverDoThisInClientCode.value ); + affirm( null!=neverEverDoThisInClientCode2.value ); + affirm( 0(){ + + private void xStepInverse(sqlite3_context cx, int v){ + this.getAggregateState(cx,0).value += v; + } + @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){ + this.xStepInverse(cx, sqlite3_value_int(args[0])); + } + @Override public void xInverse(sqlite3_context cx, sqlite3_value[] args){ + this.xStepInverse(cx, -sqlite3_value_int(args[0])); + } + + private void xFinalValue(sqlite3_context cx, Integer v){ + if(null == v) sqlite3_result_null(cx); + else sqlite3_result_int(cx, v); + } + @Override public void xFinal(sqlite3_context cx){ + xFinalValue(cx, this.takeAggregateState(cx)); + } + @Override public void xValue(sqlite3_context cx){ + xFinalValue(cx, this.getAggregateState(cx,null).value); + } + }; + int rc = sqlite3_create_function(db, "winsumint", 1, SQLITE_UTF8, func); + affirm( 0 == rc ); + execSql(db, new String[] { + "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", + "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" + }); + final sqlite3_stmt stmt = prepare(db, + "SELECT x, winsumint(y) OVER ("+ + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING"+ + ") AS sum_y "+ + "FROM twin ORDER BY x;"); + int n = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final String s = sqlite3_column_text16(stmt, 0); + final int i = sqlite3_column_int(stmt, 1); + switch(++n){ + case 1: affirm( "a".equals(s) && 9==i ); break; + case 2: affirm( "b".equals(s) && 12==i ); break; + case 3: affirm( "c".equals(s) && 16==i ); break; + case 4: affirm( "d".equals(s) && 12==i ); break; + case 5: affirm( "e".equals(s) && 9==i ); break; + default: affirm( false /* cannot happen */ ); + } + } + sqlite3_finalize(stmt); + affirm( 5 == n ); + sqlite3_close_v2(db); + } + + private void listBoundMethods(){ + if(false){ + final java.lang.reflect.Field[] declaredFields = + CApi.class.getDeclaredFields(); + outln("Bound constants:\n"); + for(java.lang.reflect.Field field : declaredFields) { + if(java.lang.reflect.Modifier.isStatic(field.getModifiers())) { + outln("\t",field.getName()); + } + } + } + final java.lang.reflect.Method[] declaredMethods = + CApi.class.getDeclaredMethods(); + final java.util.List funcList = new java.util.ArrayList<>(); + for(java.lang.reflect.Method m : declaredMethods){ + if((m.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0){ + final String name = m.getName(); + if(name.startsWith("sqlite3_")){ + funcList.add(name); + } + } + } + int count = 0; + java.util.Collections.sort(funcList); + for(String n : funcList){ + ++count; + outln("\t",n,"()"); + } + outln(count," functions named sqlite3_*."); + } + + private void testTrace(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + /* Ensure that characters outside of the UTF BMP survive the trip + from Java to sqlite3 and back to Java. (At no small efficiency + penalty.) */ + final String nonBmpChar = "😃"; + int rc = sqlite3_trace_v2( + db, SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE + | SQLITE_TRACE_ROW | SQLITE_TRACE_CLOSE, + new TraceV2Callback(){ + @Override public int call(int traceFlag, Object pNative, Object x){ + ++counter.value; + //outln("TRACE "+traceFlag+" pNative = "+pNative.getClass().getName()); + switch(traceFlag){ + case SQLITE_TRACE_STMT: + affirm(pNative instanceof sqlite3_stmt); + //outln("TRACE_STMT sql = "+x); + affirm(x instanceof String); + affirm( ((String)x).indexOf(nonBmpChar) > 0 ); + break; + case SQLITE_TRACE_PROFILE: + affirm(pNative instanceof sqlite3_stmt); + affirm(x instanceof Long); + //outln("TRACE_PROFILE time = "+x); + break; + case SQLITE_TRACE_ROW: + affirm(pNative instanceof sqlite3_stmt); + affirm(null == x); + //outln("TRACE_ROW = "+sqlite3_column_text16((sqlite3_stmt)pNative, 0)); + break; + case SQLITE_TRACE_CLOSE: + affirm(pNative instanceof sqlite3); + affirm(null == x); + break; + default: + affirm(false /*cannot happen*/); + break; + } + return 0; + } + }); + affirm( 0==rc ); + execSql(db, "SELECT coalesce(null,null,'"+nonBmpChar+"'); "+ + "SELECT 'w"+nonBmpChar+"orld'"); + affirm( 6 == counter.value ); + sqlite3_close_v2(db); + affirm( 7 == counter.value ); + } + + @SingleThreadOnly /* because threads inherently break this test */ + private static void testBusy(){ + final String dbName = "_busy-handler.db"; + try{ + final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3(); + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + + int rc = sqlite3_open(dbName, outDb); + ++metrics.dbOpen; + affirm( 0 == rc ); + final sqlite3 db1 = outDb.get(); + execSql(db1, "CREATE TABLE IF NOT EXISTS t(a)"); + rc = sqlite3_open(dbName, outDb); + ++metrics.dbOpen; + affirm( 0 == rc ); + affirm( outDb.get() != db1 ); + final sqlite3 db2 = outDb.get(); + + affirm( "main".equals( sqlite3_db_name(db1, 0) ) ); + rc = sqlite3_db_config(db1, SQLITE_DBCONFIG_MAINDBNAME, "foo"); + affirm( sqlite3_db_filename(db1, "foo").endsWith(dbName) ); + affirm( "foo".equals( sqlite3_db_name(db1, 0) ) ); + affirm( SQLITE_MISUSE == sqlite3_db_config(db1, 0, 0, null) ); + + final ValueHolder xBusyCalled = new ValueHolder<>(0); + BusyHandlerCallback handler = new BusyHandlerCallback(){ + @Override public int call(int n){ + //outln("busy handler #"+n); + return n > 2 ? 0 : ++xBusyCalled.value; + } + }; + rc = sqlite3_busy_handler(db2, handler); + affirm(0 == rc); + + // Force a locked condition... + execSql(db1, "BEGIN EXCLUSIVE"); + rc = sqlite3_prepare_v2(db2, "SELECT * from t", outStmt); + affirm( SQLITE_BUSY == rc); + affirm( null == outStmt.get() ); + affirm( 3 == xBusyCalled.value ); + sqlite3_close_v2(db1); + sqlite3_close_v2(db2); + }finally{ + try{(new java.io.File(dbName)).delete();} + catch(Exception e){/* ignore */} + } + } + + private void testProgress(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + sqlite3_progress_handler(db, 1, new ProgressHandlerCallback(){ + @Override public int call(){ + ++counter.value; + return 0; + } + }); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( counter.value > 0 ); + int nOld = counter.value; + sqlite3_progress_handler(db, 0, null); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( nOld == counter.value ); + sqlite3_close_v2(db); + } + + private void testCommitHook(){ + final sqlite3 db = createNewDb(); + sqlite3_extended_result_codes(db, true); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder hookResult = new ValueHolder<>(0); + final CommitHookCallback theHook = new CommitHookCallback(){ + @Override public int call(){ + ++counter.value; + return hookResult.value; + } + }; + CommitHookCallback oldHook = sqlite3_commit_hook(db, theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 2 == counter.value ); + execSql(db, "BEGIN; SELECT 1; SELECT 2; COMMIT;"); + affirm( 2 == counter.value /* NOT invoked if no changes are made */ ); + execSql(db, "BEGIN; update t set a='d' where a='c'; COMMIT;"); + affirm( 3 == counter.value ); + oldHook = sqlite3_commit_hook(db, theHook); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='e' where a='d'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = sqlite3_commit_hook(db, null); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='f' where a='e'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = sqlite3_commit_hook(db, null); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='g' where a='f'; COMMIT;"); + affirm( 4 == counter.value ); + + final CommitHookCallback newHook = new CommitHookCallback(){ + @Override public int call(){return 0;} + }; + oldHook = sqlite3_commit_hook(db, newHook); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='h' where a='g'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = sqlite3_commit_hook(db, theHook); + affirm( newHook == oldHook ); + execSql(db, "BEGIN; update t set a='i' where a='h'; COMMIT;"); + affirm( 5 == counter.value ); + hookResult.value = SQLITE_ERROR; + int rc = execSql(db, false, "BEGIN; update t set a='j' where a='i'; COMMIT;"); + affirm( SQLITE_CONSTRAINT_COMMITHOOK == rc ); + affirm( 6 == counter.value ); + sqlite3_close_v2(db); + } + + private void testUpdateHook(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder expectedOp = new ValueHolder<>(0); + final UpdateHookCallback theHook = new UpdateHookCallback(){ + @Override + public void call(int opId, String dbName, String tableName, long rowId){ + ++counter.value; + if( 0!=expectedOp.value ){ + affirm( expectedOp.value == opId ); + } + } + }; + UpdateHookCallback oldHook = sqlite3_update_hook(db, theHook); + affirm( null == oldHook ); + expectedOp.value = SQLITE_INSERT; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 3 == counter.value ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='d' where a='c';"); + affirm( 4 == counter.value ); + oldHook = sqlite3_update_hook(db, theHook); + affirm( theHook == oldHook ); + expectedOp.value = SQLITE_DELETE; + execSql(db, "DELETE FROM t where a='d'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_update_hook(db, null); + affirm( theHook == oldHook ); + execSql(db, "update t set a='e' where a='b';"); + affirm( 5 == counter.value ); + oldHook = sqlite3_update_hook(db, null); + affirm( null == oldHook ); + + final UpdateHookCallback newHook = new UpdateHookCallback(){ + @Override public void call(int opId, String dbName, String tableName, long rowId){ + } + }; + oldHook = sqlite3_update_hook(db, newHook); + affirm( null == oldHook ); + execSql(db, "update t set a='h' where a='a'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_update_hook(db, theHook); + affirm( newHook == oldHook ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='i' where a='h'"); + affirm( 6 == counter.value ); + sqlite3_close_v2(db); + } + + /** + This test is functionally identical to testUpdateHook(), only with a + different callback type. + */ + private void testPreUpdateHook(){ + if( !sqlite3_compileoption_used("ENABLE_PREUPDATE_HOOK") ){ + //outln("Skipping testPreUpdateHook(): no pre-update hook support."); + return; + } + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder expectedOp = new ValueHolder<>(0); + final PreupdateHookCallback theHook = new PreupdateHookCallback(){ + @Override + public void call(sqlite3 db, int opId, String dbName, String dbTable, + long iKey1, long iKey2 ){ + ++counter.value; + switch( opId ){ + case SQLITE_UPDATE: + affirm( 0 < sqlite3_preupdate_count(db) ); + affirm( null != sqlite3_preupdate_new(db, 0) ); + affirm( null != sqlite3_preupdate_old(db, 0) ); + break; + case SQLITE_INSERT: + affirm( null != sqlite3_preupdate_new(db, 0) ); + break; + case SQLITE_DELETE: + affirm( null != sqlite3_preupdate_old(db, 0) ); + break; + default: + break; + } + if( 0!=expectedOp.value ){ + affirm( expectedOp.value == opId ); + } + } + }; + PreupdateHookCallback oldHook = sqlite3_preupdate_hook(db, theHook); + affirm( null == oldHook ); + expectedOp.value = SQLITE_INSERT; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 3 == counter.value ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='d' where a='c';"); + affirm( 4 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, theHook); + affirm( theHook == oldHook ); + expectedOp.value = SQLITE_DELETE; + execSql(db, "DELETE FROM t where a='d'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, null); + affirm( theHook == oldHook ); + execSql(db, "update t set a='e' where a='b';"); + affirm( 5 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, null); + affirm( null == oldHook ); + + final PreupdateHookCallback newHook = new PreupdateHookCallback(){ + @Override + public void call(sqlite3 db, int opId, String dbName, + String tableName, long iKey1, long iKey2){ + } + }; + oldHook = sqlite3_preupdate_hook(db, newHook); + affirm( null == oldHook ); + execSql(db, "update t set a='h' where a='a'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, theHook); + affirm( newHook == oldHook ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='i' where a='h'"); + affirm( 6 == counter.value ); + + sqlite3_close_v2(db); + } + + private void testRollbackHook(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final RollbackHookCallback theHook = new RollbackHookCallback(){ + @Override public void call(){ + ++counter.value; + } + }; + RollbackHookCallback oldHook = sqlite3_rollback_hook(db, theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 0 == counter.value ); + execSql(db, false, "BEGIN; SELECT 1; SELECT 2; ROLLBACK;"); + affirm( 1 == counter.value /* contra to commit hook, is invoked if no changes are made */ ); + + final RollbackHookCallback newHook = new RollbackHookCallback(){ + @Override public void call(){return;} + }; + oldHook = sqlite3_rollback_hook(db, newHook); + affirm( theHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 1 == counter.value ); + oldHook = sqlite3_rollback_hook(db, theHook); + affirm( newHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 2 == counter.value ); + int rc = execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 0 == rc ); + affirm( 3 == counter.value ); + sqlite3_close_v2(db); + } + + /** + If FTS5 is available, runs FTS5 tests, else returns with no side + effects. If it is available but loading of the FTS5 bits fails, + it throws. + */ + @SuppressWarnings("unchecked") + @SingleThreadOnly /* because the Fts5 parts are not yet known to be + thread-safe */ + private void testFts5() throws Exception { + if( !sqlite3_compileoption_used("ENABLE_FTS5") ){ + //outln("SQLITE_ENABLE_FTS5 is not set. Skipping FTS5 tests."); + return; + } + Exception err = null; + try { + Class t = Class.forName("org.sqlite.jni.fts5.TesterFts5"); + java.lang.reflect.Constructor ctor = t.getConstructor(); + ctor.setAccessible(true); + final long timeStart = System.currentTimeMillis(); + ctor.newInstance() /* will run all tests */; + final long timeEnd = System.currentTimeMillis(); + outln("FTS5 Tests done in ",(timeEnd - timeStart),"ms"); + }catch(ClassNotFoundException e){ + outln("FTS5 classes not loaded."); + err = e; + }catch(NoSuchMethodException e){ + outln("FTS5 tester ctor not found."); + err = e; + }catch(Exception e){ + outln("Instantiation of FTS5 tester threw."); + err = e; + } + if( null != err ){ + outln("Exception: "+err); + err.printStackTrace(); + throw err; + } + } + + private void testAuthorizer(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder authRc = new ValueHolder<>(0); + final AuthorizerCallback auth = new AuthorizerCallback(){ + public int call(int op, String s0, String s1, String s2, String s3){ + ++counter.value; + //outln("xAuth(): "+s0+" "+s1+" "+s2+" "+s3); + return authRc.value; + } + }; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + sqlite3_set_authorizer(db, auth); + execSql(db, "UPDATE t SET a=1"); + affirm( 1 == counter.value ); + authRc.value = SQLITE_DENY; + int rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( SQLITE_AUTH==rc ); + sqlite3_set_authorizer(db, null); + rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( 0==rc ); + // TODO: expand these tests considerably + sqlite3_close(db); + } + + @SingleThreadOnly /* because multiple threads legitimately make these + results unpredictable */ + private synchronized void testAutoExtension(){ + final ValueHolder val = new ValueHolder<>(0); + final ValueHolder toss = new ValueHolder<>(null); + final AutoExtensionCallback ax = new AutoExtensionCallback(){ + @Override public int call(sqlite3 db){ + ++val.value; + if( null!=toss.value ){ + throw new RuntimeException(toss.value); + } + return 0; + } + }; + int rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + sqlite3_close(createNewDb()); + affirm( 1==val.value ); + sqlite3_close(createNewDb()); + affirm( 2==val.value ); + sqlite3_reset_auto_extension(); + sqlite3_close(createNewDb()); + affirm( 2==val.value ); + rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + // Must not add a new entry + rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + sqlite3_close( createNewDb() ); + affirm( 3==val.value ); + + sqlite3 db = createNewDb(); + affirm( 4==val.value ); + execSql(db, "ATTACH ':memory:' as foo"); + affirm( 4==val.value, "ATTACH uses the same connection, not sub-connections." ); + sqlite3_close(db); + db = null; + + affirm( sqlite3_cancel_auto_extension(ax) ); + affirm( !sqlite3_cancel_auto_extension(ax) ); + sqlite3_close(createNewDb()); + affirm( 4==val.value ); + rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + Exception err = null; + toss.value = "Throwing from auto_extension."; + try{ + sqlite3_close(createNewDb()); + }catch(Exception e){ + err = e; + } + affirm( err!=null ); + affirm( err.getMessage().indexOf(toss.value)>0 ); + toss.value = null; + + val.value = 0; + final AutoExtensionCallback ax2 = new AutoExtensionCallback(){ + @Override public int call(sqlite3 db){ + ++val.value; + return 0; + } + }; + rc = sqlite3_auto_extension( ax2 ); + affirm( 0 == rc ); + sqlite3_close(createNewDb()); + affirm( 2 == val.value ); + affirm( sqlite3_cancel_auto_extension(ax) ); + affirm( !sqlite3_cancel_auto_extension(ax) ); + sqlite3_close(createNewDb()); + affirm( 3 == val.value ); + rc = sqlite3_auto_extension( ax ); + affirm( 0 == rc ); + sqlite3_close(createNewDb()); + affirm( 5 == val.value ); + affirm( sqlite3_cancel_auto_extension(ax2) ); + affirm( !sqlite3_cancel_auto_extension(ax2) ); + sqlite3_close(createNewDb()); + affirm( 6 == val.value ); + rc = sqlite3_auto_extension( ax2 ); + affirm( 0 == rc ); + sqlite3_close(createNewDb()); + affirm( 8 == val.value ); + + sqlite3_reset_auto_extension(); + sqlite3_close(createNewDb()); + affirm( 8 == val.value ); + affirm( !sqlite3_cancel_auto_extension(ax) ); + affirm( !sqlite3_cancel_auto_extension(ax2) ); + sqlite3_close(createNewDb()); + affirm( 8 == val.value ); + } + + + private void testColumnMetadata(){ + final sqlite3 db = createNewDb(); + execSql(db, new String[] { + "CREATE TABLE t(a duck primary key not null collate noCase); ", + "INSERT INTO t(a) VALUES(1),(2),(3);" + }); + OutputPointer.Bool bNotNull = new OutputPointer.Bool(); + OutputPointer.Bool bPrimaryKey = new OutputPointer.Bool(); + OutputPointer.Bool bAutoinc = new OutputPointer.Bool(); + OutputPointer.String zCollSeq = new OutputPointer.String(); + OutputPointer.String zDataType = new OutputPointer.String(); + int rc = sqlite3_table_column_metadata( + db, "main", "t", "a", zDataType, zCollSeq, + bNotNull, bPrimaryKey, bAutoinc); + affirm( 0==rc ); + affirm( bPrimaryKey.value ); + affirm( !bAutoinc.value ); + affirm( bNotNull.value ); + affirm( "noCase".equals(zCollSeq.value) ); + affirm( "duck".equals(zDataType.value) ); + + TableColumnMetadata m = + sqlite3_table_column_metadata(db, "main", "t", "a"); + affirm( null != m ); + affirm( bPrimaryKey.value == m.isPrimaryKey() ); + affirm( bAutoinc.value == m.isAutoincrement() ); + affirm( bNotNull.value == m.isNotNull() ); + affirm( zCollSeq.value.equals(m.getCollation()) ); + affirm( zDataType.value.equals(m.getDataType()) ); + + affirm( null == sqlite3_table_column_metadata(db, "nope", "t", "a") ); + affirm( null == sqlite3_table_column_metadata(db, "main", "nope", "a") ); + + m = sqlite3_table_column_metadata(db, "main", "t", null) + /* Check only for existence of table */; + affirm( null != m ); + affirm( m.isPrimaryKey() ); + affirm( !m.isAutoincrement() ); + affirm( !m.isNotNull() ); + affirm( "BINARY".equalsIgnoreCase(m.getCollation()) ); + affirm( "INTEGER".equalsIgnoreCase(m.getDataType()) ); + + sqlite3_close_v2(db); + } + + private void testTxnState(){ + final sqlite3 db = createNewDb(); + affirm( SQLITE_TXN_NONE == sqlite3_txn_state(db, null) ); + affirm( sqlite3_get_autocommit(db) ); + execSql(db, "BEGIN;"); + affirm( !sqlite3_get_autocommit(db) ); + affirm( SQLITE_TXN_NONE == sqlite3_txn_state(db, null) ); + execSql(db, "SELECT * FROM sqlite_schema;"); + affirm( SQLITE_TXN_READ == sqlite3_txn_state(db, "main") ); + execSql(db, "CREATE TABLE t(a);"); + affirm( SQLITE_TXN_WRITE == sqlite3_txn_state(db, null) ); + execSql(db, "ROLLBACK;"); + affirm( SQLITE_TXN_NONE == sqlite3_txn_state(db, null) ); + sqlite3_close_v2(db); + } + + + private void testExplain(){ + final sqlite3 db = createNewDb(); + sqlite3_stmt stmt = prepare(db,"SELECT 1"); + + affirm( 0 == sqlite3_stmt_isexplain(stmt) ); + int rc = sqlite3_stmt_explain(stmt, 1); + affirm( 1 == sqlite3_stmt_isexplain(stmt) ); + rc = sqlite3_stmt_explain(stmt, 2); + affirm( 2 == sqlite3_stmt_isexplain(stmt) ); + sqlite3_finalize(stmt); + sqlite3_close_v2(db); + } + + private void testLimit(){ + final sqlite3 db = createNewDb(); + int v; + + v = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + affirm( v > 0 ); + affirm( v == sqlite3_limit(db, SQLITE_LIMIT_LENGTH, v-1) ); + affirm( v-1 == sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1) ); + sqlite3_close_v2(db); + } + + private void testComplete(){ + affirm( 0==sqlite3_complete("select 1") ); + affirm( 0!=sqlite3_complete("select 1;") ); + affirm( 0!=sqlite3_complete("nope 'nope' 'nope' 1;"), "Yup" ); + } + + private void testKeyword(){ + final int n = sqlite3_keyword_count(); + affirm( n>0 ); + affirm( !sqlite3_keyword_check("_nope_") ); + affirm( sqlite3_keyword_check("seLect") ); + affirm( null!=sqlite3_keyword_name(0) ); + affirm( null!=sqlite3_keyword_name(n-1) ); + affirm( null==sqlite3_keyword_name(n) ); + } + + private void testBackup(){ + final sqlite3 dbDest = createNewDb(); + + try (sqlite3 dbSrc = createNewDb()) { + execSql(dbSrc, new String[]{ + "pragma page_size=512; VACUUM;", + "create table t(a);", + "insert into t(a) values(1),(2),(3);" + }); + affirm( null==sqlite3_backup_init(dbSrc,"main",dbSrc,"main") ); + try (sqlite3_backup b = sqlite3_backup_init(dbDest,"main",dbSrc,"main")) { + affirm( null!=b ); + affirm( b.getNativePointer()!=0 ); + int rc; + while( SQLITE_DONE!=(rc = sqlite3_backup_step(b, 1)) ){ + affirm( 0==rc ); + } + affirm( sqlite3_backup_pagecount(b) > 0 ); + rc = sqlite3_backup_finish(b); + affirm( 0==rc ); + affirm( b.getNativePointer()==0 ); + } + } + + try (sqlite3_stmt stmt = prepare(dbDest,"SELECT sum(a) from t")) { + sqlite3_step(stmt); + affirm( sqlite3_column_int(stmt,0) == 6 ); + } + sqlite3_close_v2(dbDest); + } + + private void testRandomness(){ + byte[] foo = new byte[20]; + int i = 0; + for( byte b : foo ){ + i += b; + } + affirm( i==0 ); + sqlite3_randomness(foo); + for( byte b : foo ){ + if(b!=0) ++i; + } + affirm( i!=0, "There's a very slight chance that 0 is actually correct." ); + } + + private void testBlobOpen(){ + final sqlite3 db = createNewDb(); + + execSql(db, "CREATE TABLE T(a BLOB);" + +"INSERT INTO t(rowid,a) VALUES(1, 'def'),(2, 'XYZ');" + ); + final OutputPointer.sqlite3_blob pOut = new OutputPointer.sqlite3_blob(); + int rc = sqlite3_blob_open(db, "main", "t", "a", + sqlite3_last_insert_rowid(db), 1, pOut); + affirm( 0==rc ); + sqlite3_blob b = pOut.take(); + affirm( null!=b ); + affirm( 0!=b.getNativePointer() ); + affirm( 3==sqlite3_blob_bytes(b) ); + rc = sqlite3_blob_write( b, new byte[] {100, 101, 102 /*"DEF"*/}, 0); + affirm( 0==rc ); + rc = sqlite3_blob_close(b); + affirm( 0==rc ); + rc = sqlite3_blob_close(b); + affirm( 0!=rc ); + affirm( 0==b.getNativePointer() ); + sqlite3_stmt stmt = prepare(db,"SELECT length(a), a FROM t ORDER BY a"); + affirm( SQLITE_ROW == sqlite3_step(stmt) ); + affirm( 3 == sqlite3_column_int(stmt,0) ); + affirm( "def".equals(sqlite3_column_text16(stmt,1)) ); + sqlite3_finalize(stmt); + + b = sqlite3_blob_open(db, "main", "t", "a", + sqlite3_last_insert_rowid(db), 0); + affirm( null!=b ); + rc = sqlite3_blob_reopen(b, 2); + affirm( 0==rc ); + final byte[] tgt = new byte[3]; + rc = sqlite3_blob_read(b, tgt, 0); + affirm( 0==rc ); + affirm( 100==tgt[0] && 101==tgt[1] && 102==tgt[2], "DEF" ); + rc = sqlite3_blob_close(b); + affirm( 0==rc ); + + if( !sqlite3_jni_supports_nio() ){ + outln("WARNING: skipping tests for ByteBuffer-using sqlite3_blob APIs ", + "because this platform lacks that support."); + sqlite3_close_v2(db); + return; + } + /* Sanity checks for the java.nio.ByteBuffer-taking overloads of + sqlite3_blob_read/write(). */ + execSql(db, "UPDATE t SET a=zeroblob(10)"); + b = sqlite3_blob_open(db, "main", "t", "a", 1, 1); + affirm( null!=b ); + java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(10); + for( byte i = 0; i < 10; ++i ){ + bb.put((int)i, (byte)(48+i & 0xff)); + } + rc = sqlite3_blob_write(b, 1, bb, 1, 10); + affirm( rc==SQLITE_ERROR, "b length < (srcOffset + bb length)" ); + rc = sqlite3_blob_write(b, -1, bb); + affirm( rc==SQLITE_ERROR, "Target offset may not be negative" ); + rc = sqlite3_blob_write(b, 0, bb, -1, -1); + affirm( rc==SQLITE_ERROR, "Source offset may not be negative" ); + rc = sqlite3_blob_write(b, 1, bb, 1, 8); + affirm( rc==0 ); + // b's contents: 0 49 50 51 52 53 54 55 56 0 + // ascii: 0 '1' '2' '3' '4' '5' '6' '7' '8' 0 + byte br[] = new byte[10]; + java.nio.ByteBuffer bbr = + java.nio.ByteBuffer.allocateDirect(bb.limit()); + rc = sqlite3_blob_read( b, br, 0 ); + affirm( rc==0 ); + rc = sqlite3_blob_read( b, bbr ); + affirm( rc==0 ); + java.nio.ByteBuffer bbr2 = sqlite3_blob_read_nio_buffer(b, 0, 12); + affirm( null==bbr2, "Read size is too big"); + bbr2 = sqlite3_blob_read_nio_buffer(b, -1, 3); + affirm( null==bbr2, "Source offset is negative"); + bbr2 = sqlite3_blob_read_nio_buffer(b, 5, 6); + affirm( null==bbr2, "Read pos+size is too big"); + bbr2 = sqlite3_blob_read_nio_buffer(b, 4, 7); + affirm( null==bbr2, "Read pos+size is too big"); + bbr2 = sqlite3_blob_read_nio_buffer(b, 4, 6); + affirm( null!=bbr2 ); + java.nio.ByteBuffer bbr3 = + java.nio.ByteBuffer.allocateDirect(2 * bb.limit()); + java.nio.ByteBuffer bbr4 = + java.nio.ByteBuffer.allocateDirect(5); + rc = sqlite3_blob_read( b, bbr3 ); + affirm( rc==0 ); + rc = sqlite3_blob_read( b, bbr4 ); + affirm( rc==0 ); + affirm( sqlite3_blob_bytes(b)==bbr3.limit() ); + affirm( 5==bbr4.limit() ); + sqlite3_blob_close(b); + affirm( 0==br[0] ); + affirm( 0==br[9] ); + affirm( 0==bbr.get(0) ); + affirm( 0==bbr.get(9) ); + affirm( bbr2.limit() == 6 ); + affirm( 0==bbr3.get(0) ); + { + Exception ex = null; + try{ bbr3.get(11); } + catch(Exception e){ex = e;} + affirm( ex instanceof IndexOutOfBoundsException, + "bbr3.limit() was reset by read()" ); + ex = null; + } + affirm( 0==bbr4.get(0) ); + for( int i = 1; i < 9; ++i ){ + affirm( br[i] == 48 + i ); + affirm( br[i] == bbr.get(i) ); + affirm( br[i] == bbr3.get(i) ); + if( i>3 ){ + affirm( br[i] == bbr2.get(i-4) ); + } + if( i < bbr4.limit() ){ + affirm( br[i] == bbr4.get(i) ); + } + } + sqlite3_close_v2(db); + } + + private void testPrepareMulti(){ + final sqlite3 db = createNewDb(); + final String[] sql = { + "create table t(","a)", + "; insert into t(a) values(1),(2),(3);", + "select a from t;" + }; + final List liStmt = new ArrayList(); + final PrepareMultiCallback proxy = new PrepareMultiCallback.StepAll(); + final ValueHolder toss = new ValueHolder<>(null); + PrepareMultiCallback m = new PrepareMultiCallback() { + @Override public int call(sqlite3_stmt st){ + liStmt.add(st); + if( null!=toss.value ){ + throw new RuntimeException(toss.value); + } + return proxy.call(st); + } + }; + int rc = sqlite3_prepare_multi(db, sql, m); + affirm( 0==rc ); + affirm( liStmt.size() == 3 ); + for( sqlite3_stmt st : liStmt ){ + sqlite3_finalize(st); + } + toss.value = "This is an exception."; + rc = sqlite3_prepare_multi(db, "SELECT 1", m); + affirm( SQLITE_ERROR==rc ); + affirm( sqlite3_errmsg(db).indexOf(toss.value)>0 ); + sqlite3_close_v2(db); + } + + /* Copy/paste/rename this to add new tests. */ + private void _testTemplate(){ + final sqlite3 db = createNewDb(); + sqlite3_stmt stmt = prepare(db,"SELECT 1"); + sqlite3_finalize(stmt); + sqlite3_close_v2(db); + } + + + @ManualTest /* we really only want to run this test manually */ + private void testSleep(){ + out("Sleeping briefly... "); + sqlite3_sleep(600); + outln("Woke up."); + } + + private void nap() throws InterruptedException { + if( takeNaps ){ + Thread.sleep(java.util.concurrent.ThreadLocalRandom.current().nextInt(3, 17), 0); + } + } + + @ManualTest /* because we only want to run this test on demand */ + private void testFail(){ + affirm( false, "Intentional failure." ); + } + + private void runTests(boolean fromThread) throws Exception { + if(false) showCompileOption(); + List mlist = testMethods; + affirm( null!=mlist ); + if( shuffle ){ + mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) ); + java.util.Collections.shuffle(mlist); + } + if( (!fromThread && listRunTests>0) || listRunTests>1 ){ + synchronized(this.getClass()){ + if( !fromThread ){ + out("Initial test"," list: "); + for(java.lang.reflect.Method m : testMethods){ + out(m.getName()+" "); + } + outln(); + outln("(That list excludes some which are hard-coded to run.)"); + } + out("Running"," tests: "); + for(java.lang.reflect.Method m : mlist){ + out(m.getName()+" "); + } + outln(); + } + } + for(java.lang.reflect.Method m : mlist){ + nap(); + try{ + m.invoke(this); + }catch(java.lang.reflect.InvocationTargetException e){ + outln("FAILURE: ",m.getName(),"(): ", e.getCause()); + throw e; + } + } + synchronized( this.getClass() ){ + ++nTestRuns; + } + } + + public void run() { + try { + runTests(0!=this.tId); + }catch(Exception e){ + synchronized( listErrors ){ + listErrors.add(e); + } + }finally{ + affirm( sqlite3_java_uncache_thread() ); + affirm( !sqlite3_java_uncache_thread() ); + } + } + + /** + Runs the basic sqlite3 JNI binding sanity-check suite. + + CLI flags: + + -q|-quiet: disables most test output. + + -t|-thread N: runs the tests in N threads + concurrently. Default=1. + + -r|-repeat N: repeats the tests in a loop N times, each one + consisting of the -thread value's threads. + + -shuffle: randomizes the order of most of the test functions. + + -naps: sleep small random intervals between tests in order to add + some chaos for cross-thread contention. + + + -list-tests: outputs the list of tests being run, minus some + which are hard-coded. In multi-threaded mode, use this twice to + to emit the list run by each thread (which may differ from the initial + list, in particular if -shuffle is used). + + -fail: forces an exception to be thrown during the test run. Use + with -shuffle to make its appearance unpredictable. + + -v: emit some developer-mode info at the end. + */ + public static void main(String[] args) throws Exception { + Integer nThread = 1; + boolean doSomethingForDev = false; + Integer nRepeat = 1; + boolean forceFail = false; + boolean sqlLog = false; + boolean configLog = false; + boolean squelchTestOutput = false; + for( int i = 0; i < args.length; ){ + String arg = args[i++]; + if(arg.startsWith("-")){ + arg = arg.replaceFirst("-+",""); + if(arg.equals("v")){ + doSomethingForDev = true; + //listBoundMethods(); + }else if(arg.equals("t") || arg.equals("thread")){ + nThread = Integer.parseInt(args[i++]); + }else if(arg.equals("r") || arg.equals("repeat")){ + nRepeat = Integer.parseInt(args[i++]); + }else if(arg.equals("shuffle")){ + shuffle = true; + }else if(arg.equals("list-tests")){ + ++listRunTests; + }else if(arg.equals("fail")){ + forceFail = true; + }else if(arg.equals("sqllog")){ + sqlLog = true; + }else if(arg.equals("configlog")){ + configLog = true; + }else if(arg.equals("naps")){ + takeNaps = true; + }else if(arg.equals("q") || arg.equals("quiet")){ + squelchTestOutput = true; + }else{ + throw new IllegalArgumentException("Unhandled flag:"+arg); + } + } + } + + if( sqlLog ){ + if( sqlite3_compileoption_used("ENABLE_SQLLOG") ){ + final ConfigSqlLogCallback log = new ConfigSqlLogCallback() { + @Override public void call(sqlite3 db, String msg, int op){ + switch(op){ + case 0: outln("Opening db: ",db); break; + case 1: outln("SQL ",db,": ",msg); break; + case 2: outln("Closing db: ",db); break; + } + } + }; + int rc = sqlite3_config( log ); + affirm( 0==rc ); + rc = sqlite3_config( (ConfigSqlLogCallback)null ); + affirm( 0==rc ); + rc = sqlite3_config( log ); + affirm( 0==rc ); + }else{ + outln("WARNING: -sqllog is not active because library was built ", + "without SQLITE_ENABLE_SQLLOG."); + } + } + if( configLog ){ + final ConfigLogCallback log = new ConfigLogCallback() { + @Override public void call(int code, String msg){ + outln("ConfigLogCallback: ",ResultCode.getEntryForInt(code),": ", msg); + }; + }; + int rc = sqlite3_config( log ); + affirm( 0==rc ); + rc = sqlite3_config( (ConfigLogCallback)null ); + affirm( 0==rc ); + rc = sqlite3_config( log ); + affirm( 0==rc ); + } + + quietMode = squelchTestOutput; + outln("If you just saw warning messages regarding CallStaticObjectMethod, ", + "you are very likely seeing the side effects of a known openjdk8 ", + "bug. It is unsightly but does not affect the library."); + + { + // Build list of tests to run from the methods named test*(). + testMethods = new ArrayList<>(); + int nSkipped = 0; + for(final java.lang.reflect.Method m : Tester1.class.getDeclaredMethods()){ + final String name = m.getName(); + if( name.equals("testFail") ){ + if( forceFail ){ + testMethods.add(m); + } + }else if( m.isAnnotationPresent( RequiresJniNio.class ) + && !sqlite3_jni_supports_nio() ){ + outln("Skipping test for lack of JNI java.nio.ByteBuffer support: ", + name,"()\n"); + ++nSkipped; + }else if( !m.isAnnotationPresent( ManualTest.class ) ){ + if( nThread>1 && m.isAnnotationPresent( SingleThreadOnly.class ) ){ + out("Skipping test in multi-thread mode: ",name,"()\n"); + ++nSkipped; + }else if( name.startsWith("test") ){ + testMethods.add(m); + } + } + } + } + + final long timeStart = System.currentTimeMillis(); + int nLoop = 0; + switch( sqlite3_threadsafe() ){ /* Sanity checking */ + case 0: + affirm( SQLITE_ERROR==sqlite3_config( SQLITE_CONFIG_SINGLETHREAD ), + "Could not switch to single-thread mode." ); + affirm( SQLITE_ERROR==sqlite3_config( SQLITE_CONFIG_MULTITHREAD ), + "Could switch to multithread mode." ); + affirm( SQLITE_ERROR==sqlite3_config( SQLITE_CONFIG_SERIALIZED ), + "Could not switch to serialized threading mode." ); + outln("This is a single-threaded build. Not using threads."); + nThread = 1; + break; + case 1: + case 2: + affirm( 0==sqlite3_config( SQLITE_CONFIG_SINGLETHREAD ), + "Could not switch to single-thread mode." ); + affirm( 0==sqlite3_config( SQLITE_CONFIG_MULTITHREAD ), + "Could not switch to multithread mode." ); + affirm( 0==sqlite3_config( SQLITE_CONFIG_SERIALIZED ), + "Could not switch to serialized threading mode." ); + break; + default: + affirm( false, "Unhandled SQLITE_THREADSAFE value." ); + } + outln("libversion_number: ", + sqlite3_libversion_number(),"\n", + sqlite3_libversion(),"\n",SQLITE_SOURCE_ID,"\n", + "SQLITE_THREADSAFE=",sqlite3_threadsafe()); + outln("JVM NIO support? ",sqlite3_jni_supports_nio() ? "YES" : "NO"); + final boolean showLoopCount = (nRepeat>1 && nThread>1); + if( showLoopCount ){ + outln("Running ",nRepeat," loop(s) with ",nThread," thread(s) each."); + } + if( takeNaps ) outln("Napping between tests is enabled."); + for( int n = 0; n < nRepeat; ++n ){ + ++nLoop; + if( showLoopCount ) out((1==nLoop ? "" : " ")+nLoop); + if( nThread<=1 ){ + new Tester1(0).runTests(false); + continue; + } + Tester1.mtMode = true; + final ExecutorService ex = Executors.newFixedThreadPool( nThread ); + for( int i = 0; i < nThread; ++i ){ + ex.submit( new Tester1(i), i ); + } + ex.shutdown(); + try{ + ex.awaitTermination(nThread*200, java.util.concurrent.TimeUnit.MILLISECONDS); + ex.shutdownNow(); + }catch (InterruptedException ie){ + ex.shutdownNow(); + Thread.currentThread().interrupt(); + } + if( !listErrors.isEmpty() ){ + quietMode = false; + outln("TEST ERRORS:"); + Exception err = null; + for( Exception e : listErrors ){ + e.printStackTrace(); + if( null==err ) err = e; + } + if( null!=err ) throw err; + } + } + if( showLoopCount ) outln(); + quietMode = false; + + final long timeEnd = System.currentTimeMillis(); + outln("Tests done. Metrics across ",nTestRuns," total iteration(s):"); + outln("\tAssertions checked: ",affirmCount); + outln("\tDatabases opened: ",metrics.dbOpen); + if( doSomethingForDev ){ + sqlite3_jni_internal_details(); + } + affirm( 0==sqlite3_release_memory(1) ); + sqlite3_shutdown(); + int nMethods = 0; + int nNatives = 0; + final java.lang.reflect.Method[] declaredMethods = + CApi.class.getDeclaredMethods(); + for(java.lang.reflect.Method m : declaredMethods){ + final int mod = m.getModifiers(); + if( 0!=(mod & java.lang.reflect.Modifier.STATIC) ){ + final String name = m.getName(); + if(name.startsWith("sqlite3_")){ + ++nMethods; + if( 0!=(mod & java.lang.reflect.Modifier.NATIVE) ){ + ++nNatives; + } + } + } + } + outln("\tCApi.sqlite3_*() methods: "+ + nMethods+" total, with "+ + nNatives+" native, "+ + (nMethods - nNatives)+" Java" + ); + outln("\tTotal test time = " + +(timeEnd - timeStart)+"ms"); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java b/ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java new file mode 100644 index 0000000000000000000000000000000000000000..56465a2c0a6713888dca42be773e7d40d0de9046 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java @@ -0,0 +1,50 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.Nullable; + +/** + Callback for use with {@link CApi#sqlite3_trace_v2}. +*/ +public interface TraceV2Callback extends CallbackProxy { + /** + Called by sqlite3 for various tracing operations, as per + sqlite3_trace_v2(). Note that this interface elides the 2nd + argument to the native trace callback, as that role is better + filled by instance-local state. + +

These callbacks may throw, in which case their exceptions are + converted to C-level error information. + +

The 2nd argument to this function, if non-null, will be a an + sqlite3 or sqlite3_stmt object, depending on the first argument + (see below). + +

The final argument to this function is the "X" argument + documented for sqlite3_trace() and sqlite3_trace_v2(). Its type + depends on value of the first argument: + +

- SQLITE_TRACE_STMT: pNative is a sqlite3_stmt. pX is a String + containing the prepared SQL. + +

- SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long + holding an approximate number of nanoseconds the statement took + to run. + +

- SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null. + +

- SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null. + */ + int call(int traceFlag, Object pNative, @Nullable Object pX); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..e3d491f67e673a66053c554ecb1e7ef425004451 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_update_hook}. +*/ +public interface UpdateHookCallback extends CallbackProxy { + /** + Must function as described for the C-level sqlite3_update_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + void call(int opId, String dbName, String tableName, long rowId); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ValueHolder.java b/ext/jni/src/org/sqlite/jni/capi/ValueHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..0a469fea9a1d025b67425a57d051ce2a60852089 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ValueHolder.java @@ -0,0 +1,27 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the ValueHolder utility class for the sqlite3 +** JNI bindings. +*/ +package org.sqlite.jni.capi; + +/** + A helper class which simply holds a single value. Its primary use + is for communicating values out of anonymous classes, as doing so + requires a "final" reference, as well as communicating aggregate + SQL function state across calls to such functions. +*/ +public class ValueHolder { + public T value; + public ValueHolder(){} + public ValueHolder(T v){value = v;} +} diff --git a/ext/jni/src/org/sqlite/jni/capi/WindowFunction.java b/ext/jni/src/org/sqlite/jni/capi/WindowFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..eaf1bb9a35132b4b9e011437e01c8f2d149abfb3 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/WindowFunction.java @@ -0,0 +1,39 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + + +/** + A SQLFunction implementation for window functions. Note that + WindowFunction inherits from {@link AggregateFunction} and each + instance is required to implement the inherited abstract methods + from that class. See {@link AggregateFunction} for information on + managing the UDF's invocation-specific state. +*/ +public abstract class WindowFunction extends AggregateFunction { + + /** + As for the xInverse() argument of the C API's + sqlite3_create_window_function(). If this function throws, the + exception is not propagated and a warning might be emitted + to a debugging channel. + */ + public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args); + + /** + As for the xValue() argument of the C API's sqlite3_create_window_function(). + See xInverse() for the fate of any exceptions this throws. + */ + public abstract void xValue(sqlite3_context cx); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java b/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..372e4ec8d0ddda320cebfad8f3d49efac31f08ce --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java @@ -0,0 +1,37 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file declares JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for a hook called by SQLite when certain client-provided + state are destroyed. It gets its name from the pervasive use of + the symbol name xDestroy() for this purpose in the C API + documentation. +*/ +public interface XDestroyCallback { + /** + Must perform any cleanup required by this object. Must not + throw. Must not call back into the sqlite3 API, else it might + invoke a deadlock. + + WARNING: as a rule, it is never safe to register individual + instances with this interface multiple times in the + library. e.g., do not register the same CollationCallback with + multiple arities or names using sqlite3_create_collation(). If + this rule is violated, the library will eventually try to free + each individual reference, leading to memory corruption or a + crash via duplicate free(). + */ + public void xDestroy(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/package-info.java b/ext/jni/src/org/sqlite/jni/capi/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..127f38067522d8aff63e267193bc1198a3e66df4 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/package-info.java @@ -0,0 +1,89 @@ +/** + This package houses a JNI binding to the SQLite3 C API. + +

The primary interfaces are in {@link + org.sqlite.jni.capi.CApi}.

+ +

API Goals and Requirements

+ +
    + +
  • A 1-to-1(-ish) mapping of the C API to Java via JNI, insofar + as cross-language semantics allow for. A closely-related goal is + that the C + documentation should be usable as-is, insofar as possible, + for most of the JNI binding. As a rule, undocumented symbols in + the Java interface behave as documented for their C API + counterpart. Only semantic differences and Java-specific features + are documented here.
  • + +
  • Support Java as far back as version 8 (2014).
  • + +
  • Environment-independent. Should work everywhere both Java and + SQLite3 do.
  • + +
  • No 3rd-party dependencies beyond the JDK. That includes no + build-level dependencies for specific IDEs and toolchains. We + welcome the addition of build files for arbitrary environments + insofar as they neither interfere with each other nor become a + maintenance burden for the sqlite developers.
  • + +
+ +

Non-Goals

+ +
    + +
  • Creation of high-level OO wrapper APIs. Clients are free to + create them off of the C-style API.
  • + +
  • Support for mixed-mode operation, where client code accesses + SQLite both via the Java-side API and the C API via their own + native code. In such cases, proxy functionalities (primarily + callback handler wrappers of all sorts) may fail because the + C-side use of the SQLite APIs will bypass those proxies.
  • + +
+ +

State of this API

+ +

As of version 3.43, this software is in "tech preview" form. We + tentatively plan to stamp it as stable with the 3.44 release.

+ +

Threading Considerations

+ +

This API is, if built with SQLITE_THREADSAFE set to 1 or 2, + thread-safe, insofar as the C API guarantees, with some addenda:

+ +
    + +
  • It is not legal to use Java-facing SQLite3 resource handles + (sqlite3, sqlite3_stmt, etc) from multiple threads concurrently, + nor to use any database-specific resources concurrently in a + thread separate from the one the database is currently in use + in. i.e. do not use a sqlite3_stmt in thread #2 when thread #1 is + using the database which prepared that handle. + +
    Violating this will eventually corrupt the JNI-level bindings + between Java's and C's view of the database. This is a limitation + of the JNI bindings, not the lower-level library. +
  • + +
  • It is legal to use a given handle, and database-specific + resources, across threads, so long as no two threads pass + resources owned by the same database into the library + concurrently. +
  • + +
+ +

Any number of threads may, of course, create and use any number + of database handles they wish. Care only needs to be taken when + those handles or their associated resources cross threads, or...

+ +

When built with SQLITE_THREADSAFE=0 then no threading guarantees + are provided and multi-threaded use of the library will provoke + undefined behavior.

+ +*/ +package org.sqlite.jni.capi; diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3.java new file mode 100644 index 0000000000000000000000000000000000000000..cc6f2e6e8d88c8278cef2699041e906e37eb3ce9 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3.java @@ -0,0 +1,43 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for communicating C-level (sqlite3*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java + and C via JNI. +*/ +public final class sqlite3 extends NativePointerHolder + implements AutoCloseable { + + // Only invoked from JNI + private sqlite3(){} + + public String toString(){ + final long ptr = getNativePointer(); + if( 0==ptr ){ + return sqlite3.class.getSimpleName()+"@null"; + } + final String fn = CApi.sqlite3_db_filename(this, "main"); + return sqlite3.class.getSimpleName() + +"@"+String.format("0x%08x",ptr) + +"["+((null == fn) ? "" : fn)+"]" + ; + } + + @Override public void close(){ + CApi.sqlite3_close_v2(this); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java new file mode 100644 index 0000000000000000000000000000000000000000..0ef75c17eb744515cae53b9a13e0f3fa69d1593c --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java @@ -0,0 +1,31 @@ +/* +** 2023-09-03 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for passing C-level (sqlite3_backup*) instances around in + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class sqlite3_backup extends NativePointerHolder + implements AutoCloseable { + // Only invoked from JNI. + private sqlite3_backup(){} + + @Override public void close(){ + CApi.sqlite3_backup_finish(this); + } + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java new file mode 100644 index 0000000000000000000000000000000000000000..bdc0200af49c373d1973fdaae01c851769cc1e67 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java @@ -0,0 +1,30 @@ +/* +** 2023-09-03 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for passing C-level (sqlite3_blob*) instances around in + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class sqlite3_blob extends NativePointerHolder + implements AutoCloseable { + // Only invoked from JNI. + private sqlite3_blob(){} + + @Override public void close(){ + CApi.sqlite3_blob_close(this); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java new file mode 100644 index 0000000000000000000000000000000000000000..82ec49af16d02d5173b0d3f59f7ab8a3306ed70a --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java @@ -0,0 +1,79 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + sqlite3_context instances are used in conjunction with user-defined + SQL functions (a.k.a. UDFs). +*/ +public final class sqlite3_context extends NativePointerHolder { + private Long aggregateContext = null; + + /** + getAggregateContext() corresponds to C's + sqlite3_aggregate_context(), with a slightly different interface + to account for cross-language differences. It serves the same + purposes in a slightly different way: it provides a key which is + stable across invocations of a UDF's callbacks, such that all + calls into those callbacks can determine which "set" of those + calls they belong to. + +

Note that use of this method is not a requirement for proper use + of this class. sqlite3_aggregate_context() can also be used. + +

If the argument is true and the aggregate context has not yet + been set up, it will be initialized and fetched on demand, else it + won't. The intent is that xStep(), xValue(), and xInverse() + methods pass true and xFinal() methods pass false. + +

This function treats numeric 0 as null, always returning null instead + of 0. + +

If this object is being used in the context of an aggregate or + window UDF, this function returns a non-0 value which is distinct + for each set of UDF callbacks from a single invocation of the + UDF, otherwise it returns 0. The returned value is only only + valid within the context of execution of a single SQL statement, + and must not be re-used by future invocations of the UDF in + different SQL statements. + +

Consider this SQL, where MYFUNC is a user-defined aggregate function: + +

{@code
+     SELECT MYFUNC(A), MYFUNC(B) FROM T;
+     }
+ +

The xStep() and xFinal() methods of the callback need to be able + to differentiate between those two invocations in order to + perform their work properly. The value returned by + getAggregateContext() will be distinct for each of those + invocations of MYFUNC() and is intended to be used as a lookup + key for mapping callback invocations to whatever client-defined + state is needed by the UDF. + +

There is one case where this will return null in the context + of an aggregate or window function: if the result set has no + rows, the UDF's xFinal() will be called without any other x...() + members having been called. In that one case, no aggregate + context key will have been generated. xFinal() implementations + need to be prepared to accept that condition as legal. + */ + public synchronized Long getAggregateContext(boolean initIfNeeded){ + if( aggregateContext==null ){ + aggregateContext = CApi.sqlite3_aggregate_context(this, initIfNeeded); + if( !initIfNeeded && null==aggregateContext ) aggregateContext = 0L; + } + return (null==aggregateContext || 0!=aggregateContext) ? aggregateContext : null; + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java new file mode 100644 index 0000000000000000000000000000000000000000..564891c72799f5b6cb2ae09d1169ef4dda5351bc --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java @@ -0,0 +1,30 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for communicating C-level (sqlite3_stmt*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class sqlite3_stmt extends NativePointerHolder + implements AutoCloseable { + // Only invoked from JNI. + private sqlite3_stmt(){} + + @Override public void close(){ + CApi.sqlite3_finalize(this); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java new file mode 100644 index 0000000000000000000000000000000000000000..a4772f0f639dbb7eccc46f33cf742dfb388b1bed --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java @@ -0,0 +1,19 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +public final class sqlite3_value extends NativePointerHolder { + //! Invoked only from JNI. + private sqlite3_value(){} +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5.java new file mode 100644 index 0000000000000000000000000000000000000000..0dceeafd2e6244a7b874462039e19020254c5dd1 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5.java @@ -0,0 +1,32 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; + +/** + INCOMPLETE AND COMPLETELY UNTESTED. + + A utility object for holding FTS5-specific types and constants + which are used by multiple FTS5 classes. +*/ +public final class Fts5 { + /* Not used */ + private Fts5(){} + + + public static final int FTS5_TOKENIZE_QUERY = 0x0001; + public static final int FTS5_TOKENIZE_PREFIX = 0x0002; + public static final int FTS5_TOKENIZE_DOCUMENT = 0x0004; + public static final int FTS5_TOKENIZE_AUX = 0x0008; + public static final int FTS5_TOKEN_COLOCATED = 0x0001; +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java new file mode 100644 index 0000000000000000000000000000000000000000..439b477910923a99cb5109d1384d9a70a65fb911 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java @@ -0,0 +1,24 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.*; + +/** + A wrapper for communicating C-level (Fts5Context*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class Fts5Context extends NativePointerHolder { +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java new file mode 100644 index 0000000000000000000000000000000000000000..594f3eaad64b7a5d105bc71379a9df9a92ffcb09 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java @@ -0,0 +1,97 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import java.nio.charset.StandardCharsets; +import org.sqlite.jni.capi.*; +import org.sqlite.jni.annotation.*; + +/** +*/ +public final class Fts5ExtensionApi extends NativePointerHolder { + //! Only called from JNI + private Fts5ExtensionApi(){} + private final int iVersion = 2; + + /* Callback type for used by xQueryPhrase(). */ + public static interface XQueryPhraseCallback { + int call(Fts5ExtensionApi fapi, Fts5Context cx); + } + + /** + Returns the singleton instance of this class. + */ + public static native Fts5ExtensionApi getInstance(); + + public native int xColumnCount(@NotNull Fts5Context fcx); + + public native int xColumnSize(@NotNull Fts5Context cx, int iCol, + @NotNull OutputPointer.Int32 pnToken); + + public native int xColumnText(@NotNull Fts5Context cx, int iCol, + @NotNull OutputPointer.String txt); + + public native int xColumnTotalSize(@NotNull Fts5Context fcx, int iCol, + @NotNull OutputPointer.Int64 pnToken); + + public native Object xGetAuxdata(@NotNull Fts5Context cx, boolean clearIt); + + public native int xInst(@NotNull Fts5Context cx, int iIdx, + @NotNull OutputPointer.Int32 piPhrase, + @NotNull OutputPointer.Int32 piCol, + @NotNull OutputPointer.Int32 piOff); + + public native int xInstCount(@NotNull Fts5Context fcx, + @NotNull OutputPointer.Int32 pnInst); + + public native int xPhraseCount(@NotNull Fts5Context fcx); + + public native int xPhraseFirst(@NotNull Fts5Context cx, int iPhrase, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol, + @NotNull OutputPointer.Int32 iOff); + + public native int xPhraseFirstColumn(@NotNull Fts5Context cx, int iPhrase, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol); + public native void xPhraseNext(@NotNull Fts5Context cx, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol, + @NotNull OutputPointer.Int32 iOff); + public native void xPhraseNextColumn(@NotNull Fts5Context cx, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol); + public native int xPhraseSize(@NotNull Fts5Context fcx, int iPhrase); + + public native int xQueryPhrase(@NotNull Fts5Context cx, int iPhrase, + @NotNull XQueryPhraseCallback callback); + public native int xRowCount(@NotNull Fts5Context fcx, + @NotNull OutputPointer.Int64 nRow); + + public native long xRowid(@NotNull Fts5Context cx); + /* Note that the JNI binding lacks the C version's xDelete() + callback argument. Instead, if pAux has an xDestroy() method, it + is called if the FTS5 API finalizes the aux state (including if + allocation of storage for the auxdata fails). Any reference to + pAux held by the JNI layer will be relinquished regardless of + whether pAux has an xDestroy() method. */ + + public native int xSetAuxdata(@NotNull Fts5Context cx, @Nullable Object pAux); + + public native int xTokenize(@NotNull Fts5Context cx, @NotNull byte[] pText, + @NotNull XTokenizeCallback callback); + + public native Object xUserData(Fts5Context cx); + //^^^ returns the pointer passed as the 3rd arg to the C-level + // fts5_api::xCreateFunction(). +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java new file mode 100644 index 0000000000000000000000000000000000000000..5774eb593609f9fb5f77feb1458d4b270e1c71e9 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java @@ -0,0 +1,25 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.NativePointerHolder; + +/** + A wrapper for C-level Fts5PhraseIter. They are only modified and + inspected by native-level code. +*/ +public final class Fts5PhraseIter extends NativePointerHolder { + //! Updated and used only by native code. + private long a; + private long b; +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java new file mode 100644 index 0000000000000000000000000000000000000000..b72e5d0fc0edf076defb95c0abdf3b834c313ed5 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java @@ -0,0 +1,31 @@ +/* +** 2023-08-05x +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.NativePointerHolder; + +/** + INCOMPLETE AND COMPLETELY UNTESTED. + + A wrapper for communicating C-level (Fts5Tokenizer*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. + + At the C level, the Fts5Tokenizer type is essentially a void + pointer used specifically for tokenizers. +*/ +public final class Fts5Tokenizer extends NativePointerHolder { + //! Only called from JNI. + private Fts5Tokenizer(){} +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java new file mode 100644 index 0000000000000000000000000000000000000000..095e649caffb37f6427c5492a52a308e62582ecd --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java @@ -0,0 +1,842 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a set of tests for the sqlite3 JNI bindings. +*/ +package org.sqlite.jni.fts5; +import static org.sqlite.jni.capi.CApi.*; +import static org.sqlite.jni.capi.Tester1.*; +import org.sqlite.jni.capi.*; +import org.sqlite.jni.fts5.*; + +import java.util.*; + +public class TesterFts5 { + + private static void test1(){ + final Fts5ExtensionApi fea = Fts5ExtensionApi.getInstance(); + affirm( null != fea ); + affirm( fea.getNativePointer() != 0 ); + affirm( fea == Fts5ExtensionApi.getInstance() )/*singleton*/; + + sqlite3 db = createNewDb(); + fts5_api fApi = fts5_api.getInstanceForDb(db); + affirm( fApi != null ); + affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ ); + + execSql(db, new String[] { + "CREATE VIRTUAL TABLE ft USING fts5(a, b);", + "INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');", + "INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');" + }); + + final String pUserData = "This is pUserData"; + final int outputs[] = {0, 0}; + final fts5_extension_function func = new fts5_extension_function(){ + @Override public void call(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]){ + final int nCols = ext.xColumnCount(fCx); + affirm( 2 == nCols ); + affirm( nCols == argv.length ); + affirm( ext.xUserData(fCx) == pUserData ); + final OutputPointer.String op = new OutputPointer.String(); + final OutputPointer.Int32 colsz = new OutputPointer.Int32(); + final OutputPointer.Int64 colTotalSz = new OutputPointer.Int64(); + for(int i = 0; i < nCols; ++i ){ + int rc = ext.xColumnText(fCx, i, op); + affirm( 0 == rc ); + final String val = op.value; + affirm( val.equals(sqlite3_value_text16(argv[i])) ); + rc = ext.xColumnSize(fCx, i, colsz); + affirm( 0==rc ); + affirm( 3==sqlite3_value_bytes(argv[i]) ); + rc = ext.xColumnTotalSize(fCx, i, colTotalSz); + affirm( 0==rc ); + } + ++outputs[0]; + } + public void xDestroy(){ + outputs[1] = 1; + } + }; + + int rc = fApi.xCreateFunction("myaux", pUserData, func); + affirm( 0==rc ); + + affirm( 0==outputs[0] ); + execSql(db, "select myaux(ft,a,b) from ft;"); + affirm( 2==outputs[0] ); + affirm( 0==outputs[1] ); + sqlite3_close_v2(db); + affirm( 1==outputs[1] ); + } + + /* + ** Argument sql is a string containing one or more SQL statements + ** separated by ";" characters. This function executes each of these + ** statements against the database passed as the first argument. If + ** no error occurs, the results of the SQL script are returned as + ** an array of strings. If an error does occur, a RuntimeException is + ** thrown. + */ + private static String[] sqlite3_exec(sqlite3 db, String sql) { + List aOut = new ArrayList(); + + /* Iterate through the list of SQL statements. For each, step through + ** it and add any results to the aOut[] array. */ + int rc = sqlite3_prepare_multi(db, sql, new PrepareMultiCallback() { + @Override public int call(sqlite3_stmt pStmt){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int ii; + for(ii=0; ii, ); + */ + class fts5_aux implements fts5_extension_function { + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length>1 ){ + throw new RuntimeException("fts5_aux: wrong number of args"); + } + + boolean bClear = (argv.length==1); + Object obj = ext.xGetAuxdata(fCx, bClear); + if( obj instanceof String ){ + sqlite3_result_text16(pCx, (String)obj); + } + + if( argv.length==1 ){ + String val = sqlite3_value_text16(argv[0]); + if( !val.equals("") ){ + ext.xSetAuxdata(fCx, val); + } + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_inst(); + ** + ** This is used to test the xInstCount() and xInst() APIs. It returns a + ** text value containing a Tcl list with xInstCount() elements. Each + ** element is itself a list of 3 integers - the phrase number, column + ** number and token offset returned by each call to xInst(). + */ + fts5_extension_function fts5_inst = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_inst: wrong number of args"); + } + + OutputPointer.Int32 pnInst = new OutputPointer.Int32(); + OutputPointer.Int32 piPhrase = new OutputPointer.Int32(); + OutputPointer.Int32 piCol = new OutputPointer.Int32(); + OutputPointer.Int32 piOff = new OutputPointer.Int32(); + String ret = new String(); + + int rc = ext.xInstCount(fCx, pnInst); + int nInst = pnInst.get(); + int ii; + + for(ii=0; rc==SQLITE_OK && ii0 ) ret += " "; + ret += "{"+piPhrase.get()+" "+piCol.get()+" "+piOff.get()+"}"; + } + + sqlite3_result_text(pCx, ret); + } + public void xDestroy(){ } + }; + + /* + ** fts5_pinst(); + ** + ** Like SQL function fts5_inst(), except using the following + ** + ** xPhraseCount + ** xPhraseFirst + ** xPhraseNext + */ + fts5_extension_function fts5_pinst = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_pinst: wrong number of args"); + } + + OutputPointer.Int32 piCol = new OutputPointer.Int32(); + OutputPointer.Int32 piOff = new OutputPointer.Int32(); + String ret = new String(); + int rc = SQLITE_OK; + + int nPhrase = ext.xPhraseCount(fCx); + int ii; + + for(ii=0; rc==SQLITE_OK && ii=0; + ext.xPhraseNext(fCx, pIter, piCol, piOff) + ){ + if( !ret.equals("") ) ret += " "; + ret += "{"+ii+" "+piCol.get()+" "+piOff.get()+"}"; + } + } + + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_pinst: rc=" + rc); + }else{ + sqlite3_result_text(pCx, ret); + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_pcolinst(); + ** + ** Like SQL function fts5_pinst(), except using the following + ** + ** xPhraseFirstColumn + ** xPhraseNextColumn + */ + fts5_extension_function fts5_pcolinst = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_pcolinst: wrong number of args"); + } + + OutputPointer.Int32 piCol = new OutputPointer.Int32(); + String ret = new String(); + int rc = SQLITE_OK; + + int nPhrase = ext.xPhraseCount(fCx); + int ii; + + for(ii=0; rc==SQLITE_OK && ii=0; + ext.xPhraseNextColumn(fCx, pIter, piCol) + ){ + if( !ret.equals("") ) ret += " "; + ret += "{"+ii+" "+piCol.get()+"}"; + } + } + + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_pcolinst: rc=" + rc); + }else{ + sqlite3_result_text(pCx, ret); + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_rowcount(); + */ + fts5_extension_function fts5_rowcount = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_rowcount: wrong number of args"); + } + OutputPointer.Int64 pnRow = new OutputPointer.Int64(); + + int rc = ext.xRowCount(fCx, pnRow); + if( rc==SQLITE_OK ){ + sqlite3_result_int64(pCx, pnRow.get()); + }else{ + throw new RuntimeException("fts5_rowcount: rc=" + rc); + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_phrasesize(); + */ + fts5_extension_function fts5_phrasesize = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=1 ){ + throw new RuntimeException("fts5_phrasesize: wrong number of args"); + } + int iPhrase = sqlite3_value_int(argv[0]); + + int sz = ext.xPhraseSize(fCx, iPhrase); + sqlite3_result_int(pCx, sz); + } + public void xDestroy(){ } + }; + + /* + ** fts5_phrasehits(, ); + ** + ** Use the xQueryPhrase() API to determine how many hits, in total, + ** there are for phrase in the database. + */ + fts5_extension_function fts5_phrasehits = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=1 ){ + throw new RuntimeException("fts5_phrasesize: wrong number of args"); + } + int iPhrase = sqlite3_value_int(argv[0]); + int rc = SQLITE_OK; + + class MyCallback implements Fts5ExtensionApi.XQueryPhraseCallback { + public int nRet = 0; + public int getRet() { return nRet; } + + @Override + public int call(Fts5ExtensionApi fapi, Fts5Context cx){ + OutputPointer.Int32 pnInst = new OutputPointer.Int32(); + int rc = fapi.xInstCount(cx, pnInst); + nRet += pnInst.get(); + return rc; + } + }; + + MyCallback xCall = new MyCallback(); + rc = ext.xQueryPhrase(fCx, iPhrase, xCall); + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_phrasehits: rc=" + rc); + } + sqlite3_result_int(pCx, xCall.getRet()); + } + public void xDestroy(){ } + }; + + /* + ** fts5_tokenize(, ) + */ + fts5_extension_function fts5_tokenize = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=1 ){ + throw new RuntimeException("fts5_tokenize: wrong number of args"); + } + byte[] utf8 = sqlite3_value_text(argv[0]); + int rc = SQLITE_OK; + + class MyCallback implements XTokenizeCallback { + private List myList = new ArrayList(); + + public String getval() { + return String.join("+", myList); + } + + @Override + public int call(int tFlags, byte[] txt, int iStart, int iEnd){ + try { + String str = new String(txt, "UTF-8"); + myList.add(str); + } catch (Exception e) { + } + return SQLITE_OK; + } + }; + + MyCallback xCall = new MyCallback(); + ext.xTokenize(fCx, utf8, xCall); + sqlite3_result_text16(pCx, xCall.getval()); + + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_tokenize: rc=" + rc); + } + } + public void xDestroy(){ } + }; + + fts5_api api = fts5_api.getInstanceForDb(db); + api.xCreateFunction("fts5_rowid", fts5_rowid); + api.xCreateFunction("fts5_columncount", fts5_columncount); + api.xCreateFunction("fts5_columnsize", fts5_columnsize); + api.xCreateFunction("fts5_columntext", fts5_columntext); + api.xCreateFunction("fts5_columntotalsize", fts5_columntsize); + + api.xCreateFunction("fts5_aux1", new fts5_aux()); + api.xCreateFunction("fts5_aux2", new fts5_aux()); + + api.xCreateFunction("fts5_inst", fts5_inst); + api.xCreateFunction("fts5_pinst", fts5_pinst); + api.xCreateFunction("fts5_pcolinst", fts5_pcolinst); + api.xCreateFunction("fts5_rowcount", fts5_rowcount); + api.xCreateFunction("fts5_phrasesize", fts5_phrasesize); + api.xCreateFunction("fts5_phrasehits", fts5_phrasehits); + api.xCreateFunction("fts5_tokenize", fts5_tokenize); + } + /* + ** Test of various Fts5ExtensionApi methods + */ + private static void test2(){ + + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(a, b);" + + "INSERT INTO ft(rowid, a, b) VALUES(-9223372036854775808, 'x', 'x');" + + "INSERT INTO ft(rowid, a, b) VALUES(0, 'x', 'x');" + + "INSERT INTO ft(rowid, a, b) VALUES(1, 'x y z', 'x y z');" + + "INSERT INTO ft(rowid, a, b) VALUES(2, 'x y z', 'x z');" + + "INSERT INTO ft(rowid, a, b) VALUES(3, 'x y z', 'x y z');" + + "INSERT INTO ft(rowid, a, b) VALUES(9223372036854775807, 'x', 'x');" + ); + + create_test_functions(db); + + /* Test that fts5_rowid() seems to work */ + do_execsql_test(db, + "SELECT rowid==fts5_rowid(ft) FROM ft('x')", + "[1, 1, 1, 1, 1, 1]" + ); + + /* Test fts5_columncount() */ + do_execsql_test(db, + "SELECT fts5_columncount(ft) FROM ft('x')", + "[2, 2, 2, 2, 2, 2]" + ); + + /* Test fts5_columnsize() */ + do_execsql_test(db, + "SELECT fts5_columnsize(ft, 0) FROM ft('x') ORDER BY rowid", + "[1, 1, 3, 3, 3, 1]" + ); + do_execsql_test(db, + "SELECT fts5_columnsize(ft, 1) FROM ft('x') ORDER BY rowid", + "[1, 1, 3, 2, 3, 1]" + ); + do_execsql_test(db, + "SELECT fts5_columnsize(ft, -1) FROM ft('x') ORDER BY rowid", + "[2, 2, 6, 5, 6, 2]" + ); + + /* Test that xColumnSize() returns SQLITE_RANGE if the column number + ** is out-of range */ + try { + do_execsql_test(db, + "SELECT fts5_columnsize(ft, 2) FROM ft('x') ORDER BY rowid" + ); + } catch( RuntimeException e ){ + affirm( e.getMessage().matches(".*column index out of range") ); + } + + /* Test fts5_columntext() */ + do_execsql_test(db, + "SELECT fts5_columntext(ft, 0) FROM ft('x') ORDER BY rowid", + "[x, x, x y z, x y z, x y z, x]" + ); + do_execsql_test(db, + "SELECT fts5_columntext(ft, 1) FROM ft('x') ORDER BY rowid", + "[x, x, x y z, x z, x y z, x]" + ); + boolean threw = false; + try{ + /* columntext() used to return NULLs when given an out-of bounds column + but now results in a range error. */ + do_execsql_test(db, + "SELECT fts5_columntext(ft, 2) FROM ft('x') ORDER BY rowid", + "[null, null, null, null, null, null]" + ); + }catch(Exception e){ + threw = true; + affirm( e.getMessage().matches(".*column index out of range") ); + } + affirm( threw ); + threw = false; + + /* Test fts5_columntotalsize() */ + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, 0) FROM ft('x') ORDER BY rowid", + "[12, 12, 12, 12, 12, 12]" + ); + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, 1) FROM ft('x') ORDER BY rowid", + "[11, 11, 11, 11, 11, 11]" + ); + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, -1) FROM ft('x') ORDER BY rowid", + "[23, 23, 23, 23, 23, 23]" + ); + + /* Test that xColumnTotalSize() returns SQLITE_RANGE if the column + ** number is out-of range */ + try { + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, 2) FROM ft('x') ORDER BY rowid" + ); + } catch( RuntimeException e ){ + affirm( e.getMessage().matches(".*column index out of range") ); + } + + do_execsql_test(db, + "SELECT rowid, fts5_rowcount(ft) FROM ft('z')", + "[1, 6, 2, 6, 3, 6]" + ); + + sqlite3_close_v2(db); + } + + /* + ** Test of various Fts5ExtensionApi methods + */ + private static void test3(){ + + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(a, b);" + + "INSERT INTO ft(a, b) VALUES('the one', 1);" + + "INSERT INTO ft(a, b) VALUES('the two', 2);" + + "INSERT INTO ft(a, b) VALUES('the three', 3);" + + "INSERT INTO ft(a, b) VALUES('the four', '');" + ); + create_test_functions(db); + + /* Test fts5_aux1() + fts5_aux2() - users of xGetAuxdata and xSetAuxdata */ + do_execsql_test(db, + "SELECT fts5_aux1(ft, a) FROM ft('the')", + "[null, the one, the two, the three]" + ); + do_execsql_test(db, + "SELECT fts5_aux2(ft, b) FROM ft('the')", + "[null, 1, 2, 3]" + ); + do_execsql_test(db, + "SELECT fts5_aux1(ft, a), fts5_aux2(ft, b) FROM ft('the')", + "[null, null, the one, 1, the two, 2, the three, 3]" + ); + do_execsql_test(db, + "SELECT fts5_aux1(ft, b), fts5_aux1(ft) FROM ft('the')", + "[null, 1, 1, 2, 2, 3, 3, null]" + ); + } + + /* + ** Test of various Fts5ExtensionApi methods + */ + private static void test4(){ + + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + create_test_functions(db); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(a, b);" + + "INSERT INTO ft(a, b) VALUES('one two three', 'two three four');" + + "INSERT INTO ft(a, b) VALUES('two three four', 'three four five');" + + "INSERT INTO ft(a, b) VALUES('three four five', 'four five six');" + ); + + + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('two')", + "[{0 0 1} {0 1 0}, {0 0 0}]" + ); + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('four')", + "[{0 1 2}, {0 0 2} {0 1 1}, {0 0 1} {0 1 0}]" + ); + + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('a OR b OR four')", + "[{2 1 2}, {2 0 2} {2 1 1}, {2 0 1} {2 1 0}]" + ); + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('two four')", + "[{0 0 1} {0 1 0} {1 1 2}, {0 0 0} {1 0 2} {1 1 1}]" + ); + + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('two')", + "[{0 0 1} {0 1 0}, {0 0 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('four')", + "[{0 1 2}, {0 0 2} {0 1 1}, {0 0 1} {0 1 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('a OR b OR four')", + "[{2 1 2}, {2 0 2} {2 1 1}, {2 0 1} {2 1 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('two four')", + "[{0 0 1} {0 1 0} {1 1 2}, {0 0 0} {1 0 2} {1 1 1}]" + ); + + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('two')", + "[{0 0} {0 1}, {0 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('four')", + "[{0 1}, {0 0} {0 1}, {0 0} {0 1}]" + ); + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('a OR b OR four')", + "[{2 1}, {2 0} {2 1}, {2 0} {2 1}]" + ); + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('two four')", + "[{0 0} {0 1} {1 1}, {0 0} {1 0} {1 1}]" + ); + + do_execsql_test(db, + "SELECT fts5_phrasesize(ft, 0) FROM ft('four five six') LIMIT 1;", + "[1]" + ); + do_execsql_test(db, + "SELECT fts5_phrasesize(ft, 0) FROM ft('four + five + six') LIMIT 1;", + "[3]" + ); + + + sqlite3_close_v2(db); + } + + private static void test5(){ + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + create_test_functions(db); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(x, b);" + + "INSERT INTO ft(x) VALUES('one two three four five six seven eight');" + + "INSERT INTO ft(x) VALUES('one two one four one six one eight');" + + "INSERT INTO ft(x) VALUES('one two three four five six seven eight');" + ); + + do_execsql_test(db, + "SELECT fts5_phrasehits(ft, 0) FROM ft('one') LIMIT 1", + "[6]" + ); + + sqlite3_close_v2(db); + } + + private static void test6(){ + sqlite3 db = createNewDb(); + create_test_functions(db); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(x, b);" + + "INSERT INTO ft(x) VALUES('one two three four five six seven eight');" + ); + + do_execsql_test(db, + "SELECT fts5_tokenize(ft, 'abc def ghi') FROM ft('one')", + "[abc+def+ghi]" + ); + do_execsql_test(db, + "SELECT fts5_tokenize(ft, 'it''s BEEN a...') FROM ft('one')", + "[it+s+been+a]" + ); + + sqlite3_close_v2(db); + } + + private static synchronized void runTests(){ + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + } + + public TesterFts5(){ + runTests(); + } +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java b/ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..3aa514f314064c600dba9ca1c5c9c8aa1505088c --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java @@ -0,0 +1,22 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; + + +/** + Callback type for use with xTokenize() variants. +*/ +public interface XTokenizeCallback { + int call(int tFlags, byte[] txt, int iStart, int iEnd); +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_api.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_api.java new file mode 100644 index 0000000000000000000000000000000000000000..d7d2da430d3299a9ede2f8e7e5dd016caeeb1482 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_api.java @@ -0,0 +1,76 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.annotation.*; +import org.sqlite.jni.capi.*; + +/** + A wrapper for communicating C-level (fts5_api*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class fts5_api extends NativePointerHolder { + /* Only invoked from JNI */ + private fts5_api(){} + + public static final int iVersion = 2; + + /** + Returns the fts5_api instance associated with the given db, or + null if something goes horribly wrong. + */ + public static synchronized native fts5_api getInstanceForDb(@NotNull sqlite3 db); + + public synchronized native int xCreateFunction(@NotNull String name, + @Nullable Object userData, + @NotNull fts5_extension_function xFunction); + + /** + Convenience overload which passes null as the 2nd argument to the + 3-parameter form. + */ + public int xCreateFunction(@NotNull String name, + @NotNull fts5_extension_function xFunction){ + return xCreateFunction(name, null, xFunction); + } + + // /* Create a new auxiliary function */ + // int (*xCreateFunction)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_extension_function xFunction, + // void (*xDestroy)(void*) + // ); + + // Still potentially todo: + + // int (*xCreateTokenizer)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_tokenizer *pTokenizer, + // void (*xDestroy)(void*) + // ); + + // /* Find an existing tokenizer */ + // int (*xFindTokenizer)( + // fts5_api *pApi, + // const char *zName, + // void **ppContext, + // fts5_tokenizer *pTokenizer + // ); + +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java new file mode 100644 index 0000000000000000000000000000000000000000..5e47633baa9bbcd0b911cf3a8612fd6137c6f107 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java @@ -0,0 +1,47 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.sqlite3_context; +import org.sqlite.jni.capi.sqlite3_value; + +/** + JNI-level wrapper for C's fts5_extension_function type. +*/ +public interface fts5_extension_function { + // typedef void (*fts5_extension_function)( + // const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + // Fts5Context *pFts, /* First arg to pass to pApi functions */ + // sqlite3_context *pCtx, /* Context for returning result/error */ + // int nVal, /* Number of values in apVal[] array */ + // sqlite3_value **apVal /* Array of trailing arguments */ + // ); + + /** + The callback implementation, corresponding to the xFunction + argument of C's fts5_api::xCreateFunction(). + */ + void call(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]); + /** + Is called when this function is destroyed by sqlite3. Typically + this function will be empty. + */ + void xDestroy(); + + public static abstract class Abstract implements fts5_extension_function { + @Override public abstract void call(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]); + @Override public void xDestroy(){} + } +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java new file mode 100644 index 0000000000000000000000000000000000000000..f4ada4dc30db020f4c575c8695681b39da5b936e --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java @@ -0,0 +1,49 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.NativePointerHolder; +import org.sqlite.jni.annotation.NotNull; + +/** + A wrapper for communicating C-level (fts5_tokenizer*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class fts5_tokenizer extends NativePointerHolder { + /* Only invoked by JNI */ + private fts5_tokenizer(){} + + // int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + // void (*xDelete)(Fts5Tokenizer*); + + public native int xTokenize(@NotNull Fts5Tokenizer t, int tokFlags, + @NotNull byte pText[], + @NotNull XTokenizeCallback callback); + + + // int (*xTokenize)(Fts5Tokenizer*, + // void *pCtx, + // int flags, /* Mask of FTS5_TOKENIZE_* flags */ + // const char *pText, int nText, + // int (*xToken)( + // void *pCtx, /* Copy of 2nd argument to xTokenize() */ + // int tflags, /* Mask of FTS5_TOKEN_* flags */ + // const char *pToken, /* Pointer to buffer containing token */ + // int nToken, /* Size of token in bytes */ + // int iStart, /* Byte offset of token within input text */ + // int iEnd /* Byte offset of end of token within input text */ + // ) + // ); +} diff --git a/ext/jni/src/org/sqlite/jni/test-script-interpreter.md b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md new file mode 100644 index 0000000000000000000000000000000000000000..939f77e1be96bc658796292c9e589ff125ab0567 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md @@ -0,0 +1,270 @@ +# Specifications For A Rudimentary SQLite Test Script Interpreter + +## Overview + +The purpose of the Test Script Interpreter is to read and interpret +script files that contain SQL commands and desired results. The +interpreter will check results and report any discrepencies found. + +The test script files are ASCII text files. The filename always ends with +".test". Each script is evaluated independently; context does not carry +forward from one script to the next. So, for example, the --null command +run in one test script does not cause any changes in the behavior of +subsequent test scripts. All open database connections are closed +at the end of each test script. All database files created by a test +script are deleted when the script finishes. + +## Parsing Rules: + + 1. The test script is read line by line, where a line is a sequence of + characters that runs up to the next '\\n' (0x0a) character or until + the end of the file. There is never a need to read ahead past the + end of the current line. + + 2. If any line contains the string " MODULE_NAME:" (with a space before + the initial "M") or "MIXED_MODULE_NAME:" then that test script is + incompatible with this spec. Processing of the test script should + end immediately. There is no need to read any more of the file. + In verbose mode, the interpreter might choose to emit an informational + messages saying that the test script was abandoned due to an + incompatible module type. + + 3. If any line contains the string "SCRIPT_MODULE_NAME:" then the input + script is known to be of the correct type for this specification and + processing may continue. The "MODULE_NAME" checking in steps 2 and 3 + may optionally be discontinued after sighting a "SCRIPT_MODULE_NAME". + + 4. If any line contains "REQUIRED_PROPERTIES:" and that substring is followed + by any non-whitespace text, then the script is not compatible with this + spec. Processing should stop immediately. In verbose mode, the + interpreter might choose to emit an information message saying that the + test script was abandoned due to unsupported requirement properties. + + 5. If any line begins with the "\|" (0x7c) character, that indicates that + the input script is not compatible with this specification. Processing + of the script should stop immediately. In verbose mode, the interpreter + might choose to emit an informational message indicating that the + test script was abandoned because it contained "a dbtotxt format database + specification". + + 6. Any line that begins with "#" is a C-preprocessor line. The interpreter + described by this spec does not know how to deal with C-preprocessor lines. + Hence, processing should be abandoned. In verbose mode, the interpreter + might emit an informational message similar to + "script NAME abandoned due to C-preprocessor line: ..." + + 7. If a line begins with exactly two minus signs followed by a + lowercase letter, that is a command. Process commands as described + below. + + 8. All other lines should be accumulated into the "input buffer". + The various commands will have access to this input buffer. + Some commands will reset the buffer. + +## Initialization + +The initial state of the interpreter at the start of processing each script +is as if the following command sequence had been run: + +> ~~~ +--close all +--db 0 +--new test.db +--null nil +~~~ + +In words, all database connections are closed except for connection 0 (the +default) which is open on an empty database named "test.db". The string +"nil" is displayed for NULL column values. + +The only context carried forward after the evaluation of one test script +into the evaluation of the next test script is the count of the number of +tests run and the number of failures seen. + +## Commands: + +Each command looks like an SQL comment. The command begins at the left +margin (no leading space) and starts with exactly 2 minus signs ("-"). +The command name consists of lowercase letters and maybe a "-" or two. +Some commands have arguments. + +The arguments are separated from the command name by one or more spaces. + +Commands have access to the input buffer and might reset the input buffer. +The command can also optionally read (and consume) additional text from +script that comes after the command. + +Unknown or unrecognized commands indicate that the script contains features +that are not (yet) supported by this specification. Processing of the +script should terminate immediately. When this happens and when the +interpreter is in a "verbose" mode, the interpreter might choose to emit +an informational message along the lines of "test script NAME abandoned +due to unsupported command: --whatever". + +The initial implemention will only recognize a few commands. Other +commands may be added later. The following is the initial set of +commands: + +### The --testcase command + +Every test case starts with a --testcase command. The --testcase +command resets both the "input buffer" and the "result buffer". The +argument to the --testcase command is the name of the test case. That +test case name is used for logging and debugging and when printing +errors. The input buffer is set to the body of the test case. + +### The --result command + +The --result command tries to execute the text in the input buffer as SQL. +For each row of result coming out of this SQL, the text of that result is +appended to the "result buffer". If a result row contains multiple columns, +the columns are processed from left to right. For each column, text is +appended to the result buffer according to the following rules: + + * If the result buffer already contains some text, append a space. + (In this way, all column values and all row values are separated from + each other by a single space.) + + * If sqlite3_column_text() returns NULL, then append "nil" - or + some other text that is specified by the --null command - and skip + all subsequent rules. + + * If sqlite3_column_text() is an empty string, append `{}` to the + result buffer and skip all subsequent rules. + + * If sqlite3_column_text() does not contain any special + characters, append it to the result buffer without any + formatting and skip all subsequent rules. Special characters are: + 0x00 to 0x20 (inclusive), double-quote (0x22), backslash (0x5c), + curly braces (0x7b and 0x7d). + + * If sqlite3_column_text() does not contains curly braces, then put + the text inside of `{...}` and append it and skip all subsequent rules. + + * Append the text within double-quotes (`"..."`) and within the text + escape '"' and '\\' by prepending a single '\\' and escape any + control characters (characters less than 0x20) using octal notation: + '\\NNN'. + +If an error is encountered while running the SQL, then append the +symbolic C-preprocessor name for the error +code (ex: "SQLITE_CONSTRAINT") as if it were a column value. Then append +the error message text as if it where a column value. Then stop processing. + +After the SQL text has been run, compare the content of the result buffer +against the argument to the --result command and report a testing error if +there are any differences. + +The --result command resets the input buffer, but it does not reset +the result buffer. This distinction does not matter for the --result +command itself, but it is important for related commands like --glob +and --notglob. Sometimes test cases will contains a bunch of SQL +followed by multiple --glob and/or --notglob statements. All of the +globs should be evaluated agains the result buffer, but the SQL should +only be run once. This is accomplished by resetting the input buffer +but not the result buffer. + +### The --glob command + +The --glob command works just like --result except that the argument to +--glob is interpreted as a TEST-GLOB pattern and the results are compared +using that glob pattern rather than using strcmp(). Other than that, +the two operate the same. + +The TEST-GLOB pattern is slightly different for a standard GLOB: + + * The '*' character matches zero or more characters. + + * The '?' character matches any single character + + * The '[...]' character sequence machines a single character + in between the brackets. + + * The '#' character matches one or more digits (This is the main + difference between standard unix-glob and TEST-GLOB. unix-glob + does not have this feature. It was added to because it comes + up a lot during SQLite testing.) + +### The --notglob command + +The --notglob command works just like --glob except that it reports an +error if the GLOB does match, rather than if the GLOB does not match. + +### The --oom command + +This command is to be used for out-of-memory testing. It means that +OOM errors should be simulated to ensure that SQLite is able to deal with +them. This command can be silently ignored for now. We might add support +for this later. + +### The --tableresult command + +The --tableresult command works like --glob except that the GLOB pattern +to be matched is taken from subsequent lines of the input script up to +the next --end. Every span of one or more whitespace characters in this +pattern text is collapsed into a single space (0x20). +Leading and trailing whitespace are removed from the pattern. +The --end that ends the GLOB pattern is not part of the GLOB pattern, but +the --end is consumed from the script input. + +### The --new and --open commands + +The --new and --open commands cause a database file to be opened. +The name of the file is the argument to the command. The --new command +opens an initially empty database (it deletes the file before opening it) +whereas the --open command opens an existing database if it already +exists. + +### The --db command + +The script interpreter can have up to 7 different SQLite database +connections open at a time. The --db command is used to switch between +them. The argument to --db is an integer between 0 and 6 that selects +which database connection to use moving forward. + +### The --close command + +The --close command causes an existing database connection to close. +This command is a no-op if the database connection is not currently +open. There can be up to 7 different database connections, numbered 0 +through 6. The number of the database connection to close is an +argument to the --close command, which will fail if an out-of-range +value is provided. Or if the argument to --close is "all" then all +open database connections are closed. If passed no argument, the +currently-active database is assumed. + +### The --null command + +The NULL command changes the text that is used to represent SQL NULL +values in the result buffer. + +### The --run command + +The --run command executes text in the input buffer as if it where SQL. +However, nothing is added to the result buffer. Any output from the SQL +is silently ignored. Errors in the SQL are silently ignored. + +The --run command normally executes the SQL in the current database +connection. However, if --run has an argument that is an integer between +0 and 6 then the SQL is run in the alternative database connection specified +by that argument. + +### The --json and --json-block commands + +The --json and --json-block commands work like --result and --tableresult, +respectively. The difference is that column values are appended to the +result buffer literally, without ever enclosing the values in `{...}` or +`"..."` and without escaping any characters in the column value and comparison +is always an exact strcmp() not a GLOB. + +### The --print command + +The --print command emits both its arguments and its body (if any) to +stdout, indenting each line of output. + +### The --column-names command + +The --column-names command requires 0 or 1 as an argument, to disable +resp. enable it, and modifies SQL execution to include column names +in output. When this option is on, each column value emitted gets +prefixed by its column name, with a single space between them. diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..fc63b53542e84f0232b689a51f036c1052ce40d3 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java @@ -0,0 +1,144 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; + +/** + EXPERIMENTAL/INCOMPLETE/UNTESTED + + A SqlFunction implementation for aggregate functions. The T type + represents the type of data accumulated by this aggregate while it + works. e.g. a SUM()-like UDF might use Integer or Long and a + CONCAT()-like UDF might use a StringBuilder or a List. +*/ +public abstract class AggregateFunction implements SqlFunction { + + /** + As for the xStep() argument of the C API's + sqlite3_create_function(). If this function throws, the + exception is reported via sqlite3_result_error(). + */ + public abstract void xStep(SqlFunction.Arguments args); + + /** + As for the xFinal() argument of the C API's + sqlite3_create_function(). If this function throws, it is + translated into sqlite3_result_error(). + + Note that the passed-in object will not actually contain any + arguments for xFinal() but will contain the context object needed + for setting the call's result or error state. + */ + public abstract void xFinal(SqlFunction.Arguments args); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. + */ + public void xDestroy() {} + + /** + PerContextState assists aggregate and window functions in + managing their accumulator state across calls to the UDF's + callbacks. + +

T must be of a type which can be legally stored as a value in + java.util.HashMap. + +

If a given aggregate or window function is called multiple times + in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)..., + then the clients need some way of knowing which call is which so + that they can map their state between their various UDF callbacks + and reset it via xFinal(). This class takes care of such + mappings. + +

This class works by mapping + sqlite3_context.getAggregateContext() to a single piece of + state, of a client-defined type (the T part of this class), which + persists across a "matching set" of the UDF's callbacks. + +

This class is a helper providing commonly-needed functionality + - it is not required for use with aggregate or window functions. + Client UDFs are free to perform such mappings using custom + approaches. The provided {@link AggregateFunction} and {@link + WindowFunction} classes use this. + */ + public static final class PerContextState { + private final java.util.Map> map + = new java.util.HashMap<>(); + + /** + Should be called from a UDF's xStep(), xValue(), and xInverse() + methods, passing it that method's first argument and an initial + value for the persistent state. If there is currently no + mapping for the given context within the map, one is created + using the given initial value, else the existing one is used + and the 2nd argument is ignored. It returns a ValueHolder + which can be used to modify that state directly without + requiring that the client update the underlying map's entry. + +

The caller is obligated to eventually call + takeAggregateState() to clear the mapping. + */ + public ValueHolder getAggregateState(SqlFunction.Arguments args, T initialValue){ + final Long key = args.getContext().getAggregateContext(true); + ValueHolder rc = null==key ? null : map.get(key); + if( null==rc ){ + map.put(key, rc = new ValueHolder<>(initialValue)); + } + return rc; + } + + /** + Should be called from a UDF's xFinal() method and passed that + method's first argument. This function removes the value + associated with with the arguments' aggregate context from the + map and returns it, returning null if no other UDF method has + been called to set up such a mapping. The latter condition will + be the case if a UDF is used in a statement which has no result + rows. + */ + public T takeAggregateState(SqlFunction.Arguments args){ + final ValueHolder h = map.remove(args.getContext().getAggregateContext(false)); + return null==h ? null : h.value; + } + } + + /** Per-invocation state for the UDF. */ + private final PerContextState map = new PerContextState<>(); + + /** + To be called from the implementation's xStep() method, as well + as the xValue() and xInverse() methods of the {@link WindowFunction} + subclass, to fetch the current per-call UDF state. On the + first call to this method for any given sqlite3_context + argument, the context is set to the given initial value. On all other + calls, the 2nd argument is ignored. + + @see SQLFunction.PerContextState#getAggregateState + */ + protected final ValueHolder getAggregateState(SqlFunction.Arguments args, T initialValue){ + return map.getAggregateState(args, initialValue); + } + + /** + To be called from the implementation's xFinal() method to fetch + the final state of the UDF and remove its mapping. + + see SQLFunction.PerContextState#takeAggregateState + */ + protected final T takeAggregateState(SqlFunction.Arguments args){ + return map.takeAggregateState(args); + } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..067a6983eb0e9c9b2560d53e532fcafb7459aced --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java @@ -0,0 +1,37 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.annotation.*; +import org.sqlite.jni.capi.sqlite3_context; +import org.sqlite.jni.capi.sqlite3_value; + +/** + The SqlFunction type for scalar SQL functions. +*/ +public abstract class ScalarFunction implements SqlFunction { + /** + As for the xFunc() argument of the C API's + sqlite3_create_function(). If this function throws, it is + translated into an sqlite3_result_error(). + */ + public abstract void xFunc(SqlFunction.Arguments args); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. This default implementation does nothing. + */ + public void xDestroy() {} + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..dcfc2ebebd7988d09d042d83815e628ca180d6cc --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java @@ -0,0 +1,318 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.capi.sqlite3_context; +import org.sqlite.jni.capi.sqlite3_value; + +/** + Base marker interface for SQLite's three types of User-Defined SQL + Functions (UDFs): Scalar, Aggregate, and Window functions. +*/ +public interface SqlFunction { + + public static final int DETERMINISTIC = CApi.SQLITE_DETERMINISTIC; + public static final int INNOCUOUS = CApi.SQLITE_INNOCUOUS; + public static final int DIRECTONLY = CApi.SQLITE_DIRECTONLY; + public static final int SUBTYPE = CApi.SQLITE_SUBTYPE; + public static final int RESULT_SUBTYPE = CApi.SQLITE_RESULT_SUBTYPE; + public static final int UTF8 = CApi.SQLITE_UTF8; + public static final int UTF16 = CApi.SQLITE_UTF16; + + /** + The Arguments type is an abstraction on top of the lower-level + UDF function argument types. It provides _most_ of the functionality + of the lower-level interface, insofar as possible without "leaking" + those types into this API. + */ + public final static class Arguments implements Iterable{ + private final sqlite3_context cx; + private final sqlite3_value args[]; + public final int length; + + /** + Must be passed the context and arguments for the UDF call this + object is wrapping. Intended to be used by internal proxy + classes which "convert" the lower-level interface into this + package's higher-level interface, e.g. ScalarAdapter and + AggregateAdapter. + + Passing null for the args is equivalent to passing a length-0 + array. + */ + Arguments(sqlite3_context cx, sqlite3_value args[]){ + this.cx = cx; + this.args = args==null ? new sqlite3_value[0] : args; + this.length = this.args.length; + } + + /** + Returns the sqlite3_value at the given argument index or throws + an IllegalArgumentException exception if ndx is out of range. + */ + private sqlite3_value valueAt(int ndx){ + if(ndx<0 || ndx>=args.length){ + throw new IllegalArgumentException( + "SQL function argument index "+ndx+" is out of range." + ); + } + return args[ndx]; + } + + //! Returns the underlying sqlite3_context for these arguments. + sqlite3_context getContext(){return cx;} + + /** + Returns the Sqlite (db) object associated with this UDF call, + or null if the UDF is somehow called without such an object or + the db has been closed in an untimely manner (e.g. closed by a + UDF call). + */ + public Sqlite getDb(){ + return Sqlite.fromNative( CApi.sqlite3_context_db_handle(cx) ); + } + + public int getArgCount(){ return args.length; } + + public int getInt(int argNdx){return CApi.sqlite3_value_int(valueAt(argNdx));} + public long getInt64(int argNdx){return CApi.sqlite3_value_int64(valueAt(argNdx));} + public double getDouble(int argNdx){return CApi.sqlite3_value_double(valueAt(argNdx));} + public byte[] getBlob(int argNdx){return CApi.sqlite3_value_blob(valueAt(argNdx));} + public byte[] getText(int argNdx){return CApi.sqlite3_value_text(valueAt(argNdx));} + public String getText16(int argNdx){return CApi.sqlite3_value_text16(valueAt(argNdx));} + public int getBytes(int argNdx){return CApi.sqlite3_value_bytes(valueAt(argNdx));} + public int getBytes16(int argNdx){return CApi.sqlite3_value_bytes16(valueAt(argNdx));} + public Object getObject(int argNdx){return CApi.sqlite3_value_java_object(valueAt(argNdx));} + public T getObject(int argNdx, Class type){ + return CApi.sqlite3_value_java_object(valueAt(argNdx), type); + } + + public int getType(int argNdx){return CApi.sqlite3_value_type(valueAt(argNdx));} + public int getSubtype(int argNdx){return CApi.sqlite3_value_subtype(valueAt(argNdx));} + public int getNumericType(int argNdx){return CApi.sqlite3_value_numeric_type(valueAt(argNdx));} + public int getNoChange(int argNdx){return CApi.sqlite3_value_nochange(valueAt(argNdx));} + public boolean getFromBind(int argNdx){return CApi.sqlite3_value_frombind(valueAt(argNdx));} + public int getEncoding(int argNdx){return CApi.sqlite3_value_encoding(valueAt(argNdx));} + + public void resultInt(int v){ CApi.sqlite3_result_int(cx, v); } + public void resultInt64(long v){ CApi.sqlite3_result_int64(cx, v); } + public void resultDouble(double v){ CApi.sqlite3_result_double(cx, v); } + public void resultError(String msg){CApi.sqlite3_result_error(cx, msg);} + public void resultError(Exception e){CApi.sqlite3_result_error(cx, e);} + public void resultErrorTooBig(){CApi.sqlite3_result_error_toobig(cx);} + public void resultErrorCode(int rc){CApi.sqlite3_result_error_code(cx, rc);} + public void resultObject(Object o){CApi.sqlite3_result_java_object(cx, o);} + public void resultNull(){CApi.sqlite3_result_null(cx);} + /** + Analog to sqlite3_result_value(), using the Value object at the + given argument index. + */ + public void resultArg(int argNdx){CApi.sqlite3_result_value(cx, valueAt(argNdx));} + public void resultSubtype(int subtype){CApi.sqlite3_result_subtype(cx, subtype);} + public void resultZeroBlob(long n){ + // Throw on error? If n is too big, + // sqlite3_result_error_toobig() is automatically called. + CApi.sqlite3_result_zeroblob64(cx, n); + } + + public void resultBlob(byte[] blob){CApi.sqlite3_result_blob(cx, blob);} + public void resultText(byte[] utf8){CApi.sqlite3_result_text(cx, utf8);} + public void resultText(String txt){CApi.sqlite3_result_text(cx, txt);} + public void resultText16(byte[] utf16){CApi.sqlite3_result_text16(cx, utf16);} + public void resultText16(String txt){CApi.sqlite3_result_text16(cx, txt);} + + /** + Callbacks should invoke this on OOM errors, instead of throwing + OutOfMemoryError, because the latter cannot be propagated + through the C API. + */ + public void resultNoMem(){CApi.sqlite3_result_error_nomem(cx);} + + /** + Analog to sqlite3_set_auxdata() but throws if argNdx is out of + range. + */ + public void setAuxData(int argNdx, Object o){ + /* From the API docs: https://www.sqlite.org/c3ref/get_auxdata.html + + The value of the N parameter to these interfaces should be + non-negative. Future enhancements may make use of negative N + values to define new kinds of function caching behavior. + */ + valueAt(argNdx); + CApi.sqlite3_set_auxdata(cx, argNdx, o); + } + + /** + Analog to sqlite3_get_auxdata() but throws if argNdx is out of + range. + */ + public Object getAuxData(int argNdx){ + valueAt(argNdx); + return CApi.sqlite3_get_auxdata(cx, argNdx); + } + + /** + Represents a single SqlFunction argument. Primarily intended + for use with the Arguments class's Iterable interface. + */ + public final static class Arg { + private final Arguments a; + private final int ndx; + /* Only for use by the Arguments class. */ + private Arg(Arguments a, int ndx){ + this.a = a; + this.ndx = ndx; + } + /** Returns this argument's index in its parent argument list. */ + public int getIndex(){return ndx;} + public int getInt(){return a.getInt(ndx);} + public long getInt64(){return a.getInt64(ndx);} + public double getDouble(){return a.getDouble(ndx);} + public byte[] getBlob(){return a.getBlob(ndx);} + public byte[] getText(){return a.getText(ndx);} + public String getText16(){return a.getText16(ndx);} + public int getBytes(){return a.getBytes(ndx);} + public int getBytes16(){return a.getBytes16(ndx);} + public Object getObject(){return a.getObject(ndx);} + public T getObject(Class type){ return a.getObject(ndx, type); } + public int getType(){return a.getType(ndx);} + public Object getAuxData(){return a.getAuxData(ndx);} + public void setAuxData(Object o){a.setAuxData(ndx, o);} + } + + @Override + public java.util.Iterator iterator(){ + final Arg[] proxies = new Arg[args.length]; + for( int i = 0; i < args.length; ++i ){ + proxies[i] = new Arg(this, i); + } + return java.util.Arrays.stream(proxies).iterator(); + } + + } + + /** + Internal-use adapter for wrapping this package's ScalarFunction + for use with the org.sqlite.jni.capi.ScalarFunction interface. + */ + static final class ScalarAdapter extends org.sqlite.jni.capi.ScalarFunction { + private final ScalarFunction impl; + ScalarAdapter(ScalarFunction impl){ + this.impl = impl; + } + /** + Proxies this.impl.xFunc(), adapting the call arguments to that + function's signature. If the proxy throws, it's translated to + sqlite_result_error() with the exception's message. + */ + public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + try{ + impl.xFunc( new SqlFunction.Arguments(cx, args) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + public void xDestroy(){ + impl.xDestroy(); + } + } + + /** + Internal-use adapter for wrapping this package's AggregateFunction + for use with the org.sqlite.jni.capi.AggregateFunction interface. + */ + static /*cannot be final without duplicating the whole body in WindowAdapter*/ + class AggregateAdapter extends org.sqlite.jni.capi.AggregateFunction { + private final AggregateFunction impl; + AggregateAdapter(AggregateFunction impl){ + this.impl = impl; + } + + /** + Proxies this.impl.xStep(), adapting the call arguments to that + function's signature. If the proxied function throws, it is + translated to sqlite_result_error() with the exception's + message. + */ + public void xStep(sqlite3_context cx, sqlite3_value[] args){ + try{ + impl.xStep( new SqlFunction.Arguments(cx, args) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + /** + As for the xFinal() argument of the C API's + sqlite3_create_function(). If the proxied function throws, it + is translated into a sqlite3_result_error(). + */ + public void xFinal(sqlite3_context cx){ + try{ + impl.xFinal( new SqlFunction.Arguments(cx, null) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + public void xDestroy(){ + impl.xDestroy(); + } + } + + /** + Internal-use adapter for wrapping this package's WindowFunction + for use with the org.sqlite.jni.capi.WindowFunction interface. + */ + static final class WindowAdapter extends AggregateAdapter { + private final WindowFunction impl; + WindowAdapter(WindowFunction impl){ + super(impl); + this.impl = impl; + } + + /** + Proxies this.impl.xInverse(), adapting the call arguments to that + function's signature. If the proxied function throws, it is + translated to sqlite_result_error() with the exception's + message. + */ + public void xInverse(sqlite3_context cx, sqlite3_value[] args){ + try{ + impl.xInverse( new SqlFunction.Arguments(cx, args) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + /** + As for the xValue() argument of the C API's sqlite3_create_window_function(). + If the proxied function throws, it is translated into a sqlite3_result_error(). + */ + public void xValue(sqlite3_context cx){ + try{ + impl.xValue( new SqlFunction.Arguments(cx, null) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + public void xDestroy(){ + impl.xDestroy(); + } + } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java new file mode 100644 index 0000000000000000000000000000000000000000..de131e85421e81bfeed30963300d3d999923a210 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java @@ -0,0 +1,1991 @@ +/* +** 2023-10-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import java.nio.charset.StandardCharsets; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.capi.sqlite3; +import org.sqlite.jni.capi.sqlite3_stmt; +import org.sqlite.jni.capi.sqlite3_backup; +import org.sqlite.jni.capi.sqlite3_blob; +import org.sqlite.jni.capi.OutputPointer; +import java.nio.ByteBuffer; + +/** + This class represents a database connection, analog to the C-side + sqlite3 class but with added argument validation, exceptions, and + similar "smoothing of sharp edges" to make the API safe to use from + Java. It also acts as a namespace for other types for which + individual instances are tied to a specific database connection. +*/ +public final class Sqlite implements AutoCloseable { + private sqlite3 db; + private static final boolean JNI_SUPPORTS_NIO = + CApi.sqlite3_jni_supports_nio(); + + // Result codes + public static final int OK = CApi.SQLITE_OK; + public static final int ERROR = CApi.SQLITE_ERROR; + public static final int INTERNAL = CApi.SQLITE_INTERNAL; + public static final int PERM = CApi.SQLITE_PERM; + public static final int ABORT = CApi.SQLITE_ABORT; + public static final int BUSY = CApi.SQLITE_BUSY; + public static final int LOCKED = CApi.SQLITE_LOCKED; + public static final int NOMEM = CApi.SQLITE_NOMEM; + public static final int READONLY = CApi.SQLITE_READONLY; + public static final int INTERRUPT = CApi.SQLITE_INTERRUPT; + public static final int IOERR = CApi.SQLITE_IOERR; + public static final int CORRUPT = CApi.SQLITE_CORRUPT; + public static final int NOTFOUND = CApi.SQLITE_NOTFOUND; + public static final int FULL = CApi.SQLITE_FULL; + public static final int CANTOPEN = CApi.SQLITE_CANTOPEN; + public static final int PROTOCOL = CApi.SQLITE_PROTOCOL; + public static final int EMPTY = CApi.SQLITE_EMPTY; + public static final int SCHEMA = CApi.SQLITE_SCHEMA; + public static final int TOOBIG = CApi.SQLITE_TOOBIG; + public static final int CONSTRAINT = CApi. SQLITE_CONSTRAINT; + public static final int MISMATCH = CApi.SQLITE_MISMATCH; + public static final int MISUSE = CApi.SQLITE_MISUSE; + public static final int NOLFS = CApi.SQLITE_NOLFS; + public static final int AUTH = CApi.SQLITE_AUTH; + public static final int FORMAT = CApi.SQLITE_FORMAT; + public static final int RANGE = CApi.SQLITE_RANGE; + public static final int NOTADB = CApi.SQLITE_NOTADB; + public static final int NOTICE = CApi.SQLITE_NOTICE; + public static final int WARNING = CApi.SQLITE_WARNING; + public static final int ROW = CApi.SQLITE_ROW; + public static final int DONE = CApi.SQLITE_DONE; + public static final int ERROR_MISSING_COLLSEQ = CApi.SQLITE_ERROR_MISSING_COLLSEQ; + public static final int ERROR_RETRY = CApi.SQLITE_ERROR_RETRY; + public static final int ERROR_SNAPSHOT = CApi.SQLITE_ERROR_SNAPSHOT; + public static final int IOERR_READ = CApi.SQLITE_IOERR_READ; + public static final int IOERR_SHORT_READ = CApi.SQLITE_IOERR_SHORT_READ; + public static final int IOERR_WRITE = CApi.SQLITE_IOERR_WRITE; + public static final int IOERR_FSYNC = CApi.SQLITE_IOERR_FSYNC; + public static final int IOERR_DIR_FSYNC = CApi.SQLITE_IOERR_DIR_FSYNC; + public static final int IOERR_TRUNCATE = CApi.SQLITE_IOERR_TRUNCATE; + public static final int IOERR_FSTAT = CApi.SQLITE_IOERR_FSTAT; + public static final int IOERR_UNLOCK = CApi.SQLITE_IOERR_UNLOCK; + public static final int IOERR_RDLOCK = CApi.SQLITE_IOERR_RDLOCK; + public static final int IOERR_DELETE = CApi.SQLITE_IOERR_DELETE; + public static final int IOERR_BLOCKED = CApi.SQLITE_IOERR_BLOCKED; + public static final int IOERR_NOMEM = CApi.SQLITE_IOERR_NOMEM; + public static final int IOERR_ACCESS = CApi.SQLITE_IOERR_ACCESS; + public static final int IOERR_CHECKRESERVEDLOCK = CApi.SQLITE_IOERR_CHECKRESERVEDLOCK; + public static final int IOERR_LOCK = CApi.SQLITE_IOERR_LOCK; + public static final int IOERR_CLOSE = CApi.SQLITE_IOERR_CLOSE; + public static final int IOERR_DIR_CLOSE = CApi.SQLITE_IOERR_DIR_CLOSE; + public static final int IOERR_SHMOPEN = CApi.SQLITE_IOERR_SHMOPEN; + public static final int IOERR_SHMSIZE = CApi.SQLITE_IOERR_SHMSIZE; + public static final int IOERR_SHMLOCK = CApi.SQLITE_IOERR_SHMLOCK; + public static final int IOERR_SHMMAP = CApi.SQLITE_IOERR_SHMMAP; + public static final int IOERR_SEEK = CApi.SQLITE_IOERR_SEEK; + public static final int IOERR_DELETE_NOENT = CApi.SQLITE_IOERR_DELETE_NOENT; + public static final int IOERR_MMAP = CApi.SQLITE_IOERR_MMAP; + public static final int IOERR_GETTEMPPATH = CApi.SQLITE_IOERR_GETTEMPPATH; + public static final int IOERR_CONVPATH = CApi.SQLITE_IOERR_CONVPATH; + public static final int IOERR_VNODE = CApi.SQLITE_IOERR_VNODE; + public static final int IOERR_AUTH = CApi.SQLITE_IOERR_AUTH; + public static final int IOERR_BEGIN_ATOMIC = CApi.SQLITE_IOERR_BEGIN_ATOMIC; + public static final int IOERR_COMMIT_ATOMIC = CApi.SQLITE_IOERR_COMMIT_ATOMIC; + public static final int IOERR_ROLLBACK_ATOMIC = CApi.SQLITE_IOERR_ROLLBACK_ATOMIC; + public static final int IOERR_DATA = CApi.SQLITE_IOERR_DATA; + public static final int IOERR_CORRUPTFS = CApi.SQLITE_IOERR_CORRUPTFS; + public static final int LOCKED_SHAREDCACHE = CApi.SQLITE_LOCKED_SHAREDCACHE; + public static final int LOCKED_VTAB = CApi.SQLITE_LOCKED_VTAB; + public static final int BUSY_RECOVERY = CApi.SQLITE_BUSY_RECOVERY; + public static final int BUSY_SNAPSHOT = CApi.SQLITE_BUSY_SNAPSHOT; + public static final int BUSY_TIMEOUT = CApi.SQLITE_BUSY_TIMEOUT; + public static final int CANTOPEN_NOTEMPDIR = CApi.SQLITE_CANTOPEN_NOTEMPDIR; + public static final int CANTOPEN_ISDIR = CApi.SQLITE_CANTOPEN_ISDIR; + public static final int CANTOPEN_FULLPATH = CApi.SQLITE_CANTOPEN_FULLPATH; + public static final int CANTOPEN_CONVPATH = CApi.SQLITE_CANTOPEN_CONVPATH; + public static final int CANTOPEN_SYMLINK = CApi.SQLITE_CANTOPEN_SYMLINK; + public static final int CORRUPT_VTAB = CApi.SQLITE_CORRUPT_VTAB; + public static final int CORRUPT_SEQUENCE = CApi.SQLITE_CORRUPT_SEQUENCE; + public static final int CORRUPT_INDEX = CApi.SQLITE_CORRUPT_INDEX; + public static final int READONLY_RECOVERY = CApi.SQLITE_READONLY_RECOVERY; + public static final int READONLY_CANTLOCK = CApi.SQLITE_READONLY_CANTLOCK; + public static final int READONLY_ROLLBACK = CApi.SQLITE_READONLY_ROLLBACK; + public static final int READONLY_DBMOVED = CApi.SQLITE_READONLY_DBMOVED; + public static final int READONLY_CANTINIT = CApi.SQLITE_READONLY_CANTINIT; + public static final int READONLY_DIRECTORY = CApi.SQLITE_READONLY_DIRECTORY; + public static final int ABORT_ROLLBACK = CApi.SQLITE_ABORT_ROLLBACK; + public static final int CONSTRAINT_CHECK = CApi.SQLITE_CONSTRAINT_CHECK; + public static final int CONSTRAINT_COMMITHOOK = CApi.SQLITE_CONSTRAINT_COMMITHOOK; + public static final int CONSTRAINT_FOREIGNKEY = CApi.SQLITE_CONSTRAINT_FOREIGNKEY; + public static final int CONSTRAINT_FUNCTION = CApi.SQLITE_CONSTRAINT_FUNCTION; + public static final int CONSTRAINT_NOTNULL = CApi.SQLITE_CONSTRAINT_NOTNULL; + public static final int CONSTRAINT_PRIMARYKEY = CApi.SQLITE_CONSTRAINT_PRIMARYKEY; + public static final int CONSTRAINT_TRIGGER = CApi.SQLITE_CONSTRAINT_TRIGGER; + public static final int CONSTRAINT_UNIQUE = CApi.SQLITE_CONSTRAINT_UNIQUE; + public static final int CONSTRAINT_VTAB = CApi.SQLITE_CONSTRAINT_VTAB; + public static final int CONSTRAINT_ROWID = CApi.SQLITE_CONSTRAINT_ROWID; + public static final int CONSTRAINT_PINNED = CApi.SQLITE_CONSTRAINT_PINNED; + public static final int CONSTRAINT_DATATYPE = CApi.SQLITE_CONSTRAINT_DATATYPE; + public static final int NOTICE_RECOVER_WAL = CApi.SQLITE_NOTICE_RECOVER_WAL; + public static final int NOTICE_RECOVER_ROLLBACK = CApi.SQLITE_NOTICE_RECOVER_ROLLBACK; + public static final int WARNING_AUTOINDEX = CApi.SQLITE_WARNING_AUTOINDEX; + public static final int AUTH_USER = CApi.SQLITE_AUTH_USER; + public static final int OK_LOAD_PERMANENTLY = CApi.SQLITE_OK_LOAD_PERMANENTLY; + + // sqlite3_open() flags + public static final int OPEN_READWRITE = CApi.SQLITE_OPEN_READWRITE; + public static final int OPEN_CREATE = CApi.SQLITE_OPEN_CREATE; + public static final int OPEN_EXRESCODE = CApi.SQLITE_OPEN_EXRESCODE; + + // transaction state + public static final int TXN_NONE = CApi.SQLITE_TXN_NONE; + public static final int TXN_READ = CApi.SQLITE_TXN_READ; + public static final int TXN_WRITE = CApi.SQLITE_TXN_WRITE; + + // sqlite3_status() ops + public static final int STATUS_MEMORY_USED = CApi.SQLITE_STATUS_MEMORY_USED; + public static final int STATUS_PAGECACHE_USED = CApi.SQLITE_STATUS_PAGECACHE_USED; + public static final int STATUS_PAGECACHE_OVERFLOW = CApi.SQLITE_STATUS_PAGECACHE_OVERFLOW; + public static final int STATUS_MALLOC_SIZE = CApi.SQLITE_STATUS_MALLOC_SIZE; + public static final int STATUS_PARSER_STACK = CApi.SQLITE_STATUS_PARSER_STACK; + public static final int STATUS_PAGECACHE_SIZE = CApi.SQLITE_STATUS_PAGECACHE_SIZE; + public static final int STATUS_MALLOC_COUNT = CApi.SQLITE_STATUS_MALLOC_COUNT; + + // sqlite3_db_status() ops + public static final int DBSTATUS_LOOKASIDE_USED = CApi.SQLITE_DBSTATUS_LOOKASIDE_USED; + public static final int DBSTATUS_CACHE_USED = CApi.SQLITE_DBSTATUS_CACHE_USED; + public static final int DBSTATUS_SCHEMA_USED = CApi.SQLITE_DBSTATUS_SCHEMA_USED; + public static final int DBSTATUS_STMT_USED = CApi.SQLITE_DBSTATUS_STMT_USED; + public static final int DBSTATUS_LOOKASIDE_HIT = CApi.SQLITE_DBSTATUS_LOOKASIDE_HIT; + public static final int DBSTATUS_LOOKASIDE_MISS_SIZE = CApi.SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE; + public static final int DBSTATUS_LOOKASIDE_MISS_FULL = CApi.SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL; + public static final int DBSTATUS_CACHE_HIT = CApi.SQLITE_DBSTATUS_CACHE_HIT; + public static final int DBSTATUS_CACHE_MISS = CApi.SQLITE_DBSTATUS_CACHE_MISS; + public static final int DBSTATUS_CACHE_WRITE = CApi.SQLITE_DBSTATUS_CACHE_WRITE; + public static final int DBSTATUS_DEFERRED_FKS = CApi.SQLITE_DBSTATUS_DEFERRED_FKS; + public static final int DBSTATUS_CACHE_USED_SHARED = CApi.SQLITE_DBSTATUS_CACHE_USED_SHARED; + public static final int DBSTATUS_CACHE_SPILL = CApi.SQLITE_DBSTATUS_CACHE_SPILL; + + // Limits + public static final int LIMIT_LENGTH = CApi.SQLITE_LIMIT_LENGTH; + public static final int LIMIT_SQL_LENGTH = CApi.SQLITE_LIMIT_SQL_LENGTH; + public static final int LIMIT_COLUMN = CApi.SQLITE_LIMIT_COLUMN; + public static final int LIMIT_EXPR_DEPTH = CApi.SQLITE_LIMIT_EXPR_DEPTH; + public static final int LIMIT_COMPOUND_SELECT = CApi.SQLITE_LIMIT_COMPOUND_SELECT; + public static final int LIMIT_VDBE_OP = CApi.SQLITE_LIMIT_VDBE_OP; + public static final int LIMIT_FUNCTION_ARG = CApi.SQLITE_LIMIT_FUNCTION_ARG; + public static final int LIMIT_ATTACHED = CApi.SQLITE_LIMIT_ATTACHED; + public static final int LIMIT_LIKE_PATTERN_LENGTH = CApi.SQLITE_LIMIT_LIKE_PATTERN_LENGTH; + public static final int LIMIT_VARIABLE_NUMBER = CApi.SQLITE_LIMIT_VARIABLE_NUMBER; + public static final int LIMIT_TRIGGER_DEPTH = CApi.SQLITE_LIMIT_TRIGGER_DEPTH; + public static final int LIMIT_WORKER_THREADS = CApi.SQLITE_LIMIT_WORKER_THREADS; + + // sqlite3_prepare_v3() flags + public static final int PREPARE_PERSISTENT = CApi.SQLITE_PREPARE_PERSISTENT; + public static final int PREPARE_NO_VTAB = CApi.SQLITE_PREPARE_NO_VTAB; + + // sqlite3_trace_v2() flags + public static final int TRACE_STMT = CApi.SQLITE_TRACE_STMT; + public static final int TRACE_PROFILE = CApi.SQLITE_TRACE_PROFILE; + public static final int TRACE_ROW = CApi.SQLITE_TRACE_ROW; + public static final int TRACE_CLOSE = CApi.SQLITE_TRACE_CLOSE; + public static final int TRACE_ALL = TRACE_STMT | TRACE_PROFILE | TRACE_ROW | TRACE_CLOSE; + + // sqlite3_db_config() ops + public static final int DBCONFIG_ENABLE_FKEY = CApi.SQLITE_DBCONFIG_ENABLE_FKEY; + public static final int DBCONFIG_ENABLE_TRIGGER = CApi.SQLITE_DBCONFIG_ENABLE_TRIGGER; + public static final int DBCONFIG_ENABLE_FTS3_TOKENIZER = CApi.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER; + public static final int DBCONFIG_ENABLE_LOAD_EXTENSION = CApi.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION; + public static final int DBCONFIG_NO_CKPT_ON_CLOSE = CApi.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE; + public static final int DBCONFIG_ENABLE_QPSG = CApi.SQLITE_DBCONFIG_ENABLE_QPSG; + public static final int DBCONFIG_TRIGGER_EQP = CApi.SQLITE_DBCONFIG_TRIGGER_EQP; + public static final int DBCONFIG_RESET_DATABASE = CApi.SQLITE_DBCONFIG_RESET_DATABASE; + public static final int DBCONFIG_DEFENSIVE = CApi.SQLITE_DBCONFIG_DEFENSIVE; + public static final int DBCONFIG_WRITABLE_SCHEMA = CApi.SQLITE_DBCONFIG_WRITABLE_SCHEMA; + public static final int DBCONFIG_LEGACY_ALTER_TABLE = CApi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE; + public static final int DBCONFIG_DQS_DML = CApi.SQLITE_DBCONFIG_DQS_DML; + public static final int DBCONFIG_DQS_DDL = CApi.SQLITE_DBCONFIG_DQS_DDL; + public static final int DBCONFIG_ENABLE_VIEW = CApi.SQLITE_DBCONFIG_ENABLE_VIEW; + public static final int DBCONFIG_LEGACY_FILE_FORMAT = CApi.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT; + public static final int DBCONFIG_TRUSTED_SCHEMA = CApi.SQLITE_DBCONFIG_TRUSTED_SCHEMA; + public static final int DBCONFIG_STMT_SCANSTATUS = CApi.SQLITE_DBCONFIG_STMT_SCANSTATUS; + public static final int DBCONFIG_REVERSE_SCANORDER = CApi.SQLITE_DBCONFIG_REVERSE_SCANORDER; + + // sqlite3_config() ops + public static final int CONFIG_SINGLETHREAD = CApi.SQLITE_CONFIG_SINGLETHREAD; + public static final int CONFIG_MULTITHREAD = CApi.SQLITE_CONFIG_MULTITHREAD; + public static final int CONFIG_SERIALIZED = CApi.SQLITE_CONFIG_SERIALIZED; + + // Encodings + public static final int UTF8 = CApi.SQLITE_UTF8; + public static final int UTF16 = CApi.SQLITE_UTF16; + public static final int UTF16LE = CApi.SQLITE_UTF16LE; + public static final int UTF16BE = CApi.SQLITE_UTF16BE; + /* We elide the UTF16_ALIGNED from this interface because it + is irrelevant for the Java interface. */ + + // SQL data type IDs + public static final int INTEGER = CApi.SQLITE_INTEGER; + public static final int FLOAT = CApi.SQLITE_FLOAT; + public static final int TEXT = CApi.SQLITE_TEXT; + public static final int BLOB = CApi.SQLITE_BLOB; + public static final int NULL = CApi.SQLITE_NULL; + + // Authorizer codes. + public static final int DENY = CApi.SQLITE_DENY; + public static final int IGNORE = CApi.SQLITE_IGNORE; + public static final int CREATE_INDEX = CApi.SQLITE_CREATE_INDEX; + public static final int CREATE_TABLE = CApi.SQLITE_CREATE_TABLE; + public static final int CREATE_TEMP_INDEX = CApi.SQLITE_CREATE_TEMP_INDEX; + public static final int CREATE_TEMP_TABLE = CApi.SQLITE_CREATE_TEMP_TABLE; + public static final int CREATE_TEMP_TRIGGER = CApi.SQLITE_CREATE_TEMP_TRIGGER; + public static final int CREATE_TEMP_VIEW = CApi.SQLITE_CREATE_TEMP_VIEW; + public static final int CREATE_TRIGGER = CApi.SQLITE_CREATE_TRIGGER; + public static final int CREATE_VIEW = CApi.SQLITE_CREATE_VIEW; + public static final int DELETE = CApi.SQLITE_DELETE; + public static final int DROP_INDEX = CApi.SQLITE_DROP_INDEX; + public static final int DROP_TABLE = CApi.SQLITE_DROP_TABLE; + public static final int DROP_TEMP_INDEX = CApi.SQLITE_DROP_TEMP_INDEX; + public static final int DROP_TEMP_TABLE = CApi.SQLITE_DROP_TEMP_TABLE; + public static final int DROP_TEMP_TRIGGER = CApi.SQLITE_DROP_TEMP_TRIGGER; + public static final int DROP_TEMP_VIEW = CApi.SQLITE_DROP_TEMP_VIEW; + public static final int DROP_TRIGGER = CApi.SQLITE_DROP_TRIGGER; + public static final int DROP_VIEW = CApi.SQLITE_DROP_VIEW; + public static final int INSERT = CApi.SQLITE_INSERT; + public static final int PRAGMA = CApi.SQLITE_PRAGMA; + public static final int READ = CApi.SQLITE_READ; + public static final int SELECT = CApi.SQLITE_SELECT; + public static final int TRANSACTION = CApi.SQLITE_TRANSACTION; + public static final int UPDATE = CApi.SQLITE_UPDATE; + public static final int ATTACH = CApi.SQLITE_ATTACH; + public static final int DETACH = CApi.SQLITE_DETACH; + public static final int ALTER_TABLE = CApi.SQLITE_ALTER_TABLE; + public static final int REINDEX = CApi.SQLITE_REINDEX; + public static final int ANALYZE = CApi.SQLITE_ANALYZE; + public static final int CREATE_VTABLE = CApi.SQLITE_CREATE_VTABLE; + public static final int DROP_VTABLE = CApi.SQLITE_DROP_VTABLE; + public static final int FUNCTION = CApi.SQLITE_FUNCTION; + public static final int SAVEPOINT = CApi.SQLITE_SAVEPOINT; + public static final int RECURSIVE = CApi.SQLITE_RECURSIVE; + + //! Used only by the open() factory functions. + private Sqlite(sqlite3 db){ + this.db = db; + } + + /** Maps org.sqlite.jni.capi.sqlite3 to Sqlite instances. */ + private static final java.util.Map nativeToWrapper + = new java.util.HashMap<>(); + + + /** + When any given thread is done using the SQLite library, calling + this will free up any native-side resources which may be + associated specifically with that thread. This is not strictly + necessary, in particular in applications which only use SQLite + from a single thread, but may help free some otherwise errant + resources. + + Calling into SQLite from a given thread after this has been + called in that thread is harmless. The library will simply start + to re-cache certain state for that thread. + + Contrariwise, failing to call this will effectively leak a small + amount of cached state for the thread, which may add up to + significant amounts if the application uses SQLite from many + threads. + + This must never be called while actively using SQLite from this + thread, e.g. from within a query loop or a callback which is + operating on behalf of the library. + */ + static void uncacheThread(){ + CApi.sqlite3_java_uncache_thread(); + } + + /** + Returns the Sqlite object associated with the given sqlite3 + object, or null if there is no such mapping. + */ + static Sqlite fromNative(sqlite3 low){ + synchronized(nativeToWrapper){ + return nativeToWrapper.get(low); + } + } + + /** + Returns a newly-opened db connection or throws SqliteException if + opening fails. All arguments are as documented for + sqlite3_open_v2(). + + Design question: do we want static factory functions or should + this be reformulated as a constructor? + */ + public static Sqlite open(String filename, int flags, String vfsName){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + final int rc = CApi.sqlite3_open_v2(filename, out, flags, vfsName); + final sqlite3 n = out.take(); + if( 0!=rc ){ + if( null==n ) throw new SqliteException(rc); + final SqliteException ex = new SqliteException(n); + n.close(); + throw ex; + } + final Sqlite rv = new Sqlite(n); + synchronized(nativeToWrapper){ + nativeToWrapper.put(n, rv); + } + runAutoExtensions(rv); + return rv; + } + + public static Sqlite open(String filename, int flags){ + return open(filename, flags, null); + } + + public static Sqlite open(String filename){ + return open(filename, OPEN_READWRITE|OPEN_CREATE, null); + } + + public static String libVersion(){ + return CApi.sqlite3_libversion(); + } + + public static int libVersionNumber(){ + return CApi.sqlite3_libversion_number(); + } + + public static String libSourceId(){ + return CApi.sqlite3_sourceid(); + } + + /** + Returns the value of the native library's build-time value of the + SQLITE_THREADSAFE build option. + */ + public static int libThreadsafe(){ + return CApi.sqlite3_threadsafe(); + } + + /** + Analog to sqlite3_compileoption_get(). + */ + public static String compileOptionGet(int n){ + return CApi.sqlite3_compileoption_get(n); + } + + /** + Analog to sqlite3_compileoption_used(). + */ + public static boolean compileOptionUsed(String optName){ + return CApi.sqlite3_compileoption_used(optName); + } + + private static boolean hasNormalizeSql = + compileOptionUsed("ENABLE_NORMALIZE"); + + private static boolean hasSqlLog = + compileOptionUsed("ENABLE_SQLLOG"); + + /** + Throws UnsupportedOperationException if check is false. + flag is expected to be the name of an SQLITE_ENABLE_... + build flag. + */ + private static void checkSupported(boolean check, String flag){ + if( !check ){ + throw new UnsupportedOperationException( + "Library was built without "+flag + ); + } + } + + /** + Analog to sqlite3_complete(). + */ + public static boolean isCompleteStatement(String sql){ + switch(CApi.sqlite3_complete(sql)){ + case 0: return false; + case CApi.SQLITE_MISUSE: + throw new IllegalArgumentException("Input may not be null."); + case CApi.SQLITE_NOMEM: + throw new OutOfMemoryError(); + default: + return true; + } + } + + public static int keywordCount(){ + return CApi.sqlite3_keyword_count(); + } + + public static boolean keywordCheck(String word){ + return CApi.sqlite3_keyword_check(word); + } + + public static String keywordName(int index){ + return CApi.sqlite3_keyword_name(index); + } + + public static boolean strglob(String glob, String txt){ + return 0==CApi.sqlite3_strglob(glob, txt); + } + + public static boolean strlike(String glob, String txt, char escChar){ + return 0==CApi.sqlite3_strlike(glob, txt, escChar); + } + + /** + Output object for use with status() and libStatus(). + */ + public static final class Status { + /** The current value for the requested status() or libStatus() metric. */ + long current; + /** The peak value for the requested status() or libStatus() metric. */ + long peak; + }; + + /** + As per sqlite3_status64(), but returns its current and high-water + results as a Status object. Throws if the first argument is + not one of the STATUS_... constants. + */ + public static Status libStatus(int op, boolean resetStats){ + org.sqlite.jni.capi.OutputPointer.Int64 pCurrent = + new org.sqlite.jni.capi.OutputPointer.Int64(); + org.sqlite.jni.capi.OutputPointer.Int64 pHighwater = + new org.sqlite.jni.capi.OutputPointer.Int64(); + checkRcStatic( CApi.sqlite3_status64(op, pCurrent, pHighwater, resetStats) ); + final Status s = new Status(); + s.current = pCurrent.value; + s.peak = pHighwater.value; + return s; + } + + /** + As per sqlite3_db_status(), but returns its current and + high-water results as a Status object. Throws if the first + argument is not one of the DBSTATUS_... constants or on any other + misuse. + */ + public Status status(int op, boolean resetStats){ + org.sqlite.jni.capi.OutputPointer.Int32 pCurrent = + new org.sqlite.jni.capi.OutputPointer.Int32(); + org.sqlite.jni.capi.OutputPointer.Int32 pHighwater = + new org.sqlite.jni.capi.OutputPointer.Int32(); + checkRc( CApi.sqlite3_db_status(thisDb(), op, pCurrent, pHighwater, resetStats) ); + final Status s = new Status(); + s.current = pCurrent.value; + s.peak = pHighwater.value; + return s; + } + + @Override public void close(){ + if(null!=this.db){ + synchronized(nativeToWrapper){ + nativeToWrapper.remove(this.db); + } + this.db.close(); + this.db = null; + } + } + + /** + Returns this object's underlying native db handle, or null if + this instance has been closed. This is very specifically not + public. + */ + sqlite3 nativeHandle(){ return this.db; } + + private sqlite3 thisDb(){ + if( null==db || 0==db.getNativePointer() ){ + throw new IllegalArgumentException("This database instance is closed."); + } + return this.db; + } + + // private byte[] stringToUtf8(String s){ + // return s==null ? null : s.getBytes(StandardCharsets.UTF_8); + // } + + /** + If rc!=0, throws an SqliteException. If this db is currently + opened and has non-0 sqlite3_errcode(), the error state is + extracted from it, else only the string form of rc is used. It is + the caller's responsibility to filter out non-error codes such as + SQLITE_ROW and SQLITE_DONE before calling this. + + As a special case, if rc is SQLITE_NOMEM, an OutOfMemoryError is + thrown. + */ + private void checkRc(int rc){ + if( 0!=rc ){ + if( CApi.SQLITE_NOMEM==rc ){ + throw new OutOfMemoryError(); + }else if( null==db || 0==CApi.sqlite3_errcode(db) ){ + throw new SqliteException(rc); + }else{ + throw new SqliteException(db); + } + } + } + + /** + Like checkRc() but behaves as if that function were + called with a null db object. + */ + private static void checkRcStatic(int rc){ + if( 0!=rc ){ + if( CApi.SQLITE_NOMEM==rc ){ + throw new OutOfMemoryError(); + }else{ + throw new SqliteException(rc); + } + } + } + + /** + Toggles the use of extended result codes on or off. By default + they are turned off, but they can be enabled by default by + including the OPEN_EXRESCODE flag when opening a database. + + Because this API reports db-side errors using exceptions, + enabling this may change the values returned by + SqliteException.errcode(). + */ + public void useExtendedResultCodes(boolean on){ + checkRc( CApi.sqlite3_extended_result_codes(thisDb(), on) ); + } + + /** + Analog to sqlite3_prepare_v3(), this prepares the first SQL + statement from the given input string and returns it as a + Stmt. It throws an SqliteException if preparation fails or an + IllegalArgumentException if the input is empty (e.g. contains + only comments or whitespace). + + The first argument must be SQL input in UTF-8 encoding. + + prepFlags must be 0 or a bitmask of the PREPARE_... constants. + + For processing multiple statements from a single input, use + prepareMulti(). + + Design note: though the C-level API succeeds with a null + statement object for empty inputs, that approach is cumbersome to + use in higher-level APIs because every prepared statement has to + be checked for null before using it. + */ + public Stmt prepare(byte utf8Sql[], int prepFlags){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + final int rc = CApi.sqlite3_prepare_v3(thisDb(), utf8Sql, prepFlags, out); + checkRc(rc); + final sqlite3_stmt q = out.take(); + if( null==q ){ + /* The C-level API treats input which is devoid of SQL + statements (e.g. all comments or an empty string) as success + but returns a NULL sqlite3_stmt object. In higher-level APIs, + wrapping a "successful NULL" object that way is tedious to + use because it forces clients and/or wrapper-level code to + check for that unusual case. In practice, higher-level + bindings are generally better-served by treating empty SQL + input as an error. */ + throw new IllegalArgumentException("Input contains no SQL statements."); + } + return new Stmt(this, q); + } + + /** + Equivalent to prepare(X, prepFlags), where X is + sql.getBytes(StandardCharsets.UTF_8). + */ + public Stmt prepare(String sql, int prepFlags){ + return prepare( sql.getBytes(StandardCharsets.UTF_8), prepFlags ); + } + + /** + Equivalent to prepare(sql, 0). + */ + public Stmt prepare(String sql){ + return prepare(sql, 0); + } + + + /** + Callback type for use with prepareMulti(). + */ + public interface PrepareMulti { + /** + Gets passed a Stmt which it may handle in arbitrary ways. + Ownership of st is passed to this function. It must throw on + error. + */ + void call(Sqlite.Stmt st); + } + + /** + A PrepareMulti implementation which calls another PrepareMulti + object and then finalizes its statement. + */ + public static class PrepareMultiFinalize implements PrepareMulti { + private final PrepareMulti pm; + /** + Proxies the given PrepareMulti via this object's call() method. + */ + public PrepareMultiFinalize(PrepareMulti proxy){ + this.pm = proxy; + } + /** + Passes st to the call() method of the object this one proxies, + then finalizes st, propagating any exceptions from call() after + finalizing st. + */ + @Override public void call(Stmt st){ + try{ pm.call(st); } + finally{ st.finalizeStmt(); } + } + } + + /** + Equivalent to prepareMulti(sql,0,visitor). + */ + public void prepareMulti(String sql, PrepareMulti visitor){ + prepareMulti( sql, 0, visitor ); + } + + /** + Equivallent to prepareMulti(X,prepFlags,visitor), where X is + sql.getBytes(StandardCharsets.UTF_8). + */ + public void prepareMulti(String sql, int prepFlags, PrepareMulti visitor){ + prepareMulti(sql.getBytes(StandardCharsets.UTF_8), prepFlags, visitor); + } + + /** + A variant of prepare() which can handle multiple SQL statements + in a single input string. For each statement in the given string, + the statement is passed to visitor.call() a single time, passing + ownership of the statement to that function. This function does + not step() or close() statements - those operations are left to + caller or the visitor function. + + Unlike prepare(), this function does not fail if the input + contains only whitespace or SQL comments. In that case it is up + to the caller to arrange for that to be an error (if desired). + + PrepareMultiFinalize offers a proxy which finalizes each + statement after it is passed to another client-defined visitor. + + Be aware that certain legal SQL constructs may fail in the + preparation phase, before the corresponding statement can be + stepped. Most notably, authorizer checks which disallow access to + something in a statement behave that way. + */ + public void prepareMulti(byte sqlUtf8[], int prepFlags, PrepareMulti visitor){ + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + final org.sqlite.jni.capi.OutputPointer.sqlite3_stmt outStmt = + new org.sqlite.jni.capi.OutputPointer.sqlite3_stmt(); + final org.sqlite.jni.capi.OutputPointer.Int32 oTail = + new org.sqlite.jni.capi.OutputPointer.Int32(); + while( pos < sqlChunk.length ){ + sqlite3_stmt stmt = null; + if( pos>0 ){ + sqlChunk = java.util.Arrays.copyOfRange(sqlChunk, pos, sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + checkRc( + CApi.sqlite3_prepare_v3(db, sqlChunk, prepFlags, outStmt, oTail) + ); + pos = oTail.value; + stmt = outStmt.take(); + if( null==stmt ){ + /* empty statement, e.g. only comments or whitespace, was parsed. */ + continue; + } + visitor.call(new Stmt(this, stmt)); + } + } + + public void createFunction(String name, int nArg, int eTextRep, ScalarFunction f){ + int rc = CApi.sqlite3_create_function(thisDb(), name, nArg, eTextRep, + new SqlFunction.ScalarAdapter(f)); + if( 0!=rc ) throw new SqliteException(db); + } + + public void createFunction(String name, int nArg, ScalarFunction f){ + this.createFunction(name, nArg, CApi.SQLITE_UTF8, f); + } + + public void createFunction(String name, int nArg, int eTextRep, AggregateFunction f){ + int rc = CApi.sqlite3_create_function(thisDb(), name, nArg, eTextRep, + new SqlFunction.AggregateAdapter(f)); + if( 0!=rc ) throw new SqliteException(db); + } + + public void createFunction(String name, int nArg, AggregateFunction f){ + this.createFunction(name, nArg, CApi.SQLITE_UTF8, f); + } + + public void createFunction(String name, int nArg, int eTextRep, WindowFunction f){ + int rc = CApi.sqlite3_create_function(thisDb(), name, nArg, eTextRep, + new SqlFunction.WindowAdapter(f)); + if( 0!=rc ) throw new SqliteException(db); + } + + public void createFunction(String name, int nArg, WindowFunction f){ + this.createFunction(name, nArg, CApi.SQLITE_UTF8, f); + } + + public long changes(){ + return CApi.sqlite3_changes64(thisDb()); + } + + public long totalChanges(){ + return CApi.sqlite3_total_changes64(thisDb()); + } + + public long lastInsertRowId(){ + return CApi.sqlite3_last_insert_rowid(thisDb()); + } + + public void setLastInsertRowId(long rowId){ + CApi.sqlite3_set_last_insert_rowid(thisDb(), rowId); + } + + public void interrupt(){ + CApi.sqlite3_interrupt(thisDb()); + } + + public boolean isInterrupted(){ + return CApi.sqlite3_is_interrupted(thisDb()); + } + + public boolean isAutoCommit(){ + return CApi.sqlite3_get_autocommit(thisDb()); + } + + /** + Analog to sqlite3_txn_state(). Returns one of TXN_NONE, TXN_READ, + or TXN_WRITE to denote this database's current transaction state + for the given schema name (or the most restrictive state of any + schema if zSchema is null). + */ + public int transactionState(String zSchema){ + return CApi.sqlite3_txn_state(thisDb(), zSchema); + } + + /** + Analog to sqlite3_db_name(). Returns null if passed an unknown + index. + */ + public String dbName(int dbNdx){ + return CApi.sqlite3_db_name(thisDb(), dbNdx); + } + + /** + Analog to sqlite3_db_filename(). Returns null if passed an + unknown db name. + */ + public String dbFileName(String dbName){ + return CApi.sqlite3_db_filename(thisDb(), dbName); + } + + /** + Analog to sqlite3_db_config() for the call forms which take one + of the boolean-type db configuration flags (namely the + DBCONFIG_... constants defined in this class). On success it + returns the result of that underlying call. Throws on error. + */ + public boolean dbConfig(int op, boolean on){ + org.sqlite.jni.capi.OutputPointer.Int32 pOut = + new org.sqlite.jni.capi.OutputPointer.Int32(); + checkRc( CApi.sqlite3_db_config(thisDb(), op, on ? 1 : 0, pOut) ); + return pOut.get()!=0; + } + + /** + Analog to the variant of sqlite3_db_config() for configuring the + SQLITE_DBCONFIG_MAINDBNAME option. Throws on error. + */ + public void setMainDbName(String name){ + checkRc( + CApi.sqlite3_db_config(thisDb(), CApi.SQLITE_DBCONFIG_MAINDBNAME, + name) + ); + } + + /** + Analog to sqlite3_db_readonly() but throws an SqliteException + with result code SQLITE_NOTFOUND if given an unknown database + name. + */ + public boolean readOnly(String dbName){ + final int rc = CApi.sqlite3_db_readonly(thisDb(), dbName); + if( 0==rc ) return false; + else if( rc>0 ) return true; + throw new SqliteException(CApi.SQLITE_NOTFOUND); + } + + /** + Analog to sqlite3_db_release_memory(). + */ + public void releaseMemory(){ + CApi.sqlite3_db_release_memory(thisDb()); + } + + /** + Analog to sqlite3_release_memory(). + */ + public static int libReleaseMemory(int n){ + return CApi.sqlite3_release_memory(n); + } + + /** + Analog to sqlite3_limit(). limitId must be one of the + LIMIT_... constants. + + Returns the old limit for the given option. If newLimit is + negative, it returns the old limit without modifying the limit. + + If sqlite3_limit() returns a negative value, this function throws + an SqliteException with the SQLITE_RANGE result code but no + further error info (because that case does not qualify as a + db-level error). Such errors may indicate an invalid argument + value or an invalid range for newLimit (the underlying function + does not differentiate between those). + */ + public int limit(int limitId, int newLimit){ + final int rc = CApi.sqlite3_limit(thisDb(), limitId, newLimit); + if( rc<0 ){ + throw new SqliteException(CApi.SQLITE_RANGE); + } + return rc; + } + + /** + Analog to sqlite3_errstr(). + */ + static String errstr(int resultCode){ + return CApi.sqlite3_errstr(resultCode); + } + + /** + A wrapper object for use with tableColumnMetadata(). They are + created and populated only via that interface. + */ + public final class TableColumnMetadata { + Boolean pNotNull = null; + Boolean pPrimaryKey = null; + Boolean pAutoinc = null; + String pzCollSeq = null; + String pzDataType = null; + + private TableColumnMetadata(){} + + public String getDataType(){ return pzDataType; } + public String getCollation(){ return pzCollSeq; } + public boolean isNotNull(){ return pNotNull; } + public boolean isPrimaryKey(){ return pPrimaryKey; } + public boolean isAutoincrement(){ return pAutoinc; } + } + + /** + Returns data about a database, table, and (optionally) column + (which may be null), as per sqlite3_table_column_metadata(). + Throws if passed invalid arguments, else returns the result as a + new TableColumnMetadata object. + */ + TableColumnMetadata tableColumnMetadata( + String zDbName, String zTableName, String zColumnName + ){ + org.sqlite.jni.capi.OutputPointer.String pzDataType + = new org.sqlite.jni.capi.OutputPointer.String(); + org.sqlite.jni.capi.OutputPointer.String pzCollSeq + = new org.sqlite.jni.capi.OutputPointer.String(); + org.sqlite.jni.capi.OutputPointer.Bool pNotNull + = new org.sqlite.jni.capi.OutputPointer.Bool(); + org.sqlite.jni.capi.OutputPointer.Bool pPrimaryKey + = new org.sqlite.jni.capi.OutputPointer.Bool(); + org.sqlite.jni.capi.OutputPointer.Bool pAutoinc + = new org.sqlite.jni.capi.OutputPointer.Bool(); + final int rc = CApi.sqlite3_table_column_metadata( + thisDb(), zDbName, zTableName, zColumnName, + pzDataType, pzCollSeq, pNotNull, pPrimaryKey, pAutoinc + ); + checkRc(rc); + TableColumnMetadata rv = new TableColumnMetadata(); + rv.pzDataType = pzDataType.value; + rv.pzCollSeq = pzCollSeq.value; + rv.pNotNull = pNotNull.value; + rv.pPrimaryKey = pPrimaryKey.value; + rv.pAutoinc = pAutoinc.value; + return rv; + } + + public interface TraceCallback { + /** + Called by sqlite3 for various tracing operations, as per + sqlite3_trace_v2(). Note that this interface elides the 2nd + argument to the native trace callback, as that role is better + filled by instance-local state. + +

These callbacks may throw, in which case their exceptions are + converted to C-level error information. + +

The 2nd argument to this function, if non-null, will be a an + Sqlite or Sqlite.Stmt object, depending on the first argument + (see below). + +

The final argument to this function is the "X" argument + documented for sqlite3_trace() and sqlite3_trace_v2(). Its type + depends on value of the first argument: + +

- SQLITE_TRACE_STMT: pNative is a Sqlite.Stmt. pX is a String + containing the prepared SQL. + +

- SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long + holding an approximate number of nanoseconds the statement took + to run. + +

- SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null. + +

- SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null. + */ + void call(int traceFlag, Object pNative, Object pX); + } + + /** + Analog to sqlite3_trace_v2(). traceMask must be a mask of the + TRACE_... constants. Pass a null callback to remove tracing. + + Throws on error. + */ + public void trace(int traceMask, TraceCallback callback){ + final Sqlite self = this; + final org.sqlite.jni.capi.TraceV2Callback tc = + (null==callback) ? null : new org.sqlite.jni.capi.TraceV2Callback(){ + @SuppressWarnings("unchecked") + @Override public int call(int flag, Object pNative, Object pX){ + switch(flag){ + case TRACE_ROW: + case TRACE_PROFILE: + case TRACE_STMT: + callback.call(flag, Sqlite.Stmt.fromNative((sqlite3_stmt)pNative), pX); + break; + case TRACE_CLOSE: + callback.call(flag, self, pX); + break; + } + return 0; + } + }; + checkRc( CApi.sqlite3_trace_v2(thisDb(), traceMask, tc) ); + }; + + /** + Corresponds to the sqlite3_stmt class. Use Sqlite.prepare() to + create new instances. + */ + public static final class Stmt implements AutoCloseable { + private Sqlite _db = null; + private sqlite3_stmt stmt = null; + + /** Only called by the prepare() factory functions. */ + Stmt(Sqlite db, sqlite3_stmt stmt){ + this._db = db; + this.stmt = stmt; + synchronized(nativeToWrapper){ + nativeToWrapper.put(this.stmt, this); + } + } + + sqlite3_stmt nativeHandle(){ + return stmt; + } + + /** Maps org.sqlite.jni.capi.sqlite3_stmt to Stmt instances. */ + private static final java.util.Map nativeToWrapper + = new java.util.HashMap<>(); + + /** + Returns the Stmt object associated with the given sqlite3_stmt + object, or null if there is no such mapping. + */ + static Stmt fromNative(sqlite3_stmt low){ + synchronized(nativeToWrapper){ + return nativeToWrapper.get(low); + } + } + + /** + If this statement is still opened, its low-level handle is + returned, else an IllegalArgumentException is thrown. + */ + private sqlite3_stmt thisStmt(){ + if( null==stmt || 0==stmt.getNativePointer() ){ + throw new IllegalArgumentException("This Stmt has been finalized."); + } + return stmt; + } + + /** Throws if n is out of range of this statement's result column + count. Intended to be used by the columnXyz() methods. */ + private sqlite3_stmt checkColIndex(int n){ + if(n<0 || n>=columnCount()){ + throw new IllegalArgumentException("Column index "+n+" is out of range."); + } + return thisStmt(); + } + + /** + Corresponds to sqlite3_finalize(), but we cannot override the + name finalize() here because this one requires a different + signature. It does not throw on error here because "destructors + do not throw." If it returns non-0, the object is still + finalized, but the result code is an indication that something + went wrong in a prior call into the statement's API, as + documented for sqlite3_finalize(). + */ + public int finalizeStmt(){ + int rc = 0; + if( null!=stmt ){ + synchronized(nativeToWrapper){ + nativeToWrapper.remove(this.stmt); + } + CApi.sqlite3_finalize(stmt); + stmt = null; + _db = null; + } + return rc; + } + + @Override public void close(){ + finalizeStmt(); + } + + /** + Throws if rc is any value other than 0, SQLITE_ROW, or + SQLITE_DONE, else returns rc. Error state for the exception is + extracted from this statement object (if it's opened) or the + string form of rc. + */ + private int checkRc(int rc){ + switch(rc){ + case 0: + case CApi.SQLITE_ROW: + case CApi.SQLITE_DONE: return rc; + default: + if( null==stmt ) throw new SqliteException(rc); + else throw new SqliteException(this); + } + } + + /** + Works like sqlite3_step() but returns true for SQLITE_ROW, + false for SQLITE_DONE, and throws SqliteException for any other + result. + */ + public boolean step(){ + switch(checkRc(CApi.sqlite3_step(thisStmt()))){ + case CApi.SQLITE_ROW: return true; + case CApi.SQLITE_DONE: return false; + default: + throw new IllegalStateException( + "This \"cannot happen\": all possible result codes were checked already." + ); + } + } + + /** + Works like sqlite3_step(), returning the same result codes as + that function unless throwOnError is true, in which case it + will throw an SqliteException for any result codes other than + Sqlite.ROW or Sqlite.DONE. + + The utility of this overload over the no-argument one is the + ability to handle BUSY and LOCKED errors more easily. + */ + public int step(boolean throwOnError){ + final int rc = (null==stmt) + ? Sqlite.MISUSE + : CApi.sqlite3_step(stmt); + return throwOnError ? checkRc(rc) : rc; + } + + /** + Returns the Sqlite which prepared this statement, or null if + this statement has been finalized. + */ + public Sqlite getDb(){ return this._db; } + + /** + Works like sqlite3_reset() but throws on error. + */ + public void reset(){ + checkRc(CApi.sqlite3_reset(thisStmt())); + } + + public boolean isBusy(){ + return CApi.sqlite3_stmt_busy(thisStmt()); + } + + public boolean isReadOnly(){ + return CApi.sqlite3_stmt_readonly(thisStmt()); + } + + public String sql(){ + return CApi.sqlite3_sql(thisStmt()); + } + + public String expandedSql(){ + return CApi.sqlite3_expanded_sql(thisStmt()); + } + + /** + Analog to sqlite3_stmt_explain() but throws if op is invalid. + */ + public void explain(int op){ + checkRc(CApi.sqlite3_stmt_explain(thisStmt(), op)); + } + + /** + Analog to sqlite3_stmt_isexplain(). + */ + public int isExplain(){ + return CApi.sqlite3_stmt_isexplain(thisStmt()); + } + + /** + Analog to sqlite3_normalized_sql(), but throws + UnsupportedOperationException if the library was built without + the SQLITE_ENABLE_NORMALIZE flag. + */ + public String normalizedSql(){ + Sqlite.checkSupported(hasNormalizeSql, "SQLITE_ENABLE_NORMALIZE"); + return CApi.sqlite3_normalized_sql(thisStmt()); + } + + public void clearBindings(){ + CApi.sqlite3_clear_bindings( thisStmt() ); + } + public void bindInt(int ndx, int val){ + checkRc(CApi.sqlite3_bind_int(thisStmt(), ndx, val)); + } + public void bindInt64(int ndx, long val){ + checkRc(CApi.sqlite3_bind_int64(thisStmt(), ndx, val)); + } + public void bindDouble(int ndx, double val){ + checkRc(CApi.sqlite3_bind_double(thisStmt(), ndx, val)); + } + public void bindObject(int ndx, Object o){ + checkRc(CApi.sqlite3_bind_java_object(thisStmt(), ndx, o)); + } + public void bindNull(int ndx){ + checkRc(CApi.sqlite3_bind_null(thisStmt(), ndx)); + } + public int bindParameterCount(){ + return CApi.sqlite3_bind_parameter_count(thisStmt()); + } + public int bindParameterIndex(String paramName){ + return CApi.sqlite3_bind_parameter_index(thisStmt(), paramName); + } + public String bindParameterName(int ndx){ + return CApi.sqlite3_bind_parameter_name(thisStmt(), ndx); + } + public void bindText(int ndx, byte[] utf8){ + checkRc(CApi.sqlite3_bind_text(thisStmt(), ndx, utf8)); + } + public void bindText(int ndx, String asUtf8){ + checkRc(CApi.sqlite3_bind_text(thisStmt(), ndx, asUtf8)); + } + public void bindText16(int ndx, byte[] utf16){ + checkRc(CApi.sqlite3_bind_text16(thisStmt(), ndx, utf16)); + } + public void bindText16(int ndx, String asUtf16){ + checkRc(CApi.sqlite3_bind_text16(thisStmt(), ndx, asUtf16)); + } + public void bindZeroBlob(int ndx, int n){ + checkRc(CApi.sqlite3_bind_zeroblob(thisStmt(), ndx, n)); + } + public void bindBlob(int ndx, byte[] bytes){ + checkRc(CApi.sqlite3_bind_blob(thisStmt(), ndx, bytes)); + } + + public byte[] columnBlob(int ndx){ + return CApi.sqlite3_column_blob( checkColIndex(ndx), ndx ); + } + public byte[] columnText(int ndx){ + return CApi.sqlite3_column_text( checkColIndex(ndx), ndx ); + } + public String columnText16(int ndx){ + return CApi.sqlite3_column_text16( checkColIndex(ndx), ndx ); + } + public int columnBytes(int ndx){ + return CApi.sqlite3_column_bytes( checkColIndex(ndx), ndx ); + } + public int columnBytes16(int ndx){ + return CApi.sqlite3_column_bytes16( checkColIndex(ndx), ndx ); + } + public int columnInt(int ndx){ + return CApi.sqlite3_column_int( checkColIndex(ndx), ndx ); + } + public long columnInt64(int ndx){ + return CApi.sqlite3_column_int64( checkColIndex(ndx), ndx ); + } + public double columnDouble(int ndx){ + return CApi.sqlite3_column_double( checkColIndex(ndx), ndx ); + } + public int columnType(int ndx){ + return CApi.sqlite3_column_type( checkColIndex(ndx), ndx ); + } + public String columnDeclType(int ndx){ + return CApi.sqlite3_column_decltype( checkColIndex(ndx), ndx ); + } + /** + Analog to sqlite3_column_count() but throws if this statement + has been finalized. + */ + public int columnCount(){ + /* We cannot reliably cache the column count in a class + member because an ALTER TABLE from a separate statement + can invalidate that count and we have no way, short of + installing a COMMIT handler or the like, of knowing when + to re-read it. We cannot install such a handler without + interfering with a client's ability to do so. */ + return CApi.sqlite3_column_count(thisStmt()); + } + public int columnDataCount(){ + return CApi.sqlite3_data_count( thisStmt() ); + } + public Object columnObject(int ndx){ + return CApi.sqlite3_column_java_object( checkColIndex(ndx), ndx ); + } + public T columnObject(int ndx, Class type){ + return CApi.sqlite3_column_java_object( checkColIndex(ndx), ndx, type ); + } + public String columnName(int ndx){ + return CApi.sqlite3_column_name( checkColIndex(ndx), ndx ); + } + public String columnDatabaseName(int ndx){ + return CApi.sqlite3_column_database_name( checkColIndex(ndx), ndx ); + } + public String columnOriginName(int ndx){ + return CApi.sqlite3_column_origin_name( checkColIndex(ndx), ndx ); + } + public String columnTableName(int ndx){ + return CApi.sqlite3_column_table_name( checkColIndex(ndx), ndx ); + } + } /* Stmt class */ + + /** + Interface for auto-extensions, as per the + sqlite3_auto_extension() API. + + Design note: the chicken/egg timing of auto-extension execution + requires that this feature be entirely re-implemented in Java + because the C-level API has no access to the Sqlite type so + cannot pass on an object of that type while the database is being + opened. One side effect of this reimplementation is that this + class's list of auto-extensions is 100% independent of the + C-level list so, e.g., clearAutoExtensions() will have no effect + on auto-extensions added via the C-level API and databases opened + from that level of API will not be passed to this level's + AutoExtension instances. + */ + public interface AutoExtension { + public void call(Sqlite db); + } + + private static final java.util.Set autoExtensions = + new java.util.LinkedHashSet<>(); + + /** + Passes db to all auto-extensions. If any one of them throws, + db.close() is called before the exception is propagated. + */ + private static void runAutoExtensions(Sqlite db){ + AutoExtension list[]; + synchronized(autoExtensions){ + /* Avoid that modifications to the AutoExtension list from within + auto-extensions affect this execution of this list. */ + list = autoExtensions.toArray(new AutoExtension[0]); + } + try { + for( AutoExtension ax : list ) ax.call(db); + }catch(Exception e){ + db.close(); + throw e; + } + } + + /** + Analog to sqlite3_auto_extension(), adds the given object to the + list of auto-extensions if it is not already in that list. The + given object will be run as part of Sqlite.open(), and passed the + being-opened database. If the extension throws then open() will + fail. + + This API does not guaranty whether or not manipulations made to + the auto-extension list from within auto-extension callbacks will + affect the current traversal of the auto-extension list. Whether + or not they do is unspecified and subject to change between + versions. e.g. if an AutoExtension calls addAutoExtension(), + whether or not the new extension will be run on the being-opened + database is undefined. + + Note that calling Sqlite.open() from an auto-extension will + necessarily result in recursion loop and (eventually) a stack + overflow. + */ + public static void addAutoExtension( AutoExtension e ){ + if( null==e ){ + throw new IllegalArgumentException("AutoExtension may not be null."); + } + synchronized(autoExtensions){ + autoExtensions.add(e); + } + } + + /** + Removes the given object from the auto-extension list if it is in + that list, otherwise this has no side-effects beyond briefly + locking that list. + */ + public static void removeAutoExtension( AutoExtension e ){ + synchronized(autoExtensions){ + autoExtensions.remove(e); + } + } + + /** + Removes all auto-extensions which were added via addAutoExtension(). + */ + public static void clearAutoExtensions(){ + synchronized(autoExtensions){ + autoExtensions.clear(); + } + } + + /** + Encapsulates state related to the sqlite3 backup API. Use + Sqlite.initBackup() to create new instances. + */ + public static final class Backup implements AutoCloseable { + private sqlite3_backup b = null; + private Sqlite dbTo = null; + private Sqlite dbFrom = null; + + Backup(Sqlite dbDest, String schemaDest,Sqlite dbSrc, String schemaSrc){ + this.dbTo = dbDest; + this.dbFrom = dbSrc; + b = CApi.sqlite3_backup_init(dbDest.nativeHandle(), schemaDest, + dbSrc.nativeHandle(), schemaSrc); + if(null==b) toss(); + } + + private void toss(){ + int rc = CApi.sqlite3_errcode(dbTo.nativeHandle()); + if(0!=rc) throw new SqliteException(dbTo); + rc = CApi.sqlite3_errcode(dbFrom.nativeHandle()); + if(0!=rc) throw new SqliteException(dbFrom); + throw new SqliteException(CApi.SQLITE_ERROR); + } + + private sqlite3_backup getNative(){ + if( null==b ) throw new IllegalStateException("This Backup is already closed."); + return b; + } + /** + If this backup is still active, this completes the backup and + frees its native resources, otherwise it this is a no-op. + */ + public void finish(){ + if( null!=b ){ + CApi.sqlite3_backup_finish(b); + b = null; + dbTo = null; + dbFrom = null; + } + } + + /** Equivalent to finish(). */ + @Override public void close(){ + this.finish(); + } + + /** + Analog to sqlite3_backup_step(). Returns 0 if stepping succeeds + or, Sqlite.DONE if the end is reached, Sqlite.BUSY if one of + the databases is busy, Sqlite.LOCKED if one of the databases is + locked, and throws for any other result code or if this object + has been closed. Note that BUSY and LOCKED are not necessarily + permanent errors, so do not trigger an exception. + */ + public int step(int pageCount){ + final int rc = CApi.sqlite3_backup_step(getNative(), pageCount); + switch(rc){ + case 0: + case Sqlite.DONE: + case Sqlite.BUSY: + case Sqlite.LOCKED: + return rc; + default: + toss(); + return CApi.SQLITE_ERROR/*not reached*/; + } + } + + /** + Analog to sqlite3_backup_pagecount(). + */ + public int pageCount(){ + return CApi.sqlite3_backup_pagecount(getNative()); + } + + /** + Analog to sqlite3_backup_remaining(). + */ + public int remaining(){ + return CApi.sqlite3_backup_remaining(getNative()); + } + } + + /** + Analog to sqlite3_backup_init(). If schemaSrc is null, "main" is + assumed. Throws if either this db or dbSrc (the source db) are + not opened, if either of schemaDest or schemaSrc are null, or if + the underlying call to sqlite3_backup_init() fails. + + The returned object must eventually be cleaned up by either + arranging for it to be auto-closed (e.g. using + try-with-resources) or by calling its finish() method. + */ + public Backup initBackup(String schemaDest, Sqlite dbSrc, String schemaSrc){ + thisDb(); + dbSrc.thisDb(); + if( null==schemaSrc || null==schemaDest ){ + throw new IllegalArgumentException( + "Neither the source nor destination schema name may be null." + ); + } + return new Backup(this, schemaDest, dbSrc, schemaSrc); + } + + + /** + Callback type for use with createCollation(). + */ + public interface Collation { + /** + Called by the SQLite core to compare inputs. Implementations + must compare its two arguments using memcmp(3) semantics. + + Warning: the SQLite core has no mechanism for reporting errors + from custom collations and its workflow does not accommodate + propagation of exceptions from callbacks. Any exceptions thrown + from collations will be silently supressed and sorting results + will be unpredictable. + */ + int call(byte[] lhs, byte[] rhs); + } + + /** + Analog to sqlite3_create_collation(). + + Throws if name is null or empty, c is null, or the encoding flag + is invalid. The encoding must be one of the UTF8, UTF16, UTF16LE, + or UTF16BE constants. + */ + public void createCollation(String name, int encoding, Collation c){ + thisDb(); + if( null==name || 0==name.length()){ + throw new IllegalArgumentException("Collation name may not be null or empty."); + } + if( null==c ){ + throw new IllegalArgumentException("Collation may not be null."); + } + switch(encoding){ + case UTF8: + case UTF16: + case UTF16LE: + case UTF16BE: + break; + default: + throw new IllegalArgumentException("Invalid Collation encoding."); + } + checkRc( + CApi.sqlite3_create_collation( + thisDb(), name, encoding, new org.sqlite.jni.capi.CollationCallback(){ + @Override public int call(byte[] lhs, byte[] rhs){ + try{return c.call(lhs, rhs);} + catch(Exception e){return 0;} + } + @Override public void xDestroy(){} + } + ) + ); + } + + /** + Callback for use with onCollationNeeded(). + */ + public interface CollationNeeded { + /** + Must behave as documented for the callback for + sqlite3_collation_needed(). + + Warning: the C API has no mechanism for reporting or + propagating errors from this callback, so any exceptions it + throws are suppressed. + */ + void call(Sqlite db, int encoding, String collationName); + } + + /** + Sets up the given object to be called by the SQLite core when it + encounters a collation name which it does not know. Pass a null + object to disconnect the object from the core. This replaces any + existing collation-needed loader, or is a no-op if the given + object is already registered. Throws if registering the loader + fails. + */ + public void onCollationNeeded( CollationNeeded cn ){ + org.sqlite.jni.capi.CollationNeededCallback cnc = null; + if( null!=cn ){ + cnc = new org.sqlite.jni.capi.CollationNeededCallback(){ + @Override public void call(sqlite3 db, int encoding, String collationName){ + final Sqlite xdb = Sqlite.fromNative(db); + if(null!=xdb) cn.call(xdb, encoding, collationName); + } + }; + } + checkRc( CApi.sqlite3_collation_needed(thisDb(), cnc) ); + } + + /** + Callback for use with busyHandler(). + */ + public interface BusyHandler { + /** + Must function as documented for the C-level + sqlite3_busy_handler() callback argument, minus the (void*) + argument the C-level function requires. + + If this function throws, it is translated to a database-level + error. + */ + int call(int n); + } + + /** + Analog to sqlite3_busy_timeout(). + */ + public void setBusyTimeout(int ms){ + checkRc(CApi.sqlite3_busy_timeout(thisDb(), ms)); + } + + /** + Analog to sqlite3_busy_handler(). If b is null then any + current handler is cleared. + */ + public void setBusyHandler( BusyHandler b ){ + org.sqlite.jni.capi.BusyHandlerCallback bhc = null; + if( null!=b ){ + bhc = new org.sqlite.jni.capi.BusyHandlerCallback(){ + @Override public int call(int n){ + return b.call(n); + } + }; + } + checkRc( CApi.sqlite3_busy_handler(thisDb(), bhc) ); + } + + public interface CommitHook { + /** + Must behave as documented for the C-level sqlite3_commit_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + int call(); + } + + /** + A level of indirection to permit setCommitHook() to have similar + semantics as the C API, returning the previous hook. The caveat + is that if the low-level API is used to install a hook, it will + have a different hook type than Sqlite.CommitHook so + setCommitHook() will return null instead of that object. + */ + private static class CommitHookProxy + implements org.sqlite.jni.capi.CommitHookCallback { + final CommitHook commitHook; + CommitHookProxy(CommitHook ch){ + this.commitHook = ch; + } + @Override public int call(){ + return commitHook.call(); + } + } + + /** + Analog to sqlite3_commit_hook(). Returns the previous hook, if + any (else null). Throws if this db is closed. + + Minor caveat: if a commit hook is set on this object's underlying + db handle using the lower-level SQLite API, this function may + return null when replacing it, despite there being a hook, + because it will have a different callback type. So long as the + handle is only manipulated via the high-level API, this caveat + does not apply. + */ + public CommitHook setCommitHook( CommitHook c ){ + CommitHookProxy chp = null; + if( null!=c ){ + chp = new CommitHookProxy(c); + } + final org.sqlite.jni.capi.CommitHookCallback rv = + CApi.sqlite3_commit_hook(thisDb(), chp); + return (rv instanceof CommitHookProxy) + ? ((CommitHookProxy)rv).commitHook + : null; + } + + + public interface RollbackHook { + /** + Must behave as documented for the C-level sqlite3_rollback_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + void call(); + } + + /** + A level of indirection to permit setRollbackHook() to have similar + semantics as the C API, returning the previous hook. The caveat + is that if the low-level API is used to install a hook, it will + have a different hook type than Sqlite.RollbackHook so + setRollbackHook() will return null instead of that object. + */ + private static class RollbackHookProxy + implements org.sqlite.jni.capi.RollbackHookCallback { + final RollbackHook rollbackHook; + RollbackHookProxy(RollbackHook ch){ + this.rollbackHook = ch; + } + @Override public void call(){rollbackHook.call();} + } + + /** + Analog to sqlite3_rollback_hook(). Returns the previous hook, if + any (else null). Throws if this db is closed. + + Minor caveat: if a rollback hook is set on this object's underlying + db handle using the lower-level SQLite API, this function may + return null when replacing it, despite there being a hook, + because it will have a different callback type. So long as the + handle is only manipulated via the high-level API, this caveat + does not apply. + */ + public RollbackHook setRollbackHook( RollbackHook c ){ + RollbackHookProxy chp = null; + if( null!=c ){ + chp = new RollbackHookProxy(c); + } + final org.sqlite.jni.capi.RollbackHookCallback rv = + CApi.sqlite3_rollback_hook(thisDb(), chp); + return (rv instanceof RollbackHookProxy) + ? ((RollbackHookProxy)rv).rollbackHook + : null; + } + + public interface UpdateHook { + /** + Must function as described for the C-level sqlite3_update_hook() + callback. + */ + void call(int opId, String dbName, String tableName, long rowId); + } + + /** + A level of indirection to permit setUpdateHook() to have similar + semantics as the C API, returning the previous hook. The caveat + is that if the low-level API is used to install a hook, it will + have a different hook type than Sqlite.UpdateHook so + setUpdateHook() will return null instead of that object. + */ + private static class UpdateHookProxy + implements org.sqlite.jni.capi.UpdateHookCallback { + final UpdateHook updateHook; + UpdateHookProxy(UpdateHook ch){ + this.updateHook = ch; + } + @Override public void call(int opId, String dbName, String tableName, long rowId){ + updateHook.call(opId, dbName, tableName, rowId); + } + } + + /** + Analog to sqlite3_update_hook(). Returns the previous hook, if + any (else null). Throws if this db is closed. + + Minor caveat: if a update hook is set on this object's underlying + db handle using the lower-level SQLite API, this function may + return null when replacing it, despite there being a hook, + because it will have a different callback type. So long as the + handle is only manipulated via the high-level API, this caveat + does not apply. + */ + public UpdateHook setUpdateHook( UpdateHook c ){ + UpdateHookProxy chp = null; + if( null!=c ){ + chp = new UpdateHookProxy(c); + } + final org.sqlite.jni.capi.UpdateHookCallback rv = + CApi.sqlite3_update_hook(thisDb(), chp); + return (rv instanceof UpdateHookProxy) + ? ((UpdateHookProxy)rv).updateHook + : null; + } + + + /** + Callback interface for use with setProgressHandler(). + */ + public interface ProgressHandler { + /** + Must behave as documented for the C-level sqlite3_progress_handler() + callback. If it throws, the exception is translated into + a db-level error. + */ + int call(); + } + + /** + Analog to sqlite3_progress_handler(), sets the current progress + handler or clears it if p is null. + + Note that this API, in contrast to setUpdateHook(), + setRollbackHook(), and setCommitHook(), cannot return the + previous handler. That inconsistency is part of the lower-level C + API. + */ + public void setProgressHandler( int n, ProgressHandler p ){ + org.sqlite.jni.capi.ProgressHandlerCallback phc = null; + if( null!=p ){ + phc = new org.sqlite.jni.capi.ProgressHandlerCallback(){ + @Override public int call(){ return p.call(); } + }; + } + CApi.sqlite3_progress_handler( thisDb(), n, phc ); + } + + + /** + Callback for use with setAuthorizer(). + */ + public interface Authorizer { + /** + Must function as described for the C-level + sqlite3_set_authorizer() callback. If it throws, the error is + converted to a db-level error and the exception is suppressed. + */ + int call(int opId, String s1, String s2, String s3, String s4); + } + + /** + Analog to sqlite3_set_authorizer(), this sets the current + authorizer callback, or clears if it passed null. + */ + public void setAuthorizer( Authorizer a ) { + org.sqlite.jni.capi.AuthorizerCallback ac = null; + if( null!=a ){ + ac = new org.sqlite.jni.capi.AuthorizerCallback(){ + @Override public int call(int opId, String s1, String s2, String s3, String s4){ + return a.call(opId, s1, s2, s3, s4); + } + }; + } + checkRc( CApi.sqlite3_set_authorizer( thisDb(), ac ) ); + } + + /** + Object type for use with blobOpen() + */ + public final class Blob implements AutoCloseable { + private Sqlite db; + private sqlite3_blob b; + Blob(Sqlite db, sqlite3_blob b){ + this.db = db; + this.b = b; + } + + /** + If this blob is still opened, its low-level handle is + returned, else an IllegalArgumentException is thrown. + */ + private sqlite3_blob thisBlob(){ + if( null==b || 0==b.getNativePointer() ){ + throw new IllegalArgumentException("This Blob has been finalized."); + } + return b; + } + + /** + Analog to sqlite3_blob_close(). + */ + @Override public void close(){ + if( null!=b ){ + CApi.sqlite3_blob_close(b); + b = null; + db = null; + } + } + + /** + Throws if the JVM does not have JNI-level support for + ByteBuffer. + */ + private void checkNio(){ + if( !Sqlite.JNI_SUPPORTS_NIO ){ + throw new UnsupportedOperationException( + "This JVM does not support JNI access to ByteBuffer." + ); + } + } + /** + Analog to sqlite3_blob_reopen() but throws on error. + */ + public void reopen(long newRowId){ + db.checkRc( CApi.sqlite3_blob_reopen(thisBlob(), newRowId) ); + } + + /** + Analog to sqlite3_blob_write() but throws on error. + */ + public void write( byte[] bytes, int atOffset ){ + db.checkRc( CApi.sqlite3_blob_write(thisBlob(), bytes, atOffset) ); + } + + /** + Analog to sqlite3_blob_read() but throws on error. + */ + public void read( byte[] dest, int atOffset ){ + db.checkRc( CApi.sqlite3_blob_read(thisBlob(), dest, atOffset) ); + } + + /** + Analog to sqlite3_blob_bytes(). + */ + public int bytes(){ + return CApi.sqlite3_blob_bytes(thisBlob()); + } + } + + /** + Analog to sqlite3_blob_open(). Returns a Blob object for the + given database, table, column, and rowid. The blob is opened for + read-write mode if writeable is true, else it is read-only. + + The returned object must eventually be freed, before this + database is closed, by either arranging for it to be auto-closed + or calling its close() method. + + Throws on error. + */ + public Blob blobOpen(String dbName, String tableName, String columnName, + long iRow, boolean writeable){ + final OutputPointer.sqlite3_blob out = new OutputPointer.sqlite3_blob(); + checkRc( + CApi.sqlite3_blob_open(thisDb(), dbName, tableName, columnName, + iRow, writeable ? 1 : 0, out) + ); + return new Blob(this, out.take()); + } + + /** + Callback for use with libConfigLog(). + */ + public interface ConfigLog { + /** + Must function as described for a C-level callback for + sqlite3_config()'s SQLITE_CONFIG_LOG callback, with the slight + signature change. Any exceptions thrown from this callback are + necessarily suppressed. + */ + void call(int errCode, String msg); + } + + /** + Analog to sqlite3_config() with the SQLITE_CONFIG_LOG option, + this sets or (if log is null) clears the current logger. + */ + public static void libConfigLog(ConfigLog log){ + final org.sqlite.jni.capi.ConfigLogCallback l = + null==log + ? null + : new org.sqlite.jni.capi.ConfigLogCallback() { + @Override public void call(int errCode, String msg){ + log.call(errCode, msg); + } + }; + checkRcStatic(CApi.sqlite3_config(l)); + } + + /** + Callback for use with libConfigSqlLog(). + */ + public interface ConfigSqlLog { + /** + Must function as described for a C-level callback for + sqlite3_config()'s SQLITE_CONFIG_SQLLOG callback, with the + slight signature change. Any exceptions thrown from this + callback are necessarily suppressed. + */ + void call(Sqlite db, String msg, int msgType); + } + + /** + Analog to sqlite3_config() with the SQLITE_CONFIG_SQLLOG option, + this sets or (if log is null) clears the current logger. + + If SQLite is built without SQLITE_ENABLE_SQLLOG defined then this + will throw an UnsupportedOperationException. + */ + public static void libConfigSqlLog(ConfigSqlLog log){ + Sqlite.checkSupported(hasNormalizeSql, "SQLITE_ENABLE_SQLLOG"); + final org.sqlite.jni.capi.ConfigSqlLogCallback l = + null==log + ? null + : new org.sqlite.jni.capi.ConfigSqlLogCallback() { + @Override public void call(sqlite3 db, String msg, int msgType){ + try{ + log.call(fromNative(db), msg, msgType); + }catch(Exception e){ + /* Suppressed */ + } + } + }; + checkRcStatic(CApi.sqlite3_config(l)); + } + + /** + Analog to the C-level sqlite3_config() with one of the + SQLITE_CONFIG_... constants defined as CONFIG_... in this + class. Throws on error, including passing of an unknown option or + if a specified option is not supported by the underlying build of + the SQLite library. + */ + public static void libConfigOp( int op ){ + checkRcStatic(CApi.sqlite3_config(op)); + } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java b/ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java new file mode 100644 index 0000000000000000000000000000000000000000..9b4440f1905e3a6edb4eb5ee259269682499fbdb --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java @@ -0,0 +1,85 @@ +/* +** 2023-10-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.capi.sqlite3; + +/** + A wrapper for communicating C-level (sqlite3*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java + and C via JNI. +*/ +public final class SqliteException extends java.lang.RuntimeException { + private int errCode = CApi.SQLITE_ERROR; + private int xerrCode = CApi.SQLITE_ERROR; + private int errOffset = -1; + private int sysErrno = 0; + + /** + Records the given error string and uses SQLITE_ERROR for both the + error code and extended error code. + */ + public SqliteException(String msg){ + super(msg); + } + + /** + Uses sqlite3_errstr(sqlite3ResultCode) for the error string and + sets both the error code and extended error code to the given + value. This approach includes no database-level information and + systemErrno() will be 0, so is intended only for use with sqlite3 + APIs for which a result code is not an error but which the + higher-level wrapper should treat as one. + */ + public SqliteException(int sqlite3ResultCode){ + super(CApi.sqlite3_errstr(sqlite3ResultCode)); + errCode = xerrCode = sqlite3ResultCode; + } + + /** + Records the current error state of db (which must not be null and + must refer to an opened db object). Note that this does not close + the db. + + Design note: closing the db on error is really only useful during + a failed db-open operation, and the place(s) where that can + happen are inside this library, not client-level code. + */ + SqliteException(sqlite3 db){ + super(CApi.sqlite3_errmsg(db)); + errCode = CApi.sqlite3_errcode(db); + xerrCode = CApi.sqlite3_extended_errcode(db); + errOffset = CApi.sqlite3_error_offset(db); + sysErrno = CApi.sqlite3_system_errno(db); + } + + /** + Records the current error state of db (which must not be null and + must refer to an open database). + */ + public SqliteException(Sqlite db){ + this(db.nativeHandle()); + } + + public SqliteException(Sqlite.Stmt stmt){ + this(stmt.getDb()); + } + + public int errcode(){ return errCode; } + public int extendedErrcode(){ return xerrCode; } + public int errorOffset(){ return errOffset; } + public int systemErrno(){ return sysErrno; } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java b/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java new file mode 100644 index 0000000000000000000000000000000000000000..5ac41323cb8f4b34dc96ca45beca8bb7266384da --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java @@ -0,0 +1,1213 @@ +/* +** 2023-10-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a set of tests for the sqlite3 JNI bindings. +*/ +package org.sqlite.jni.wrapper1; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.sqlite.jni.capi.CApi; + +/** + An annotation for Tester2 tests which we do not want to run in + reflection-driven test mode because either they are not suitable + for multi-threaded threaded mode or we have to control their execution + order. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface ManualTest{} +/** + Annotation for Tester2 tests which mark those which must be skipped + in multi-threaded mode. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface SingleThreadOnly{} + +public class Tester2 implements Runnable { + //! True when running in multi-threaded mode. + private static boolean mtMode = false; + //! True to sleep briefly between tests. + private static boolean takeNaps = false; + //! True to shuffle the order of the tests. + private static boolean shuffle = false; + //! True to dump the list of to-run tests to stdout. + private static int listRunTests = 0; + //! True to squelch all out() and outln() output. + private static boolean quietMode = false; + //! Total number of runTests() calls. + private static int nTestRuns = 0; + //! List of test*() methods to run. + private static List testMethods = null; + //! List of exceptions collected by run() + private static List listErrors = new ArrayList<>(); + private static final class Metrics { + //! Number of times createNewDb() (or equivalent) is invoked. + volatile int dbOpen = 0; + } + + //! Instance ID. + private Integer tId; + + Tester2(Integer id){ + tId = id; + } + + static final Metrics metrics = new Metrics(); + + public static synchronized void outln(){ + if( !quietMode ){ + System.out.println(""); + } + } + + public static synchronized void outPrefix(){ + if( !quietMode ){ + System.out.print(Thread.currentThread().getName()+": "); + } + } + + public static synchronized void outln(Object val){ + if( !quietMode ){ + outPrefix(); + System.out.println(val); + } + } + + public static synchronized void out(Object val){ + if( !quietMode ){ + System.out.print(val); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void out(Object... vals){ + if( !quietMode ){ + outPrefix(); + for(Object v : vals) out(v); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void outln(Object... vals){ + if( !quietMode ){ + out(vals); out("\n"); + } + } + + static volatile int affirmCount = 0; + public static synchronized int affirm(Boolean v, String comment){ + ++affirmCount; + if( false ) assert( v /* prefer assert over exception if it's enabled because + the JNI layer sometimes has to suppress exceptions, + so they might be squelched on their way back to the + top. */); + if( !v ) throw new RuntimeException(comment); + return affirmCount; + } + + public static void affirm(Boolean v){ + affirm(v, "Affirmation failed."); + } + + + public static void execSql(Sqlite db, String sql[]){ + execSql(db, String.join("", sql)); + } + + /** + Executes all SQL statements in the given string. If throwOnError + is true then it will throw for any prepare/step errors, else it + will return the corresponding non-0 result code. + */ + public static int execSql(Sqlite dbw, boolean throwOnError, String sql){ + final ValueHolder rv = new ValueHolder<>(0); + final Sqlite.PrepareMulti pm = new Sqlite.PrepareMulti(){ + @Override public void call(Sqlite.Stmt stmt){ + try{ + while( Sqlite.ROW == (rv.value = stmt.step(throwOnError)) ){} + } + finally{ stmt.finalizeStmt(); } + } + }; + try { + dbw.prepareMulti(sql, pm); + }catch(SqliteException se){ + if( throwOnError ){ + throw se; + }else{ + /* This error (likely) happened in the prepare() phase and we + need to preempt it. */ + rv.value = se.errcode(); + } + } + return (rv.value==Sqlite.DONE) ? 0 : rv.value; + } + + static void execSql(Sqlite db, String sql){ + execSql(db, true, sql); + } + + @SingleThreadOnly /* because it's thread-agnostic */ + private void test1(){ + affirm(Sqlite.libVersionNumber() == CApi.SQLITE_VERSION_NUMBER); + } + + private void nap() throws InterruptedException { + if( takeNaps ){ + Thread.sleep(java.util.concurrent.ThreadLocalRandom.current().nextInt(3, 17), 0); + } + } + + Sqlite openDb(String name){ + final Sqlite db = Sqlite.open(name, Sqlite.OPEN_READWRITE| + Sqlite.OPEN_CREATE| + Sqlite.OPEN_EXRESCODE); + ++metrics.dbOpen; + return db; + } + + Sqlite openDb(){ return openDb(":memory:"); } + + void testOpenDb1(){ + Sqlite db = openDb(); + affirm( 0!=db.nativeHandle().getNativePointer() ); + affirm( "main".equals( db.dbName(0) ) ); + db.setMainDbName("foo"); + affirm( "foo".equals( db.dbName(0) ) ); + affirm( db.dbConfig(Sqlite.DBCONFIG_DEFENSIVE, true) + /* The underlying function has different mangled names in jdk8 + vs jdk19, and this call is here to ensure that the build + fails if it cannot find both names. */ ); + affirm( !db.dbConfig(Sqlite.DBCONFIG_DEFENSIVE, false) ); + SqliteException ex = null; + try{ db.dbConfig(0, false); } + catch(SqliteException e){ ex = e; } + affirm( null!=ex ); + ex = null; + db.close(); + affirm( null==db.nativeHandle() ); + + try{ db = openDb("/no/such/dir/.../probably"); } + catch(SqliteException e){ ex = e; } + affirm( ex!=null ); + affirm( ex.errcode() != 0 ); + affirm( ex.extendedErrcode() != 0 ); + affirm( ex.errorOffset() < 0 ); + // there's no reliable way to predict what ex.systemErrno() might be + } + + void testPrepare1(){ + try (Sqlite db = openDb()) { + Sqlite.Stmt stmt = db.prepare("SELECT ?1"); + Exception e = null; + affirm( null!=stmt.nativeHandle() ); + affirm( db == stmt.getDb() ); + affirm( 1==stmt.bindParameterCount() ); + affirm( "?1".equals(stmt.bindParameterName(1)) ); + affirm( null==stmt.bindParameterName(2) ); + stmt.bindInt64(1, 1); + stmt.bindDouble(1, 1.1); + stmt.bindObject(1, db); + stmt.bindNull(1); + stmt.bindText(1, new byte[] {32,32,32}); + stmt.bindText(1, "123"); + stmt.bindText16(1, "123".getBytes(StandardCharsets.UTF_16)); + stmt.bindText16(1, "123"); + stmt.bindZeroBlob(1, 8); + stmt.bindBlob(1, new byte[] {1,2,3,4}); + stmt.bindInt(1, 17); + try{ stmt.bindInt(2,1); } + catch(Exception ex){ e = ex; } + affirm( null!=e ); + e = null; + affirm( stmt.step() ); + try{ stmt.columnInt(1); } + catch(Exception ex){ e = ex; } + affirm( null!=e ); + e = null; + affirm( 17 == stmt.columnInt(0) ); + affirm( 17L == stmt.columnInt64(0) ); + affirm( 17.0 == stmt.columnDouble(0) ); + affirm( "17".equals(stmt.columnText16(0)) ); + affirm( !stmt.step() ); + stmt.reset(); + affirm( Sqlite.ROW==stmt.step(false) ); + affirm( !stmt.step() ); + affirm( 0 == stmt.finalizeStmt() ); + affirm( null==stmt.nativeHandle() ); + + stmt = db.prepare("SELECT ?"); + stmt.bindObject(1, db); + affirm( Sqlite.ROW == stmt.step(false) ); + affirm( db==stmt.columnObject(0) ); + affirm( db==stmt.columnObject(0, Sqlite.class ) ); + affirm( null==stmt.columnObject(0, Sqlite.Stmt.class ) ); + affirm( 0==stmt.finalizeStmt() ) + /* getting a non-0 out of sqlite3_finalize() is tricky */; + affirm( null==stmt.nativeHandle() ); + } + } + + void testUdfScalar(){ + final ValueHolder xDestroyCalled = new ValueHolder<>(0); + try (Sqlite db = openDb()) { + execSql(db, "create table t(a); insert into t(a) values(1),(2),(3)"); + final ValueHolder vh = new ValueHolder<>(0); + final ScalarFunction f = new ScalarFunction(){ + public void xFunc(SqlFunction.Arguments args){ + affirm( db == args.getDb() ); + for( SqlFunction.Arguments.Arg arg : args ){ + vh.value += arg.getInt(); + } + args.resultInt(vh.value); + } + public void xDestroy(){ + ++xDestroyCalled.value; + } + }; + db.createFunction("myfunc", -1, f); + Sqlite.Stmt q = db.prepare("select myfunc(1,2,3)"); + affirm( q.step() ); + affirm( 6 == vh.value ); + affirm( 6 == q.columnInt(0) ); + q.finalizeStmt(); + affirm( 0 == xDestroyCalled.value ); + vh.value = 0; + q = db.prepare("select myfunc(-1,-2,-3)"); + affirm( q.step() ); + affirm( -6 == vh.value ); + affirm( -6 == q.columnInt(0) ); + affirm( 0 == xDestroyCalled.value ); + q.finalizeStmt(); + } + affirm( 1 == xDestroyCalled.value ); + } + + void testUdfAggregate(){ + final ValueHolder xDestroyCalled = new ValueHolder<>(0); + Sqlite.Stmt q = null; + try (Sqlite db = openDb()) { + execSql(db, "create table t(a); insert into t(a) values(1),(2),(3)"); + final AggregateFunction f = new AggregateFunction(){ + public void xStep(SqlFunction.Arguments args){ + final ValueHolder agg = this.getAggregateState(args, 0); + for( SqlFunction.Arguments.Arg arg : args ){ + agg.value += arg.getInt(); + } + } + public void xFinal(SqlFunction.Arguments args){ + final Integer v = this.takeAggregateState(args); + if( null==v ) args.resultNull(); + else args.resultInt(v); + } + public void xDestroy(){ + ++xDestroyCalled.value; + } + }; + db.createFunction("summer", 1, f); + q = db.prepare( + "with cte(v) as ("+ + "select 3 union all select 5 union all select 7"+ + ") select summer(v), summer(v+1) from cte" + /* ------------------^^^^^^^^^^^ ensures that we're handling + sqlite3_aggregate_context() properly. */ + ); + affirm( q.step() ); + affirm( 15==q.columnInt(0) ); + q.finalizeStmt(); + q = null; + affirm( 0 == xDestroyCalled.value ); + db.createFunction("summerN", -1, f); + + q = db.prepare("select summerN(1,8,9), summerN(2,3,4)"); + affirm( q.step() ); + affirm( 18==q.columnInt(0) ); + affirm( 9==q.columnInt(1) ); + q.finalizeStmt(); + q = null; + + }/*db*/ + finally{ + if( null!=q ) q.finalizeStmt(); + } + affirm( 2 == xDestroyCalled.value + /* because we've bound the same instance twice */ ); + } + + private void testUdfWindow(){ + final Sqlite db = openDb(); + /* Example window function, table, and results taken from: + https://sqlite.org/windowfunctions.html#udfwinfunc */ + final WindowFunction func = new WindowFunction(){ + //! Impl of xStep() and xInverse() + private void xStepInverse(SqlFunction.Arguments args, int v){ + this.getAggregateState(args,0).value += v; + } + @Override public void xStep(SqlFunction.Arguments args){ + this.xStepInverse(args, args.getInt(0)); + } + @Override public void xInverse(SqlFunction.Arguments args){ + this.xStepInverse(args, -args.getInt(0)); + } + //! Impl of xFinal() and xValue() + private void xFinalValue(SqlFunction.Arguments args, Integer v){ + if(null == v) args.resultNull(); + else args.resultInt(v); + } + @Override public void xFinal(SqlFunction.Arguments args){ + xFinalValue(args, this.takeAggregateState(args)); + affirm( null == this.getAggregateState(args,null).value ); + } + @Override public void xValue(SqlFunction.Arguments args){ + xFinalValue(args, this.getAggregateState(args,null).value); + } + }; + db.createFunction("winsumint", 1, func); + execSql(db, new String[] { + "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", + "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" + }); + final Sqlite.Stmt stmt = db.prepare( + "SELECT x, winsumint(y) OVER ("+ + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING"+ + ") AS sum_y "+ + "FROM twin ORDER BY x;" + ); + int n = 0; + while( stmt.step() ){ + final String s = stmt.columnText16(0); + final int i = stmt.columnInt(1); + switch(++n){ + case 1: affirm( "a".equals(s) && 9==i ); break; + case 2: affirm( "b".equals(s) && 12==i ); break; + case 3: affirm( "c".equals(s) && 16==i ); break; + case 4: affirm( "d".equals(s) && 12==i ); break; + case 5: affirm( "e".equals(s) && 9==i ); break; + default: affirm( false /* cannot happen */ ); + } + } + stmt.close(); + affirm( 5 == n ); + db.close(); + } + + private void testKeyword(){ + final int n = Sqlite.keywordCount(); + affirm( n>0 ); + affirm( !Sqlite.keywordCheck("_nope_") ); + affirm( Sqlite.keywordCheck("seLect") ); + affirm( null!=Sqlite.keywordName(0) ); + affirm( null!=Sqlite.keywordName(n-1) ); + affirm( null==Sqlite.keywordName(n) ); + } + + + private void testExplain(){ + final Sqlite db = openDb(); + Sqlite.Stmt q = db.prepare("SELECT 1"); + affirm( 0 == q.isExplain() ); + q.explain(0); + affirm( 0 == q.isExplain() ); + q.explain(1); + affirm( 1 == q.isExplain() ); + q.explain(2); + affirm( 2 == q.isExplain() ); + Exception ex = null; + try{ + q.explain(-1); + }catch(Exception e){ + ex = e; + } + affirm( ex instanceof SqliteException ); + q.finalizeStmt(); + db.close(); + } + + + private void testTrace(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + /* Ensure that characters outside of the UTF BMP survive the trip + from Java to sqlite3 and back to Java. (At no small efficiency + penalty.) */ + final String nonBmpChar = "😃"; + db.trace( + Sqlite.TRACE_ALL, + new Sqlite.TraceCallback(){ + @Override public void call(int traceFlag, Object pNative, Object x){ + ++counter.value; + //outln("TRACE "+traceFlag+" pNative = "+pNative.getClass().getName()); + switch(traceFlag){ + case Sqlite.TRACE_STMT: + affirm(pNative instanceof Sqlite.Stmt); + //outln("TRACE_STMT sql = "+x); + affirm(x instanceof String); + affirm( ((String)x).indexOf(nonBmpChar) > 0 ); + break; + case Sqlite.TRACE_PROFILE: + affirm(pNative instanceof Sqlite.Stmt); + affirm(x instanceof Long); + //outln("TRACE_PROFILE time = "+x); + break; + case Sqlite.TRACE_ROW: + affirm(pNative instanceof Sqlite.Stmt); + affirm(null == x); + //outln("TRACE_ROW = "+sqlite3_column_text16((sqlite3_stmt)pNative, 0)); + break; + case Sqlite.TRACE_CLOSE: + affirm(pNative instanceof Sqlite); + affirm(null == x); + break; + default: + affirm(false /*cannot happen*/); + break; + } + } + }); + execSql(db, "SELECT coalesce(null,null,'"+nonBmpChar+"'); "+ + "SELECT 'w"+nonBmpChar+"orld'"); + affirm( 6 == counter.value ); + db.close(); + affirm( 7 == counter.value ); + } + + private void testStatus(){ + final Sqlite db = openDb(); + execSql(db, "create table t(a); insert into t values(1),(2),(3)"); + + Sqlite.Status s = Sqlite.libStatus(Sqlite.STATUS_MEMORY_USED, false); + affirm( s.current > 0 ); + affirm( s.peak >= s.current ); + + s = db.status(Sqlite.DBSTATUS_SCHEMA_USED, false); + affirm( s.current > 0 ); + affirm( s.peak == 0 /* always 0 for SCHEMA_USED */ ); + + db.close(); + } + + @SingleThreadOnly /* because multiple threads legitimately make these + results unpredictable */ + private synchronized void testAutoExtension(){ + final ValueHolder val = new ValueHolder<>(0); + final ValueHolder toss = new ValueHolder<>(null); + final Sqlite.AutoExtension ax = new Sqlite.AutoExtension(){ + @Override public void call(Sqlite db){ + ++val.value; + if( null!=toss.value ){ + throw new RuntimeException(toss.value); + } + } + }; + Sqlite.addAutoExtension(ax); + openDb().close(); + affirm( 1==val.value ); + openDb().close(); + affirm( 2==val.value ); + Sqlite.clearAutoExtensions(); + openDb().close(); + affirm( 2==val.value ); + + Sqlite.addAutoExtension( ax ); + Sqlite.addAutoExtension( ax ); // Must not add a second entry + Sqlite.addAutoExtension( ax ); // or a third one + openDb().close(); + affirm( 3==val.value ); + + Sqlite db = openDb(); + affirm( 4==val.value ); + execSql(db, "ATTACH ':memory:' as foo"); + affirm( 4==val.value, "ATTACH uses the same connection, not sub-connections." ); + db.close(); + db = null; + + Sqlite.removeAutoExtension(ax); + openDb().close(); + affirm( 4==val.value ); + Sqlite.addAutoExtension(ax); + Exception err = null; + toss.value = "Throwing from auto_extension."; + try{ + openDb(); + }catch(Exception e){ + err = e; + } + affirm( err!=null ); + affirm( err.getMessage().indexOf(toss.value)>=0 ); + toss.value = null; + + val.value = 0; + final Sqlite.AutoExtension ax2 = new Sqlite.AutoExtension(){ + @Override public void call(Sqlite db){ + ++val.value; + } + }; + Sqlite.addAutoExtension(ax2); + openDb().close(); + affirm( 2 == val.value ); + Sqlite.removeAutoExtension(ax); + openDb().close(); + affirm( 3 == val.value ); + Sqlite.addAutoExtension(ax); + openDb().close(); + affirm( 5 == val.value ); + Sqlite.removeAutoExtension(ax2); + openDb().close(); + affirm( 6 == val.value ); + Sqlite.addAutoExtension(ax2); + openDb().close(); + affirm( 8 == val.value ); + + Sqlite.clearAutoExtensions(); + openDb().close(); + affirm( 8 == val.value ); + } + + private void testBackup(){ + final Sqlite dbDest = openDb(); + + try (Sqlite dbSrc = openDb()) { + execSql(dbSrc, new String[]{ + "pragma page_size=512; VACUUM;", + "create table t(a);", + "insert into t(a) values(1),(2),(3);" + }); + Exception e = null; + try { + dbSrc.initBackup("main",dbSrc,"main"); + }catch(Exception x){ + e = x; + } + affirm( e instanceof SqliteException ); + e = null; + try (Sqlite.Backup b = dbDest.initBackup("main",dbSrc,"main")) { + affirm( null!=b ); + int rc; + while( Sqlite.DONE!=(rc = b.step(1)) ){ + affirm( 0==rc ); + } + affirm( b.pageCount() > 0 ); + b.finish(); + } + } + + try (Sqlite.Stmt q = dbDest.prepare("SELECT sum(a) from t")) { + q.step(); + affirm( q.columnInt(0) == 6 ); + } + dbDest.close(); + } + + private void testCollation(){ + final Sqlite db = openDb(); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + final Sqlite.Collation myCollation = new Sqlite.Collation() { + private String myState = + "this is local state. There is much like it, but this is mine."; + @Override + // Reverse-sorts its inputs... + public int call(byte[] lhs, byte[] rhs){ + int len = lhs.length > rhs.length ? rhs.length : lhs.length; + int c = 0, i = 0; + for(i = 0; i < len; ++i){ + c = lhs[i] - rhs[i]; + if(0 != c) break; + } + if(0==c){ + if(i < lhs.length) c = 1; + else if(i < rhs.length) c = -1; + } + return -c; + } + }; + final Sqlite.CollationNeeded collLoader = new Sqlite.CollationNeeded(){ + @Override + public void call(Sqlite dbArg, int eTextRep, String collationName){ + affirm(dbArg == db); + db.createCollation("reversi", eTextRep, myCollation); + } + }; + db.onCollationNeeded(collLoader); + Sqlite.Stmt stmt = db.prepare("SELECT a FROM t ORDER BY a COLLATE reversi"); + int counter = 0; + while( stmt.step() ){ + final String val = stmt.columnText16(0); + ++counter; + switch(counter){ + case 1: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 3: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + stmt.finalizeStmt(); + stmt = db.prepare("SELECT a FROM t ORDER BY a"); + counter = 0; + while( stmt.step() ){ + final String val = stmt.columnText16(0); + ++counter; + //outln("Non-REVERSI'd row#"+counter+": "+val); + switch(counter){ + case 3: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 1: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + stmt.finalizeStmt(); + db.onCollationNeeded(null); + db.close(); + } + + @SingleThreadOnly /* because threads inherently break this test */ + private void testBusy(){ + final String dbName = "_busy-handler.db"; + try{ + Sqlite db1 = openDb(dbName); + ++metrics.dbOpen; + execSql(db1, "CREATE TABLE IF NOT EXISTS t(a)"); + Sqlite db2 = openDb(dbName); + ++metrics.dbOpen; + + final ValueHolder xBusyCalled = new ValueHolder<>(0); + Sqlite.BusyHandler handler = new Sqlite.BusyHandler(){ + @Override public int call(int n){ + return n > 2 ? 0 : ++xBusyCalled.value; + } + }; + db2.setBusyHandler(handler); + + // Force a locked condition... + execSql(db1, "BEGIN EXCLUSIVE"); + int rc = 0; + SqliteException ex = null; + try{ + db2.prepare("SELECT * from t"); + }catch(SqliteException x){ + ex = x; + } + affirm( null!=ex ); + affirm( Sqlite.BUSY == ex.errcode() ); + affirm( 3 == xBusyCalled.value ); + db1.close(); + db2.close(); + }finally{ + try{(new java.io.File(dbName)).delete();} + catch(Exception e){/* ignore */} + } + } + + private void testCommitHook(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder hookResult = new ValueHolder<>(0); + final Sqlite.CommitHook theHook = new Sqlite.CommitHook(){ + @Override public int call(){ + ++counter.value; + return hookResult.value; + } + }; + Sqlite.CommitHook oldHook = db.setCommitHook(theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 2 == counter.value ); + execSql(db, "BEGIN; SELECT 1; SELECT 2; COMMIT;"); + affirm( 2 == counter.value /* NOT invoked if no changes are made */ ); + execSql(db, "BEGIN; update t set a='d' where a='c'; COMMIT;"); + affirm( 3 == counter.value ); + oldHook = db.setCommitHook(theHook); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='e' where a='d'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = db.setCommitHook(null); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='f' where a='e'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = db.setCommitHook(null); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='g' where a='f'; COMMIT;"); + affirm( 4 == counter.value ); + + final Sqlite.CommitHook newHook = new Sqlite.CommitHook(){ + @Override public int call(){return 0;} + }; + oldHook = db.setCommitHook(newHook); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='h' where a='g'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = db.setCommitHook(theHook); + affirm( newHook == oldHook ); + execSql(db, "BEGIN; update t set a='i' where a='h'; COMMIT;"); + affirm( 5 == counter.value ); + hookResult.value = Sqlite.ERROR; + int rc = execSql(db, false, "BEGIN; update t set a='j' where a='i'; COMMIT;"); + affirm( Sqlite.CONSTRAINT_COMMITHOOK == rc ); + affirm( 6 == counter.value ); + db.close(); + } + + private void testRollbackHook(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final Sqlite.RollbackHook theHook = new Sqlite.RollbackHook(){ + @Override public void call(){ + ++counter.value; + } + }; + Sqlite.RollbackHook oldHook = db.setRollbackHook(theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 0 == counter.value ); + execSql(db, false, "BEGIN; SELECT 1; SELECT 2; ROLLBACK;"); + affirm( 1 == counter.value /* contra to commit hook, is invoked if no changes are made */ ); + + final Sqlite.RollbackHook newHook = new Sqlite.RollbackHook(){ + @Override public void call(){} + }; + oldHook = db.setRollbackHook(newHook); + affirm( theHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 1 == counter.value ); + oldHook = db.setRollbackHook(theHook); + affirm( newHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 2 == counter.value ); + int rc = execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 0 == rc ); + affirm( 3 == counter.value ); + db.close(); + } + + private void testUpdateHook(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder expectedOp = new ValueHolder<>(0); + final Sqlite.UpdateHook theHook = new Sqlite.UpdateHook(){ + @Override + public void call(int opId, String dbName, String tableName, long rowId){ + ++counter.value; + if( 0!=expectedOp.value ){ + affirm( expectedOp.value == opId ); + } + } + }; + Sqlite.UpdateHook oldHook = db.setUpdateHook(theHook); + affirm( null == oldHook ); + expectedOp.value = Sqlite.INSERT; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 3 == counter.value ); + expectedOp.value = Sqlite.UPDATE; + execSql(db, "update t set a='d' where a='c';"); + affirm( 4 == counter.value ); + oldHook = db.setUpdateHook(theHook); + affirm( theHook == oldHook ); + expectedOp.value = Sqlite.DELETE; + execSql(db, "DELETE FROM t where a='d'"); + affirm( 5 == counter.value ); + oldHook = db.setUpdateHook(null); + affirm( theHook == oldHook ); + execSql(db, "update t set a='e' where a='b';"); + affirm( 5 == counter.value ); + oldHook = db.setUpdateHook(null); + affirm( null == oldHook ); + + final Sqlite.UpdateHook newHook = new Sqlite.UpdateHook(){ + @Override public void call(int opId, String dbName, String tableName, long rowId){ + } + }; + oldHook = db.setUpdateHook(newHook); + affirm( null == oldHook ); + execSql(db, "update t set a='h' where a='a'"); + affirm( 5 == counter.value ); + oldHook = db.setUpdateHook(theHook); + affirm( newHook == oldHook ); + expectedOp.value = Sqlite.UPDATE; + execSql(db, "update t set a='i' where a='h'"); + affirm( 6 == counter.value ); + db.close(); + } + + private void testProgress(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + db.setProgressHandler(1, new Sqlite.ProgressHandler(){ + @Override public int call(){ + ++counter.value; + return 0; + } + }); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( counter.value > 0 ); + int nOld = counter.value; + db.setProgressHandler(0, null); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( nOld == counter.value ); + db.close(); + } + + private void testAuthorizer(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder authRc = new ValueHolder<>(0); + final Sqlite.Authorizer auth = new Sqlite.Authorizer(){ + public int call(int op, String s0, String s1, String s2, String s3){ + ++counter.value; + //outln("xAuth(): "+s0+" "+s1+" "+s2+" "+s3); + return authRc.value; + } + }; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + db.setAuthorizer(auth); + execSql(db, "UPDATE t SET a=1"); + affirm( 1 == counter.value ); + authRc.value = Sqlite.DENY; + int rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( Sqlite.AUTH==rc ); + db.setAuthorizer(null); + rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( 0==rc ); + db.close(); + } + + private void testBlobOpen(){ + final Sqlite db = openDb(); + + execSql(db, "CREATE TABLE T(a BLOB);" + +"INSERT INTO t(rowid,a) VALUES(1, 'def'),(2, 'XYZ');" + ); + Sqlite.Blob b = db.blobOpen("main", "t", "a", + db.lastInsertRowId(), true); + affirm( 3==b.bytes() ); + b.write(new byte[] {100, 101, 102 /*"DEF"*/}, 0); + b.close(); + Sqlite.Stmt stmt = db.prepare("SELECT length(a), a FROM t ORDER BY a"); + affirm( stmt.step() ); + affirm( 3 == stmt.columnInt(0) ); + affirm( "def".equals(stmt.columnText16(1)) ); + stmt.finalizeStmt(); + + b = db.blobOpen("main", "t", "a", db.lastInsertRowId(), false); + final byte[] tgt = new byte[3]; + b.read( tgt, 0 ); + affirm( 100==tgt[0] && 101==tgt[1] && 102==tgt[2], "DEF" ); + execSql(db,"UPDATE t SET a=zeroblob(10) WHERE rowid=2"); + b.close(); + b = db.blobOpen("main", "t", "a", db.lastInsertRowId(), true); + byte[] bw = new byte[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + }; + b.write(bw, 0); + byte[] br = new byte[10]; + b.read(br, 0); + for( int i = 0; i < br.length; ++i ){ + affirm(bw[i] == br[i]); + } + b.close(); + db.close(); + } + + void testPrepareMulti(){ + final ValueHolder fCount = new ValueHolder<>(0); + final ValueHolder mCount = new ValueHolder<>(0); + try (Sqlite db = openDb()) { + execSql(db, "create table t(a); insert into t(a) values(1),(2),(3)"); + db.createFunction("counter", -1, new ScalarFunction(){ + @Override public void xFunc(SqlFunction.Arguments args){ + ++fCount.value; + args.resultNull(); + } + } + ); + final Sqlite.PrepareMulti pm = new Sqlite.PrepareMultiFinalize( + new Sqlite.PrepareMulti() { + @Override public void call(Sqlite.Stmt q){ + ++mCount.value; + while(q.step()){} + } + } + ); + final String sql = "select counter(*) from t;"+ + "select counter(*) from t; /* comment */"+ + "select counter(*) from t; -- comment\n" + ; + db.prepareMulti(sql, pm); + } + affirm( 3 == mCount.value ); + affirm( 9 == fCount.value ); + } + + + /* Copy/paste/rename this to add new tests. */ + private void _testTemplate(){ + try (Sqlite db = openDb()) { + Sqlite.Stmt stmt = db.prepare("SELECT 1"); + stmt.finalizeStmt(); + } + } + + private void runTests(boolean fromThread) throws Exception { + List mlist = testMethods; + affirm( null!=mlist ); + if( shuffle ){ + mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) ); + java.util.Collections.shuffle(mlist); + } + if( (!fromThread && listRunTests>0) || listRunTests>1 ){ + synchronized(this.getClass()){ + if( !fromThread ){ + out("Initial test"," list: "); + for(java.lang.reflect.Method m : testMethods){ + out(m.getName()+" "); + } + outln(); + outln("(That list excludes some which are hard-coded to run.)"); + } + out("Running"," tests: "); + for(java.lang.reflect.Method m : mlist){ + out(m.getName()+" "); + } + outln(); + } + } + for(java.lang.reflect.Method m : mlist){ + nap(); + try{ + m.invoke(this); + }catch(java.lang.reflect.InvocationTargetException e){ + outln("FAILURE: ",m.getName(),"(): ", e.getCause()); + throw e; + } + } + synchronized( this.getClass() ){ + ++nTestRuns; + } + } + + public void run() { + try { + runTests(0!=this.tId); + }catch(Exception e){ + synchronized( listErrors ){ + listErrors.add(e); + } + }finally{ + Sqlite.uncacheThread(); + } + } + + /** + Runs the basic sqlite3 JNI binding sanity-check suite. + + CLI flags: + + -q|-quiet: disables most test output. + + -t|-thread N: runs the tests in N threads + concurrently. Default=1. + + -r|-repeat N: repeats the tests in a loop N times, each one + consisting of the -thread value's threads. + + -shuffle: randomizes the order of most of the test functions. + + -naps: sleep small random intervals between tests in order to add + some chaos for cross-thread contention. + + -list-tests: outputs the list of tests being run, minus some + which are hard-coded. In multi-threaded mode, use this twice to + to emit the list run by each thread (which may differ from the initial + list, in particular if -shuffle is used). + + -fail: forces an exception to be thrown during the test run. Use + with -shuffle to make its appearance unpredictable. + + -v: emit some developer-mode info at the end. + */ + public static void main(String[] args) throws Exception { + Integer nThread = 1; + boolean doSomethingForDev = false; + Integer nRepeat = 1; + boolean forceFail = false; + boolean sqlLog = false; + boolean configLog = false; + boolean squelchTestOutput = false; + for( int i = 0; i < args.length; ){ + String arg = args[i++]; + if(arg.startsWith("-")){ + arg = arg.replaceFirst("-+",""); + if(arg.equals("v")){ + doSomethingForDev = true; + //listBoundMethods(); + }else if(arg.equals("t") || arg.equals("thread")){ + nThread = Integer.parseInt(args[i++]); + }else if(arg.equals("r") || arg.equals("repeat")){ + nRepeat = Integer.parseInt(args[i++]); + }else if(arg.equals("shuffle")){ + shuffle = true; + }else if(arg.equals("list-tests")){ + ++listRunTests; + }else if(arg.equals("fail")){ + forceFail = true; + }else if(arg.equals("sqllog")){ + sqlLog = true; + }else if(arg.equals("configlog")){ + configLog = true; + }else if(arg.equals("naps")){ + takeNaps = true; + }else if(arg.equals("q") || arg.equals("quiet")){ + squelchTestOutput = true; + }else{ + throw new IllegalArgumentException("Unhandled flag:"+arg); + } + } + } + + if( sqlLog ){ + if( Sqlite.compileOptionUsed("ENABLE_SQLLOG") ){ + Sqlite.libConfigSqlLog( new Sqlite.ConfigSqlLog() { + @Override public void call(Sqlite db, String msg, int op){ + switch(op){ + case 0: outln("Opening db: ",db); break; + case 1: outln("SQL ",db,": ",msg); break; + case 2: outln("Closing db: ",db); break; + } + } + } + ); + }else{ + outln("WARNING: -sqllog is not active because library was built ", + "without SQLITE_ENABLE_SQLLOG."); + } + } + if( configLog ){ + Sqlite.libConfigLog( new Sqlite.ConfigLog() { + @Override public void call(int code, String msg){ + outln("ConfigLog: ",Sqlite.errstr(code),": ", msg); + }; + } + ); + } + + quietMode = squelchTestOutput; + outln("If you just saw warning messages regarding CallStaticObjectMethod, ", + "you are very likely seeing the side effects of a known openjdk8 ", + "bug. It is unsightly but does not affect the library."); + + { + // Build list of tests to run from the methods named test*(). + testMethods = new ArrayList<>(); + int nSkipped = 0; + for(final java.lang.reflect.Method m : Tester2.class.getDeclaredMethods()){ + final String name = m.getName(); + if( name.equals("testFail") ){ + if( forceFail ){ + testMethods.add(m); + } + }else if( !m.isAnnotationPresent( ManualTest.class ) ){ + if( nThread>1 && m.isAnnotationPresent( SingleThreadOnly.class ) ){ + if( 0==nSkipped++ ){ + out("Skipping tests in multi-thread mode:"); + } + out(" "+name+"()"); + }else if( name.startsWith("test") ){ + testMethods.add(m); + } + } + } + if( nSkipped>0 ) out("\n"); + } + + final long timeStart = System.currentTimeMillis(); + outln("libversion_number: ", + Sqlite.libVersionNumber(),"\n", + Sqlite.libVersion(),"\n",Sqlite.libSourceId(),"\n", + "SQLITE_THREADSAFE=",CApi.sqlite3_threadsafe()); + final boolean showLoopCount = (nRepeat>1 && nThread>1); + if( showLoopCount ){ + outln("Running ",nRepeat," loop(s) with ",nThread," thread(s) each."); + } + if( takeNaps ) outln("Napping between tests is enabled."); + int nLoop = 0; + for( int n = 0; n < nRepeat; ++n ){ + ++nLoop; + if( showLoopCount ) out((1==nLoop ? "" : " ")+nLoop); + if( nThread<=1 ){ + new Tester2(0).runTests(false); + continue; + } + Tester2.mtMode = true; + final ExecutorService ex = Executors.newFixedThreadPool( nThread ); + for( int i = 0; i < nThread; ++i ){ + ex.submit( new Tester2(i), i ); + } + ex.shutdown(); + try{ + ex.awaitTermination(nThread*200, java.util.concurrent.TimeUnit.MILLISECONDS); + ex.shutdownNow(); + }catch (InterruptedException ie){ + ex.shutdownNow(); + Thread.currentThread().interrupt(); + } + if( !listErrors.isEmpty() ){ + quietMode = false; + outln("TEST ERRORS:"); + Exception err = null; + for( Exception e : listErrors ){ + e.printStackTrace(); + if( null==err ) err = e; + } + if( null!=err ) throw err; + } + } + if( showLoopCount ) outln(); + quietMode = false; + + final long timeEnd = System.currentTimeMillis(); + outln("Tests done. Metrics across ",nTestRuns," total iteration(s):"); + outln("\tAssertions checked: ",affirmCount); + outln("\tDatabases opened: ",metrics.dbOpen); + if( doSomethingForDev ){ + CApi.sqlite3_jni_internal_details(); + } + affirm( 0==Sqlite.libReleaseMemory(1) ); + CApi.sqlite3_shutdown(); + int nMethods = 0; + int nNatives = 0; + int nCanonical = 0; + final java.lang.reflect.Method[] declaredMethods = + CApi.class.getDeclaredMethods(); + for(java.lang.reflect.Method m : declaredMethods){ + final int mod = m.getModifiers(); + if( 0!=(mod & java.lang.reflect.Modifier.STATIC) ){ + final String name = m.getName(); + if(name.startsWith("sqlite3_")){ + ++nMethods; + if( 0!=(mod & java.lang.reflect.Modifier.NATIVE) ){ + ++nNatives; + } + } + } + } + outln("\tCApi.sqlite3_*() methods: "+ + nMethods+" total, with "+ + nNatives+" native, "+ + (nMethods - nNatives)+" Java" + ); + outln("\tTotal test time = " + +(timeEnd - timeStart)+"ms"); + } +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java b/ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..7549bb97b28e2a1fd350a40988c3dbc492cfad12 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java @@ -0,0 +1,25 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the ValueHolder utility class. +*/ +package org.sqlite.jni.wrapper1; + +/** + A helper class which simply holds a single value. Its primary use + is for communicating values out of anonymous callbacks, as doing so + requires a "final" reference. +*/ +public class ValueHolder { + public T value; + public ValueHolder(){} + public ValueHolder(T v){value = v;} +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..a3905567d468cda90270a35e57b75031085911f9 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java @@ -0,0 +1,42 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; + +/** + A SqlFunction implementation for window functions. The T type + represents the type of data accumulated by this function while it + works. e.g. a SUM()-like UDF might use Integer or Long and a + CONCAT()-like UDF might use a StringBuilder or a List. +*/ +public abstract class WindowFunction extends AggregateFunction { + + /** + As for the xInverse() argument of the C API's + sqlite3_create_window_function(). If this function throws, the + exception is reported via sqlite3_result_error(). + */ + public abstract void xInverse(SqlFunction.Arguments args); + + /** + As for the xValue() argument of the C API's + sqlite3_create_window_function(). If this function throws, it is + translated into sqlite3_result_error(). + + Note that the passed-in object will not actually contain any + arguments for xValue() but will contain the context object needed + for setting the call's result or error state. + */ + public abstract void xValue(SqlFunction.Arguments args); + +} diff --git a/ext/jni/src/tests/000-000-sanity.test b/ext/jni/src/tests/000-000-sanity.test new file mode 100644 index 0000000000000000000000000000000000000000..4ccbece31c3ae6c64d119756558ae49fefcd15b5 --- /dev/null +++ b/ext/jni/src/tests/000-000-sanity.test @@ -0,0 +1,53 @@ +/* +** This is a comment. There are many like it but this one is mine. +** +** SCRIPT_MODULE_NAME: sanity-check +** xMIXED_MODULE_NAME: mixed-module +** xMODULE_NAME: module-name +** xREQUIRED_PROPERTIES: small fast reliable +** xREQUIRED_PROPERTIES: RECURSIVE_TRIGGERS +** xREQUIRED_PROPERTIES: TEMPSTORE_FILE TEMPSTORE_MEM +** xREQUIRED_PROPERTIES: AUTOVACUUM INCRVACUUM +** +*/ +--print starting up 😃 +--close all +--oom +--db 0 +--new my.db +--null zilch +--testcase 1.0 +SELECT 1, null; +--result 1 zilch +--glob *zil* +--notglob *ZIL* +SELECT 1, 2; +intentional error; +--run +--testcase json-1 +SELECT json_array(1,2,3) +--json [1,2,3] +--testcase tableresult-1 + select 1, 'a'; + select 2, 'b'; +--tableresult + # [a-z] + 2 b +--end +--testcase json-block-1 + select json_array(1,2,3); + select json_object('a',1,'b',2); +--json-block + [1,2,3] + {"a":1,"b":2} +--end +--testcase col-names-on +--column-names 1 + select 1 as 'a', 2 as 'b'; +--result a 1 b 2 +--testcase col-names-off +--column-names 0 + select 1 as 'a', 2 as 'b'; +--result 1 2 +--close +--print reached the end 😃 diff --git a/ext/jni/src/tests/000-001-ignored.test b/ext/jni/src/tests/000-001-ignored.test new file mode 100644 index 0000000000000000000000000000000000000000..5af852e197318b63649e1cc042a158d10ff7d2e3 --- /dev/null +++ b/ext/jni/src/tests/000-001-ignored.test @@ -0,0 +1,9 @@ +/* +** This script must be marked as ignored because it contains +** content which triggers that condition. +** +** SCRIPT_MODULE_NAME: ignored +** +*/ + +| diff --git a/ext/jni/src/tests/900-001-fts.test b/ext/jni/src/tests/900-001-fts.test new file mode 100644 index 0000000000000000000000000000000000000000..65285e86b06d995e3bbd7de6cd0e603ba8466210 --- /dev/null +++ b/ext/jni/src/tests/900-001-fts.test @@ -0,0 +1,12 @@ +/* +** SCRIPT_MODULE_NAME: fts5-sanity-checks +** xREQUIRED_PROPERTIES: FTS5 +** +*/ + +--testcase 1.0 +CREATE VIRTUAL TABLE email USING fts5(sender, title, body); +insert into email values('fred','Help!','Dear Sir...'); +insert into email values('barney','Assistance','Dear Madam...'); +select * from email where email match 'assistance'; +--result barney Assistance {Dear Madam...} diff --git a/ext/lsm1/Makefile b/ext/lsm1/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d497a1d133eff6bf09d029a1cf819d2de7fca3fb --- /dev/null +++ b/ext/lsm1/Makefile @@ -0,0 +1,56 @@ +# +# This Makefile is designed for use with main.mk in the root directory of +# this project. After including main.mk, the users makefile should contain: +# +# LSMDIR=$(TOP)/ext/lsm1/ +# LSMOPTS=-fPIC +# include $(LSMDIR)/Makefile +# +# The most useful targets are [lsmtest] and [lsm.so]. +# + +LSMOBJ = \ + lsm_ckpt.o \ + lsm_file.o \ + lsm_log.o \ + lsm_main.o \ + lsm_mem.o \ + lsm_mutex.o \ + lsm_shared.o \ + lsm_sorted.o \ + lsm_str.o \ + lsm_tree.o \ + lsm_unix.o \ + lsm_win32.o \ + lsm_varint.o \ + lsm_vtab.o + +LSMHDR = \ + $(LSMDIR)/lsm.h \ + $(LSMDIR)/lsmInt.h + +LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \ + $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c \ + $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c \ + $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c \ + $(LSMDIR)/lsm-test/lsmtest9.c \ + $(LSMDIR)/lsm-test/lsmtest_datasource.c \ + $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c \ + $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \ + $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \ + $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c + + +# all: lsm.so + +LSMOPTS += -fPIC -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB + +lsm.so: $(LSMOBJ) + $(T.link) -shared -fPIC -o lsm.so $(LSMOBJ) + +%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h + $(T.link) $(LSMOPTS) -c $< + +lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o + # $(T.link) -c $(TOP)/lsm-test/lsmtest_tdb2.cc + $(T.link) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz diff --git a/ext/lsm1/Makefile.msc b/ext/lsm1/Makefile.msc new file mode 100644 index 0000000000000000000000000000000000000000..3e5a3b3310e66f75dbbb0624f13b1e416ee3f87f --- /dev/null +++ b/ext/lsm1/Makefile.msc @@ -0,0 +1,102 @@ +# +# This Makefile is designed for use with Makefile.msc in the root directory +# of this project. The Makefile.msc should contain: +# +# LSMDIR=$(TOP)\ext\lsm1 +# !INCLUDE $(LSMDIR)\Makefile.msc +# +# The most useful targets are [lsmtest.exe] and [lsm.dll]. +# + +LSMOBJ = \ + lsm_ckpt.lo \ + lsm_file.lo \ + lsm_log.lo \ + lsm_main.lo \ + lsm_mem.lo \ + lsm_mutex.lo \ + lsm_shared.lo \ + lsm_sorted.lo \ + lsm_str.lo \ + lsm_tree.lo \ + lsm_unix.lo \ + lsm_win32.lo \ + lsm_varint.lo \ + lsm_vtab.lo + +LSMHDR = \ + $(LSMDIR)\lsm.h \ + $(LSMDIR)\lsmInt.h + +LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c \ + $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c \ + $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c \ + $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c \ + $(LSMDIR)\lsm-test\lsmtest9.c \ + $(LSMDIR)\lsm-test\lsmtest_datasource.c \ + $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c \ + $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \ + $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \ + $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c + +# all: lsm.dll lsmtest.exe + +LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR) + +!IF $(DEBUG)>2 +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1 +!ENDIF + +!IF $(MEMDEBUG)!=0 +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1 +!ENDIF + +lsm_ckpt.lo: $(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c + +lsm_file.lo: $(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c + +lsm_log.lo: $(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c + +lsm_main.lo: $(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c + +lsm_mem.lo: $(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c + +lsm_mutex.lo: $(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c + +lsm_shared.lo: $(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c + +lsm_sorted.lo: $(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c + +lsm_str.lo: $(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c + +lsm_tree.lo: $(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c + +lsm_unix.lo: $(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c + +lsm_win32.lo: $(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c + +lsm_varint.lo: $(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c + +lsm_vtab.lo: $(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H) + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c + +lsm.dll: $(LSMOBJ) + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ) + copy /Y $@ $(LSMDIR)\$@ + +lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ) + $(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ) + copy /Y $@ $(LSMDIR)\$@ diff --git a/ext/lsm1/lsm-test/README b/ext/lsm1/lsm-test/README new file mode 100644 index 0000000000000000000000000000000000000000..80654ee97ec12f3a935a1b28a6dac600e8098335 --- /dev/null +++ b/ext/lsm1/lsm-test/README @@ -0,0 +1,40 @@ + + +Organization of test case files: + + lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a + database file, then verify that the contents of the database can + be queried. + + lsmtest2.c: Crash tests. Tests that attempt to verify that the database + recovers correctly following an application or system crash. + + lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of + transactions and sub-transactions. + + lsmtest4.c: Multi-client tests. + + lsmtest5.c: Multi-client tests with a different thread for each client. + + lsmtest6.c: OOM injection tests. + + lsmtest7.c: API tests. + + lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that + the system recovers and other clients proceed unaffected if + a process fails in the middle of a write transaction. + + The difference from lsmtest2.c is that this file tests + live-recovery (recovery from a failure that occurs while other + clients are still running) whereas lsmtest2.c tests recovery + from a system or power failure. + + lsmtest9.c: More data tests. These focus on testing that calling + lsm_work(nMerge=1) to compact the database does not corrupt it. + In other words, that databases containing block-redirects + can be read and written. + + + + + diff --git a/ext/lsm1/lsm-test/lsmtest.h b/ext/lsm1/lsm-test/lsmtest.h new file mode 100644 index 0000000000000000000000000000000000000000..ca60424adde9a60bcfd90acce739db44908c0338 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest.h @@ -0,0 +1,303 @@ + +#ifndef __WRAPPER_INT_H_ +#define __WRAPPER_INT_H_ + +#include "lsmtest_tdb.h" +#include "sqlite3.h" +#include "lsm.h" + +#include +#include +#include +#include +#include +#ifndef _WIN32 +# include +#endif +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _WIN32 +# include "windows.h" +# define gettimeofday win32GetTimeOfDay +# define F_OK (0) +# define sleep(sec) Sleep(1000 * (sec)) +# define usleep(usec) Sleep(((usec) + 999) / 1000) +# ifdef _MSC_VER +# include +# define snprintf _snprintf +# define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) +# define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) +# define __va_copy(dst,src) ((dst) = (src)) +# define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1) +# else +# error Unsupported C compiler for Windows. +# endif +int win32GetTimeOfDay(struct timeval *, void *); +#endif + +#ifndef _LSM_INT_H +typedef unsigned int u32; +typedef unsigned char u8; +typedef long long int i64; +typedef unsigned long long int u64; +#endif + + +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) + +#define MIN(x,y) ((x)<(y) ? (x) : (y)) +#define MAX(x,y) ((x)>(y) ? (x) : (y)) + +#define unused_parameter(x) (void)(x) + +#define TESTDB_DEFAULT_PAGE_SIZE 4096 +#define TESTDB_DEFAULT_CACHE_SIZE 2048 + +#ifndef _O_BINARY +# define _O_BINARY (0) +#endif + +/* +** Ideally, these should be in wrapper.c. But they are here instead so that +** they can be used by the C++ database wrappers in wrapper2.cc. +*/ +typedef struct DatabaseMethods DatabaseMethods; +struct TestDb { + DatabaseMethods const *pMethods; /* Database methods */ + const char *zLibrary; /* Library name for tdb_open() */ +}; +struct DatabaseMethods { + int (*xClose)(TestDb *); + int (*xWrite)(TestDb *, void *, int , void *, int); + int (*xDelete)(TestDb *, void *, int); + int (*xDeleteRange)(TestDb *, void *, int, void *, int); + int (*xFetch)(TestDb *, void *, int, void **, int *); + int (*xScan)(TestDb *, void *, int, void *, int, void *, int, + void (*)(void *, void *, int , void *, int) + ); + int (*xBegin)(TestDb *, int); + int (*xCommit)(TestDb *, int); + int (*xRollback)(TestDb *, int); +}; + +/* +** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the +** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but +** the primary interface is the C++ API. +*/ +int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb); +int test_kc_close(TestDb *); +int test_kc_write(TestDb *, void *, int , void *, int); +int test_kc_delete(TestDb *, void *, int); +int test_kc_delete_range(TestDb *, void *, int, void *, int); +int test_kc_fetch(TestDb *, void *, int, void **, int *); +int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, + void (*)(void *, void *, int , void *, int) +); + +int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb); +int test_mdb_close(TestDb *); +int test_mdb_write(TestDb *, void *, int , void *, int); +int test_mdb_delete(TestDb *, void *, int); +int test_mdb_fetch(TestDb *, void *, int, void **, int *); +int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, + void (*)(void *, void *, int , void *, int) +); + +/* +** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. +** The wrapper for lsm is a bit more involved than the others, as it +** includes code for a couple of different lsm configurations, and for +** various types of fault injection and robustness testing. +*/ +int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb); +int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb); +int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb); +int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb); +int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb); +int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb); +int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb); + +int tdb_lsm_configure(lsm_db *, const char *); + +/* Functions in lsmtest_tdb4.c */ +int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); +int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); +int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb); + + +/* Functions in testutil.c. */ +int testPrngInit(void); +u32 testPrngValue(u32 iVal); +void testPrngArray(u32 iVal, u32 *aOut, int nOut); +void testPrngString(u32 iVal, char *aOut, int nOut); + +void testErrorInit(int argc, char **); +void testPrintError(const char *zFormat, ...); +void testPrintUsage(const char *zArgs); +void testPrintFUsage(const char *zFormat, ...); +void testTimeInit(void); +int testTimeGet(void); + +/* Functions in testmem.c. */ +void testMallocInstall(lsm_env *pEnv); +void testMallocUninstall(lsm_env *pEnv); +void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *); +void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *); +void testMallocOomEnable(lsm_env *pEnv, int); + +/* lsmtest.c */ +TestDb *testOpen(const char *zSystem, int, int *pRc); +void testReopen(TestDb **ppDb, int *pRc); +void testClose(TestDb **ppDb); + +void testFetch(TestDb *, void *, int, void *, int, int *); +void testWrite(TestDb *, void *, int, void *, int, int *); +void testDelete(TestDb *, void *, int, int *); +void testDeleteRange(TestDb *, void *, int, void *, int, int *); +void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc); +void testFetchStr(TestDb *, const char *, const char *, int *pRc); + +void testBegin(TestDb *pDb, int iTrans, int *pRc); +void testCommit(TestDb *pDb, int iTrans, int *pRc); + +void test_failed(void); + +char *testMallocPrintf(const char *zFormat, ...); +char *testMallocVPrintf(const char *zFormat, va_list ap); +int testGlobMatch(const char *zPattern, const char *zStr); + +void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *); +void testFetchCompare(TestDb *, TestDb *, void *, int, int *); + +void *testMalloc(int); +void *testMallocCopy(void *pCopy, int nByte); +void *testRealloc(void *, int); +void testFree(void *); + +/* lsmtest_bt.c */ +int do_bt(int nArg, char **azArg); + +/* testio.c */ +int testVfsConfigureDb(TestDb *pDb); + +/* testfunc.c */ +int do_show(int nArg, char **azArg); +int do_work(int nArg, char **azArg); + +/* testio.c */ +int do_io(int nArg, char **azArg); + +/* lsmtest2.c */ +void do_crash_test(const char *zPattern, int *pRc); +int do_rollback_test(int nArg, char **azArg); + +/* test3.c */ +void test_rollback(const char *zSystem, const char *zPattern, int *pRc); + +/* test4.c */ +void test_mc(const char *zSystem, const char *zPattern, int *pRc); + +/* test5.c */ +void test_mt(const char *zSystem, const char *zPattern, int *pRc); + +/* lsmtest6.c */ +void test_oom(const char *zPattern, int *pRc); +void testDeleteLsmdb(const char *zFile); + +void testSaveDb(const char *zFile, const char *zAuxExt); +void testRestoreDb(const char *zFile, const char *zAuxExt); +void testCopyLsmdb(const char *zFrom, const char *zTo); + +/* lsmtest7.c */ +void test_api(const char *zPattern, int *pRc); + +/* lsmtest8.c */ +void do_writer_crash_test(const char *zPattern, int *pRc); + +/************************************************************************* +** Interface to functionality in test_datasource.c. +*/ +typedef struct Datasource Datasource; +typedef struct DatasourceDefn DatasourceDefn; + +struct DatasourceDefn { + int eType; /* A TEST_DATASOURCE_* value */ + int nMinKey; /* Minimum key size */ + int nMaxKey; /* Maximum key size */ + int nMinVal; /* Minimum value size */ + int nMaxVal; /* Maximum value size */ +}; + +#define TEST_DATASOURCE_RANDOM 1 +#define TEST_DATASOURCE_SEQUENCE 2 + +char *testDatasourceName(const DatasourceDefn *); +Datasource *testDatasourceNew(const DatasourceDefn *); +void testDatasourceFree(Datasource *); +void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *); +/* End of test_datasource.c interface. +*************************************************************************/ +void testDatasourceFetch( + TestDb *pDb, /* Database handle */ + Datasource *pData, + int iKey, + int *pRc /* IN/OUT: Error code */ +); + +void testWriteDatasource(TestDb *, Datasource *, int, int *); +void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *); +void testDeleteDatasource(TestDb *, Datasource *, int, int *); +void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *); + + +/* test1.c */ +void test_data_1(const char *, const char *, int *pRc); +void test_data_2(const char *, const char *, int *pRc); +void test_data_3(const char *, const char *, int *pRc); +void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); +void testCaseProgress(int, int, int, int *); +int testCaseNDot(void); + +void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); +int testControlDb(TestDb **ppDb); + +typedef struct CksumDb CksumDb; +CksumDb *testCksumArrayNew(Datasource *, int, int, int); +char *testCksumArrayGet(CksumDb *, int); +void testCksumArrayFree(CksumDb *); +void testCaseStart(int *pRc, char *zFmt, ...); +void testCaseFinish(int rc); +void testCaseSkip(void); +int testCaseBegin(int *, const char *, const char *, ...); + +#define TEST_CKSUM_BYTES 29 +int testCksumDatabase(TestDb *pDb, char *zOut); +int testCountDatabase(TestDb *pDb); +void testCompareInt(int, int, int *); +void testCompareStr(const char *z1, const char *z2, int *pRc); + +/* lsmtest9.c */ +void test_data_4(const char *, const char *, int *pRc); + + +/* +** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. +*/ +#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) +int testArgSelectX(void *, const char *, int, const char *, int *); + +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif + +#endif diff --git a/ext/lsm1/lsm-test/lsmtest1.c b/ext/lsm1/lsm-test/lsmtest1.c new file mode 100644 index 0000000000000000000000000000000000000000..1ce2cc05881093b8dd9dc784c39c1da4e0685f23 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest1.c @@ -0,0 +1,656 @@ + +#include "lsmtest.h" + +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE +#define DATA_RANDOM TEST_DATASOURCE_RANDOM + +typedef struct Datatest1 Datatest1; +typedef struct Datatest2 Datatest2; + +/* +** An instance of the following structure contains parameters used to +** customize the test function in this file. Test procedure: +** +** 1. Create a data-source based on the "datasource definition" vars. +** +** 2. Insert nRow key value pairs into the database. +** +** 3. Delete all keys from the database. Deletes are done in the same +** order as the inserts. +** +** During steps 2 and 3 above, after each Datatest1.nVerify inserts or +** deletes, the following: +** +** a. Run Datasource.nTest key lookups and check the results are as expected. +** +** b. If Datasource.bTestScan is true, run a handful (8) of range +** queries (scanning forwards and backwards). Check that the results +** are as expected. +** +** c. Close and reopen the database. Then run (a) and (b) again. +*/ +struct Datatest1 { + /* Datasource definition */ + DatasourceDefn defn; + + /* Test procedure parameters */ + int nRow; /* Number of rows to insert then delete */ + int nVerify; /* How often to verify the db contents */ + int nTest; /* Number of keys to test (0==all) */ + int bTestScan; /* True to do scan tests */ +}; + +/* +** An instance of the following data structure is used to describe the +** second type of test case in this file. The chief difference between +** these tests and those described by Datatest1 is that these tests also +** experiment with range-delete operations. Tests proceed as follows: +** +** 1. Open the datasource described by Datatest2.defn. +** +** 2. Open a connection on an empty database. +** +** 3. Do this Datatest2.nIter times: +** +** a) Insert Datatest2.nWrite key-value pairs from the datasource. +** +** b) Select two pseudo-random keys and use them as the start +** and end points of a range-delete operation. +** +** c) Verify that the contents of the database are as expected (see +** below for details). +** +** d) Close and then reopen the database handle. +** +** e) Verify that the contents of the database are still as expected. +** +** The inserts and range deletes are run twice - once on the database being +** tested and once using a control system (sqlite3, kc etc. - something that +** works). In order to verify that the contents of the db being tested are +** correct, the test runs a bunch of scans and lookups on both the test and +** control databases. If the results are the same, the test passes. +*/ +struct Datatest2 { + DatasourceDefn defn; + int nRange; + int nWrite; /* Number of writes per iteration */ + int nIter; /* Total number of iterations to run */ +}; + +/* +** Generate a unique name for the test case pTest with database system +** zSystem. +*/ +static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){ + char *zRet; + char *zData; + zData = testDatasourceName(&pTest->defn); + zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", + zSystem, zData, bRecover, pTest->nRow, pTest->nVerify + ); + testFree(zData); + return zRet; +} + +int testControlDb(TestDb **ppDb){ +#ifdef HAVE_KYOTOCABINET + return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); +#else + return tdb_open("sqlite3", "", 1, ppDb); +#endif +} + +void testDatasourceFetch( + TestDb *pDb, /* Database handle */ + Datasource *pData, + int iKey, + int *pRc /* IN/OUT: Error code */ +){ + void *pKey; int nKey; /* Database key to query for */ + void *pVal; int nVal; /* Expected result of query */ + + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); + testFetch(pDb, pKey, nKey, pVal, nVal, pRc); +} + +/* +** This function is called to test that the contents of database pDb +** are as expected. In this case, expected is defined as containing +** key-value pairs iFirst through iLast, inclusive, from data source +** pData. In other words, a loop like the following could be used to +** construct a database with identical contents from scratch. +** +** for(i=iFirst; i<=iLast; i++){ +** testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); +** // insert (pKey, nKey) -> (pVal, nVal) into database +** } +** +** The key domain consists of keys 0 to (nRow-1), inclusive, from +** data source pData. For both scan and lookup tests, keys are selected +** pseudo-randomly from within this set. +** +** This function runs nLookupTest lookup tests and nScanTest scan tests. +** +** A lookup test consists of selecting a key from the domain and querying +** pDb for it. The test fails if the presence of the key and, if present, +** the associated value do not match the expectations defined above. +** +** A scan test involves selecting a key from the domain and running +** the following queries: +** +** 1. Scan all keys equal to or greater than the key, in ascending order. +** 2. Scan all keys equal to or smaller than the key, in descending order. +** +** Additionally, if nLookupTest is greater than zero, the following are +** run once: +** +** 1. Scan all keys in the db, in ascending order. +** 2. Scan all keys in the db, in descending order. +** +** As you would assume, the test fails if the returned values do not match +** expectations. +*/ +void testDbContents( + TestDb *pDb, /* Database handle being tested */ + Datasource *pData, /* pDb contains data from here */ + int nRow, /* Size of key domain */ + int iFirst, /* Index of first key from pData in pDb */ + int iLast, /* Index of last key from pData in pDb */ + int nLookupTest, /* Number of lookup tests to run */ + int nScanTest, /* Number of scan tests to run */ + int *pRc /* IN/OUT: Error code */ +){ + int j; + int rc = *pRc; + + if( rc==0 && nScanTest ){ + TestDb *pDb2 = 0; + + /* Open a control db (i.e. one that we assume works) */ + rc = testControlDb(&pDb2); + + for(j=iFirst; rc==0 && j<=iLast; j++){ + void *pKey; int nKey; /* Database key to insert */ + void *pVal; int nVal; /* Database value to insert */ + testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal); + rc = tdb_write(pDb2, pKey, nKey, pVal, nVal); + } + + if( rc==0 ){ + int iKey1; + int iKey2; + void *pKey1; int nKey1; /* Start key */ + void *pKey2; int nKey2; /* Final key */ + + iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow; + iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow; + testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); + pKey1 = testMalloc(nKey1+1); + memcpy(pKey1, pKey2, nKey1+1); + testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); + + testScanCompare(pDb2, pDb, 0, 0, 0, 0, 0, &rc); + testScanCompare(pDb2, pDb, 0, 0, 0, pKey2, nKey2, &rc); + testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0, &rc); + testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc); + testScanCompare(pDb2, pDb, 1, 0, 0, 0, 0, &rc); + testScanCompare(pDb2, pDb, 1, 0, 0, pKey2, nKey2, &rc); + testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0, &rc); + testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc); + testFree(pKey1); + } + tdb_close(pDb2); + } + + /* Test some lookups. */ + for(j=0; rc==0 && j=nRow ){ + iKey = j; + }else{ + iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow; + } + + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); + if( iFirst>iKey || iKey>iLast ){ + pVal = 0; + nVal = -1; + } + + testFetch(pDb, pKey, nKey, pVal, nVal, &rc); + } + + *pRc = rc; +} + +/* +** This function should be called during long running test cases to output +** the progress dots (...) to stdout. +*/ +void testCaseProgress(int i, int n, int nDot, int *piDot){ + int iDot = *piDot; + while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){ + printf("."); + fflush(stdout); + iDot++; + } + *piDot = iDot; +} + +int testCaseNDot(void){ return 20; } + +#if 0 +static void printScanCb( + void *pCtx, void *pKey, int nKey, void *pVal, int nVal +){ + printf("%s\n", (char *)pKey); + fflush(stdout); +} +#endif + +void testReopenRecover(TestDb **ppDb, int *pRc){ + if( *pRc==0 ){ + const char *zLib = tdb_library_name(*ppDb); + const char *zDflt = tdb_default_db(zLib); + testCopyLsmdb(zDflt, "bak.db"); + testClose(ppDb); + testCopyLsmdb("bak.db", zDflt); + *pRc = tdb_open(zLib, 0, 0, ppDb); + } +} + + +static void doDataTest1( + const char *zSystem, /* Database system to test */ + int bRecover, + Datatest1 *p, /* Structure containing test parameters */ + int *pRc /* OUT: Error code */ +){ + int i; + int iDot; + int rc = LSM_OK; + Datasource *pData; + TestDb *pDb; + int iToggle = 0; + + /* Start the test case, open a database and allocate the datasource. */ + pDb = testOpen(zSystem, 1, &rc); + pData = testDatasourceNew(&p->defn); + + i = 0; + iDot = 0; + while( rc==LSM_OK && inRow ){ + + /* Insert some data */ + testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc); + i += p->nVerify; + + if( iToggle ) testBegin(pDb, 1, &rc); + /* Check that the db content is correct. */ + testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); + if( iToggle ) testCommit(pDb, 0, &rc); + iToggle = (iToggle+1)%2; + + if( bRecover ){ + testReopenRecover(&pDb, &rc); + }else{ + testReopen(&pDb, &rc); + } + + /* Check that the db content is still correct. */ + testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); + + /* Update the progress dots... */ + testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); + } + + i = 0; + iDot = 0; + while( rc==LSM_OK && inRow ){ + + /* Delete some entries */ + testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc); + i += p->nVerify; + + /* Check that the db content is correct. */ + testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); + + /* Close and reopen the database. */ + if( bRecover ){ + testReopenRecover(&pDb, &rc); + }else{ + testReopen(&pDb, &rc); + } + + /* Check that the db content is still correct. */ + testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); + + /* Update the progress dots... */ + testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); + } + + /* Free the datasource, close the database and finish the test case. */ + testDatasourceFree(pData); + tdb_close(pDb); + testCaseFinish(rc); + *pRc = rc; +} + + +void test_data_1( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + Datatest1 aTest[] = { + { {DATA_RANDOM, 500,600, 1000,2000}, 1000, 100, 10, 0}, + { {DATA_RANDOM, 20,25, 100,200}, 1000, 250, 1000, 1}, + { {DATA_RANDOM, 8,10, 100,200}, 1000, 250, 1000, 1}, + { {DATA_RANDOM, 8,10, 10,20}, 1000, 250, 1000, 1}, + { {DATA_RANDOM, 8,10, 1000,2000}, 1000, 250, 1000, 1}, + { {DATA_RANDOM, 8,100, 10000,20000}, 100, 25, 100, 1}, + { {DATA_RANDOM, 80,100, 10,20}, 1000, 250, 1000, 1}, + { {DATA_RANDOM, 5000,6000, 10,20}, 100, 25, 100, 1}, + { {DATA_SEQUENTIAL, 5,10, 10,20}, 1000, 250, 1000, 1}, + { {DATA_SEQUENTIAL, 5,10, 100,200}, 1000, 250, 1000, 1}, + { {DATA_SEQUENTIAL, 5,10, 1000,2000}, 1000, 250, 1000, 1}, + { {DATA_SEQUENTIAL, 5,100, 10000,20000}, 100, 25, 100, 1}, + { {DATA_RANDOM, 10,10, 100,100}, 100000, 1000, 100, 0}, + { {DATA_SEQUENTIAL, 10,10, 100,100}, 100000, 1000, 100, 0}, + }; + + int i; + int bRecover; + + for(bRecover=0; bRecover<2; bRecover++){ + if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; + for(i=0; *pRc==LSM_OK && idefn); + rc = testControlDb(&pControl); + + if( tdb_lsm(pDb) ){ + int nBuf = 32 * 1024 * 1024; + lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); + } + + for(i=0; rc==0 && inIter; i++){ + void *pKey1; int nKey1; + void *pKey2; int nKey2; + int ii; + int nRange = MIN(p->nIter*p->nWrite, p->nRange); + + for(ii=0; rc==0 && iinWrite; ii++){ + int iKey = (i*p->nWrite + ii) % p->nRange; + testWriteDatasource(pControl, pData, iKey, &rc); + testWriteDatasource(pDb, pData, iKey, &rc); + } + + testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0); + pKey1 = testMallocCopy(pKey1, nKey1); + testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0); + + testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc); + testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc); + testFree(pKey1); + + testCompareDb(pData, nRange, i, pControl, pDb, &rc); + if( bRecover ){ + testReopenRecover(&pDb, &rc); + }else{ + testReopen(&pDb, &rc); + } + testCompareDb(pData, nRange, i, pControl, pDb, &rc); + + /* Update the progress dots... */ + testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); + } + + testClose(&pDb); + testClose(&pControl); + testDatasourceFree(pData); + testCaseFinish(rc); + *pRc = rc; +} + +static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){ + char *zRet; + char *zData; + zData = testDatasourceName(&pTest->defn); + zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", + zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter + ); + testFree(zData); + return zRet; +} + +void test_data_2( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + Datatest2 aTest[] = { + /* defn, nRange, nWrite, nIter */ + { {DATA_RANDOM, 20,25, 100,200}, 10000, 10, 50 }, + { {DATA_RANDOM, 20,25, 100,200}, 10000, 200, 50 }, + { {DATA_RANDOM, 20,25, 100,200}, 100, 10, 1000 }, + { {DATA_RANDOM, 20,25, 100,200}, 100, 200, 50 }, + }; + + int i; + int bRecover; + + for(bRecover=0; bRecover<2; bRecover++){ + if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; + for(i=0; *pRc==LSM_OK && i> 24) & 0xFF; + aBuf[1] = (iVal >> 16) & 0xFF; + aBuf[2] = (iVal >> 8) & 0xFF; + aBuf[3] = (iVal >> 0) & 0xFF; +} + +void dt3PutKey(u8 *aBuf, int iKey){ + assert( iKey<100000 && iKey>=0 ); + sprintf((char *)aBuf, "%.5d", iKey); +} + +static void doDataTest3( + const char *zSystem, /* Database system to test */ + Datatest3 *p, /* Structure containing test parameters */ + int *pRc /* OUT: Error code */ +){ + int iDot = 0; + int rc = *pRc; + TestDb *pDb; + u8 *abPresent; /* Array of boolean */ + char *aVal; /* Buffer to hold values */ + int i; + u32 iSeq = 10; /* prng counter */ + + abPresent = (u8 *)testMalloc(p->nRange+1); + aVal = (char *)testMalloc(p->nValMax+1); + pDb = testOpen(zSystem, 1, &rc); + + for(i=0; inIter && rc==0; i++){ + int ii; + + testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); + + /* Perform nWrite inserts */ + for(ii=0; iinWrite; ii++){ + u8 aKey[6]; + u32 iKey; + int nVal; + + iKey = (testPrngValue(iSeq++) % p->nRange) + 1; + nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin; + testPrngString(testPrngValue(iSeq++), aVal, nVal); + dt3PutKey(aKey, iKey); + + testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc); + abPresent[iKey] = 1; + } + + /* Perform nDelete deletes */ + for(ii=0; iinDelete; ii++){ + u8 aKey1[6]; + u8 aKey2[6]; + u32 iKey; + + iKey = (testPrngValue(iSeq++) % p->nRange) + 1; + dt3PutKey(aKey1, iKey-1); + dt3PutKey(aKey2, iKey+1); + + testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc); + abPresent[iKey] = 0; + } + + testReopen(&pDb, &rc); + + for(ii=1; rc==0 && ii<=p->nRange; ii++){ + int nDbVal; + void *pDbVal; + u8 aKey[6]; + int dbrc; + + dt3PutKey(aKey, ii); + dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal); + testCompareInt(0, dbrc, &rc); + + if( abPresent[ii] ){ + testCompareInt(1, (nDbVal>0), &rc); + }else{ + testCompareInt(1, (nDbVal<0), &rc); + } + } + } + + testClose(&pDb); + testCaseFinish(rc); + *pRc = rc; +} + +static char *getName3(const char *zSystem, Datatest3 *p){ + return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)", + zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, + p->nValMin, p->nValMax + ); +} + +void test_data_3( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + Datatest3 aTest[] = { + /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */ + { 100, 1000, 5, 5, 50, 100 }, + { 100, 1000, 2, 2, 5, 10 }, + }; + + int i; + + for(i=0; *pRc==LSM_OK && inRow++; + for(i=0; icksum1 += ((u8 *)pKey)[i]; + p->cksum2 += p->cksum1; + } + for(i=0; icksum1 += ((u8 *)pVal)[i]; + p->cksum2 += p->cksum1; + } +} + +/* +** tdb_scan() callback used by testCountDatabase() +*/ +static void scanCountDb( + void *pCtx, + void *pKey, int nKey, + void *pVal, int nVal +){ + Cksum *p = (Cksum *)pCtx; + p->nRow++; + + unused_parameter(pKey); + unused_parameter(nKey); + unused_parameter(pVal); + unused_parameter(nVal); +} + + +/* +** Iterate through the entire contents of database pDb. Write a checksum +** string based on the db contents into buffer zOut before returning. A +** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size: +** +** * 32-bit integer (10 bytes) +** * 1 space (1 byte) +** * 32-bit hex (8 bytes) +** * 1 space (1 byte) +** * 32-bit hex (8 bytes) +** * nul-terminator (1 byte) +** +** The number of entries in the database is returned. +*/ +int testCksumDatabase( + TestDb *pDb, /* Database handle */ + char *zOut /* Buffer to write checksum to */ +){ + Cksum cksum; + memset(&cksum, 0, sizeof(Cksum)); + tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb); + sprintf(zOut, "%d %x %x", + cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2 + ); + assert( strlen(zOut)0 ); */ + if( testrc==0 ) testrc = lsm_checkpoint(db, 0); + } + tdb_close(pDb); + + /* Check that the database content is still correct */ + testCompareCksumLsmdb(DBNAME, + bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc); + } + + testCksumArrayFree(pCksumDb); + testDatasourceFree(pData); +} + +/* +** This test verifies that if a system crash occurs while committing a +** transaction to the log file, no earlier transactions are lost or damaged. +*/ +static void crash_test2(int bCompress, int *pRc){ + const char *DBNAME = "testdb.lsm"; + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; + + const int nIter = 200; + const int nInsert = 20; + + int i; + int iDot = 0; + Datasource *pData; + CksumDb *pCksumDb; + TestDb *pDb; + + /* Allocate datasource. And calculate the expected checksums. */ + pData = testDatasourceNew(&defn); + pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1); + + /* Setup and save the initial database. */ + testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); + + for(i=0; izTest) ){ + p->x(p->bCompress, pRc); + testCaseFinish(*pRc); + } + } +} diff --git a/ext/lsm1/lsm-test/lsmtest3.c b/ext/lsm1/lsm-test/lsmtest3.c new file mode 100644 index 0000000000000000000000000000000000000000..760dec300f464a5d042f2f6e483ed66215a5325d --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest3.c @@ -0,0 +1,238 @@ + + +/* +** This file contains tests related to the explicit rollback of database +** transactions and sub-transactions. +*/ + + +/* +** Repeat 2000 times (until the db contains 100,000 entries): +** +** 1. Open a transaction and insert 500 rows, opening a nested +** sub-transaction each 100 rows. +** +** 2. Roll back to each sub-transaction savepoint. Check the database +** checksum looks Ok. +** +** 3. Every second iteration, roll back the main transaction. Check the +** db checksum is correct. Every other iteration, commit the main +** transaction (increasing the size of the db by 100 rows). +*/ + + +#include "lsmtest.h" + +struct CksumDb { + int nFirst; + int nLast; + int nStep; + char **azCksum; +}; + +CksumDb *testCksumArrayNew( + Datasource *pData, + int nFirst, + int nLast, + int nStep +){ + TestDb *pDb; + CksumDb *pRet; + int i; + int nEntry; + int rc = 0; + + assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 ); + + pRet = malloc(sizeof(CksumDb)); + memset(pRet, 0, sizeof(CksumDb)); + pRet->nFirst = nFirst; + pRet->nLast = nLast; + pRet->nStep = nStep; + nEntry = 1 + ((nLast - nFirst) / nStep); + + /* Allocate space so that azCksum is an array of nEntry pointers to + ** buffers each TEST_CKSUM_BYTES in size. */ + pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES)); + for(i=0; iazCksum[nEntry]); + pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES]; + } + + tdb_open("lsm", "tempdb.lsm", 1, &pDb); + testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc); + for(i=0; iazCksum[i]); + if( i==nEntry ) break; + testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc); + } + + tdb_close(pDb); + + return pRet; +} + +char *testCksumArrayGet(CksumDb *p, int nRow){ + int i; + assert( nRow>=p->nFirst ); + assert( nRow<=p->nLast ); + assert( ((nRow-p->nFirst) % p->nStep)==0 ); + + i = (nRow - p->nFirst) / p->nStep; + return p->azCksum[i]; +} + +void testCksumArrayFree(CksumDb *p){ + free(p->azCksum); + memset(p, 0x55, sizeof(*p)); + free(p); +} + +/* End of CksumDb code. +**************************************************************************/ + +/* +** Test utility function. Write key-value pair $i from datasource pData +** into database pDb. +*/ +void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ + void *pKey; int nKey; + void *pVal; int nVal; + testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); + testWrite(pDb, pKey, nKey, pVal, nVal, pRc); +} + +/* +** Test utility function. Delete datasource pData key $i from database pDb. +*/ +void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ + void *pKey; int nKey; + testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); + testDelete(pDb, pKey, nKey, pRc); +} + +/* +** This function inserts nWrite key/value pairs into database pDb - the +** nWrite key value pairs starting at iFirst from data source pData. +*/ +void testWriteDatasourceRange( + TestDb *pDb, /* Database to write to */ + Datasource *pData, /* Data source to read values from */ + int iFirst, /* Index of first key/value pair */ + int nWrite, /* Number of key/value pairs to write */ + int *pRc /* IN/OUT: Error code */ +){ + int i; + for(i=0; i2 && rc==0; iTrans--){ + tdb_rollback(pDb, iTrans); + nCurrent -= 100; + testCksumDatabase(pDb, zCksum); + testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); + } + + if( i%2 ){ + tdb_rollback(pDb, 0); + nCurrent -= 100; + testCksumDatabase(pDb, zCksum); + testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); + }else{ + tdb_commit(pDb, 0); + } + } + testCaseFinish(rc); + + skip_rollback_test: + tdb_close(pDb); + testCksumArrayFree(pCksum); + return rc; +} + +void test_rollback( + const char *zSystem, + const char *zPattern, + int *pRc +){ + if( *pRc==0 ){ + int bRun = 1; + + if( zPattern ){ + char *zName = getName(zSystem); + bRun = testGlobMatch(zPattern, zName); + testFree(zName); + } + + if( bRun ){ + DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 }; + Datasource *pData = testDatasourceNew(&defn); + *pRc = rollback_test_1(zSystem, pData); + testDatasourceFree(pData); + } + } +} diff --git a/ext/lsm1/lsm-test/lsmtest4.c b/ext/lsm1/lsm-test/lsmtest4.c new file mode 100644 index 0000000000000000000000000000000000000000..a47241db929504e918b3310927249401132e4261 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest4.c @@ -0,0 +1,127 @@ + +/* +** This file contains test cases involving multiple database clients. +*/ + +#include "lsmtest.h" + +/* +** The following code implements test cases "mc1.*". +** +** This test case uses one writer and $nReader readers. All connections +** are driven by a single thread. All connections are opened at the start +** of the test and remain open until the test is finished. +** +** The test consists of $nStep steps. Each step the following is performed: +** +** 1. The writer inserts $nWriteStep records into the db. +** +** 2. The writer checks that the contents of the db are as expected. +** +** 3. Each reader that currently has an open read transaction also checks +** that the contents of the db are as expected (according to the snapshot +** the read transaction is reading - see below). +** +** After step 1, reader 1 opens a read transaction. After step 2, reader +** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1 +** closes the current read transaction and opens a new one. And so on. +** The result is that at step N (for N > $nReader), there exists a reader +** with an open read transaction reading the snapshot committed following +** steps (N-$nReader-1) to N. +*/ +typedef struct Mctest Mctest; +struct Mctest { + DatasourceDefn defn; /* Datasource to use */ + int nStep; /* Total number of steps in test */ + int nWriteStep; /* Number of rows to insert each step */ + int nReader; /* Number of read connections */ +}; +static void do_mc_test( + const char *zSystem, /* Database system to test */ + Mctest *pTest, + int *pRc /* IN/OUT: return code */ +){ + const int nDomain = pTest->nStep * pTest->nWriteStep; + Datasource *pData; /* Source of data */ + TestDb *pDb; /* First database connection (writer) */ + int iReader; /* Used to iterate through aReader */ + int iStep; /* Current step in test */ + int iDot = 0; /* Current step in test */ + + /* Array of reader connections */ + struct Reader { + TestDb *pDb; /* Connection handle */ + int iLast; /* Current snapshot contains keys 0..iLast */ + } *aReader; + + /* Create a data source */ + pData = testDatasourceNew(&pTest->defn); + + /* Open the writer connection */ + pDb = testOpen(zSystem, 1, pRc); + + /* Allocate aReader */ + aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader); + for(iReader=0; iReadernReader; iReader++){ + aReader[iReader].pDb = testOpen(zSystem, 0, pRc); + } + + for(iStep=0; iStepnStep; iStep++){ + int iLast; + int iBegin; /* Start read trans using aReader[iBegin] */ + + /* Insert nWriteStep more records into the database */ + int iFirst = iStep*pTest->nWriteStep; + testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc); + + /* Check that the db is Ok according to the writer */ + iLast = (iStep+1) * pTest->nWriteStep - 1; + testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc); + + /* Have reader (iStep % nReader) open a read transaction here. */ + iBegin = (iStep % pTest->nReader); + if( iBeginnReader && aReader[iReader].iLast; iReader++){ + iLast = aReader[iReader].iLast; + testDbContents( + aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc + ); + } + + /* Report progress */ + testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot); + } + + /* Close all readers */ + for(iReader=0; iReadernReader; iReader++){ + testClose(&aReader[iReader].pDb); + } + testFree(aReader); + + /* Close the writer-connection and free the datasource */ + testClose(&pDb); + testDatasourceFree(pData); +} + + +void test_mc( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + int i; + Mctest aTest[] = { + { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 }, + }; + + for(i=0; i "k.0000000045". +** +** As well as the key/value pairs, the database also contains checksum +** entries. The checksums form a hierarchy - for every F key/value +** entries there is one level 1 checksum. And for each F level 1 checksums +** there is one level 2 checksum. And so on. +** +** Checksum keys are encoded as the two byte "c." followed by the +** checksum level, followed by a 10 digit decimal number containing +** the value of the first key that contributes to the checksum value. +** For example, assuming F==10, the level 1 checksum that spans keys +** 10 to 19 is "c.1.0000000010". +** +** Clients may perform one of two operations on the database: a read +** or a write. +** +** READ OPERATIONS: +** +** A read operation scans a range of F key/value pairs. It computes +** the expected checksum and then compares the computed value to the +** actual value stored in the level 1 checksum entry. It then scans +** the group of F level 1 checksums, and compares the computed checksum +** to the associated level 2 checksum value, and so on until the +** highest level checksum value has been verified. +** +** If a checksum ever fails to match the expected value, the test +** has failed. +** +** WRITE OPERATIONS: +** +** A write operation involves writing (possibly clobbering) a single +** key/value pair. The associated level 1 checksum is then recalculated +** updated. Then the level 2 checksum, and so on until the highest +** level checksum has been modified. +** +** All updates occur inside a single transaction. +** +** INTERFACE: +** +** The interface used by test cases to read and write the db consists +** of type DbParameters and the following functions: +** +** dbReadOperation() +** dbWriteOperation() +*/ + +#include "lsmtest.h" + +typedef struct DbParameters DbParameters; +struct DbParameters { + int nFanout; /* Checksum fanout (F) */ + int nKey; /* Size of key space (N) */ +}; + +#define DB_KEY_BYTES (2+5+10+1) + +/* +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. +** This function populates the buffer with a nul-terminated key string +** corresponding to key iKey. +*/ +static void dbFormatKey( + DbParameters *pParam, + int iLevel, + int iKey, /* Key value */ + char *aBuf /* Write key string here */ +){ + if( iLevel==0 ){ + snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey); + }else{ + int f = 1; + int i; + for(i=0; inFanout; + snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f)); + } +} + +/* +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. +** This function populates the buffer with the string representation of +** checksum value iVal. +*/ +static void dbFormatCksumValue(u32 iVal, char *aBuf){ + snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal); +} + +/* +** Return the highest level of checksum in the database described +** by *pParam. +*/ +static int dbMaxLevel(DbParameters *pParam){ + int iMax; + int n = 1; + for(iMax=0; nnKey; iMax++){ + n = n * pParam->nFanout; + } + return iMax; +} + +static void dbCksum( + void *pCtx, /* IN/OUT: Pointer to u32 containing cksum */ + void *pKey, int nKey, /* Database key. Unused. */ + void *pVal, int nVal /* Database value. Checksum this. */ +){ + u8 *aVal = (u8 *)pVal; + u32 *pCksum = (u32 *)pCtx; + u32 cksum = *pCksum; + int i; + + unused_parameter(pKey); + unused_parameter(nKey); + + for(i=0; inFanout entries at level +** iLevel-1. +*/ +static u32 dbComputeCksum( + DbParameters *pParam, /* Database parameters */ + TestDb *pDb, /* Database connection handle */ + int iLevel, /* Level of checksum to compute */ + int iKey, /* Compute checksum for this key */ + int *pRc /* IN/OUT: Error code */ +){ + u32 cksum = 0; + if( *pRc==0 ){ + int nFirst; + int nLast; + int iFirst = 0; + int iLast = 0; + int i; + int f = 1; + char zFirst[DB_KEY_BYTES]; + char zLast[DB_KEY_BYTES]; + + assert( iLevel>=1 ); + for(i=0; inFanout; + + iFirst = f*(iKey/f); + iLast = iFirst + f - 1; + dbFormatKey(pParam, iLevel-1, iFirst, zFirst); + dbFormatKey(pParam, iLevel-1, iLast, zLast); + nFirst = strlen(zFirst); + nLast = strlen(zLast); + + *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum); + } + + return cksum; +} + +static void dbReadOperation( + DbParameters *pParam, /* Database parameters */ + TestDb *pDb, /* Database connection handle */ + void (*xDelay)(void *), + void *pDelayCtx, + int iKey, /* Key to read */ + int *pRc /* IN/OUT: Error code */ +){ + const int iMax = dbMaxLevel(pParam); + int i; + + if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc); + for(i=1; *pRc==0 && i<=iMax; i++){ + char zCksum[DB_KEY_BYTES]; + char zKey[DB_KEY_BYTES]; + u32 iCksum = 0; + + iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); + if( iCksum ){ + if( xDelay && i==1 ) xDelay(pDelayCtx); + dbFormatCksumValue(iCksum, zCksum); + dbFormatKey(pParam, i, iKey, zKey); + testFetchStr(pDb, zKey, zCksum, pRc); + } + } + if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); +} + +static int dbWriteOperation( + DbParameters *pParam, /* Database parameters */ + TestDb *pDb, /* Database connection handle */ + int iKey, /* Key to write to */ + const char *zValue, /* Nul-terminated value to write */ + int *pRc /* IN/OUT: Error code */ +){ + const int iMax = dbMaxLevel(pParam); + char zKey[DB_KEY_BYTES]; + int i; + int rc; + + assert( iKey>=0 && iKeynKey ); + dbFormatKey(pParam, 0, iKey, zKey); + + /* Open a write transaction. This may fail - SQLITE4_BUSY */ + if( *pRc==0 && tdb_transaction_support(pDb) ){ + rc = tdb_begin(pDb, 2); + if( rc==5 ) return 0; + *pRc = rc; + } + + testWriteStr(pDb, zKey, zValue, pRc); + for(i=1; i<=iMax; i++){ + char zCksum[DB_KEY_BYTES]; + u32 iCksum = 0; + + iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); + dbFormatCksumValue(iCksum, zCksum); + dbFormatKey(pParam, i, iKey, zKey); + testWriteStr(pDb, zKey, zCksum, pRc); + } + if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); + return 1; +} + +/************************************************************************* +** The following block contains testXXX() functions that implement a +** wrapper around the systems native multi-thread support. There are no +** synchronization primitives - just functions to launch and join +** threads. Wrapper functions are: +** +** testThreadSupport() +** +** testThreadInit() +** testThreadShutdown() +** testThreadLaunch() +** testThreadWait() +** +** testThreadSetHalt() +** testThreadGetHalt() +** testThreadSetResult() +** testThreadGetResult() +** +** testThreadEnterMutex() +** testThreadLeaveMutex() +*/ +typedef struct ThreadSet ThreadSet; +#ifdef LSM_MUTEX_PTHREADS + +#include +#include + +typedef struct Thread Thread; +struct Thread { + int rc; + char *zMsg; + pthread_t id; + void (*xMain)(ThreadSet *, int, void *); + void *pCtx; + ThreadSet *pThreadSet; +}; + +struct ThreadSet { + int bHalt; /* Halt flag */ + int nThread; /* Number of threads */ + Thread *aThread; /* Array of Thread structures */ + pthread_mutex_t mutex; /* Mutex used for cheating */ +}; + +/* +** Return true if this build supports threads, or false otherwise. If +** this function returns false, no other testThreadXXX() functions should +** be called. +*/ +static int testThreadSupport(){ return 1; } + +/* +** Allocate and return a thread-set handle with enough space allocated +** to handle up to nMax threads. Each call to this function should be +** matched by a call to testThreadShutdown() to delete the object. +*/ +static ThreadSet *testThreadInit(int nMax){ + int nByte; /* Total space to allocate */ + ThreadSet *p; /* Return value */ + + nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax; + p = (ThreadSet *)testMalloc(nByte); + p->nThread = nMax; + p->aThread = (Thread *)&p[1]; + pthread_mutex_init(&p->mutex, 0); + + return p; +} + +/* +** Delete a thread-set object and release all resources held by it. +*/ +static void testThreadShutdown(ThreadSet *p){ + int i; + for(i=0; inThread; i++){ + testFree(p->aThread[i].zMsg); + } + pthread_mutex_destroy(&p->mutex); + testFree(p); +} + +static void *ttMain(void *pArg){ + Thread *pThread = (Thread *)pArg; + int iThread; + iThread = (pThread - pThread->pThreadSet->aThread); + pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx); + return 0; +} + +/* +** Launch a new thread. +*/ +static int testThreadLaunch( + ThreadSet *p, + int iThread, + void (*xMain)(ThreadSet *, int, void *), + void *pCtx +){ + int rc; + Thread *pThread; + + assert( iThread>=0 && iThreadnThread ); + + pThread = &p->aThread[iThread]; + assert( pThread->pThreadSet==0 ); + pThread->xMain = xMain; + pThread->pCtx = pCtx; + pThread->pThreadSet = p; + rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread); + + return rc; +} + +/* +** Set the thread-set "halt" flag. +*/ +static void testThreadSetHalt(ThreadSet *pThreadSet){ + pThreadSet->bHalt = 1; +} + +/* +** Return the current value of the thread-set "halt" flag. +*/ +static int testThreadGetHalt(ThreadSet *pThreadSet){ + return pThreadSet->bHalt; +} + +static void testThreadSleep(ThreadSet *pThreadSet, int nMs){ + int nRem = nMs; + while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){ + usleep(50000); + nRem -= 50; + } +} + +/* +** Wait for all threads launched to finish before returning. If nMs +** is greater than zero, set the "halt" flag to tell all threads +** to halt after waiting nMs milliseconds. +*/ +static void testThreadWait(ThreadSet *pThreadSet, int nMs){ + int i; + + testThreadSleep(pThreadSet, nMs); + testThreadSetHalt(pThreadSet); + for(i=0; inThread; i++){ + Thread *pThread = &pThreadSet->aThread[i]; + if( pThread->xMain ){ + pthread_join(pThread->id, 0); + } + } +} + +/* +** Set the result for thread iThread. +*/ +static void testThreadSetResult( + ThreadSet *pThreadSet, /* Thread-set handle */ + int iThread, /* Set result for this thread */ + int rc, /* Result error code */ + char *zFmt, /* Result string format */ + ... /* Result string formatting args... */ +){ + va_list ap; + + testFree(pThreadSet->aThread[iThread].zMsg); + pThreadSet->aThread[iThread].rc = rc; + pThreadSet->aThread[iThread].zMsg = 0; + if( zFmt ){ + va_start(ap, zFmt); + pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap); + va_end(ap); + } +} + +/* +** Retrieve the result for thread iThread. +*/ +static int testThreadGetResult( + ThreadSet *pThreadSet, /* Thread-set handle */ + int iThread, /* Get result for this thread */ + const char **pzRes /* OUT: Pointer to result string */ +){ + if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg; + return pThreadSet->aThread[iThread].rc; +} + +/* +** Enter and leave the test case mutex. +*/ +#if 0 +static void testThreadEnterMutex(ThreadSet *p){ + pthread_mutex_lock(&p->mutex); +} +static void testThreadLeaveMutex(ThreadSet *p){ + pthread_mutex_unlock(&p->mutex); +} +#endif +#endif + +#if !defined(LSM_MUTEX_PTHREADS) +static int testThreadSupport(){ return 0; } + +#define testThreadInit(a) 0 +#define testThreadShutdown(a) +#define testThreadLaunch(a,b,c,d) 0 +#define testThreadWait(a,b) +#define testThreadSetHalt(a) +#define testThreadGetHalt(a) 0 +#define testThreadGetResult(a,b,c) 0 +#define testThreadSleep(a,b) 0 + +static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){ + unused_parameter(a); + unused_parameter(b); + unused_parameter(c); + unused_parameter(d); +} +#endif +/* End of threads wrapper. +*************************************************************************/ + +/************************************************************************* +** Below this point is the third part of this file - the implementation +** of the mt1.* tests. +*/ +typedef struct Mt1Test Mt1Test; +struct Mt1Test { + DbParameters param; /* Description of database to read/write */ + int nReadwrite; /* Number of read/write threads */ + int nFastReader; /* Number of fast reader threads */ + int nSlowReader; /* Number of slow reader threads */ + int nMs; /* How long to run for */ + const char *zSystem; /* Database system to test */ +}; + +typedef struct Mt1DelayCtx Mt1DelayCtx; +struct Mt1DelayCtx { + ThreadSet *pSet; /* Threadset to sleep within */ + int nMs; /* Sleep in ms */ +}; + +static void xMt1Delay(void *pCtx){ + Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx; + testThreadSleep(p->pSet, p->nMs); +} + +#define MT1_THREAD_RDWR 0 +#define MT1_THREAD_SLOW 1 +#define MT1_THREAD_FAST 2 + +static void xMt1Work(lsm_db *pDb, void *pCtx){ +#if 0 + char *z = 0; + lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z); + printf("%s\n", z); + fflush(stdout); +#endif +} + +/* +** This is the main() proc for all threads in test case "mt1". +*/ +static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){ + Mt1Test *p = (Mt1Test *)pCtx; /* Test parameters */ + Mt1DelayCtx delay; + int nRead = 0; /* Number of calls to dbReadOperation() */ + int nWrite = 0; /* Number of completed database writes */ + int rc = 0; /* Error code */ + int iPrng; /* Prng argument variable */ + TestDb *pDb; /* Database handle */ + int eType; + + delay.pSet = pThreadSet; + delay.nMs = 0; + if( iThreadnReadwrite ){ + eType = MT1_THREAD_RDWR; + }else if( iThread<(p->nReadwrite+p->nFastReader) ){ + eType = MT1_THREAD_FAST; + }else{ + eType = MT1_THREAD_SLOW; + delay.nMs = (p->nMs / 20); + } + + /* Open a new database connection. Initialize the pseudo-random number + ** argument based on the thread number. */ + iPrng = testPrngValue(iThread); + pDb = testOpen(p->zSystem, 0, &rc); + + if( rc==0 ){ + tdb_lsm_config_work_hook(pDb, xMt1Work, 0); + } + + /* Loop until either an error occurs or some other thread sets the + ** halt flag. */ + while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){ + int iKey; + + /* Perform a read operation on an arbitrarily selected key. */ + iKey = (testPrngValue(iPrng++) % p->param.nKey); + dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc); + if( rc ) continue; + nRead++; + + /* Attempt to write an arbitrary key value pair (and update the associated + ** checksum entries). dbWriteOperation() returns 1 if the write is + ** successful, or 0 if it failed with an LSM_BUSY error. */ + if( eType==MT1_THREAD_RDWR ){ + char aValue[50]; + char aRnd[25]; + + iKey = (testPrngValue(iPrng++) % p->param.nKey); + testPrngString(iPrng, aRnd, sizeof(aRnd)); + iPrng += sizeof(aRnd); + snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd); + nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc); + } + } + testClose(&pDb); + + /* If an error has occured, set the thread error code and the threadset + ** halt flag to tell the other test threads to halt. Otherwise, set the + ** thread error code to 0 and post a message with the number of read + ** and write operations completed. */ + if( rc ){ + testThreadSetResult(pThreadSet, iThread, rc, 0); + testThreadSetHalt(pThreadSet); + }else{ + testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite); + } +} + +static void do_test_mt1( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + Mt1Test aTest[] = { + /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */ + { {10, 1000}, 4, 0, 0, 10000, 0 }, + { {10, 1000}, 4, 4, 2, 100000, 0 }, + { {10, 100000}, 4, 0, 0, 10000, 0 }, + { {10, 100000}, 4, 4, 2, 100000, 0 }, + }; + int i; + + for(i=0; *pRc==0 && iparam.nFanout, p->param.nKey, + p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader + ); + if( bRun ){ + TestDb *pDb; + ThreadSet *pSet; + int iThread; + int nThread; + + p->zSystem = zSystem; + pDb = testOpen(zSystem, 1, pRc); + + nThread = p->nReadwrite + p->nFastReader + p->nSlowReader; + pSet = testThreadInit(nThread); + for(iThread=0; *pRc==0 && iThreadnMs); + for(iThread=0; *pRc==0 && iThreadiNext = 1; + p->bEnable = 1; + p->nFail = 1; + p->pEnv = tdb_lsm_env(); +} + +static void xOomHook(OomTest *p){ + p->nFail++; +} + +static int testOomContinue(OomTest *p){ + if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){ + return 0; + } + p->nFail = 0; + testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p); + return 1; +} + +static void testOomEnable(OomTest *p, int bEnable){ + p->bEnable = bEnable; + testMallocOomEnable(p->pEnv, bEnable); +} + +static void testOomNext(OomTest *p){ + p->iNext++; +} + +static int testOomHit(OomTest *p){ + return (p->nFail>0); +} + +static int testOomFinish(OomTest *p){ + return p->rc; +} + +static void testOomAssert(OomTest *p, int bVal){ + if( bVal==0 ){ + test_failed(); + p->rc = 1; + } +} + +/* +** Test that the error code matches the state of the OomTest object passed +** as the first argument. Specifically, check that rc is LSM_NOMEM if an +** OOM error has already been injected, or LSM_OK if not. +*/ +static void testOomAssertRc(OomTest *p, int rc){ + testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM); + testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 ); +} + +static void testOomOpen( + OomTest *pOom, + const char *zName, + lsm_db **ppDb, + int *pRc +){ + if( *pRc==LSM_OK ){ + int rc; + rc = lsm_new(tdb_lsm_env(), ppDb); + if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName); + testOomAssertRc(pOom, rc); + *pRc = rc; + } +} + +static void testOomFetch( + OomTest *pOom, + lsm_db *pDb, + void *pKey, int nKey, + void *pVal, int nVal, + int *pRc +){ + testOomAssertRc(pOom, *pRc); + if( *pRc==LSM_OK ){ + lsm_cursor *pCsr; + int rc; + + rc = lsm_csr_open(pDb, &pCsr); + if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0); + testOomAssertRc(pOom, rc); + + if( rc==LSM_OK ){ + const void *p; int n; + testOomAssert(pOom, lsm_csr_valid(pCsr)); + + rc = lsm_csr_key(pCsr, &p, &n); + testOomAssertRc(pOom, rc); + testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) ); + } + + if( rc==LSM_OK ){ + const void *p; int n; + testOomAssert(pOom, lsm_csr_valid(pCsr)); + + rc = lsm_csr_value(pCsr, &p, &n); + testOomAssertRc(pOom, rc); + testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) ); + } + + lsm_csr_close(pCsr); + *pRc = rc; + } +} + +static void testOomWrite( + OomTest *pOom, + lsm_db *pDb, + void *pKey, int nKey, + void *pVal, int nVal, + int *pRc +){ + testOomAssertRc(pOom, *pRc); + if( *pRc==LSM_OK ){ + int rc; + + rc = lsm_insert(pDb, pKey, nKey, pVal, nVal); + testOomAssertRc(pOom, rc); + + *pRc = rc; + } +} + + +static void testOomFetchStr( + OomTest *pOom, + lsm_db *pDb, + const char *zKey, + const char *zVal, + int *pRc +){ + int nKey = strlen(zKey); + int nVal = strlen(zVal); + testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); +} + +static void testOomFetchData( + OomTest *pOom, + lsm_db *pDb, + Datasource *pData, + int iKey, + int *pRc +){ + void *pKey; int nKey; + void *pVal; int nVal; + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); + testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc); +} + +static void testOomWriteStr( + OomTest *pOom, + lsm_db *pDb, + const char *zKey, + const char *zVal, + int *pRc +){ + int nKey = strlen(zKey); + int nVal = strlen(zVal); + testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); +} + +static void testOomWriteData( + OomTest *pOom, + lsm_db *pDb, + Datasource *pData, + int iKey, + int *pRc +){ + void *pKey; int nKey; + void *pVal; int nVal; + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); + testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc); +} + +static void testOomScan( + OomTest *pOom, + lsm_db *pDb, + int bReverse, + const void *pKey, int nKey, + int nScan, + int *pRc +){ + if( *pRc==0 ){ + int rc; + int iScan = 0; + lsm_cursor *pCsr; + int (*xAdvance)(lsm_cursor *) = 0; + + + rc = lsm_csr_open(pDb, &pCsr); + testOomAssertRc(pOom, rc); + + if( rc==LSM_OK ){ + if( bReverse ){ + rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE); + xAdvance = lsm_csr_prev; + }else{ + rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE); + xAdvance = lsm_csr_next; + } + } + testOomAssertRc(pOom, rc); + + while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan "one" +** "two" -> "four" +** "three" -> "nine" +** "four" -> "sixteen" +** "five" -> "twentyfive" +** "six" -> "thirtysix" +** "seven" -> "fourtynine" +** "eight" -> "sixtyfour" +*/ +static void setup_populate_db(void){ + const char *azStr[] = { + "one", "one", + "two", "four", + "three", "nine", + "four", "sixteen", + "five", "twentyfive", + "six", "thirtysix", + "seven", "fourtynine", + "eight", "sixtyfour", + }; + int rc; + int ii; + lsm_db *pDb; + + testDeleteLsmdb(LSMTEST6_TESTDB); + + rc = lsm_new(tdb_lsm_env(), &pDb); + if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); + + for(ii=0; rc==LSM_OK && iiiInsStart, pStep->nIns, pRc); + testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc); + if( *pRc==0 ){ + int nSave = -1; + int nBuf = 64; + lsm_db *db = tdb_lsm(pDb); + + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); + lsm_begin(db, 1); + lsm_commit(db, 0); + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); + + *pRc = lsm_work(db, 0, 0, 0); + if( *pRc==0 ){ + *pRc = lsm_checkpoint(db, 0); + } + } +} + +static void doSetupStepArray( + TestDb *pDb, + Datasource *pData, + const SetupStep *aStep, + int nStep +){ + int i; + for(i=0; i +void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){ + if( *pRc==0 ){ + FILE *fd; + fd = fopen(zFile, "rb"); + if( fd==0 ){ + *pRc = 1; + }else{ + if( 0!=fseek(fd, iOff, SEEK_SET) ){ + *pRc = 1; + }else{ + assert( nByte>=0 ); + if( (size_t)nByte!=fread(pOut, 1, nByte, fd) ){ + *pRc = 1; + } + } + fclose(fd); + } + } +} + +void testWriteFile( + const char *zFile, + int iOff, + void *pOut, + int nByte, + int *pRc +){ + if( *pRc==0 ){ + FILE *fd; + fd = fopen(zFile, "r+b"); + if( fd==0 ){ + *pRc = 1; + }else{ + if( 0!=fseek(fd, iOff, SEEK_SET) ){ + *pRc = 1; + }else{ + assert( nByte>=0 ); + if( (size_t)nByte!=fwrite(pOut, 1, nByte, fd) ){ + *pRc = 1; + } + } + fclose(fd); + } + } +} + +static ShmHeader *getShmHeader(const char *zDb){ + int rc = 0; + char *zShm = testMallocPrintf("%s-shm", zDb); + ShmHeader *pHdr; + + pHdr = testMalloc(sizeof(ShmHeader)); + testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc); + assert( rc==0 ); + + return pHdr; +} + +/* +** This function makes a copy of the three files associated with LSM +** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db", +** "test.db-log" and "test.db-shm"). +** +** It then opens a new database connection to the copy with the xLock() call +** instrumented so that it appears that some other process already connected +** to the db (holding a shared lock on DMS2). This prevents recovery from +** running. Then: +** +** 1) Check that the checksum of the database is zCksum. +** 2) Write a few keys to the database. Then delete the same keys. +** 3) Check that the checksum is zCksum. +** 4) Flush the db to disk and run a checkpoint. +** 5) Check once more that the checksum is still zCksum. +*/ +static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){ + if( *pRc==LSM_OK ){ + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500}; + Datasource *pData; + const char *zCopy = "testcopy.lsm"; + char zCksum2[TEST_CKSUM_BYTES]; + TestDb *pDb = 0; + int rc; + + pData = testDatasourceNew(&defn); + + testCopyLsmdb(zDb, zCopy); + rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb); + if( rc==0 ){ + ShmHeader *pHdr; + lsm_db *db; + testCksumDatabase(pDb, zCksum2); + testCompareStr(zCksum, zCksum2, &rc); + + testWriteDatasourceRange(pDb, pData, 1, 10, &rc); + testDeleteDatasourceRange(pDb, pData, 1, 10, &rc); + + /* Test that the two tree-headers are now consistent. */ + pHdr = getShmHeader(zCopy); + if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){ + rc = 1; + } + testFree(pHdr); + + if( rc==0 ){ + int nBuf = 64; + db = tdb_lsm(pDb); + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); + lsm_begin(db, 1); + lsm_commit(db, 0); + rc = lsm_work(db, 0, 0, 0); + } + + testCksumDatabase(pDb, zCksum2); + testCompareStr(zCksum, zCksum2, &rc); + } + + testDatasourceFree(pData); + testClose(&pDb); + testDeleteLsmdb(zCopy); + *pRc = rc; + } +} + +static void doWriterCrash1(int *pRc){ + const int nWrite = 2000; + const int nStep = 10; + const int iWriteStart = 20000; + int rc = 0; + TestDb *pDb = 0; + Datasource *pData = 0; + + rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); + if( rc==0 ){ + int iDot = 0; + char zCksum[TEST_CKSUM_BYTES]; + int i; + setupDatabase1(pDb, &pData); + testCksumDatabase(pDb, zCksum); + testBegin(pDb, 2, &rc); + for(i=0; rc==0 && ihdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); + pHdr2->bWriter = 1; + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); + doLiveRecovery("testdb.lsm", zCksum1, &rc); + + /* If both tree-headers are valid, tree-header-1 is used. */ + memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); + memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); + pHdr2->bWriter = 1; + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); + doLiveRecovery("testdb.lsm", zCksum2, &rc); + + /* If tree-header 1 is invalid, tree-header-2 is used */ + memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1)); + pHdr2->hdr1.aCksum[0] = 5; + pHdr2->hdr1.aCksum[0] = 6; + pHdr2->bWriter = 1; + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); + doLiveRecovery("testdb.lsm", zCksum2, &rc); + + /* If tree-header 2 is invalid, tree-header-1 is used */ + memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); + pHdr2->hdr2.aCksum[0] = 5; + pHdr2->hdr2.aCksum[0] = 6; + pHdr2->bWriter = 1; + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); + doLiveRecovery("testdb.lsm", zCksum2, &rc); + + testFree(pHdr1); + testFree(pHdr2); + testClose(&pDb); + } + + *pRc = rc; +} + +void do_writer_crash_test(const char *zPattern, int *pRc){ + struct Test { + const char *zName; + void (*xFunc)(int *); + } aTest[] = { + { "writercrash1.lsm", doWriterCrash1 }, + { "writercrash2.lsm", doWriterCrash2 }, + }; + int i; + for(i=0; izName) ){ + p->xFunc(pRc); + testCaseFinish(*pRc); + } + } + +} diff --git a/ext/lsm1/lsm-test/lsmtest9.c b/ext/lsm1/lsm-test/lsmtest9.c new file mode 100644 index 0000000000000000000000000000000000000000..b01de0d4e5323137583b71c4caef9d5e4b281ead --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest9.c @@ -0,0 +1,140 @@ + +#include "lsmtest.h" + +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE +#define DATA_RANDOM TEST_DATASOURCE_RANDOM + +typedef struct Datatest4 Datatest4; + +/* +** Test overview: +** +** 1. Insert (Datatest4.nRec) records into a database. +** +** 2. Repeat (Datatest4.nRepeat) times: +** +** 2a. Delete 2/3 of the records in the database. +** +** 2b. Run lsm_work(nMerge=1). +** +** 2c. Insert as many records as were deleted in 2a. +** +** 2d. Check database content is as expected. +** +** 2e. If (Datatest4.bReopen) is true, close and reopen the database. +*/ +struct Datatest4 { + /* Datasource definition */ + DatasourceDefn defn; + + int nRec; + int nRepeat; + int bReopen; +}; + +static void doDataTest4( + const char *zSystem, /* Database system to test */ + Datatest4 *p, /* Structure containing test parameters */ + int *pRc /* OUT: Error code */ +){ + lsm_db *db = 0; + TestDb *pDb; + TestDb *pControl; + Datasource *pData; + int i; + int rc = 0; + int iDot = 0; + int bMultiThreaded = 0; /* True for MT LSM database */ + + int nRecOn3 = (p->nRec / 3); + int iData = 0; + + /* Start the test case, open a database and allocate the datasource. */ + rc = testControlDb(&pControl); + pDb = testOpen(zSystem, 1, &rc); + pData = testDatasourceNew(&p->defn); + if( rc==0 ){ + db = tdb_lsm(pDb); + bMultiThreaded = tdb_lsm_multithread(pDb); + } + + testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); + testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); + + for(i=0; rc==0 && inRepeat; i++){ + + testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); + testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); + + if( db ){ + int nDone; +#if 0 + fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); +#endif + do { + nDone = 0; + rc = lsm_work(db, 1, (1<<30), &nDone); + }while( rc==0 && nDone>0 ); + if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK; +#if 0 + fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); +#endif + } + +if( i+1nRepeat ){ + iData += (nRecOn3*2); + testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); + testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); + + testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); + + /* If Datatest4.bReopen is true, close and reopen the database */ + if( p->bReopen ){ + testReopen(&pDb, &rc); + if( rc==0 ) db = tdb_lsm(pDb); + } +} + + /* Update the progress dots... */ + testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); + } + + testClose(&pDb); + testClose(&pControl); + testDatasourceFree(pData); + testCaseFinish(rc); + *pRc = rc; +} + +static char *getName4(const char *zSystem, Datatest4 *pTest){ + char *zRet; + char *zData; + zData = testDatasourceName(&pTest->defn); + zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", + zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen + ); + testFree(zData); + return zRet; +} + +void test_data_4( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + Datatest4 aTest[] = { + /* defn, nRec, nRepeat, bReopen */ + { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, + { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, + }; + + int i; + + for(i=0; *pRc==LSM_OK && ieType ){ + case TEST_DATASOURCE_RANDOM: { + int nRange = (1 + p->nMaxKey - p->nMinKey); + nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; + testPrngString((u32)iData, p->aKey, nKey); + break; + } + case TEST_DATASOURCE_SEQUENCE: + nKey = sprintf(p->aKey, "%012d", iData); + break; + } + *ppKey = p->aKey; + *pnKey = nKey; + } + if( ppVal ){ + u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal; + testPrngString((u32)~iData, p->aVal, (int)nVal); + *ppVal = p->aVal; + *pnVal = (int)nVal; + } +} + +void testDatasourceFree(Datasource *p){ + testFree(p); +} + +/* +** Return a pointer to a nul-terminated string that corresponds to the +** contents of the datasource-definition passed as the first argument. +** The caller should eventually free the returned pointer using testFree(). +*/ +char *testDatasourceName(const DatasourceDefn *p){ + char *zRet; + zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)", + (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"), + p->nMinKey, p->nMaxKey, + p->nMinVal, p->nMaxVal + ); + return zRet; +} + +Datasource *testDatasourceNew(const DatasourceDefn *pDefn){ + Datasource *p; + int nMinKey; + int nMaxKey; + int nMinVal; + int nMaxVal; + + if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){ + nMinKey = 128; + nMaxKey = 128; + }else{ + nMinKey = MAX(0, pDefn->nMinKey); + nMaxKey = MAX(nMinKey, pDefn->nMaxKey); + } + nMinVal = MAX(0, pDefn->nMinVal); + nMaxVal = MAX(nMinVal, pDefn->nMaxVal); + + p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1); + p->eType = pDefn->eType; + p->nMinKey = nMinKey; + p->nMinVal = nMinVal; + p->nMaxKey = nMaxKey; + p->nMaxVal = nMaxVal; + + p->aKey = (char *)&p[1]; + p->aVal = &p->aKey[nMaxKey]; + return p; +}; diff --git a/ext/lsm1/lsm-test/lsmtest_func.c b/ext/lsm1/lsm-test/lsmtest_func.c new file mode 100644 index 0000000000000000000000000000000000000000..eb8346aa8361ee0b1d5e75faa9e228f31914844c --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_func.c @@ -0,0 +1,177 @@ + +#include "lsmtest.h" + + +int do_work(int nArg, char **azArg){ + struct Option { + const char *zName; + } aOpt [] = { + { "-nmerge" }, + { "-nkb" }, + { 0 } + }; + + lsm_db *pDb; + int rc; + int i; + const char *zDb; + int nMerge = 1; + int nKB = (1<<30); + + if( nArg==0 ) goto usage; + zDb = azArg[nArg-1]; + for(i=0; i<(nArg-1); i++){ + int iSel; + rc = testArgSelect(aOpt, "option", azArg[i], &iSel); + if( rc ) return rc; + switch( iSel ){ + case 0: + i++; + if( i==(nArg-1) ) goto usage; + nMerge = atoi(azArg[i]); + break; + case 1: + i++; + if( i==(nArg-1) ) goto usage; + nKB = atoi(azArg[i]); + break; + } + } + + rc = lsm_new(0, &pDb); + if( rc!=LSM_OK ){ + testPrintError("lsm_open(): rc=%d\n", rc); + }else{ + rc = lsm_open(pDb, zDb); + if( rc!=LSM_OK ){ + testPrintError("lsm_open(): rc=%d\n", rc); + }else{ + int n = -1; + lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n); + n = n*2; + lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n); + + rc = lsm_work(pDb, nMerge, nKB, 0); + if( rc!=LSM_OK ){ + testPrintError("lsm_work(): rc=%d\n", rc); + } + } + } + if( rc==LSM_OK ){ + rc = lsm_checkpoint(pDb, 0); + } + + lsm_close(pDb); + return rc; + + usage: + testPrintUsage("?-optimize? ?-n N? DATABASE"); + return -1; +} + + +/* +** lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO?? +*/ +int do_show(int nArg, char **azArg){ + lsm_db *pDb; + int rc; + const char *zDb; + + int eOpt = LSM_INFO_DB_STRUCTURE; + unsigned int iPg = 0; + int bConfig = 0; + const char *zConfig = ""; + + struct Option { + const char *zName; + int bConfig; + int eOpt; + } aOpt [] = { + { "array", 0, LSM_INFO_ARRAY_STRUCTURE }, + { "array-pages", 0, LSM_INFO_ARRAY_PAGES }, + { "blocksize", 1, LSM_CONFIG_BLOCK_SIZE }, + { "pagesize", 1, LSM_CONFIG_PAGE_SIZE }, + { "freelist", 0, LSM_INFO_FREELIST }, + { "page-ascii", 0, LSM_INFO_PAGE_ASCII_DUMP }, + { "page-hex", 0, LSM_INFO_PAGE_HEX_DUMP }, + { 0, 0 } + }; + + char *z = 0; + int iDb = 0; /* Index of DATABASE in azArg[] */ + + /* Check if there is a "-config" option: */ + if( nArg>2 && strlen(azArg[0])>1 + && memcmp(azArg[0], "-config", strlen(azArg[0]))==0 + ){ + zConfig = azArg[1]; + iDb = 2; + } + if( nArg<(iDb+1) ) goto usage; + + if( nArg>(iDb+1) ){ + rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt); + if( rc!=0 ) return rc; + bConfig = aOpt[eOpt].bConfig; + eOpt = aOpt[eOpt].eOpt; + if( (bConfig==0 && eOpt==LSM_INFO_FREELIST) + || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE) + || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE) + ){ + if( nArg!=(iDb+2) ) goto usage; + }else{ + if( nArg!=(iDb+3) ) goto usage; + iPg = atoi(azArg[iDb+2]); + } + } + zDb = azArg[iDb]; + + rc = lsm_new(0, &pDb); + tdb_lsm_configure(pDb, zConfig); + if( rc!=LSM_OK ){ + testPrintError("lsm_new(): rc=%d\n", rc); + }else{ + rc = lsm_open(pDb, zDb); + if( rc!=LSM_OK ){ + testPrintError("lsm_open(): rc=%d\n", rc); + } + } + + if( rc==LSM_OK ){ + if( bConfig==0 ){ + switch( eOpt ){ + case LSM_INFO_DB_STRUCTURE: + case LSM_INFO_FREELIST: + rc = lsm_info(pDb, eOpt, &z); + break; + case LSM_INFO_ARRAY_STRUCTURE: + case LSM_INFO_ARRAY_PAGES: + case LSM_INFO_PAGE_ASCII_DUMP: + case LSM_INFO_PAGE_HEX_DUMP: + rc = lsm_info(pDb, eOpt, iPg, &z); + break; + default: + assert( !"no chance" ); + } + + if( rc==LSM_OK ){ + printf("%s\n", z ? z : ""); + fflush(stdout); + } + lsm_free(lsm_get_env(pDb), z); + }else{ + int iRes = -1; + lsm_config(pDb, eOpt, &iRes); + printf("%d\n", iRes); + fflush(stdout); + } + } + + lsm_close(pDb); + return rc; + + usage: + testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?"); + return -1; +} diff --git a/ext/lsm1/lsm-test/lsmtest_io.c b/ext/lsm1/lsm-test/lsmtest_io.c new file mode 100644 index 0000000000000000000000000000000000000000..7aa5d10948315b36f4bad80d3b1938c4334f1431 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_io.c @@ -0,0 +1,248 @@ + +/* +** SUMMARY +** +** This file implements the 'io' subcommand of the test program. It is used +** for testing the performance of various combinations of write() and fsync() +** system calls. All operations occur on a single file, which may or may not +** exist when a test is started. +** +** A test consists of a series of commands. Each command is either a write +** or an fsync. A write is specified as "@", where +** is the amount of data written, and is the offset of the file +** to write to. An or an is specified as an integer number +** of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of +** KB, MB or GB, respectively. An fsync is simply "S". All commands are +** case-insensitive. +** +** Example test program: +** +** 2M@6M 1492K@4M S 4096@4K S +** +** This program writes 2 MB of data starting at the offset 6MB offset of +** the file, followed by 1492 KB of data written at the 4MB offset of the +** file, followed by a call to fsync(), a write of 4KB of data at byte +** offset 4096, and finally another call to fsync(). +** +** Commands may either be specified on the command line (one command per +** command line argument) or read from stdin. Commands read from stdin +** must be separated by white-space. +** +** COMMAND LINE INVOCATION +** +** The sub-command implemented in this file must be invoked with at least +** two arguments - the path to the file to write to and the page-size to +** use for writing. If there are more than two arguments, then each +** subsequent argument is assumed to be a test command. If there are exactly +** two arguments, the test commands are read from stdin. +** +** A write command does not result in a single call to system call write(). +** Instead, the specified region is written sequentially using one or +** more calls to write(), each of which writes not more than one page of +** data. For example, if the page-size is 4KB, the command "2M@6M" results +** in 512 calls to write(), each of which writes 4KB of data. +** +** EXAMPLES +** +** Two equivalent examples: +** +** $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S +** 3544K written in 129 ms +** $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 +** 3544K written in 127 ms +** +*/ + +#include "lsmtest.h" + +typedef struct IoContext IoContext; + +struct IoContext { + int fd; + int nWrite; +}; + +/* +** As isspace(3) +*/ +static int safe_isspace(char c){ + if( c&0x80) return 0; + return isspace(c); +} + +/* +** As isdigit(3) +*/ +static int safe_isdigit(char c){ + if( c&0x80) return 0; + return isdigit(c); +} + +static i64 getNextSize(char *zIn, char **pzOut, int *pRc){ + i64 iRet = 0; + if( *pRc==0 ){ + char *z = zIn; + + if( !safe_isdigit(*z) ){ + *pRc = 1; + return 0; + } + + /* Process digits */ + while( safe_isdigit(*z) ){ + iRet = iRet*10 + (*z - '0'); + z++; + } + + /* Process suffix */ + switch( *z ){ + case 'k': case 'K': + iRet = iRet * 1024; + z++; + break; + + case 'm': case 'M': + iRet = iRet * 1024 * 1024; + z++; + break; + + case 'g': case 'G': + iRet = iRet * 1024 * 1024 * 1024; + z++; + break; + } + + if( pzOut ) *pzOut = z; + } + return iRet; +} + +static int doOneCmd( + IoContext *pCtx, + u8 *aData, + int pgsz, + char *zCmd, + char **pzOut +){ + char c; + char *z = zCmd; + + while( safe_isspace(*z) ) z++; + c = *z; + + if( c==0 ){ + if( pzOut ) *pzOut = z; + return 0; + } + + if( c=='s' || c=='S' ){ + if( pzOut ) *pzOut = &z[1]; + return fdatasync(pCtx->fd); + } + + if( safe_isdigit(c) ){ + i64 iOff = 0; + int nByte = 0; + int rc = 0; + int nPg; + int iPg; + + nByte = (int)getNextSize(z, &z, &rc); + if( rc || *z!='@' ) goto bad_command; + z++; + iOff = getNextSize(z, &z, &rc); + if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command; + if( pzOut ) *pzOut = z; + + nPg = (nByte+pgsz-1) / pgsz; + lseek(pCtx->fd, (off_t)iOff, SEEK_SET); + for(iPg=0; iPgfd, aData, pgsz); + } + pCtx->nWrite += nByte/1024; + + return 0; + } + + bad_command: + testPrintError("unrecognized command: %s", zCmd); + return 1; +} + +static int readStdin(char **pzOut){ + int nAlloc = 128; + char *zOut = 0; + int nOut = 0; + + while( !feof(stdin) ){ + int nRead; + + nAlloc = nAlloc*2; + zOut = realloc(zOut, nAlloc); + nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin); + + if( nRead==0 ) break; + nOut += nRead; + zOut[nOut] = '\0'; + } + + *pzOut = zOut; + return 0; +} + +int do_io(int nArg, char **azArg){ + IoContext ctx; + int pgsz; + char *zFile; + char *zPgsz; + int i; + int rc = 0; + + char *zStdin = 0; + char *z; + + u8 *aData; + + memset(&ctx, 0, sizeof(IoContext)); + if( nArg<2 ){ + testPrintUsage("FILE PGSZ ?CMD-1 ...?"); + return -1; + } + zFile = azArg[0]; + zPgsz = azArg[1]; + + pgsz = (int)getNextSize(zPgsz, 0, &rc); + if( pgsz<=0 ){ + testPrintError("Ridiculous page size: %d", pgsz); + return -1; + } + aData = malloc(pgsz); + memset(aData, 0x77, pgsz); + + ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644); + if( ctx.fd<0 ){ + perror("open: "); + return -1; + } + + if( nArg==2 ){ + readStdin(&zStdin); + testTimeInit(); + z = zStdin; + while( *z && rc==0 ){ + rc = doOneCmd(&ctx, aData, pgsz, z, &z); + } + }else{ + testTimeInit(); + for(i=2; i + +void test_failed(){ + assert( 0 ); + return; +} + +#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__) +static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){ + if( rc ){ + *pRc = rc; + fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc); + test_failed(); + } +} + +static int lsm_memcmp(u8 *a, u8 *b, int c){ + int i; + for(i=0; i0 && lsm_memcmp(pVal, pDbVal, nVal))) ){ + testSetError(1); + } + } +} + +void testWrite( + TestDb *pDb, /* Database handle */ + void *pKey, int nKey, /* Key to query database for */ + void *pVal, int nVal, /* Value to write */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==0 ){ + int rc; +static int nCall = 0; +nCall++; + rc = tdb_write(pDb, pKey, nKey, pVal, nVal); + testSetError(rc); + } +} +void testDelete( + TestDb *pDb, /* Database handle */ + void *pKey, int nKey, /* Key to query database for */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==0 ){ + int rc; + *pRc = rc = tdb_delete(pDb, pKey, nKey); + testSetError(rc); + } +} +void testDeleteRange( + TestDb *pDb, /* Database handle */ + void *pKey1, int nKey1, + void *pKey2, int nKey2, + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==0 ){ + int rc; + *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2); + testSetError(rc); + } +} + +void testBegin(TestDb *pDb, int iTrans, int *pRc){ + if( *pRc==0 ){ + int rc; + rc = tdb_begin(pDb, iTrans); + testSetError(rc); + } +} +void testCommit(TestDb *pDb, int iTrans, int *pRc){ + if( *pRc==0 ){ + int rc; + rc = tdb_commit(pDb, iTrans); + testSetError(rc); + } +} +#if 0 /* unused */ +static void testRollback(TestDb *pDb, int iTrans, int *pRc){ + if( *pRc==0 ){ + int rc; + rc = tdb_rollback(pDb, iTrans); + testSetError(rc); + } +} +#endif + +void testWriteStr( + TestDb *pDb, /* Database handle */ + const char *zKey, /* Key to query database for */ + const char *zVal, /* Value to write */ + int *pRc /* IN/OUT: Error code */ +){ + int nVal = (zVal ? strlen(zVal) : 0); + testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); +} + +#if 0 /* unused */ +static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){ + testDelete(pDb, (void *)zKey, strlen(zKey), pRc); +} +#endif +void testFetchStr( + TestDb *pDb, /* Database handle */ + const char *zKey, /* Key to query database for */ + const char *zVal, /* Value to write */ + int *pRc /* IN/OUT: Error code */ +){ + int nVal = (zVal ? strlen(zVal) : 0); + testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); +} + +void testFetchCompare( + TestDb *pControl, + TestDb *pDb, + void *pKey, int nKey, + int *pRc +){ + int rc; + void *pDbVal1; + void *pDbVal2; + int nDbVal1; + int nDbVal2; + + static int nCall = 0; + nCall++; + + rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1); + testSetError(rc); + + rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2); + testSetError(rc); + + if( *pRc==0 + && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1))) + ){ + testSetError(1); + } +} + +typedef struct ScanResult ScanResult; +struct ScanResult { + TestDb *pDb; + + int nRow; + u32 cksum1; + u32 cksum2; + void *pKey1; int nKey1; + void *pKey2; int nKey2; + + int bReverse; + int nPrevKey; + u8 aPrevKey[256]; +}; + +static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){ + int res; + res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2)); + if( res==0 ){ + res = nKey1 - nKey2; + } + return res; +} + +int test_scan_debug = 0; + +static void scanCompareCb( + void *pCtx, + void *pKey, int nKey, + void *pVal, int nVal +){ + ScanResult *p = (ScanResult *)pCtx; + u8 *aKey = (u8 *)pKey; + u8 *aVal = (u8 *)pVal; + int i; + + if( test_scan_debug ){ + printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey); + fflush(stdout); + } +#if 0 + if( test_scan_debug ) printf("%.20s\n", (char *)pVal); +#endif + +#if 0 + /* Check tdb_fetch() matches */ + int rc = 0; + testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc); + assert( rc==0 ); +#endif + + /* Update the checksum data */ + p->nRow++; + for(i=0; icksum1 += ((int)aKey[i] << (i&0x0F)); + p->cksum2 += p->cksum1; + } + for(i=0; icksum1 += ((int)aVal[i] << (i&0x0F)); + p->cksum2 += p->cksum1; + } + + /* Check that the delivered row is not out of order. */ + if( nKey<(int)sizeof(p->aPrevKey) ){ + if( p->nPrevKey ){ + int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey); + if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){ + testPrintError("Returned key out of order at %s:%d\n", + __FILE__, __LINE__ + ); + } + } + + p->nPrevKey = nKey; + memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey)); + } + + /* Check that the delivered row is within range. */ + if( p->pKey1 && ( + (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0) + || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey) + )){ + testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__); + } + if( p->pKey2 && ( + (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0) + || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2=0 ); + zRet = (char *)testMalloc(nByte+1); + vsnprintf(zRet, nByte+1, zFormat, ap); + return zRet; +} + +char *testMallocPrintf(const char *zFormat, ...){ + va_list ap; + char *zRet; + + va_start(ap, zFormat); + zRet = testMallocVPrintf(zFormat, ap); + va_end(ap); + + return zRet; +} + + +/* +** A wrapper around malloc(3). +** +** This function should be used for all allocations made by test procedures. +** It has the following properties: +** +** * Test code may assume that allocations may not fail. +** * Returned memory is always zeroed. +** +** Allocations made using testMalloc() should be freed using testFree(). +*/ +void *testMalloc(int n){ + u8 *p = (u8*)malloc(n + 8); + memset(p, 0, n+8); + *(int*)p = n; + return (void*)&p[8]; +} + +void *testMallocCopy(void *pCopy, int nByte){ + void *pRet = testMalloc(nByte); + memcpy(pRet, pCopy, nByte); + return pRet; +} + +void *testRealloc(void *ptr, int n){ + if( ptr ){ + u8 *p = (u8*)ptr - 8; + int nOrig = *(int*)p; + p = (u8*)realloc(p, n+8); + if( nOrig1 ){ + testPrintError("Usage: test ?PATTERN?\n"); + return 1; + } + if( nArg==1 ){ + zPattern = azArg[0]; + } + + for(j=0; tdb_system_name(j); j++){ + rc = 0; + + test_data_1(tdb_system_name(j), zPattern, &rc); + test_data_2(tdb_system_name(j), zPattern, &rc); + test_data_3(tdb_system_name(j), zPattern, &rc); + test_data_4(tdb_system_name(j), zPattern, &rc); + test_rollback(tdb_system_name(j), zPattern, &rc); + test_mc(tdb_system_name(j), zPattern, &rc); + test_mt(tdb_system_name(j), zPattern, &rc); + + if( rc ) nFail++; + } + + rc = 0; + test_oom(zPattern, &rc); + if( rc ) nFail++; + + rc = 0; + test_api(zPattern, &rc); + if( rc ) nFail++; + + rc = 0; + do_crash_test(zPattern, &rc); + if( rc ) nFail++; + + rc = 0; + do_writer_crash_test(zPattern, &rc); + if( rc ) nFail++; + + return (nFail!=0); +} + +static lsm_db *configure_lsm_db(TestDb *pDb){ + lsm_db *pLsm; + pLsm = tdb_lsm(pDb); + if( pLsm ){ + tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4"); + } + return pLsm; +} + +typedef struct WriteHookEvent WriteHookEvent; +struct WriteHookEvent { + i64 iOff; + int nData; + int nUs; +}; +WriteHookEvent prev = {0, 0, 0}; + +static void flushPrev(FILE *pOut){ + if( prev.nData ){ + fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs); + prev.nData = 0; + } +} + +#if 0 /* unused */ +static void do_speed_write_hook2( + void *pCtx, + int bLog, + i64 iOff, + int nData, + int nUs +){ + FILE *pOut = (FILE *)pCtx; + if( bLog ) return; + + if( prev.nData && nData && iOff==prev.iOff+prev.nData ){ + prev.nData += nData; + prev.nUs += nUs; + }else{ + flushPrev(pOut); + if( nData==0 ){ + fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs); + }else{ + prev.iOff = iOff; + prev.nData = nData; + prev.nUs = nUs; + } + } +} +#endif + +#define ST_REPEAT 0 +#define ST_WRITE 1 +#define ST_PAUSE 2 +#define ST_FETCH 3 +#define ST_SCAN 4 +#define ST_NSCAN 5 +#define ST_KEYSIZE 6 +#define ST_VALSIZE 7 +#define ST_TRANS 8 + + +static void print_speed_test_help(){ + printf( +"\n" +"Repeat the following $repeat times:\n" +" 1. Insert $write key-value pairs. One transaction for each write op.\n" +" 2. Pause for $pause ms.\n" +" 3. Perform $fetch queries on the database.\n" +"\n" +" Keys are $keysize bytes in size. Values are $valsize bytes in size\n" +" Both keys and values are pseudo-randomly generated\n" +"\n" +"Options are:\n" +" -repeat $repeat (default value 10)\n" +" -write $write (default value 10000)\n" +" -pause $pause (default value 0)\n" +" -fetch $fetch (default value 0)\n" +" -keysize $keysize (default value 12)\n" +" -valsize $valsize (default value 100)\n" +" -system $system (default value \"lsm\")\n" +" -trans $trans (default value 0)\n" +"\n" +); +} + +int do_speed_test2(int nArg, char **azArg){ + struct Option { + const char *zOpt; + int eVal; + int iDefault; + } aOpt[] = { + { "-repeat", ST_REPEAT, 10}, + { "-write", ST_WRITE, 10000}, + { "-pause", ST_PAUSE, 0}, + { "-fetch", ST_FETCH, 0}, + { "-scan", ST_SCAN, 0}, + { "-nscan", ST_NSCAN, 0}, + { "-keysize", ST_KEYSIZE, 12}, + { "-valsize", ST_VALSIZE, 100}, + { "-trans", ST_TRANS, 0}, + { "-system", -1, 0}, + { "help", -2, 0}, + {0, 0, 0} + }; + int i; + int aParam[9]; + int rc = 0; + int bReadonly = 0; + int nContent = 0; + + TestDb *pDb; + Datasource *pData; + DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 }; + char *zSystem = ""; + int bLsm = 1; + FILE *pLog = 0; + +#ifdef NDEBUG + /* If NDEBUG is defined, disable the dynamic memory related checks in + ** lsmtest_mem.c. They slow things down. */ + testMallocUninstall(tdb_lsm_env()); +#endif + + /* Initialize aParam[] with default values. */ + for(i=0; i=0 ){ + aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]); + }else{ + zSystem = azArg[i+1]; + bLsm = 0; +#if 0 + for(j=0; zSystem[j]; j++){ + if( zSystem[j]=='=' ) bLsm = 1; + } +#endif + } + } + + printf("#"); + for(i=0; i=0 ){ + printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]); + }else if( aOpt[i].eVal==-1 ){ + printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem); + } + } + } + printf("\n"); + + defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE]; + defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE]; + pData = testDatasourceNew(&defn); + + if( aParam[ST_WRITE]==0 ){ + bReadonly = 1; + } + + if( bLsm ){ + rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb); + }else{ + pDb = testOpen(zSystem, !bReadonly, &rc); + } + if( rc!=0 ) return rc; + if( bReadonly ){ + nContent = testCountDatabase(pDb); + } + +#if 0 + pLog = fopen("/tmp/speed.log", "w"); + tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); +#endif + + for(i=0; i=nArg ){ + testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); + return 1; + } + if( aOpt[iSel].isSwitch==1 ){ + nRow = atoi(azArg[i]); + } + if( aOpt[iSel].isSwitch==2 ){ + nSleep = atoi(azArg[i]); + } + if( aOpt[iSel].isSwitch==3 ){ + struct Mode { + const char *zMode; + int doReadTest; + int doWriteTest; + } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}}; + int iMode; + rc = testArgSelect(aMode, "option", azArg[i], &iMode); + if( rc ) return rc; + doReadTest = aMode[iMode].doReadTest; + doWriteTest = aMode[iMode].doWriteTest; + } + if( aOpt[iSel].isSwitch==4 ){ + /* The "-out FILE" switch. This option is used to specify a file to + ** write the gnuplot script to. */ + zOut = azArg[i]; + } + }else{ + /* A db name */ + rc = testArgSelect(aOpt, "system", azArg[i], &iSel); + if( rc ) return rc; + sys_mask |= (1< 100000) ? 100000 : nSelStep; + + aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep); + aWrite = malloc(sizeof(int) * nRow/nStep); + aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep); + + /* This loop collects the INSERT speed data. */ + if( doWriteTest ){ + printf("Writing output to file \"%s\".\n", zOut); + + for(j=0; aSys[j].zLibrary; j++){ + FILE *pLog = 0; + TestDb *pDb; /* Database being tested */ + lsm_db *pLsm; + int iDot = 0; + + if( ((1<nData ){ + fprintf(pHook->pOut, "write %s %d %d\n", + (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData + ); + pHook->nData = 0; + fflush(pHook->pOut); + } +} + +static void do_insert_write_hook( + void *pCtx, + int bLog, + i64 iOff, + int nData, + int nUs +){ + InsertWriteHook *pHook = (InsertWriteHook *)pCtx; + if( bLog ) return; + + if( nData==0 ){ + flushHook(pHook); + fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db")); + }else if( pHook->nData + && bLog==pHook->bLog + && iOff==(pHook->iOff+pHook->nData) + ){ + pHook->nData += nData; + }else{ + flushHook(pHook); + pHook->bLog = bLog; + pHook->iOff = iOff; + pHook->nData = nData; + } +} + +static int do_replay(int nArg, char **azArg){ + char aBuf[4096]; + FILE *pInput; + FILE *pClose = 0; + const char *zDb; + + lsm_env *pEnv; + lsm_file *pOut; + int rc; + + if( nArg!=2 ){ + testPrintError("Usage: replay WRITELOG FILE\n"); + return 1; + } + + if( strcmp(azArg[0], "-")==0 ){ + pInput = stdin; + }else{ + pClose = pInput = fopen(azArg[0], "r"); + } + zDb = azArg[1]; + pEnv = tdb_lsm_env(); + rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); + if( rc!=LSM_OK ) return rc; + + while( feof(pInput)==0 ){ + char zLine[80]; + fgets(zLine, sizeof(zLine)-1, pInput); + zLine[sizeof(zLine)-1] = '\0'; + + if( 0==memcmp("sync db", zLine, 7) ){ + rc = pEnv->xSync(pOut); + if( rc!=0 ) break; + }else{ + int iOff; + int nData; + int nMatch; + nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData); + if( nMatch==2 ){ + int i; + for(i=0; ixWrite(pOut, iOff+i, aBuf, sizeof(aBuf)); + if( rc!=0 ) break; + } + } + } + } + if( pClose ) fclose(pClose); + pEnv->xClose(pOut); + + return rc; +} + +static int do_insert(int nArg, char **azArg){ + const char *zDb = "lsm"; + TestDb *pDb = 0; + int i; + int rc; + const int nRow = 1 * 1000 * 1000; + + DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 }; + Datasource *pData = 0; + + if( nArg>1 ){ + testPrintError("Usage: insert ?DATABASE?\n"); + return 1; + } + if( nArg==1 ){ zDb = azArg[0]; } + + testMallocUninstall(tdb_lsm_env()); + for(i=0; zDb[i] && zDb[i]!='='; i++); + if( zDb[i] ){ + rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb); + }else{ + rc = tdb_open(zDb, 0, 1, &pDb); + } + + if( rc!=0 ){ + testPrintError("Error opening db \"%s\": %d\n", zDb, rc); + }else{ + InsertWriteHook hook; + memset(&hook, 0, sizeof(hook)); + hook.pOut = fopen("writelog.txt", "w"); + + pData = testDatasourceNew(&defn); + tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0); + tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook); + + if( rc==0 ){ + for(i=0; i +#include + +static void lsmtest_rusage_report(void){ + struct rusage r; + memset(&r, 0, sizeof(r)); + + getrusage(RUSAGE_SELF, &r); + printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", + (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock + ); +} +#else +static void lsmtest_rusage_report(void){ + /* no-op */ +} +#endif + +int main(int argc, char **argv){ + struct TestFunc { + const char *zName; + int bRusageReport; + int (*xFunc)(int, char **); + } aTest[] = { + {"random", 1, do_random_tests}, + {"writespeed", 1, do_writer_test}, + {"io", 1, st_do_io}, + + {"insert", 1, do_insert}, + {"replay", 1, do_replay}, + + {"speed", 1, do_speed_tests}, + {"speed2", 1, do_speed_test2}, + {"show", 0, st_do_show}, + {"work", 1, st_do_work}, + {"test", 1, do_test}, + + {0, 0} + }; + int rc; /* Return Code */ + int iFunc; /* Index into aTest[] */ + + int nLeakAlloc = 0; /* Allocations leaked by lsm */ + int nLeakByte = 0; /* Bytes leaked by lsm */ + +#ifdef LSM_DEBUG_MEM + FILE *pReport = 0; /* lsm malloc() report file */ + const char *zReport = "malloc.txt generated"; +#else + const char *zReport = "malloc.txt NOT generated"; +#endif + + testMallocInstall(tdb_lsm_env()); + + if( argc<2 ){ + testPrintError("Usage: %s sub-command ?args...?\n", argv[0]); + return -1; + } + + /* Initialize error reporting */ + testErrorInit(argc, argv); + + /* Initialize PRNG system */ + testPrngInit(); + + rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc); + if( rc==0 ){ + rc = aTest[iFunc].xFunc(argc-2, &argv[2]); + } + +#ifdef LSM_DEBUG_MEM + pReport = fopen("malloc.txt", "w"); + testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport); + fclose(pReport); +#else + testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0); +#endif + + if( nLeakAlloc ){ + testPrintError("Leaked %d bytes in %d allocations (%s)\n", + nLeakByte, nLeakAlloc, zReport + ); + if( rc==0 ) rc = -1; + } + testMallocUninstall(tdb_lsm_env()); + + if( aTest[iFunc].bRusageReport ){ + lsmtest_rusage_report(); + } + return rc; +} diff --git a/ext/lsm1/lsm-test/lsmtest_mem.c b/ext/lsm1/lsm-test/lsmtest_mem.c new file mode 100644 index 0000000000000000000000000000000000000000..4c35e849f2504a73584bf0385f28cbc0daa9a966 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_mem.c @@ -0,0 +1,409 @@ + +#include +#include +#include + +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) + +#define MIN(x,y) ((x)<(y) ? (x) : (y)) + +typedef unsigned int u32; +typedef unsigned char u8; +typedef long long int i64; +typedef unsigned long long int u64; + +#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM) + extern int backtrace(void**,int); + extern void backtrace_symbols_fd(void*const*,int,int); +# define TM_BACKTRACE 12 +#else +# define backtrace(A,B) 1 +# define backtrace_symbols_fd(A,B,C) +#endif + + +typedef struct TmBlockHdr TmBlockHdr; +typedef struct TmAgg TmAgg; +typedef struct TmGlobal TmGlobal; + +struct TmGlobal { + /* Linked list of all currently outstanding allocations. And a table of + ** all allocations, past and present, indexed by backtrace() info. */ + TmBlockHdr *pFirst; +#ifdef TM_BACKTRACE + TmAgg *aHash[10000]; +#endif + + /* Underlying malloc/realloc/free functions */ + void *(*xMalloc)(int); /* underlying malloc(3) function */ + void *(*xRealloc)(void *, int); /* underlying realloc(3) function */ + void (*xFree)(void *); /* underlying free(3) function */ + + /* Mutex to protect pFirst and aHash */ + void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */ + void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */ + void (*xDelMutex)(TmGlobal*); /* Call this to delete mutex */ + void *pMutex; /* Mutex handle */ + + void *(*xSaveMalloc)(void *, size_t); + void *(*xSaveRealloc)(void *, void *, size_t); + void (*xSaveFree)(void *, void *); + + /* OOM injection scheduling. If nCountdown is greater than zero when a + ** malloc attempt is made, it is decremented. If this means nCountdown + ** transitions from 1 to 0, then the allocation fails. If bPersist is true + ** when this happens, nCountdown is then incremented back to 1 (so that the + ** next attempt fails too). + */ + int nCountdown; + int bPersist; + int bEnable; + void (*xHook)(void *); + void *pHookCtx; +}; + +struct TmBlockHdr { + TmBlockHdr *pNext; + TmBlockHdr *pPrev; + int nByte; +#ifdef TM_BACKTRACE + TmAgg *pAgg; +#endif + u32 iForeGuard; +}; + +#ifdef TM_BACKTRACE +struct TmAgg { + int nAlloc; /* Number of allocations at this path */ + int nByte; /* Total number of bytes allocated */ + int nOutAlloc; /* Number of outstanding allocations */ + int nOutByte; /* Number of outstanding bytes */ + void *aFrame[TM_BACKTRACE]; /* backtrace() output */ + TmAgg *pNext; /* Next object in hash-table collision */ +}; +#endif + +#define FOREGUARD 0x80F5E153 +#define REARGUARD 0xE4676B53 +static const u32 rearguard = REARGUARD; + +#define ROUND8(x) (((x)+7)&~7) + +#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) )) + +static void lsmtest_oom_error(void){ + static int nErr = 0; + nErr++; +} + +static void tmEnterMutex(TmGlobal *pTm){ + pTm->xEnterMutex(pTm); +} +static void tmLeaveMutex(TmGlobal *pTm){ + pTm->xLeaveMutex(pTm); +} + +static void *tmMalloc(TmGlobal *pTm, int nByte){ + TmBlockHdr *pNew; /* New allocation header block */ + u8 *pUser; /* Return value */ + int nReq; /* Total number of bytes requested */ + + assert( sizeof(rearguard)==4 ); + nReq = BLOCK_HDR_SIZE + nByte + 4; + pNew = (TmBlockHdr *)pTm->xMalloc(nReq); + memset(pNew, 0, sizeof(TmBlockHdr)); + + tmEnterMutex(pTm); + assert( pTm->nCountdown>=0 ); + assert( pTm->bPersist==0 || pTm->bPersist==1 ); + + if( pTm->bEnable && pTm->nCountdown==1 ){ + /* Simulate an OOM error. */ + lsmtest_oom_error(); + pTm->xFree(pNew); + pTm->nCountdown = pTm->bPersist; + if( pTm->xHook ) pTm->xHook(pTm->pHookCtx); + pUser = 0; + }else{ + if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--; + + pNew->iForeGuard = FOREGUARD; + pNew->nByte = nByte; + pNew->pNext = pTm->pFirst; + + if( pTm->pFirst ){ + pTm->pFirst->pPrev = pNew; + } + pTm->pFirst = pNew; + + pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE]; + memset(pUser, 0x56, nByte); + memcpy(&pUser[nByte], &rearguard, 4); + +#ifdef TM_BACKTRACE + { + TmAgg *pAgg; + int i; + u32 iHash = 0; + void *aFrame[TM_BACKTRACE]; + memset(aFrame, 0, sizeof(aFrame)); + backtrace(aFrame, TM_BACKTRACE); + + for(i=0; iaHash); + + for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){ + if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break; + } + if( !pAgg ){ + pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg)); + memset(pAgg, 0, sizeof(TmAgg)); + memcpy(pAgg->aFrame, aFrame, sizeof(aFrame)); + pAgg->pNext = pTm->aHash[iHash]; + pTm->aHash[iHash] = pAgg; + } + pAgg->nAlloc++; + pAgg->nByte += nByte; + pAgg->nOutAlloc++; + pAgg->nOutByte += nByte; + pNew->pAgg = pAgg; + } +#endif + } + + tmLeaveMutex(pTm); + return pUser; +} + +static void tmFree(TmGlobal *pTm, void *p){ + if( p ){ + TmBlockHdr *pHdr; + u8 *pUser = (u8 *)p; + + tmEnterMutex(pTm); + pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); + assert( pHdr->iForeGuard==FOREGUARD ); + assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) ); + + if( pHdr->pPrev ){ + assert( pHdr->pPrev->pNext==pHdr ); + pHdr->pPrev->pNext = pHdr->pNext; + }else{ + assert( pHdr==pTm->pFirst ); + pTm->pFirst = pHdr->pNext; + } + if( pHdr->pNext ){ + assert( pHdr->pNext->pPrev==pHdr ); + pHdr->pNext->pPrev = pHdr->pPrev; + } + +#ifdef TM_BACKTRACE + pHdr->pAgg->nOutAlloc--; + pHdr->pAgg->nOutByte -= pHdr->nByte; +#endif + + tmLeaveMutex(pTm); + memset(pUser, 0x58, pHdr->nByte); + memset(pHdr, 0x57, sizeof(TmBlockHdr)); + pTm->xFree(pHdr); + } +} + +static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){ + void *pNew; + + pNew = tmMalloc(pTm, nByte); + if( pNew && p ){ + TmBlockHdr *pHdr; + u8 *pUser = (u8 *)p; + pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); + memcpy(pNew, p, MIN(nByte, pHdr->nByte)); + tmFree(pTm, p); + } + return pNew; +} + +static void tmMallocOom( + TmGlobal *pTm, + int nCountdown, + int bPersist, + void (*xHook)(void *), + void *pHookCtx +){ + assert( nCountdown>=0 ); + assert( bPersist==0 || bPersist==1 ); + pTm->nCountdown = nCountdown; + pTm->bPersist = bPersist; + pTm->xHook = xHook; + pTm->pHookCtx = pHookCtx; + pTm->bEnable = 1; +} + +static void tmMallocOomEnable( + TmGlobal *pTm, + int bEnable +){ + pTm->bEnable = bEnable; +} + +static void tmMallocCheck( + TmGlobal *pTm, + int *pnLeakAlloc, + int *pnLeakByte, + FILE *pFile +){ + TmBlockHdr *pHdr; + int nLeak = 0; + int nByte = 0; + + if( pTm==0 ) return; + + for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){ + nLeak++; + nByte += pHdr->nByte; + } + if( pnLeakAlloc ) *pnLeakAlloc = nLeak; + if( pnLeakByte ) *pnLeakByte = nByte; + +#ifdef TM_BACKTRACE + if( pFile ){ + int i; + fprintf(pFile, "LEAKS\n"); + for(i=0; iaHash); i++){ + TmAgg *pAgg; + for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ + if( pAgg->nOutAlloc ){ + int j; + fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc); + for(j=0; jaFrame[j]); + } + fprintf(pFile, "\n"); + } + } + } + fprintf(pFile, "\nALLOCATIONS\n"); + for(i=0; iaHash); i++){ + TmAgg *pAgg; + for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ + int j; + fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc); + for(j=0; jaFrame[j]); + fprintf(pFile, "\n"); + } + } + } +#else + (void)pFile; +#endif +} + + +#include "lsm.h" +#include "stdlib.h" + +typedef struct LsmMutex LsmMutex; +struct LsmMutex { + lsm_env *pEnv; + lsm_mutex *pMutex; +}; + +static void tmLsmMutexEnter(TmGlobal *pTm){ + LsmMutex *p = (LsmMutex *)pTm->pMutex; + p->pEnv->xMutexEnter(p->pMutex); +} +static void tmLsmMutexLeave(TmGlobal *pTm){ + LsmMutex *p = (LsmMutex *)(pTm->pMutex); + p->pEnv->xMutexLeave(p->pMutex); +} +static void tmLsmMutexDel(TmGlobal *pTm){ + LsmMutex *p = (LsmMutex *)pTm->pMutex; + pTm->xFree(p); +} +static void *tmLsmMalloc(int n){ return malloc(n); } +static void tmLsmFree(void *ptr){ free(ptr); } +static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); } + +static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ + return tmMalloc((TmGlobal *)(p->pMemCtx), n); +} +static void tmLsmEnvFree(lsm_env *p, void *ptr){ + tmFree((TmGlobal *)(p->pMemCtx), ptr); +} +static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ + return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n); +} + +void testMallocInstall(lsm_env *pEnv){ + TmGlobal *pGlobal; + LsmMutex *pMutex; + assert( pEnv->pMemCtx==0 ); + + /* Allocate and populate a TmGlobal structure. */ + pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal)); + memset(pGlobal, 0, sizeof(TmGlobal)); + pGlobal->xMalloc = tmLsmMalloc; + pGlobal->xRealloc = tmLsmRealloc; + pGlobal->xFree = tmLsmFree; + pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex)); + pMutex->pEnv = pEnv; + pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex); + pGlobal->xEnterMutex = tmLsmMutexEnter; + pGlobal->xLeaveMutex = tmLsmMutexLeave; + pGlobal->xDelMutex = tmLsmMutexDel; + pGlobal->pMutex = (void *)pMutex; + + pGlobal->xSaveMalloc = pEnv->xMalloc; + pGlobal->xSaveRealloc = pEnv->xRealloc; + pGlobal->xSaveFree = pEnv->xFree; + + /* Set up pEnv to the use the new TmGlobal */ + pEnv->pMemCtx = (void *)pGlobal; + pEnv->xMalloc = tmLsmEnvMalloc; + pEnv->xRealloc = tmLsmEnvRealloc; + pEnv->xFree = tmLsmEnvFree; +} + +void testMallocUninstall(lsm_env *pEnv){ + TmGlobal *p = (TmGlobal *)pEnv->pMemCtx; + pEnv->pMemCtx = 0; + if( p ){ + pEnv->xMalloc = p->xSaveMalloc; + pEnv->xRealloc = p->xSaveRealloc; + pEnv->xFree = p->xSaveFree; + p->xDelMutex(p); + tmLsmFree(p); + } +} + +void testMallocCheck( + lsm_env *pEnv, + int *pnLeakAlloc, + int *pnLeakByte, + FILE *pFile +){ + if( pEnv->pMemCtx==0 ){ + *pnLeakAlloc = 0; + *pnLeakByte = 0; + }else{ + tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile); + } +} + +void testMallocOom( + lsm_env *pEnv, + int nCountdown, + int bPersist, + void (*xHook)(void *), + void *pHookCtx +){ + TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); + tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx); +} + +void testMallocOomEnable(lsm_env *pEnv, int bEnable){ + TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); + tmMallocOomEnable(pTm, bEnable); +} diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.c b/ext/lsm1/lsm-test/lsmtest_tdb.c new file mode 100644 index 0000000000000000000000000000000000000000..8f63f64acb7c7f084f62ada7c2ac1d7ad4711a45 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_tdb.c @@ -0,0 +1,846 @@ + +/* +** This program attempts to test the correctness of some facets of the +** LSM database library. Specifically, that the contents of the database +** are maintained correctly during a series of inserts and deletes. +*/ + + +#include "lsmtest_tdb.h" +#include "lsm.h" + +#include "lsmtest.h" + +#include +#include +#include +#ifndef _WIN32 +# include +#endif +#include + + +typedef struct SqlDb SqlDb; + +static int error_transaction_function(TestDb *p, int iLevel){ + unused_parameter(p); + unused_parameter(iLevel); + return -1; +} + + +/************************************************************************* +** Begin wrapper for LevelDB. +*/ +#ifdef HAVE_LEVELDB + +#include + +typedef struct LevelDb LevelDb; +struct LevelDb { + TestDb base; + leveldb_t *db; + leveldb_options_t *pOpt; + leveldb_writeoptions_t *pWriteOpt; + leveldb_readoptions_t *pReadOpt; + + char *pVal; +}; + +static int test_leveldb_close(TestDb *pTestDb){ + LevelDb *pDb = (LevelDb *)pTestDb; + + leveldb_close(pDb->db); + leveldb_writeoptions_destroy(pDb->pWriteOpt); + leveldb_readoptions_destroy(pDb->pReadOpt); + leveldb_options_destroy(pDb->pOpt); + free(pDb->pVal); + free(pDb); + + return 0; +} + +static int test_leveldb_write( + TestDb *pTestDb, + void *pKey, + int nKey, + void *pVal, + int nVal +){ + LevelDb *pDb = (LevelDb *)pTestDb; + char *zErr = 0; + leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr); + return (zErr!=0); +} + +static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){ + LevelDb *pDb = (LevelDb *)pTestDb; + char *zErr = 0; + leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr); + return (zErr!=0); +} + +static int test_leveldb_fetch( + TestDb *pTestDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + LevelDb *pDb = (LevelDb *)pTestDb; + char *zErr = 0; + size_t nVal = 0; + + if( pKey==0 ) return 0; + free(pDb->pVal); + pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr); + *ppVal = (void *)(pDb->pVal); + if( pDb->pVal==0 ){ + *pnVal = -1; + }else{ + *pnVal = (int)nVal; + } + + return (zErr!=0); +} + +static int test_leveldb_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *, void *, int , void *, int) +){ + LevelDb *pDb = (LevelDb *)pTestDb; + leveldb_iterator_t *iter; + + iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt); + + if( bReverse==0 ){ + if( pKey1 ){ + leveldb_iter_seek(iter, pKey1, nKey1); + }else{ + leveldb_iter_seek_to_first(iter); + } + }else{ + if( pKey2 ){ + leveldb_iter_seek(iter, pKey2, nKey2); + + if( leveldb_iter_valid(iter)==0 ){ + leveldb_iter_seek_to_last(iter); + }else{ + const char *k; size_t n; + int res; + k = leveldb_iter_key(iter, &n); + res = memcmp(k, pKey2, MIN(n, nKey2)); + if( res==0 ) res = n - nKey2; + assert( res>=0 ); + if( res>0 ){ + leveldb_iter_prev(iter); + } + } + }else{ + leveldb_iter_seek_to_last(iter); + } + } + + + while( leveldb_iter_valid(iter) ){ + const char *k; size_t n; + const char *v; size_t n2; + int res; + + k = leveldb_iter_key(iter, &n); + if( bReverse==0 && pKey2 ){ + res = memcmp(k, pKey2, MIN(n, nKey2)); + if( res==0 ) res = n - nKey2; + if( res>0 ) break; + } + if( bReverse!=0 && pKey1 ){ + res = memcmp(k, pKey1, MIN(n, nKey1)); + if( res==0 ) res = n - nKey1; + if( res<0 ) break; + } + + v = leveldb_iter_value(iter, &n2); + + xCallback(pCtx, (void *)k, n, (void *)v, n2); + + if( bReverse==0 ){ + leveldb_iter_next(iter); + }else{ + leveldb_iter_prev(iter); + } + } + + leveldb_iter_destroy(iter); + return 0; +} + +static int test_leveldb_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + static const DatabaseMethods LeveldbMethods = { + test_leveldb_close, + test_leveldb_write, + test_leveldb_delete, + 0, + test_leveldb_fetch, + test_leveldb_scan, + error_transaction_function, + error_transaction_function, + error_transaction_function + }; + + LevelDb *pLevelDb; + char *zErr = 0; + + if( bClear ){ + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); + system(zCmd); + sqlite3_free(zCmd); + } + + pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); + memset(pLevelDb, 0, sizeof(LevelDb)); + + pLevelDb->pOpt = leveldb_options_create(); + leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1); + pLevelDb->pWriteOpt = leveldb_writeoptions_create(); + pLevelDb->pReadOpt = leveldb_readoptions_create(); + + pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr); + + if( zErr ){ + test_leveldb_close((TestDb *)pLevelDb); + *ppDb = 0; + return 1; + } + + *ppDb = (TestDb *)pLevelDb; + pLevelDb->base.pMethods = &LeveldbMethods; + return 0; +} +#endif /* HAVE_LEVELDB */ +/* +** End wrapper for LevelDB. +*************************************************************************/ + +#ifdef HAVE_KYOTOCABINET +static int kc_close(TestDb *pTestDb){ + return test_kc_close(pTestDb); +} + +static int kc_write( + TestDb *pTestDb, + void *pKey, + int nKey, + void *pVal, + int nVal +){ + return test_kc_write(pTestDb, pKey, nKey, pVal, nVal); +} + +static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){ + return test_kc_delete(pTestDb, pKey, nKey); +} + +static int kc_delete_range( + TestDb *pTestDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2); +} + +static int kc_fetch( + TestDb *pTestDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + if( pKey==0 ) return LSM_OK; + return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal); +} + +static int kc_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pFirst, int nFirst, + void *pLast, int nLast, + void (*xCallback)(void *, void *, int , void *, int) +){ + return test_kc_scan( + pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback + ); +} + +static int kc_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + static const DatabaseMethods KcdbMethods = { + kc_close, + kc_write, + kc_delete, + kc_delete_range, + kc_fetch, + kc_scan, + error_transaction_function, + error_transaction_function, + error_transaction_function + }; + + int rc; + TestDb *pTestDb = 0; + + rc = test_kc_open(zFilename, bClear, &pTestDb); + if( rc!=0 ){ + *ppDb = 0; + return rc; + } + pTestDb->pMethods = &KcdbMethods; + *ppDb = pTestDb; + return 0; +} +#endif /* HAVE_KYOTOCABINET */ +/* +** End wrapper for Kyoto cabinet. +*************************************************************************/ + +#ifdef HAVE_MDB +static int mdb_close(TestDb *pTestDb){ + return test_mdb_close(pTestDb); +} + +static int mdb_write( + TestDb *pTestDb, + void *pKey, + int nKey, + void *pVal, + int nVal +){ + return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal); +} + +static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){ + return test_mdb_delete(pTestDb, pKey, nKey); +} + +static int mdb_fetch( + TestDb *pTestDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + if( pKey==0 ) return LSM_OK; + return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal); +} + +static int mdb_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pFirst, int nFirst, + void *pLast, int nLast, + void (*xCallback)(void *, void *, int , void *, int) +){ + return test_mdb_scan( + pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback + ); +} + +static int mdb_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + static const DatabaseMethods KcdbMethods = { + mdb_close, + mdb_write, + mdb_delete, + 0, + mdb_fetch, + mdb_scan, + error_transaction_function, + error_transaction_function, + error_transaction_function + }; + + int rc; + TestDb *pTestDb = 0; + + rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb); + if( rc!=0 ){ + *ppDb = 0; + return rc; + } + pTestDb->pMethods = &KcdbMethods; + *ppDb = pTestDb; + return 0; +} +#endif /* HAVE_MDB */ + +/************************************************************************* +** Begin wrapper for SQLite. +*/ + +/* +** nOpenTrans: +** The number of open nested transactions, in the same sense as used +** by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this +** value is 0, there are no transactions open at all. If it is 1, then +** there is a read transaction. If it is 2 or greater, then there are +** (nOpenTrans-1) nested write transactions open. +*/ +struct SqlDb { + TestDb base; + sqlite3 *db; + sqlite3_stmt *pInsert; + sqlite3_stmt *pDelete; + sqlite3_stmt *pDeleteRange; + sqlite3_stmt *pFetch; + sqlite3_stmt *apScan[8]; + + int nOpenTrans; + + /* Used by sql_fetch() to allocate space for results */ + int nAlloc; + u8 *aAlloc; +}; + +static int sql_close(TestDb *pTestDb){ + SqlDb *pDb = (SqlDb *)pTestDb; + sqlite3_finalize(pDb->pInsert); + sqlite3_finalize(pDb->pDelete); + sqlite3_finalize(pDb->pDeleteRange); + sqlite3_finalize(pDb->pFetch); + sqlite3_finalize(pDb->apScan[0]); + sqlite3_finalize(pDb->apScan[1]); + sqlite3_finalize(pDb->apScan[2]); + sqlite3_finalize(pDb->apScan[3]); + sqlite3_finalize(pDb->apScan[4]); + sqlite3_finalize(pDb->apScan[5]); + sqlite3_finalize(pDb->apScan[6]); + sqlite3_finalize(pDb->apScan[7]); + sqlite3_close(pDb->db); + free((char *)pDb->aAlloc); + free((char *)pDb); + return SQLITE_OK; +} + +static int sql_write( + TestDb *pTestDb, + void *pKey, + int nKey, + void *pVal, + int nVal +){ + SqlDb *pDb = (SqlDb *)pTestDb; + sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC); + sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC); + sqlite3_step(pDb->pInsert); + return sqlite3_reset(pDb->pInsert); +} + +static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){ + SqlDb *pDb = (SqlDb *)pTestDb; + sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC); + sqlite3_step(pDb->pDelete); + return sqlite3_reset(pDb->pDelete); +} + +static int sql_delete_range( + TestDb *pTestDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + SqlDb *pDb = (SqlDb *)pTestDb; + sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC); + sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC); + sqlite3_step(pDb->pDeleteRange); + return sqlite3_reset(pDb->pDeleteRange); +} + +static int sql_fetch( + TestDb *pTestDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + SqlDb *pDb = (SqlDb *)pTestDb; + int rc; + + sqlite3_reset(pDb->pFetch); + if( pKey==0 ){ + assert( ppVal==0 ); + assert( pnVal==0 ); + return LSM_OK; + } + + sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC); + rc = sqlite3_step(pDb->pFetch); + if( rc==SQLITE_ROW ){ + int nVal = sqlite3_column_bytes(pDb->pFetch, 0); + u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0); + + if( nVal>pDb->nAlloc ){ + free(pDb->aAlloc); + pDb->aAlloc = (u8 *)malloc(nVal*2); + pDb->nAlloc = nVal*2; + } + memcpy(pDb->aAlloc, aVal, nVal); + *pnVal = nVal; + *ppVal = (void *)pDb->aAlloc; + }else{ + *pnVal = -1; + *ppVal = 0; + } + + rc = sqlite3_reset(pDb->pFetch); + return rc; +} + +static int sql_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pFirst, int nFirst, + void *pLast, int nLast, + void (*xCallback)(void *, void *, int , void *, int) +){ + SqlDb *pDb = (SqlDb *)pTestDb; + sqlite3_stmt *pScan; + + assert( bReverse==1 || bReverse==0 ); + pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4]; + + if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC); + if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC); + + while( SQLITE_ROW==sqlite3_step(pScan) ){ + void *pKey; int nKey; + void *pVal; int nVal; + + nKey = sqlite3_column_bytes(pScan, 0); + pKey = (void *)sqlite3_column_blob(pScan, 0); + nVal = sqlite3_column_bytes(pScan, 1); + pVal = (void *)sqlite3_column_blob(pScan, 1); + + xCallback(pCtx, pKey, nKey, pVal, nVal); + } + return sqlite3_reset(pScan); +} + +static int sql_begin(TestDb *pTestDb, int iLevel){ + int i; + SqlDb *pDb = (SqlDb *)pTestDb; + + /* iLevel==0 is a no-op */ + if( iLevel==0 ) return 0; + + /* If there are no transactions at all open, open a read transaction. */ + if( pDb->nOpenTrans==0 ){ + int rc = sqlite3_exec(pDb->db, + "BEGIN; SELECT * FROM sqlite_schema LIMIT 1;" , 0, 0, 0 + ); + if( rc!=0 ) return rc; + pDb->nOpenTrans = 1; + } + + /* Open any required write transactions */ + for(i=pDb->nOpenTrans; idb, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ) return rc; + } + + pDb->nOpenTrans = iLevel; + return 0; +} + +static int sql_commit(TestDb *pTestDb, int iLevel){ + SqlDb *pDb = (SqlDb *)pTestDb; + assert( iLevel>=0 ); + + /* Close the read transaction if requested. */ + if( pDb->nOpenTrans>=1 && iLevel==0 ){ + int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0); + if( rc!=0 ) return rc; + pDb->nOpenTrans = 0; + } + + /* Close write transactions as required */ + if( pDb->nOpenTrans>iLevel ){ + char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel); + int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc!=0 ) return rc; + } + + pDb->nOpenTrans = iLevel; + return 0; +} + +static int sql_rollback(TestDb *pTestDb, int iLevel){ + SqlDb *pDb = (SqlDb *)pTestDb; + assert( iLevel>=0 ); + + if( pDb->nOpenTrans>=1 && iLevel==0 ){ + /* Close the read transaction if requested. */ + int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0); + if( rc!=0 ) return rc; + }else if( pDb->nOpenTrans>1 && iLevel==1 ){ + /* Or, rollback and close the top-level write transaction */ + int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0); + if( rc!=0 ) return rc; + }else{ + /* Or, just roll back some nested transactions */ + char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1); + int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc!=0 ) return rc; + } + + pDb->nOpenTrans = iLevel; + return 0; +} + +static int sql_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + static const DatabaseMethods SqlMethods = { + sql_close, + sql_write, + sql_delete, + sql_delete_range, + sql_fetch, + sql_scan, + sql_begin, + sql_commit, + sql_rollback + }; + const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)"; + const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)"; + const char *zDelete = "DELETE FROM t1 WHERE k = ?"; + const char *zRange = "DELETE FROM t1 WHERE k>? AND k= ?1 ORDER BY k"; + const char *zScan3 = "SELECT * FROM t1 ORDER BY k"; + + const char *zScan4 = + "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC"; + const char *zScan5 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC"; + const char *zScan6 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC"; + const char *zScan7 = "SELECT * FROM t1 ORDER BY k DESC"; + + int rc; + SqlDb *pDb; + char *zPragma; + + if( bClear && zFilename && zFilename[0] ){ + unlink(zFilename); + } + + pDb = (SqlDb *)malloc(sizeof(SqlDb)); + memset(pDb, 0, sizeof(SqlDb)); + pDb->base.pMethods = &SqlMethods; + + if( 0!=(rc = sqlite3_open(zFilename, &pDb->db)) + || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0)) + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0)) + ){ + *ppDb = 0; + sql_close((TestDb *)pDb); + return rc; + } + + zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE); + sqlite3_exec(pDb->db, zPragma, 0, 0, 0); + sqlite3_free(zPragma); + zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE); + sqlite3_exec(pDb->db, zPragma, 0, 0, 0); + sqlite3_free(zPragma); + + /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */ + sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); + sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0); + sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0); + if( zSpec ){ + rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0); + if( rc!=SQLITE_OK ){ + sql_close((TestDb *)pDb); + return rc; + } + } + + *ppDb = (TestDb *)pDb; + return 0; +} +/* +** End wrapper for SQLite. +*************************************************************************/ + +/************************************************************************* +** Begin exported functions. +*/ +static struct Lib { + const char *zName; + const char *zDefaultDb; + int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb); +} aLib[] = { + { "sqlite3", "testdb.sqlite", sql_open }, + { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, + { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, + { "lsm_lomem2", "testdb.lsm_lomem2", test_lsm_lomem2_open }, +#ifdef HAVE_ZLIB + { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, +#endif + { "lsm", "testdb.lsm", test_lsm_open }, +#ifdef LSM_MUTEX_PTHREADS + { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, + { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, +#endif +#ifdef HAVE_LEVELDB + { "leveldb", "testdb.leveldb", test_leveldb_open }, +#endif +#ifdef HAVE_KYOTOCABINET + { "kyotocabinet", "testdb.kc", kc_open }, +#endif +#ifdef HAVE_MDB + { "mdb", "./testdb.mdb", mdb_open } +#endif +}; + +const char *tdb_system_name(int i){ + if( i<0 || i>=ArraySize(aLib) ) return 0; + return aLib[i].zName; +} + +const char *tdb_default_db(const char *zSys){ + int i; + for(i=0; izLibrary = aLib[i].zName; + } + break; + } + } + + if( rc ){ + /* Failed to find the requested database library. Return an error. */ + *ppDb = 0; + } + return rc; +} + +int tdb_close(TestDb *pDb){ + if( pDb ){ + return pDb->pMethods->xClose(pDb); + } + return 0; +} + +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ + return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal); +} + +int tdb_delete(TestDb *pDb, void *pKey, int nKey){ + return pDb->pMethods->xDelete(pDb, pKey, nKey); +} + +int tdb_delete_range( + TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2 +){ + return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2); +} + +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){ + return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal); +} + +int tdb_scan( + TestDb *pDb, /* Database handle */ + void *pCtx, /* Context pointer to pass to xCallback */ + int bReverse, /* True to scan in reverse order */ + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) +){ + return pDb->pMethods->xScan( + pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback + ); +} + +int tdb_begin(TestDb *pDb, int iLevel){ + return pDb->pMethods->xBegin(pDb, iLevel); +} +int tdb_commit(TestDb *pDb, int iLevel){ + return pDb->pMethods->xCommit(pDb, iLevel); +} +int tdb_rollback(TestDb *pDb, int iLevel){ + return pDb->pMethods->xRollback(pDb, iLevel); +} + +int tdb_transaction_support(TestDb *pDb){ + return (pDb->pMethods->xBegin != error_transaction_function); +} + +const char *tdb_library_name(TestDb *pDb){ + return pDb->zLibrary; +} + +/* +** End exported functions. +*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.h b/ext/lsm1/lsm-test/lsmtest_tdb.h new file mode 100644 index 0000000000000000000000000000000000000000..c55b6e2f802653a64bf43df1c2a23c801d8316f3 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_tdb.h @@ -0,0 +1,174 @@ + +/* +** This file is the interface to a very simple database library used for +** testing. The interface is similar to that of the LSM. The main virtue +** of this library is that the same API may be used to access a key-value +** store implemented by LSM, SQLite or another database system. Which +** makes it easy to use for correctness and performance tests. +*/ + +#ifndef __WRAPPER_H_ +#define __WRAPPER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "lsm.h" + +typedef struct TestDb TestDb; + +/* +** Open a new database connection. The first argument is the name of the +** database library to use. e.g. something like: +** +** "sqlite3" +** "lsm" +** +** See function tdb_system_name() for a list of available database systems. +** +** The second argument is the name of the database to open (e.g. a filename). +** +** If the third parameter is non-zero, then any existing database by the +** name of zDb is removed before opening a new one. If it is zero, then an +** existing database may be opened. +*/ +int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb); + +/* +** Close a database handle. +*/ +int tdb_close(TestDb *pDb); + +/* +** Write a new key/value into the database. +*/ +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal); + +/* +** Delete a key from the database. +*/ +int tdb_delete(TestDb *pDb, void *pKey, int nKey); + +/* +** Delete a range of keys from the database. +*/ +int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2); + +/* +** Query the database for key (pKey/nKey). If no entry is found, set *ppVal +** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal +** to a pointer to and size of the value associated with (pKey/nKey). +*/ +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal); + +/* +** Open and close nested transactions. Currently, these functions only +** work for SQLite3 and LSM systems. Use the tdb_transaction_support() +** function to determine if a given TestDb handle supports these methods. +** +** These functions and the iLevel parameter follow the same conventions as +** the SQLite 4 transaction interface. Note that this is slightly different +** from the way LSM does things. As follows: +** +** tdb_begin(): +** A successful call to tdb_begin() with (iLevel>1) guarantees that +** there are at least (iLevel-1) write transactions open. If iLevel==1, +** then it guarantees that at least a read-transaction is open. Calling +** tdb_begin() with iLevel==0 is a no-op. +** +** tdb_commit(): +** A successful call to tdb_commit() with (iLevel>1) guarantees that +** there are at most (iLevel-1) write transactions open. If iLevel==1, +** then it guarantees that there are no write transactions open (although +** a read-transaction may remain open). Calling tdb_commit() with +** iLevel==0 ensures that all transactions, read or write, have been +** closed and committed. +** +** tdb_rollback(): +** This call is similar to tdb_commit(), except that instead of committing +** transactions, it reverts them. For example, calling tdb_rollback() with +** iLevel==2 ensures that there is at most one write transaction open, and +** restores the database to the state that it was in when that transaction +** was opened. +** +** In other words, tdb_commit() just closes transactions - tdb_rollback() +** closes transactions and then restores the database to the state it +** was in before those transactions were even opened. +*/ +int tdb_begin(TestDb *pDb, int iLevel); +int tdb_commit(TestDb *pDb, int iLevel); +int tdb_rollback(TestDb *pDb, int iLevel); + +/* +** Return true if transactions are supported, or false otherwise. +*/ +int tdb_transaction_support(TestDb *pDb); + +/* +** Return the name of the database library (as passed to tdb_open()) used +** by the handled passed as the first argument. +*/ +const char *tdb_library_name(TestDb *pDb); + +/* +** Scan a range of database keys. Invoke the callback function for each +** key visited. +*/ +int tdb_scan( + TestDb *pDb, /* Database handle */ + void *pCtx, /* Context pointer to pass to xCallback */ + int bReverse, /* True to scan in reverse order */ + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) +); + +const char *tdb_system_name(int i); +const char *tdb_default_db(const char *zSys); + +int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb); + +/* +** If the TestDb handle passed as an argument is a wrapper around an LSM +** database, return the LSM handle. Otherwise, if the argument is some other +** database system, return NULL. +*/ +lsm_db *tdb_lsm(TestDb *pDb); + +/* +** Return true if the db passed as an argument is a multi-threaded LSM +** connection. +*/ +int tdb_lsm_multithread(TestDb *pDb); + +/* +** Return a pointer to the lsm_env object used by all lsm database +** connections initialized as a copy of the object returned by +** lsm_default_env(). It may be modified (e.g. to override functions) +** if the caller can guarantee that it is not already in use. +*/ +lsm_env *tdb_lsm_env(void); + +/* +** The following functions only work with LSM database handles. It is +** illegal to call them with any other type of database handle specified +** as an argument. +*/ +void tdb_lsm_enable_log(TestDb *pDb, int bEnable); +void tdb_lsm_application_crash(TestDb *pDb); +void tdb_lsm_prepare_system_crash(TestDb *pDb); +void tdb_lsm_system_crash(TestDb *pDb); +void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync); + + +void tdb_lsm_safety(TestDb *pDb, int eMode); +void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *); +void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*); +int tdb_lsm_config_str(TestDb *pDb, const char *zStr); + +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif + +#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb2.cc b/ext/lsm1/lsm-test/lsmtest_tdb2.cc new file mode 100644 index 0000000000000000000000000000000000000000..86ebb49583ffb0dcfe7e60121906e7dbba8a9a01 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_tdb2.cc @@ -0,0 +1,369 @@ + + +#include "lsmtest.h" +#include + +#ifdef HAVE_KYOTOCABINET +#include "kcpolydb.h" +extern "C" { + struct KcDb { + TestDb base; + kyotocabinet::TreeDB* db; + char *pVal; + }; +} + +int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){ + KcDb *pKcDb; + int ok; + int rc = 0; + + if( bClear ){ + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); + system(zCmd); + sqlite3_free(zCmd); + } + + pKcDb = (KcDb *)malloc(sizeof(KcDb)); + memset(pKcDb, 0, sizeof(KcDb)); + + + pKcDb->db = new kyotocabinet::TreeDB(); + pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE); + pKcDb->db->tune_page_cache( + TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE + ); + ok = pKcDb->db->open(zFilename, + kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE + ); + if( ok==0 ){ + free(pKcDb); + pKcDb = 0; + rc = 1; + } + + *ppDb = (TestDb *)pKcDb; + return rc; +} + +int test_kc_close(TestDb *pDb){ + KcDb *pKcDb = (KcDb *)pDb; + if( pKcDb->pVal ){ + delete [] pKcDb->pVal; + } + pKcDb->db->close(); + delete pKcDb->db; + free(pKcDb); + return 0; +} + +int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ + KcDb *pKcDb = (KcDb *)pDb; + int ok; + + ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal); + return (ok ? 0 : 1); +} + +int test_kc_delete(TestDb *pDb, void *pKey, int nKey){ + KcDb *pKcDb = (KcDb *)pDb; + int ok; + + ok = pKcDb->db->remove((const char *)pKey, nKey); + return (ok ? 0 : 1); +} + +int test_kc_delete_range( + TestDb *pDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + int res; + KcDb *pKcDb = (KcDb *)pDb; + kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); + + if( pKey1 ){ + res = pCur->jump((const char *)pKey1, nKey1); + }else{ + res = pCur->jump(); + } + + while( 1 ){ + const char *pKey; size_t nKey; + const char *pVal; size_t nVal; + + pKey = pCur->get(&nKey, &pVal, &nVal); + if( pKey==0 ) break; + +#ifndef NDEBUG + if( pKey1 ){ + res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); + assert( res>0 || (res==0 && nKey>nKey1) ); + } +#endif + + if( pKey2 ){ + res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); + if( res>0 || (res==0 && (size_t)nKey2remove(); + delete [] pKey; + } + + delete pCur; + return 0; +} + +int test_kc_fetch( + TestDb *pDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + KcDb *pKcDb = (KcDb *)pDb; + size_t nVal; + + if( pKcDb->pVal ){ + delete [] pKcDb->pVal; + pKcDb->pVal = 0; + } + + pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal); + if( pKcDb->pVal ){ + *ppVal = pKcDb->pVal; + *pnVal = nVal; + }else{ + *ppVal = 0; + *pnVal = -1; + } + + return 0; +} + +int test_kc_scan( + TestDb *pDb, /* Database handle */ + void *pCtx, /* Context pointer to pass to xCallback */ + int bReverse, /* True for a reverse order scan */ + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) +){ + KcDb *pKcDb = (KcDb *)pDb; + kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); + int res; + + if( bReverse==0 ){ + if( pKey1 ){ + res = pCur->jump((const char *)pKey1, nKey1); + }else{ + res = pCur->jump(); + } + }else{ + if( pKey2 ){ + res = pCur->jump_back((const char *)pKey2, nKey2); + }else{ + res = pCur->jump_back(); + } + } + + while( res ){ + const char *pKey; size_t nKey; + const char *pVal; size_t nVal; + pKey = pCur->get(&nKey, &pVal, &nVal); + + if( bReverse==0 && pKey2 ){ + res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); + if( res>0 || (res==0 && (size_t)nKey2nKey) ){ + delete [] pKey; + break; + } + } + + xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal); + delete [] pKey; + + if( bReverse ){ + res = pCur->step_back(); + }else{ + res = pCur->step(); + } + } + + delete pCur; + return 0; +} +#endif /* HAVE_KYOTOCABINET */ + +#ifdef HAVE_MDB +#include "lmdb.h" + +extern "C" { + struct MdbDb { + TestDb base; + MDB_env *env; + MDB_dbi dbi; + }; +} + +int test_mdb_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + MDB_txn *txn; + MdbDb *pMdb; + int rc; + + if( bClear ){ + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); + system(zCmd); + sqlite3_free(zCmd); + } + + pMdb = (MdbDb *)malloc(sizeof(MdbDb)); + memset(pMdb, 0, sizeof(MdbDb)); + + rc = mdb_env_create(&pMdb->env); + if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); + if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); + if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); + if( rc==0 ){ + rc = mdb_open(txn, NULL, 0, &pMdb->dbi); + mdb_txn_commit(txn); + } + + *ppDb = (TestDb *)pMdb; + return rc; +} + +int test_mdb_close(TestDb *pDb){ + MdbDb *pMdb = (MdbDb *)pDb; + + mdb_close(pMdb->env, pMdb->dbi); + mdb_env_close(pMdb->env); + free(pMdb); + return 0; +} + +int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ + int rc; + MdbDb *pMdb = (MdbDb *)pDb; + MDB_val val; + MDB_val key; + MDB_txn *txn; + + val.mv_size = nVal; + val.mv_data = pVal; + key.mv_size = nKey; + key.mv_data = pKey; + + rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); + if( rc==0 ){ + rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); + if( rc==0 ){ + rc = mdb_txn_commit(txn); + }else{ + mdb_txn_abort(txn); + } + } + + return rc; +} + +int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ + int rc; + MdbDb *pMdb = (MdbDb *)pDb; + MDB_val key; + MDB_txn *txn; + + key.mv_size = nKey; + key.mv_data = pKey; + rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); + if( rc==0 ){ + rc = mdb_del(txn, pMdb->dbi, &key, 0); + if( rc==0 ){ + rc = mdb_txn_commit(txn); + }else{ + mdb_txn_abort(txn); + } + } + + return rc; +} + +int test_mdb_fetch( + TestDb *pDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + int rc; + MdbDb *pMdb = (MdbDb *)pDb; + MDB_val key; + MDB_txn *txn; + + key.mv_size = nKey; + key.mv_data = pKey; + + rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); + if( rc==0 ){ + MDB_val val = {0, 0}; + rc = mdb_get(txn, pMdb->dbi, &key, &val); + if( rc==MDB_NOTFOUND ){ + rc = 0; + *ppVal = 0; + *pnVal = -1; + }else{ + *ppVal = val.mv_data; + *pnVal = val.mv_size; + } + mdb_txn_commit(txn); + } + + return rc; +} + +int test_mdb_scan( + TestDb *pDb, /* Database handle */ + void *pCtx, /* Context pointer to pass to xCallback */ + int bReverse, /* True for a reverse order scan */ + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) +){ + MdbDb *pMdb = (MdbDb *)pDb; + int rc; + MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; + MDB_txn *txn; + + rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); + if( rc==0 ){ + MDB_cursor *csr; + MDB_val key = {0, 0}; + MDB_val val = {0, 0}; + + rc = mdb_cursor_open(txn, pMdb->dbi, &csr); + if( rc==0 ){ + while( mdb_cursor_get(csr, &key, &val, op)==0 ){ + xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); + } + mdb_cursor_close(csr); + } + } + + return rc; +} + +#endif /* HAVE_MDB */ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb3.c b/ext/lsm1/lsm-test/lsmtest_tdb3.c new file mode 100644 index 0000000000000000000000000000000000000000..e29497af204dc6078e44b975a20953203995d8b5 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_tdb3.c @@ -0,0 +1,1429 @@ + +#include "lsmtest_tdb.h" +#include "lsm.h" +#include "lsmtest.h" + +#include +#include +#include +#ifndef _WIN32 +# include +#endif +#include + +#ifndef _WIN32 +# include +#endif + +typedef struct LsmDb LsmDb; +typedef struct LsmWorker LsmWorker; +typedef struct LsmFile LsmFile; + +#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) +#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) + +#ifdef LSM_MUTEX_PTHREADS +#include + +#define LSMTEST_THREAD_CKPT 1 +#define LSMTEST_THREAD_WORKER 2 +#define LSMTEST_THREAD_WORKER_AC 3 + +/* +** There are several different types of worker threads that run in different +** test configurations, depending on the value of LsmWorker.eType. +** +** 1. Checkpointer. +** 2. Worker with auto-checkpoint. +** 3. Worker without auto-checkpoint. +*/ +struct LsmWorker { + LsmDb *pDb; /* Main database structure */ + lsm_db *pWorker; /* Worker database handle */ + pthread_t worker_thread; /* Worker thread */ + pthread_cond_t worker_cond; /* Condition var the worker waits on */ + pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ + int bDoWork; /* Set to true by client when there is work */ + int worker_rc; /* Store error code here */ + int eType; /* LSMTEST_THREAD_XXX constant */ + int bBlock; +}; +#else +struct LsmWorker { int worker_rc; int bBlock; }; +#endif + +static void mt_shutdown(LsmDb *); + +lsm_env *tdb_lsm_env(void){ + static int bInit = 0; + static lsm_env env; + if( bInit==0 ){ + memcpy(&env, lsm_default_env(), sizeof(env)); + bInit = 1; + } + return &env; +} + +typedef struct FileSector FileSector; +typedef struct FileData FileData; + +struct FileSector { + u8 *aOld; /* Old data for this sector */ +}; + +struct FileData { + int nSector; /* Allocated size of apSector[] array */ + FileSector *aSector; /* Array of file sectors */ +}; + +/* +** bPrepareCrash: +** If non-zero, the file wrappers maintain enough in-memory data to +** simulate the effect of a power-failure on the file-system (i.e. that +** unsynced sectors may be written, not written, or overwritten with +** arbitrary data when the crash occurs). +** +** bCrashed: +** Set to true after a crash is simulated. Once this variable is true, all +** VFS methods other than xClose() return LSM_IOERR as soon as they are +** called (without affecting the contents of the file-system). +** +** env: +** The environment object used by all lsm_db* handles opened by this +** object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx +** always points to the containing LsmDb structure. +*/ +struct LsmDb { + TestDb base; /* Base class - methods table */ + lsm_env env; /* Environment used by connection db */ + char *zName; /* Database file name */ + lsm_db *db; /* LSM database handle */ + + lsm_cursor *pCsr; /* Cursor held open during read transaction */ + void *pBuf; /* Buffer for tdb_fetch() output */ + int nBuf; /* Allocated (not used) size of pBuf */ + + /* Crash testing related state */ + int bCrashed; /* True once a crash has occurred */ + int nAutoCrash; /* Number of syncs until a crash */ + int bPrepareCrash; /* True to store writes in memory */ + + /* Unsynced data (while crash testing) */ + int szSector; /* Assumed size of disk sectors (512B) */ + FileData aFile[2]; /* Database and log file data */ + + /* Other test instrumentation */ + int bNoRecovery; /* If true, assume DMS2 is locked */ + + /* Work hook redirection */ + void (*xWork)(lsm_db *, void *); + void *pWorkCtx; + + /* IO logging hook */ + void (*xWriteHook)(void *, int, lsm_i64, int, int); + void *pWriteCtx; + + /* Worker threads (for lsm_mt) */ + int nMtMinCkpt; + int nMtMaxCkpt; + int eMode; + int nWorker; + LsmWorker *aWorker; +}; + +#define LSMTEST_MODE_SINGLETHREAD 1 +#define LSMTEST_MODE_BACKGROUND_CKPT 2 +#define LSMTEST_MODE_BACKGROUND_WORK 3 +#define LSMTEST_MODE_BACKGROUND_BOTH 4 + +/************************************************************************* +************************************************************************** +** Begin test VFS code. +*/ + +struct LsmFile { + lsm_file *pReal; /* Real underlying file */ + int bLog; /* True for log file. False for db file */ + LsmDb *pDb; /* Database handle that uses this file */ +}; + +static int testEnvFullpath( + lsm_env *pEnv, /* Environment for current LsmDb */ + const char *zFile, /* Relative path name */ + char *zOut, /* Output buffer */ + int *pnOut /* IN/OUT: Size of output buffer */ +){ + lsm_env *pRealEnv = tdb_lsm_env(); + return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut); +} + +static int testEnvOpen( + lsm_env *pEnv, /* Environment for current LsmDb */ + const char *zFile, /* Name of file to open */ + int flags, + lsm_file **ppFile /* OUT: New file handle object */ +){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; + int rc; /* Return Code */ + LsmFile *pRet; /* The new file handle */ + int nFile; /* Length of string zFile in bytes */ + + nFile = strlen(zFile); + pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); + pRet->pDb = pDb; + pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); + + rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); + if( rc!=LSM_OK ){ + testFree(pRet); + pRet = 0; + } + + *ppFile = (lsm_file *)pRet; + return rc; +} + +static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + if( p->pDb->bCrashed ) return LSM_IOERR; + return pRealEnv->xRead(p->pReal, iOff, pData, nData); +} + +static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + LsmDb *pDb = p->pDb; + + if( pDb->bCrashed ) return LSM_IOERR; + + if( pDb->bPrepareCrash ){ + FileData *pData2 = &pDb->aFile[p->bLog]; + int iFirst; + int iLast; + int iSector; + + iFirst = (int)(iOff / pDb->szSector); + iLast = (int)((iOff + nData - 1) / pDb->szSector); + + if( pData2->nSector<(iLast+1) ){ + int nNew = ( ((iLast + 1) + 63) / 64 ) * 64; + assert( nNew>iLast ); + pData2->aSector = (FileSector *)testRealloc( + pData2->aSector, nNew*sizeof(FileSector) + ); + memset(&pData2->aSector[pData2->nSector], + 0, (nNew - pData2->nSector) * sizeof(FileSector) + ); + pData2->nSector = nNew; + } + + for(iSector=iFirst; iSector<=iLast; iSector++){ + if( pData2->aSector[iSector].aOld==0 ){ + u8 *aOld = (u8 *)testMalloc(pDb->szSector); + pRealEnv->xRead( + p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector + ); + pData2->aSector[iSector].aOld = aOld; + } + } + } + + if( pDb->xWriteHook ){ + int rc; + int nUs; + struct timeval t1; + struct timeval t2; + + gettimeofday(&t1, 0); + assert( nData>0 ); + rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData); + gettimeofday(&t2, 0); + + nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); + pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs); + return rc; + } + + return pRealEnv->xWrite(p->pReal, iOff, pData, nData); +} + +static void doSystemCrash(LsmDb *pDb); + +static int testEnvSync(lsm_file *pFile){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + LsmDb *pDb = p->pDb; + FileData *pData = &pDb->aFile[p->bLog]; + int i; + + if( pDb->bCrashed ) return LSM_IOERR; + + if( pDb->nAutoCrash ){ + pDb->nAutoCrash--; + if( pDb->nAutoCrash==0 ){ + doSystemCrash(pDb); + pDb->bCrashed = 1; + return LSM_IOERR; + } + } + + if( pDb->bPrepareCrash ){ + for(i=0; inSector; i++){ + testFree(pData->aSector[i].aOld); + pData->aSector[i].aOld = 0; + } + } + + if( pDb->xWriteHook ){ + int rc; + int nUs; + struct timeval t1; + struct timeval t2; + + gettimeofday(&t1, 0); + rc = pRealEnv->xSync(p->pReal); + gettimeofday(&t2, 0); + + nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); + pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs); + return rc; + } + + return pRealEnv->xSync(p->pReal); +} + +static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + if( p->pDb->bCrashed ) return LSM_IOERR; + return pRealEnv->xTruncate(p->pReal, iOff); +} + +static int testEnvSectorSize(lsm_file *pFile){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + return pRealEnv->xSectorSize(p->pReal); +} + +static int testEnvRemap( + lsm_file *pFile, + lsm_i64 iMin, + void **ppOut, + lsm_i64 *pnOut +){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut); +} + +static int testEnvFileid( + lsm_file *pFile, + void *ppOut, + int *pnOut +){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + return pRealEnv->xFileid(p->pReal, ppOut, pnOut); +} + +static int testEnvClose(lsm_file *pFile){ + lsm_env *pRealEnv = tdb_lsm_env(); + LsmFile *p = (LsmFile *)pFile; + + pRealEnv->xClose(p->pReal); + testFree(p); + return LSM_OK; +} + +static int testEnvUnlink(lsm_env *pEnv, const char *zFile){ + lsm_env *pRealEnv = tdb_lsm_env(); + unused_parameter(pEnv); + return pRealEnv->xUnlink(pRealEnv, zFile); +} + +static int testEnvLock(lsm_file *pFile, int iLock, int eType){ + LsmFile *p = (LsmFile *)pFile; + lsm_env *pRealEnv = tdb_lsm_env(); + + if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ + return LSM_BUSY; + } + return pRealEnv->xLock(p->pReal, iLock, eType); +} + +static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ + LsmFile *p = (LsmFile *)pFile; + lsm_env *pRealEnv = tdb_lsm_env(); + + if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ + return LSM_BUSY; + } + return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType); +} + +static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){ + LsmFile *p = (LsmFile *)pFile; + lsm_env *pRealEnv = tdb_lsm_env(); + return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp); +} + +static void testEnvShmBarrier(void){ +} + +static int testEnvShmUnmap(lsm_file *pFile, int bDel){ + LsmFile *p = (LsmFile *)pFile; + lsm_env *pRealEnv = tdb_lsm_env(); + return pRealEnv->xShmUnmap(p->pReal, bDel); +} + +static int testEnvSleep(lsm_env *pEnv, int us){ + lsm_env *pRealEnv = tdb_lsm_env(); + return pRealEnv->xSleep(pRealEnv, us); +} + +static void doSystemCrash(LsmDb *pDb){ + lsm_env *pEnv = tdb_lsm_env(); + int iFile; + int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector; + + char *zFile = pDb->zName; + char *zFree = 0; + + for(iFile=0; iFile<2; iFile++){ + lsm_file *pFile = 0; + int i; + + pEnv->xOpen(pEnv, zFile, 0, &pFile); + for(i=0; iaFile[iFile].nSector; i++){ + u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; + if( aOld ){ + int iOpt = testPrngValue(iSeed++) % 3; + switch( iOpt ){ + case 0: + break; + + case 1: + testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4); + /* Fall-through */ + + case 2: + pEnv->xWrite( + pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector + ); + break; + } + testFree(aOld); + pDb->aFile[iFile].aSector[i].aOld = 0; + } + } + pEnv->xClose(pFile); + zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName); + } + + sqlite3_free(zFree); +} +/* +** End test VFS code. +************************************************************************** +*************************************************************************/ + +/************************************************************************* +************************************************************************** +** Begin test compression hooks. +*/ + +#ifdef HAVE_ZLIB +#include + +static int testZipBound(void *pCtx, int nSrc){ + return compressBound(nSrc); +} + +static int testZipCompress( + void *pCtx, /* Context pointer */ + char *aOut, int *pnOut, /* OUT: Buffer containing compressed data */ + const char *aIn, int nIn /* Buffer containing input data */ +){ + uLongf n = *pnOut; /* In/out buffer size for compress() */ + int rc; /* compress() return code */ + + rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); + *pnOut = n; + return (rc==Z_OK ? 0 : LSM_ERROR); +} + +static int testZipUncompress( + void *pCtx, /* Context pointer */ + char *aOut, int *pnOut, /* OUT: Buffer containing uncompressed data */ + const char *aIn, int nIn /* Buffer containing input data */ +){ + uLongf n = *pnOut; /* In/out buffer size for uncompress() */ + int rc; /* uncompress() return code */ + + rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); + *pnOut = n; + return (rc==Z_OK ? 0 : LSM_ERROR); +} + +static int testConfigureCompression(lsm_db *pDb){ + static lsm_compress zip = { + 0, /* Context pointer (unused) */ + 1, /* Id value */ + testZipBound, /* xBound method */ + testZipCompress, /* xCompress method */ + testZipUncompress /* xUncompress method */ + }; + return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); +} +#endif /* ifdef HAVE_ZLIB */ + +/* +** End test compression hooks. +************************************************************************** +*************************************************************************/ + +static int test_lsm_close(TestDb *pTestDb){ + int i; + int rc = LSM_OK; + LsmDb *pDb = (LsmDb *)pTestDb; + + lsm_csr_close(pDb->pCsr); + lsm_close(pDb->db); + + /* If this is a multi-threaded database, wait on the worker threads. */ + mt_shutdown(pDb); + for(i=0; inWorker && rc==LSM_OK; i++){ + rc = pDb->aWorker[i].worker_rc; + } + + for(i=0; iaFile[0].nSector; i++){ + testFree(pDb->aFile[0].aSector[i].aOld); + } + testFree(pDb->aFile[0].aSector); + for(i=0; iaFile[1].nSector; i++){ + testFree(pDb->aFile[1].aSector[i].aOld); + } + testFree(pDb->aFile[1].aSector); + + memset(pDb, sizeof(LsmDb), 0x11); + testFree((char *)pDb->pBuf); + testFree((char *)pDb); + return rc; +} + +static void mt_signal_worker(LsmDb*, int); + +static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ + int nSleep = 0; + int nKB; + int rc; + + do { + nKB = 0; + rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); + if( rc!=LSM_OK || nKBnMtMaxCkpt ) break; +#ifdef LSM_MUTEX_PTHREADS + mt_signal_worker(pDb, + (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1) + ); +#endif + usleep(5000); + nSleep += 5; + }while( 1 ); + +#if 0 + if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); +#endif + + return rc; +} + +static int waitOnWorker(LsmDb *pDb){ + int rc; + int nLimit = -1; + int nSleep = 0; + + rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); + do { + int nOld, nNew, rc2; + rc2 = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); + if( rc2!=LSM_OK ) return rc2; + if( nOld==0 || nNew<(nLimit/2) ) break; +#ifdef LSM_MUTEX_PTHREADS + mt_signal_worker(pDb, 0); +#endif + usleep(5000); + nSleep += 5; + }while( 1 ); + +#if 0 + if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); +#endif + + return rc; +} + +static int test_lsm_write( + TestDb *pTestDb, + void *pKey, + int nKey, + void *pVal, + int nVal +){ + LsmDb *pDb = (LsmDb *)pTestDb; + int rc = LSM_OK; + + if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ + rc = waitOnCheckpointer(pDb, pDb->db); + }else if( + pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK + || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH + ){ + rc = waitOnWorker(pDb); + } + + if( rc==LSM_OK ){ + rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); + } + return rc; +} + +static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ + LsmDb *pDb = (LsmDb *)pTestDb; + return lsm_delete(pDb->db, pKey, nKey); +} + +static int test_lsm_delete_range( + TestDb *pTestDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + LsmDb *pDb = (LsmDb *)pTestDb; + return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2); +} + +static int test_lsm_fetch( + TestDb *pTestDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + int rc; + LsmDb *pDb = (LsmDb *)pTestDb; + lsm_cursor *csr; + + if( pKey==0 ) return LSM_OK; + + if( pDb->pCsr==0 ){ + rc = lsm_csr_open(pDb->db, &csr); + if( rc!=LSM_OK ) return rc; + }else{ + csr = pDb->pCsr; + } + + rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ); + if( rc==LSM_OK ){ + if( lsm_csr_valid(csr) ){ + const void *pVal; int nVal; + rc = lsm_csr_value(csr, &pVal, &nVal); + if( nVal>pDb->nBuf ){ + testFree(pDb->pBuf); + pDb->pBuf = testMalloc(nVal*2); + pDb->nBuf = nVal*2; + } + memcpy(pDb->pBuf, pVal, nVal); + *ppVal = pDb->pBuf; + *pnVal = nVal; + }else{ + *ppVal = 0; + *pnVal = -1; + } + } + if( pDb->pCsr==0 ){ + lsm_csr_close(csr); + } + return rc; +} + +static int test_lsm_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pFirst, int nFirst, + void *pLast, int nLast, + void (*xCallback)(void *, void *, int , void *, int) +){ + LsmDb *pDb = (LsmDb *)pTestDb; + lsm_cursor *csr; + lsm_cursor *csr2 = 0; + int rc; + + if( pDb->pCsr==0 ){ + rc = lsm_csr_open(pDb->db, &csr); + if( rc!=LSM_OK ) return rc; + }else{ + rc = LSM_OK; + csr = pDb->pCsr; + } + + /* To enhance testing, if both pLast and pFirst are defined, seek the + ** cursor to the "end" boundary here. Then the next block seeks it to + ** the "start" ready for the scan. The point is to test that cursors + ** can be reused. */ + if( pLast && pFirst ){ + if( bReverse ){ + rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE); + }else{ + rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE); + } + } + + if( bReverse ){ + if( pLast ){ + rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE); + }else{ + rc = lsm_csr_last(csr); + } + }else{ + if( pFirst ){ + rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_GE); + }else{ + rc = lsm_csr_first(csr); + } + } + + while( rc==LSM_OK && lsm_csr_valid(csr) ){ + const void *pKey; int nKey; + const void *pVal; int nVal; + int cmp; + + lsm_csr_key(csr, &pKey, &nKey); + lsm_csr_value(csr, &pVal, &nVal); + + if( bReverse && pFirst ){ + cmp = memcmp(pFirst, pKey, MIN(nKey, nFirst)); + if( cmp>0 || (cmp==0 && nFirst>nKey) ) break; + }else if( bReverse==0 && pLast ){ + cmp = memcmp(pLast, pKey, MIN(nKey, nLast)); + if( cmp<0 || (cmp==0 && nLastpCsr==0 ){ + lsm_csr_close(csr); + } + return rc; +} + +static int test_lsm_begin(TestDb *pTestDb, int iLevel){ + int rc = LSM_OK; + LsmDb *pDb = (LsmDb *)pTestDb; + + /* iLevel==0 is a no-op. */ + if( iLevel==0 ) return 0; + + if( pDb->pCsr==0 ) rc = lsm_csr_open(pDb->db, &pDb->pCsr); + if( rc==LSM_OK && iLevel>1 ){ + rc = lsm_begin(pDb->db, iLevel-1); + } + + return rc; +} +static int test_lsm_commit(TestDb *pTestDb, int iLevel){ + LsmDb *pDb = (LsmDb *)pTestDb; + + /* If iLevel==0, close any open read transaction */ + if( iLevel==0 && pDb->pCsr ){ + lsm_csr_close(pDb->pCsr); + pDb->pCsr = 0; + } + + /* If iLevel==0, close any open read transaction */ + return lsm_commit(pDb->db, MAX(0, iLevel-1)); +} +static int test_lsm_rollback(TestDb *pTestDb, int iLevel){ + LsmDb *pDb = (LsmDb *)pTestDb; + + /* If iLevel==0, close any open read transaction */ + if( iLevel==0 && pDb->pCsr ){ + lsm_csr_close(pDb->pCsr); + pDb->pCsr = 0; + } + + return lsm_rollback(pDb->db, MAX(0, iLevel-1)); +} + +/* +** A log message callback registered with lsm connections. Prints all +** messages to stderr. +*/ +static void xLog(void *pCtx, int rc, const char *z){ + unused_parameter(rc); + /* fprintf(stderr, "lsm: rc=%d \"%s\"\n", rc, z); */ + if( pCtx ) fprintf(stderr, "%s: ", (char *)pCtx); + fprintf(stderr, "%s\n", z); + fflush(stderr); +} + +static void xWorkHook(lsm_db *db, void *pArg){ + LsmDb *p = (LsmDb *)pArg; + if( p->xWork ) p->xWork(db, p->pWorkCtx); +} + +#define TEST_NO_RECOVERY -1 +#define TEST_COMPRESSION -3 + +#define TEST_MT_MODE -2 +#define TEST_MT_MIN_CKPT -4 +#define TEST_MT_MAX_CKPT -5 + + +int test_lsm_config_str( + LsmDb *pLsm, + lsm_db *db, + int bWorker, + const char *zStr, + int *pnThread +){ + struct CfgParam { + const char *zParam; + int bWorker; + int eParam; + } aParam[] = { + { "autoflush", 0, LSM_CONFIG_AUTOFLUSH }, + { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, + { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, + { "safety", 0, LSM_CONFIG_SAFETY }, + { "autowork", 0, LSM_CONFIG_AUTOWORK }, + { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, + { "mmap", 0, LSM_CONFIG_MMAP }, + { "use_log", 0, LSM_CONFIG_USE_LOG }, + { "automerge", 0, LSM_CONFIG_AUTOMERGE }, + { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, + { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, + { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE }, + { "test_no_recovery", 0, TEST_NO_RECOVERY }, + { "bg_min_ckpt", 0, TEST_NO_RECOVERY }, + + { "mt_mode", 0, TEST_MT_MODE }, + { "mt_min_ckpt", 0, TEST_MT_MIN_CKPT }, + { "mt_max_ckpt", 0, TEST_MT_MAX_CKPT }, + +#ifdef HAVE_ZLIB + { "compression", 0, TEST_COMPRESSION }, +#endif + { 0, 0 } + }; + const char *z = zStr; + int nThread = 1; + + if( zStr==0 ) return 0; + + assert( db ); + while( z[0] ){ + const char *zStart; + + /* Skip whitespace */ + while( *z==' ' ) z++; + zStart = z; + + while( *z && *z!='=' ) z++; + if( *z ){ + int eParam; + int i; + int iVal; + int iMul = 1; + int rc; + char zParam[32]; + int nParam = z-zStart; + if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; + + memcpy(zParam, zStart, nParam); + zParam[nParam] = '\0'; + rc = testArgSelect(aParam, "param", zParam, &i); + if( rc!=0 ) return rc; + eParam = aParam[i].eParam; + + z++; + zStart = z; + while( *z>='0' && *z<='9' ) z++; + if( *z=='k' || *z=='K' ){ + iMul = 1; + z++; + }else if( *z=='M' || *z=='M' ){ + iMul = 1024; + z++; + } + nParam = z-zStart; + if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; + memcpy(zParam, zStart, nParam); + zParam[nParam] = '\0'; + iVal = atoi(zParam) * iMul; + + if( eParam>0 ){ + if( bWorker || aParam[i].bWorker==0 ){ + lsm_config(db, eParam, &iVal); + } + }else{ + switch( eParam ){ + case TEST_NO_RECOVERY: + if( pLsm ) pLsm->bNoRecovery = iVal; + break; + case TEST_MT_MODE: + if( pLsm ) nThread = iVal; + break; + case TEST_MT_MIN_CKPT: + if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal*1024; + break; + case TEST_MT_MAX_CKPT: + if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal*1024; + break; +#ifdef HAVE_ZLIB + case TEST_COMPRESSION: + testConfigureCompression(db); + break; +#endif + } + } + }else if( z!=zStart ){ + goto syntax_error; + } + } + + if( pnThread ) *pnThread = nThread; + if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){ + pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt; + } + + return 0; + syntax_error: + testPrintError("syntax error at: \"%s\"\n", z); + return 1; +} + +int tdb_lsm_config_str(TestDb *pDb, const char *zStr){ + int rc = 0; + if( tdb_lsm(pDb) ){ +#ifdef LSM_MUTEX_PTHREADS + int i; +#endif + LsmDb *pLsm = (LsmDb *)pDb; + + rc = test_lsm_config_str(pLsm, pLsm->db, 0, zStr, 0); +#ifdef LSM_MUTEX_PTHREADS + for(i=0; rc==0 && inWorker; i++){ + rc = test_lsm_config_str(0, pLsm->aWorker[i].pWorker, 1, zStr, 0); + } +#endif + } + return rc; +} + +int tdb_lsm_configure(lsm_db *db, const char *zConfig){ + return test_lsm_config_str(0, db, 0, zConfig, 0); +} + +static int testLsmStartWorkers(LsmDb *, int, const char *, const char *); + +static int testLsmOpen( + const char *zCfg, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + static const DatabaseMethods LsmMethods = { + test_lsm_close, + test_lsm_write, + test_lsm_delete, + test_lsm_delete_range, + test_lsm_fetch, + test_lsm_scan, + test_lsm_begin, + test_lsm_commit, + test_lsm_rollback + }; + + int rc; + int nFilename; + LsmDb *pDb; + + /* If the bClear flag is set, delete any existing database. */ + assert( zFilename); + if( bClear ) testDeleteLsmdb(zFilename); + nFilename = strlen(zFilename); + + pDb = (LsmDb *)testMalloc(sizeof(LsmDb) + nFilename + 1); + memset(pDb, 0, sizeof(LsmDb)); + pDb->base.pMethods = &LsmMethods; + pDb->zName = (char *)&pDb[1]; + memcpy(pDb->zName, zFilename, nFilename + 1); + + /* Default the sector size used for crash simulation to 512 bytes. + ** Todo: There should be an OS method to obtain this value - just as + ** there is in SQLite. For now, LSM assumes that it is smaller than + ** the page size (default 4KB). + */ + pDb->szSector = 256; + + /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ + pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; + pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; + + memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); + pDb->env.pVfsCtx = (void *)pDb; + pDb->env.xFullpath = testEnvFullpath; + pDb->env.xOpen = testEnvOpen; + pDb->env.xRead = testEnvRead; + pDb->env.xWrite = testEnvWrite; + pDb->env.xTruncate = testEnvTruncate; + pDb->env.xSync = testEnvSync; + pDb->env.xSectorSize = testEnvSectorSize; + pDb->env.xRemap = testEnvRemap; + pDb->env.xFileid = testEnvFileid; + pDb->env.xClose = testEnvClose; + pDb->env.xUnlink = testEnvUnlink; + pDb->env.xLock = testEnvLock; + pDb->env.xTestLock = testEnvTestLock; + pDb->env.xShmBarrier = testEnvShmBarrier; + pDb->env.xShmMap = testEnvShmMap; + pDb->env.xShmUnmap = testEnvShmUnmap; + pDb->env.xSleep = testEnvSleep; + + rc = lsm_new(&pDb->env, &pDb->db); + if( rc==LSM_OK ){ + int nThread = 1; + lsm_config_log(pDb->db, xLog, 0); + lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); + + rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); + if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); + + pDb->eMode = nThread; +#ifdef LSM_MUTEX_PTHREADS + if( rc==LSM_OK && nThread>1 ){ + testLsmStartWorkers(pDb, nThread, zFilename, zCfg); + } +#endif + + if( rc!=LSM_OK ){ + test_lsm_close((TestDb *)pDb); + pDb = 0; + } + } + + *ppDb = (TestDb *)pDb; + return rc; +} + +int test_lsm_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + return testLsmOpen(zSpec, zFilename, bClear, ppDb); +} + +int test_lsm_small_open( + const char *zSpec, + const char *zFile, + int bClear, + TestDb **ppDb +){ + const char *zCfg = "page_size=256 block_size=64 mmap=1024"; + return testLsmOpen(zCfg, zFile, bClear, ppDb); +} + +int test_lsm_lomem_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + /* "max_freelist=4 autocheckpoint=32" */ + const char *zCfg = + "page_size=256 block_size=64 autoflush=16 " + "autocheckpoint=32" + "mmap=0 " + ; + return testLsmOpen(zCfg, zFilename, bClear, ppDb); +} + +int test_lsm_lomem2_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + /* "max_freelist=4 autocheckpoint=32" */ + const char *zCfg = + "page_size=512 block_size=64 autoflush=0 mmap=0 " + ; + return testLsmOpen(zCfg, zFilename, bClear, ppDb); +} + +int test_lsm_zip_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + const char *zCfg = + "page_size=256 block_size=64 autoflush=16 " + "autocheckpoint=32 compression=1 mmap=0 " + ; + return testLsmOpen(zCfg, zFilename, bClear, ppDb); +} + +lsm_db *tdb_lsm(TestDb *pDb){ + if( pDb->pMethods->xClose==test_lsm_close ){ + return ((LsmDb *)pDb)->db; + } + return 0; +} + +int tdb_lsm_multithread(TestDb *pDb){ + int ret = 0; + if( tdb_lsm(pDb) ){ + ret = ((LsmDb*)pDb)->eMode!=LSMTEST_MODE_SINGLETHREAD; + } + return ret; +} + +void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ + lsm_db *db = tdb_lsm(pDb); + if( db ){ + lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); + } +} + +void tdb_lsm_application_crash(TestDb *pDb){ + if( tdb_lsm(pDb) ){ + LsmDb *p = (LsmDb *)pDb; + p->bCrashed = 1; + } +} + +void tdb_lsm_prepare_system_crash(TestDb *pDb){ + if( tdb_lsm(pDb) ){ + LsmDb *p = (LsmDb *)pDb; + p->bPrepareCrash = 1; + } +} + +void tdb_lsm_system_crash(TestDb *pDb){ + if( tdb_lsm(pDb) ){ + LsmDb *p = (LsmDb *)pDb; + p->bCrashed = 1; + doSystemCrash(p); + } +} + +void tdb_lsm_safety(TestDb *pDb, int eMode){ + assert( eMode==LSM_SAFETY_OFF + || eMode==LSM_SAFETY_NORMAL + || eMode==LSM_SAFETY_FULL + ); + if( tdb_lsm(pDb) ){ + int iParam = eMode; + LsmDb *p = (LsmDb *)pDb; + lsm_config(p->db, LSM_CONFIG_SAFETY, &iParam); + } +} + +void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync){ + assert( iSync>0 ); + if( tdb_lsm(pDb) ){ + LsmDb *p = (LsmDb *)pDb; + p->nAutoCrash = iSync; + p->bPrepareCrash = 1; + } +} + +void tdb_lsm_config_work_hook( + TestDb *pDb, + void (*xWork)(lsm_db *, void *), + void *pWorkCtx +){ + if( tdb_lsm(pDb) ){ + LsmDb *p = (LsmDb *)pDb; + p->xWork = xWork; + p->pWorkCtx = pWorkCtx; + } +} + +void tdb_lsm_write_hook( + TestDb *pDb, + void (*xWrite)(void *, int, lsm_i64, int, int), + void *pWriteCtx +){ + if( tdb_lsm(pDb) ){ + LsmDb *p = (LsmDb *)pDb; + p->xWriteHook = xWrite; + p->pWriteCtx = pWriteCtx; + } +} + +int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb){ + return testLsmOpen(zCfg, zDb, bClear, ppDb); +} + +#ifdef LSM_MUTEX_PTHREADS + +/* +** Signal worker thread iWorker that there may be work to do. +*/ +static void mt_signal_worker(LsmDb *pDb, int iWorker){ + LsmWorker *p = &pDb->aWorker[iWorker]; + pthread_mutex_lock(&p->worker_mutex); + p->bDoWork = 1; + pthread_cond_signal(&p->worker_cond); + pthread_mutex_unlock(&p->worker_mutex); +} + +/* +** This routine is used as the main() for all worker threads. +*/ +static void *worker_main(void *pArg){ + LsmWorker *p = (LsmWorker *)pArg; + lsm_db *pWorker; /* Connection to access db through */ + + pthread_mutex_lock(&p->worker_mutex); + while( (pWorker = p->pWorker) ){ + int rc = LSM_OK; + + /* Do some work. If an error occurs, exit. */ + + pthread_mutex_unlock(&p->worker_mutex); + if( p->eType==LSMTEST_THREAD_CKPT ){ + int nKB = 0; + rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); + if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ + rc = lsm_checkpoint(pWorker, 0); + } + }else{ + int nWrite; + do { + + if( p->eType==LSMTEST_THREAD_WORKER ){ + waitOnCheckpointer(p->pDb, pWorker); + } + + nWrite = 0; + rc = lsm_work(pWorker, 0, 256, &nWrite); + + if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){ + mt_signal_worker(p->pDb, 1); + } + }while( nWrite && p->pWorker ); + } + pthread_mutex_lock(&p->worker_mutex); + + if( rc!=LSM_OK && rc!=LSM_BUSY ){ + p->worker_rc = rc; + break; + } + + /* The thread will wake up when it is signaled either because another + ** thread has created some work for this one or because the connection + ** is being closed. */ + if( p->pWorker && p->bDoWork==0 ){ + pthread_cond_wait(&p->worker_cond, &p->worker_mutex); + } + p->bDoWork = 0; + } + pthread_mutex_unlock(&p->worker_mutex); + + return 0; +} + + +static void mt_stop_worker(LsmDb *pDb, int iWorker){ + LsmWorker *p = &pDb->aWorker[iWorker]; + if( p->pWorker ){ + void *pDummy; + lsm_db *pWorker; + + /* Signal the worker to stop */ + pthread_mutex_lock(&p->worker_mutex); + pWorker = p->pWorker; + p->pWorker = 0; + pthread_cond_signal(&p->worker_cond); + pthread_mutex_unlock(&p->worker_mutex); + + /* Join the worker thread. */ + pthread_join(p->worker_thread, &pDummy); + + /* Free resources allocated in mt_start_worker() */ + pthread_cond_destroy(&p->worker_cond); + pthread_mutex_destroy(&p->worker_mutex); + lsm_close(pWorker); + } +} + +static void mt_shutdown(LsmDb *pDb){ + int i; + for(i=0; inWorker; i++){ + mt_stop_worker(pDb, i); + } +} + +/* +** This callback is invoked by LSM when the client database writes to +** the database file (i.e. to flush the contents of the in-memory tree). +** This implies there may be work to do on the database, so signal +** the worker threads. +*/ +static void mt_client_work_hook(lsm_db *db, void *pArg){ + LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ + + /* Invoke the user level work-hook, if any. */ + if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); + + /* Wake up worker thread 0. */ + mt_signal_worker(pDb, 0); +} + +static void mt_worker_work_hook(lsm_db *db, void *pArg){ + LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ + + /* Invoke the user level work-hook, if any. */ + if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); +} + +/* +** Launch worker thread iWorker for database connection pDb. +*/ +static int mt_start_worker( + LsmDb *pDb, /* Main database structure */ + int iWorker, /* Worker number to start */ + const char *zFilename, /* File name of database to open */ + const char *zCfg, /* Connection configuration string */ + int eType /* Type of worker thread */ +){ + int rc = 0; /* Return code */ + LsmWorker *p; /* Object to initialize */ + + assert( iWorkernWorker ); + assert( eType==LSMTEST_THREAD_CKPT + || eType==LSMTEST_THREAD_WORKER + || eType==LSMTEST_THREAD_WORKER_AC + ); + + p = &pDb->aWorker[iWorker]; + p->eType = eType; + p->pDb = pDb; + + /* Open the worker connection */ + if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); + if( zCfg ){ + test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0); + } + if( rc==0 ) rc = lsm_open(p->pWorker, zFilename); + lsm_config_log(p->pWorker, xLog, (void *)"worker"); + + /* Configure the work-hook */ + if( rc==0 ){ + lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb); + } + + if( eType==LSMTEST_THREAD_WORKER ){ + test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0); + } + + /* Kick off the worker thread. */ + if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0); + if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0); + if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); + + return rc; +} + + +static int testLsmStartWorkers( + LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg +){ + int rc; + + if( eModel<1 || eModel>4 ) return 1; + if( eModel==1 ) return 0; + + /* Configure a work-hook for the client connection. Worker 0 is signalled + ** every time the users connection writes to the database. */ + lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); + + /* Allocate space for two worker connections. They may not both be + ** used, but both are allocated. */ + pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); + memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); + + switch( eModel ){ + case LSMTEST_MODE_BACKGROUND_CKPT: + pDb->nWorker = 1; + test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); + rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); + break; + + case LSMTEST_MODE_BACKGROUND_WORK: + pDb->nWorker = 1; + test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); + rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC); + break; + + case LSMTEST_MODE_BACKGROUND_BOTH: + pDb->nWorker = 2; + test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); + rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER); + if( rc==0 ){ + rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT); + } + break; + } + + return rc; +} + + +int test_lsm_mt2( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + const char *zCfg = "mt_mode=2"; + return testLsmOpen(zCfg, zFilename, bClear, ppDb); +} + +int test_lsm_mt3( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + const char *zCfg = "mt_mode=4"; + return testLsmOpen(zCfg, zFilename, bClear, ppDb); +} + +#else +static void mt_shutdown(LsmDb *pDb) { + unused_parameter(pDb); +} +int test_lsm_mt(const char *zFilename, int bClear, TestDb **ppDb){ + unused_parameter(zFilename); + unused_parameter(bClear); + unused_parameter(ppDb); + testPrintError("threads unavailable - recompile with LSM_MUTEX_PTHREADS\n"); + return 1; +} +#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb4.c b/ext/lsm1/lsm-test/lsmtest_tdb4.c new file mode 100644 index 0000000000000000000000000000000000000000..1f92928522d4c79f8575f993601e9e8621c06fb7 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_tdb4.c @@ -0,0 +1,980 @@ + +/* +** This file contains the TestDb bt wrapper. +*/ + +#include "lsmtest_tdb.h" +#include "lsmtest.h" +#include +#include "bt.h" + +#include + +typedef struct BtDb BtDb; +typedef struct BtFile BtFile; + +/* Background checkpointer interface (see implementations below). */ +typedef struct bt_ckpter bt_ckpter; +static int bgc_attach(BtDb *pDb, const char*); +static int bgc_detach(BtDb *pDb); + +/* +** Each database or log file opened by a database handle is wrapped by +** an object of the following type. +*/ +struct BtFile { + BtDb *pBt; /* Database handle that opened this file */ + bt_env *pVfs; /* Underlying VFS */ + bt_file *pFile; /* File handle belonging to underlying VFS */ + int nSectorSize; /* Size of sectors in bytes */ + int nSector; /* Allocated size of nSector array */ + u8 **apSector; /* Original sector data */ +}; + +/* +** nCrashSync: +** If this value is non-zero, then a "crash-test" is running. If +** nCrashSync==1, then the crash is simulated during the very next +** call to the xSync() VFS method (on either the db or log file). +** If nCrashSync==2, the following call to xSync(), and so on. +** +** bCrash: +** After a crash is simulated, this variable is set. Any subsequent +** attempts to write to a file or modify the file system in any way +** fail once this is set. All the caller can do is close the connection. +** +** bFastInsert: +** If this variable is set to true, then a BT_CONTROL_FAST_INSERT_OP +** control is issued before each callto BtReplace() or BtCsrOpen(). +*/ +struct BtDb { + TestDb base; /* Base class */ + bt_db *pBt; /* bt database handle */ + sqlite4_env *pEnv; /* SQLite environment (for malloc/free) */ + bt_env *pVfs; /* Underlying VFS */ + int bFastInsert; /* True to use fast-insert */ + + /* Space for bt_fetch() results */ + u8 *aBuffer; /* Space to store results */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + int nRef; + + /* Background checkpointer used by mt connections */ + bt_ckpter *pCkpter; + + /* Stuff used for crash test simulation */ + BtFile *apFile[2]; /* Database and log files used by pBt */ + bt_env env; /* Private VFS for this object */ + int nCrashSync; /* Number of syncs until crash (see above) */ + int bCrash; /* True once a crash has been simulated */ +}; + +static int btVfsFullpath( + sqlite4_env *pEnv, + bt_env *pVfs, + const char *z, + char **pzOut +){ + BtDb *pBt = (BtDb*)pVfs->pVfsCtx; + if( pBt->bCrash ) return SQLITE4_IOERR; + return pBt->pVfs->xFullpath(pEnv, pBt->pVfs, z, pzOut); +} + +static int btVfsOpen( + sqlite4_env *pEnv, + bt_env *pVfs, + const char *zFile, + int flags, bt_file **ppFile +){ + BtFile *p; + BtDb *pBt = (BtDb*)pVfs->pVfsCtx; + int rc; + + if( pBt->bCrash ) return SQLITE4_IOERR; + + p = (BtFile*)testMalloc(sizeof(BtFile)); + if( !p ) return SQLITE4_NOMEM; + if( flags & BT_OPEN_DATABASE ){ + pBt->apFile[0] = p; + }else if( flags & BT_OPEN_LOG ){ + pBt->apFile[1] = p; + } + if( (flags & BT_OPEN_SHARED)==0 ){ + p->pBt = pBt; + } + p->pVfs = pBt->pVfs; + + rc = pBt->pVfs->xOpen(pEnv, pVfs, zFile, flags, &p->pFile); + if( rc!=SQLITE4_OK ){ + testFree(p); + p = 0; + }else{ + pBt->nRef++; + } + + *ppFile = (bt_file*)p; + return rc; +} + +static int btVfsSize(bt_file *pFile, sqlite4_int64 *piRes){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xSize(p->pFile, piRes); +} + +static int btVfsRead(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xRead(p->pFile, iOff, pBuf, nBuf); +} + +static int btFlushSectors(BtFile *p, int iFile){ + sqlite4_int64 iSz; + int rc; + int i; + u8 *aTmp = 0; + + rc = p->pBt->pVfs->xSize(p->pFile, &iSz); + for(i=0; rc==SQLITE4_OK && inSector; i++){ + if( p->pBt->bCrash && p->apSector[i] ){ + + /* The system is simulating a crash. There are three choices for + ** this sector: + ** + ** 1) Leave it as it is (simulating a successful write), + ** 2) Restore the original data (simulating a lost write), + ** 3) Populate the disk sector with garbage data. + */ + sqlite4_int64 iSOff = p->nSectorSize*i; + int nWrite = MIN(p->nSectorSize, iSz - iSOff); + + if( nWrite ){ + u8 *aWrite = 0; + int iOpt = (testPrngValue(i) % 3) + 1; + if( iOpt==1 ){ + aWrite = p->apSector[i]; + }else if( iOpt==3 ){ + if( aTmp==0 ) aTmp = testMalloc(p->nSectorSize); + aWrite = aTmp; + testPrngArray(i*13, (u32*)aWrite, nWrite/sizeof(u32)); + } + +#if 0 +fprintf(stderr, "handle sector %d of %s with %s\n", i, + iFile==0 ? "db" : "log", + iOpt==1 ? "rollback" : iOpt==2 ? "write" : "omit" +); +fflush(stderr); +#endif + + if( aWrite ){ + rc = p->pBt->pVfs->xWrite(p->pFile, iSOff, aWrite, nWrite); + } + } + } + testFree(p->apSector[i]); + p->apSector[i] = 0; + } + + testFree(aTmp); + return rc; +} + +static int btSaveSectors(BtFile *p, sqlite4_int64 iOff, int nBuf){ + int rc; + sqlite4_int64 iSz; /* Size of file on disk */ + int iFirst; /* First sector affected */ + int iSector; /* Current sector */ + int iLast; /* Last sector affected */ + + if( p->nSectorSize==0 ){ + p->nSectorSize = p->pBt->pVfs->xSectorSize(p->pFile); + if( p->nSectorSize<512 ) p->nSectorSize = 512; + } + iLast = (iOff+nBuf-1) / p->nSectorSize; + iFirst = iOff / p->nSectorSize; + + rc = p->pBt->pVfs->xSize(p->pFile, &iSz); + for(iSector=iFirst; rc==SQLITE4_OK && iSector<=iLast; iSector++){ + int nRead; + sqlite4_int64 iSOff = iSector * p->nSectorSize; + u8 *aBuf = testMalloc(p->nSectorSize); + nRead = MIN(p->nSectorSize, (iSz - iSOff)); + if( nRead>0 ){ + rc = p->pBt->pVfs->xRead(p->pFile, iSOff, aBuf, nRead); + } + + while( rc==SQLITE4_OK && iSector>=p->nSector ){ + int nNew = p->nSector + 32; + u8 **apNew = (u8**)testMalloc(nNew * sizeof(u8*)); + memcpy(apNew, p->apSector, p->nSector*sizeof(u8*)); + testFree(p->apSector); + p->apSector = apNew; + p->nSector = nNew; + } + + p->apSector[iSector] = aBuf; + } + + return rc; +} + +static int btVfsWrite(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + if( p->pBt && p->pBt->nCrashSync ){ + btSaveSectors(p, iOff, nBuf); + } + return p->pVfs->xWrite(p->pFile, iOff, pBuf, nBuf); +} + +static int btVfsTruncate(bt_file *pFile, sqlite4_int64 iOff){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xTruncate(p->pFile, iOff); +} + +static int btVfsSync(bt_file *pFile){ + int rc = SQLITE4_OK; + BtFile *p = (BtFile*)pFile; + BtDb *pBt = p->pBt; + + if( pBt ){ + if( pBt->bCrash ) return SQLITE4_IOERR; + if( pBt->nCrashSync ){ + pBt->nCrashSync--; + pBt->bCrash = (pBt->nCrashSync==0); + if( pBt->bCrash ){ + btFlushSectors(pBt->apFile[0], 0); + btFlushSectors(pBt->apFile[1], 1); + rc = SQLITE4_IOERR; + }else{ + btFlushSectors(p, 0); + } + } + } + + if( rc==SQLITE4_OK ){ + rc = p->pVfs->xSync(p->pFile); + } + return rc; +} + +static int btVfsSectorSize(bt_file *pFile){ + BtFile *p = (BtFile*)pFile; + return p->pVfs->xSectorSize(p->pFile); +} + +static void btDeref(BtDb *p){ + p->nRef--; + assert( p->nRef>=0 ); + if( p->nRef<=0 ) testFree(p); +} + +static int btVfsClose(bt_file *pFile){ + BtFile *p = (BtFile*)pFile; + BtDb *pBt = p->pBt; + int rc; + if( pBt ){ + btFlushSectors(p, 0); + if( p==pBt->apFile[0] ) pBt->apFile[0] = 0; + if( p==pBt->apFile[1] ) pBt->apFile[1] = 0; + } + testFree(p->apSector); + rc = p->pVfs->xClose(p->pFile); +#if 0 + btDeref(p->pBt); +#endif + testFree(p); + return rc; +} + +static int btVfsUnlink(sqlite4_env *pEnv, bt_env *pVfs, const char *zFile){ + BtDb *pBt = (BtDb*)pVfs->pVfsCtx; + if( pBt->bCrash ) return SQLITE4_IOERR; + return pBt->pVfs->xUnlink(pEnv, pBt->pVfs, zFile); +} + +static int btVfsLock(bt_file *pFile, int iLock, int eType){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xLock(p->pFile, iLock, eType); +} + +static int btVfsTestLock(bt_file *pFile, int iLock, int nLock, int eType){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xTestLock(p->pFile, iLock, nLock, eType); +} + +static int btVfsShmMap(bt_file *pFile, int iChunk, int sz, void **ppOut){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xShmMap(p->pFile, iChunk, sz, ppOut); +} + +static void btVfsShmBarrier(bt_file *pFile){ + BtFile *p = (BtFile*)pFile; + return p->pVfs->xShmBarrier(p->pFile); +} + +static int btVfsShmUnmap(bt_file *pFile, int bDelete){ + BtFile *p = (BtFile*)pFile; + if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; + return p->pVfs->xShmUnmap(p->pFile, bDelete); +} + +static int bt_close(TestDb *pTestDb){ + BtDb *p = (BtDb*)pTestDb; + int rc = sqlite4BtClose(p->pBt); + free(p->aBuffer); + if( p->apFile[0] ) p->apFile[0]->pBt = 0; + if( p->apFile[1] ) p->apFile[1]->pBt = 0; + bgc_detach(p); + testFree(p); + return rc; +} + +static int btMinTransaction(BtDb *p, int iMin, int *piLevel){ + int iLevel; + int rc = SQLITE4_OK; + + iLevel = sqlite4BtTransactionLevel(p->pBt); + if( iLevelpBt, iMin); + *piLevel = iLevel; + }else{ + *piLevel = -1; + } + + return rc; +} +static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){ + int rc = rcin; + if( iLevel>=0 ){ + if( rc==SQLITE4_OK ){ + rc = sqlite4BtCommit(p->pBt, iLevel); + }else{ + sqlite4BtRollback(p->pBt, iLevel); + } + assert( iLevel==sqlite4BtTransactionLevel(p->pBt) ); + } + return rc; +} + +static int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){ + BtDb *p = (BtDb*)pTestDb; + int iLevel; + int rc; + + rc = btMinTransaction(p, 2, &iLevel); + if( rc==SQLITE4_OK ){ + if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); + rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV); + rc = btRestoreTransaction(p, iLevel, rc); + } + return rc; +} + +static int bt_delete(TestDb *pTestDb, void *pK, int nK){ + return bt_write(pTestDb, pK, nK, 0, -1); +} + +static int bt_delete_range( + TestDb *pTestDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + BtDb *p = (BtDb*)pTestDb; + bt_cursor *pCsr = 0; + int rc = SQLITE4_OK; + int iLevel; + + rc = btMinTransaction(p, 2, &iLevel); + if( rc==SQLITE4_OK ){ + if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); + rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); + } + while( rc==SQLITE4_OK ){ + const void *pK; + int n; + int nCmp; + int res; + + rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE); + if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; + if( rc!=SQLITE4_OK ) break; + + rc = sqlite4BtCsrKey(pCsr, &pK, &n); + if( rc!=SQLITE4_OK ) break; + + nCmp = MIN(n, nKey1); + res = memcmp(pKey1, pK, nCmp); + assert( res<0 || (res==0 && nKey1<=n) ); + if( res==0 && nKey1==n ){ + rc = sqlite4BtCsrNext(pCsr); + if( rc!=SQLITE4_OK ) break; + rc = sqlite4BtCsrKey(pCsr, &pK, &n); + if( rc!=SQLITE4_OK ) break; + } + + nCmp = MIN(n, nKey2); + res = memcmp(pKey2, pK, nCmp); + if( res<0 || (res==0 && nKey2<=n) ) break; + + rc = sqlite4BtDelete(pCsr); + } + if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; + + sqlite4BtCsrClose(pCsr); + + rc = btRestoreTransaction(p, iLevel, rc); + return rc; +} + +static int bt_fetch( + TestDb *pTestDb, + void *pK, int nK, + void **ppVal, int *pnVal +){ + BtDb *p = (BtDb*)pTestDb; + bt_cursor *pCsr = 0; + int iLevel; + int rc = SQLITE4_OK; + + iLevel = sqlite4BtTransactionLevel(p->pBt); + if( iLevel==0 ){ + rc = sqlite4BtBegin(p->pBt, 1); + if( rc!=SQLITE4_OK ) return rc; + } + + if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); + rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); + if( rc==SQLITE4_OK ){ + rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ); + if( rc==SQLITE4_OK ){ + const void *pV = 0; + int nV = 0; + rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); + if( rc==SQLITE4_OK ){ + if( nV>p->nBuffer ){ + free(p->aBuffer); + p->aBuffer = (u8*)malloc(nV*2); + p->nBuffer = nV*2; + } + memcpy(p->aBuffer, pV, nV); + *pnVal = nV; + *ppVal = (void*)(p->aBuffer); + } + + }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){ + *ppVal = 0; + *pnVal = -1; + rc = SQLITE4_OK; + } + sqlite4BtCsrClose(pCsr); + } + + if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0); + return rc; +} + +static int bt_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pFirst, int nFirst, + void *pLast, int nLast, + void (*xCallback)(void *, void *, int , void *, int) +){ + BtDb *p = (BtDb*)pTestDb; + bt_cursor *pCsr = 0; + int rc; + int iLevel; + + rc = btMinTransaction(p, 1, &iLevel); + + if( rc==SQLITE4_OK ){ + if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); + rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); + } + if( rc==SQLITE4_OK ){ + if( bReverse ){ + if( pLast ){ + rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE); + }else{ + rc = sqlite4BtCsrLast(pCsr); + } + }else{ + rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE); + } + if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; + + while( rc==SQLITE4_OK ){ + const void *pK = 0; int nK = 0; + const void *pV = 0; int nV = 0; + + rc = sqlite4BtCsrKey(pCsr, &pK, &nK); + if( rc==SQLITE4_OK ){ + rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); + } + + if( rc!=SQLITE4_OK ) break; + if( bReverse ){ + if( pFirst ){ + int res; + int nCmp = MIN(nK, nFirst); + res = memcmp(pFirst, pK, nCmp); + if( res>0 || (res==0 && nKnLast) ) break; + } + } + + xCallback(pCtx, (void*)pK, nK, (void*)pV, nV); + if( bReverse ){ + rc = sqlite4BtCsrPrev(pCsr); + }else{ + rc = sqlite4BtCsrNext(pCsr); + } + } + if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; + + sqlite4BtCsrClose(pCsr); + } + + rc = btRestoreTransaction(p, iLevel, rc); + return rc; +} + +static int bt_begin(TestDb *pTestDb, int iLvl){ + BtDb *p = (BtDb*)pTestDb; + int rc = sqlite4BtBegin(p->pBt, iLvl); + return rc; +} + +static int bt_commit(TestDb *pTestDb, int iLvl){ + BtDb *p = (BtDb*)pTestDb; + int rc = sqlite4BtCommit(p->pBt, iLvl); + return rc; +} + +static int bt_rollback(TestDb *pTestDb, int iLvl){ + BtDb *p = (BtDb*)pTestDb; + int rc = sqlite4BtRollback(p->pBt, iLvl); + return rc; +} + +static int testParseOption( + const char **pzIn, /* IN/OUT: pointer to next option */ + const char **pzOpt, /* OUT: nul-terminated option name */ + const char **pzArg, /* OUT: nul-terminated option argument */ + char *pSpace /* Temporary space for output params */ +){ + const char *p = *pzIn; + const char *pStart; + int n; + + char *pOut = pSpace; + + while( *p==' ' ) p++; + pStart = p; + while( *p && *p!='=' ) p++; + if( *p==0 ) return 1; + + n = (p - pStart); + memcpy(pOut, pStart, n); + *pzOpt = pOut; + pOut += n; + *pOut++ = '\0'; + + p++; + pStart = p; + while( *p && *p!=' ' ) p++; + n = (p - pStart); + + memcpy(pOut, pStart, n); + *pzArg = pOut; + pOut += n; + *pOut++ = '\0'; + + *pzIn = p; + return 0; +} + +static int testParseInt(const char *z, int *piVal){ + int i = 0; + const char *p = z; + + while( *p>='0' && *p<='9' ){ + i = i*10 + (*p - '0'); + p++; + } + if( *p=='K' || *p=='k' ){ + i = i * 1024; + p++; + }else if( *p=='M' || *p=='m' ){ + i = i * 1024 * 1024; + p++; + } + + if( *p ) return SQLITE4_ERROR; + *piVal = i; + return SQLITE4_OK; +} + +static int testBtConfigure(BtDb *pDb, const char *zCfg, int *pbMt){ + int rc = SQLITE4_OK; + + if( zCfg ){ + struct CfgParam { + const char *zParam; + int eParam; + } aParam[] = { + { "safety", BT_CONTROL_SAFETY }, + { "autockpt", BT_CONTROL_AUTOCKPT }, + { "multiproc", BT_CONTROL_MULTIPROC }, + { "blksz", BT_CONTROL_BLKSZ }, + { "pagesz", BT_CONTROL_PAGESZ }, + { "mt", -1 }, + { "fastinsert", -2 }, + { 0, 0 } + }; + const char *z = zCfg; + int n = strlen(z); + char *aSpace; + const char *zOpt; + const char *zArg; + + aSpace = (char*)testMalloc(n+2); + while( rc==SQLITE4_OK && 0==testParseOption(&z, &zOpt, &zArg, aSpace) ){ + int i; + int iVal; + rc = testArgSelect(aParam, "param", zOpt, &i); + if( rc!=SQLITE4_OK ) break; + + rc = testParseInt(zArg, &iVal); + if( rc!=SQLITE4_OK ) break; + + switch( aParam[i].eParam ){ + case -1: + *pbMt = iVal; + break; + case -2: + pDb->bFastInsert = 1; + break; + default: + rc = sqlite4BtControl(pDb->pBt, aParam[i].eParam, (void*)&iVal); + break; + } + } + testFree(aSpace); + } + + return rc; +} + + +int test_bt_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + + static const DatabaseMethods SqlMethods = { + bt_close, + bt_write, + bt_delete, + bt_delete_range, + bt_fetch, + bt_scan, + bt_begin, + bt_commit, + bt_rollback + }; + BtDb *p = 0; + bt_db *pBt = 0; + int rc; + sqlite4_env *pEnv = sqlite4_env_default(); + + if( bClear && zFilename && zFilename[0] ){ + char *zLog = sqlite3_mprintf("%s-wal", zFilename); + unlink(zFilename); + unlink(zLog); + sqlite3_free(zLog); + } + + rc = sqlite4BtNew(pEnv, 0, &pBt); + if( rc==SQLITE4_OK ){ + int mt = 0; /* True for multi-threaded connection */ + + p = (BtDb*)testMalloc(sizeof(BtDb)); + p->base.pMethods = &SqlMethods; + p->pBt = pBt; + p->pEnv = pEnv; + p->nRef = 1; + + p->env.pVfsCtx = (void*)p; + p->env.xFullpath = btVfsFullpath; + p->env.xOpen = btVfsOpen; + p->env.xSize = btVfsSize; + p->env.xRead = btVfsRead; + p->env.xWrite = btVfsWrite; + p->env.xTruncate = btVfsTruncate; + p->env.xSync = btVfsSync; + p->env.xSectorSize = btVfsSectorSize; + p->env.xClose = btVfsClose; + p->env.xUnlink = btVfsUnlink; + p->env.xLock = btVfsLock; + p->env.xTestLock = btVfsTestLock; + p->env.xShmMap = btVfsShmMap; + p->env.xShmBarrier = btVfsShmBarrier; + p->env.xShmUnmap = btVfsShmUnmap; + + sqlite4BtControl(pBt, BT_CONTROL_GETVFS, (void*)&p->pVfs); + sqlite4BtControl(pBt, BT_CONTROL_SETVFS, (void*)&p->env); + + rc = testBtConfigure(p, zSpec, &mt); + if( rc==SQLITE4_OK ){ + rc = sqlite4BtOpen(pBt, zFilename); + } + + if( rc==SQLITE4_OK && mt ){ + int nAuto = 0; + rc = bgc_attach(p, zSpec); + sqlite4BtControl(pBt, BT_CONTROL_AUTOCKPT, (void*)&nAuto); + } + } + + if( rc!=SQLITE4_OK && p ){ + bt_close(&p->base); + } + + *ppDb = &p->base; + return rc; +} + +int test_fbt_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + return test_bt_open("fast=1", zFilename, bClear, ppDb); +} + +int test_fbts_open( + const char *zSpec, + const char *zFilename, + int bClear, + TestDb **ppDb +){ + return test_bt_open("fast=1 blksz=32K pagesz=512", zFilename, bClear, ppDb); +} + + +void tdb_bt_prepare_sync_crash(TestDb *pTestDb, int iSync){ + BtDb *p = (BtDb*)pTestDb; + assert( pTestDb->pMethods->xClose==bt_close ); + assert( p->bCrash==0 ); + p->nCrashSync = iSync; +} + +bt_db *tdb_bt(TestDb *pDb){ + if( pDb->pMethods->xClose==bt_close ){ + return ((BtDb *)pDb)->pBt; + } + return 0; +} + +/************************************************************************* +** Beginning of code for background checkpointer. +*/ + +struct bt_ckpter { + sqlite4_buffer file; /* File name */ + sqlite4_buffer spec; /* Options */ + int nLogsize; /* Minimum log size to checkpoint */ + int nRef; /* Number of clients */ + + int bDoWork; /* Set by client threads */ + pthread_t ckpter_thread; /* Checkpointer thread */ + pthread_cond_t ckpter_cond; /* Condition var the ckpter waits on */ + pthread_mutex_t ckpter_mutex; /* Mutex used with ckpter_cond */ + + bt_ckpter *pNext; /* Next object in list at gBgc.pCkpter */ +}; + +static struct GlobalBackgroundCheckpointer { + bt_ckpter *pCkpter; /* Linked list of checkpointers */ +} gBgc; + +static void *bgc_main(void *pArg){ + BtDb *pDb = 0; + int rc; + int mt; + bt_ckpter *pCkpter = (bt_ckpter*)pArg; + + rc = test_bt_open("", (char*)pCkpter->file.p, 0, (TestDb**)&pDb); + assert( rc==SQLITE4_OK ); + rc = testBtConfigure(pDb, (char*)pCkpter->spec.p, &mt); + + while( pCkpter->nRef>0 ){ + bt_db *db = pDb->pBt; + int nLog = 0; + + sqlite4BtBegin(db, 1); + sqlite4BtCommit(db, 0); + sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); + + if( nLog>=pCkpter->nLogsize ){ + int rc; + bt_checkpoint ckpt; + memset(&ckpt, 0, sizeof(bt_checkpoint)); + ckpt.nFrameBuffer = nLog/2; + rc = sqlite4BtControl(db, BT_CONTROL_CHECKPOINT, (void*)&ckpt); + assert( rc==SQLITE4_OK ); + sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); + } + + /* The thread will wake up when it is signaled either because another + ** thread has created some work for this one or because the connection + ** is being closed. */ + pthread_mutex_lock(&pCkpter->ckpter_mutex); + if( pCkpter->bDoWork==0 ){ + pthread_cond_wait(&pCkpter->ckpter_cond, &pCkpter->ckpter_mutex); + } + pCkpter->bDoWork = 0; + pthread_mutex_unlock(&pCkpter->ckpter_mutex); + } + + if( pDb ) bt_close((TestDb*)pDb); + return 0; +} + +static void bgc_logsize_cb(void *pCtx, int nLogsize){ + bt_ckpter *p = (bt_ckpter*)pCtx; + if( nLogsize>=p->nLogsize ){ + pthread_mutex_lock(&p->ckpter_mutex); + p->bDoWork = 1; + pthread_cond_signal(&p->ckpter_cond); + pthread_mutex_unlock(&p->ckpter_mutex); + } +} + +static int bgc_attach(BtDb *pDb, const char *zSpec){ + int rc; + int n; + bt_info info; + bt_ckpter *pCkpter; + + /* Figure out the full path to the database opened by handle pDb. */ + info.eType = BT_INFO_FILENAME; + info.pgno = 0; + sqlite4_buffer_init(&info.output, 0); + rc = sqlite4BtControl(pDb->pBt, BT_CONTROL_INFO, (void*)&info); + if( rc!=SQLITE4_OK ) return rc; + + sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); + + /* Search for an existing bt_ckpter object. */ + n = info.output.n; + for(pCkpter=gBgc.pCkpter; pCkpter; pCkpter=pCkpter->pNext){ + if( n==pCkpter->file.n && 0==memcmp(info.output.p, pCkpter->file.p, n) ){ + break; + } + } + + /* Failed to find a suitable checkpointer. Create a new one. */ + if( pCkpter==0 ){ + bt_logsizecb cb; + + pCkpter = testMalloc(sizeof(bt_ckpter)); + memcpy(&pCkpter->file, &info.output, sizeof(sqlite4_buffer)); + info.output.p = 0; + pCkpter->pNext = gBgc.pCkpter; + pCkpter->nLogsize = 1000; + gBgc.pCkpter = pCkpter; + pCkpter->nRef = 1; + + sqlite4_buffer_init(&pCkpter->spec, 0); + rc = sqlite4_buffer_set(&pCkpter->spec, zSpec, strlen(zSpec)+1); + assert( rc==SQLITE4_OK ); + + /* Kick off the checkpointer thread. */ + if( rc==0 ) rc = pthread_cond_init(&pCkpter->ckpter_cond, 0); + if( rc==0 ) rc = pthread_mutex_init(&pCkpter->ckpter_mutex, 0); + if( rc==0 ){ + rc = pthread_create(&pCkpter->ckpter_thread, 0, bgc_main, (void*)pCkpter); + } + assert( rc==0 ); /* todo: Fix this */ + + /* Set up the logsize callback for the client thread */ + cb.pCtx = (void*)pCkpter; + cb.xLogsize = bgc_logsize_cb; + sqlite4BtControl(pDb->pBt, BT_CONTROL_LOGSIZECB, (void*)&cb); + }else{ + pCkpter->nRef++; + } + + /* Assuming a checkpointer was encountered or effected, attach the + ** connection to it. */ + if( pCkpter ){ + pDb->pCkpter = pCkpter; + } + + sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); + sqlite4_buffer_clear(&info.output); + return rc; +} + +static int bgc_detach(BtDb *pDb){ + int rc = SQLITE4_OK; + bt_ckpter *pCkpter = pDb->pCkpter; + if( pCkpter ){ + int bShutdown = 0; /* True if this is the last reference */ + + sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); + pCkpter->nRef--; + if( pCkpter->nRef==0 ){ + bt_ckpter **pp; + + *pp = pCkpter->pNext; + for(pp=&gBgc.pCkpter; *pp!=pCkpter; pp=&((*pp)->pNext)); + bShutdown = 1; + } + sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); + + if( bShutdown ){ + void *pDummy; + + /* Signal the checkpointer thread. */ + pthread_mutex_lock(&pCkpter->ckpter_mutex); + pCkpter->bDoWork = 1; + pthread_cond_signal(&pCkpter->ckpter_cond); + pthread_mutex_unlock(&pCkpter->ckpter_mutex); + + /* Join the checkpointer thread. */ + pthread_join(pCkpter->ckpter_thread, &pDummy); + pthread_cond_destroy(&pCkpter->ckpter_cond); + pthread_mutex_destroy(&pCkpter->ckpter_mutex); + + sqlite4_buffer_clear(&pCkpter->file); + sqlite4_buffer_clear(&pCkpter->spec); + testFree(pCkpter); + } + + pDb->pCkpter = 0; + } + return rc; +} + +/* +** End of background checkpointer. +*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_util.c b/ext/lsm1/lsm-test/lsmtest_util.c new file mode 100644 index 0000000000000000000000000000000000000000..adab8a53e81cda1cae74bcfbcd35c74593efc731 --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_util.c @@ -0,0 +1,223 @@ + +#include "lsmtest.h" +#include +#include +#include +#ifndef _WIN32 +# include +#endif + +/* +** Global variables used within this module. +*/ +static struct TestutilGlobal { + char **argv; + int argc; +} g = {0, 0}; + +static struct TestutilRnd { + unsigned int aRand1[2048]; /* Bits 0..10 */ + unsigned int aRand2[2048]; /* Bits 11..21 */ + unsigned int aRand3[1024]; /* Bits 22..31 */ +} r; + +/************************************************************************* +** The following block is a copy of the implementation of SQLite function +** sqlite3_randomness. This version has two important differences: +** +** 1. It always uses the same seed. So the sequence of random data output +** is the same for every run of the program. +** +** 2. It is not threadsafe. +*/ +static struct sqlite3PrngType { + unsigned char i, j; /* State variables */ + unsigned char s[256]; /* State variables */ +} sqlite3Prng = { + 0xAF, 0x28, + { + 0x71, 0xF5, 0xB4, 0x6E, 0x80, 0xAB, 0x1D, 0xB8, + 0xFB, 0xB7, 0x49, 0xBF, 0xFF, 0x72, 0x2D, 0x14, + 0x79, 0x09, 0xE3, 0x78, 0x76, 0xB0, 0x2C, 0x0A, + 0x8E, 0x23, 0xEE, 0xDF, 0xE0, 0x9A, 0x2F, 0x67, + 0xE1, 0xBE, 0x0E, 0xA7, 0x08, 0x97, 0xEB, 0x77, + 0x78, 0xBA, 0x9D, 0xCA, 0x49, 0x4C, 0x60, 0x9A, + 0xF6, 0xBD, 0xDA, 0x7F, 0xBC, 0x48, 0x58, 0x52, + 0xE5, 0xCD, 0x83, 0x72, 0x23, 0x52, 0xFF, 0x6D, + 0xEF, 0x0F, 0x82, 0x29, 0xA0, 0x83, 0x3F, 0x7D, + 0xA4, 0x88, 0x31, 0xE7, 0x88, 0x92, 0x3B, 0x9B, + 0x3B, 0x2C, 0xC2, 0x4C, 0x71, 0xA2, 0xB0, 0xEA, + 0x36, 0xD0, 0x00, 0xF1, 0xD3, 0x39, 0x17, 0x5D, + 0x2A, 0x7A, 0xE4, 0xAD, 0xE1, 0x64, 0xCE, 0x0F, + 0x9C, 0xD9, 0xF5, 0xED, 0xB0, 0x22, 0x5E, 0x62, + 0x97, 0x02, 0xA3, 0x8C, 0x67, 0x80, 0xFC, 0x88, + 0x14, 0x0B, 0x15, 0x10, 0x0F, 0xC7, 0x40, 0xD4, + 0xF1, 0xF9, 0x0E, 0x1A, 0xCE, 0xB9, 0x1E, 0xA1, + 0x72, 0x8E, 0xD7, 0x78, 0x39, 0xCD, 0xF4, 0x5D, + 0x2A, 0x59, 0x26, 0x34, 0xF2, 0x73, 0x0B, 0xA0, + 0x02, 0x51, 0x2C, 0x03, 0xA3, 0xA7, 0x43, 0x13, + 0xE8, 0x98, 0x2B, 0xD2, 0x53, 0xF8, 0xEE, 0x91, + 0x7D, 0xE7, 0xE3, 0xDA, 0xD5, 0xBB, 0xC0, 0x92, + 0x9D, 0x98, 0x01, 0x2C, 0xF9, 0xB9, 0xA0, 0xEB, + 0xCF, 0x32, 0xFA, 0x01, 0x49, 0xA5, 0x1D, 0x9A, + 0x76, 0x86, 0x3F, 0x40, 0xD4, 0x89, 0x8F, 0x9C, + 0xE2, 0xE3, 0x11, 0x31, 0x37, 0xB2, 0x49, 0x28, + 0x35, 0xC0, 0x99, 0xB6, 0xD0, 0xBC, 0x66, 0x35, + 0xF7, 0x83, 0x5B, 0xD7, 0x37, 0x1A, 0x2B, 0x18, + 0xA6, 0xFF, 0x8D, 0x7C, 0x81, 0xA8, 0xFC, 0x9E, + 0xC4, 0xEC, 0x80, 0xD0, 0x98, 0xA7, 0x76, 0xCC, + 0x9C, 0x2F, 0x7B, 0xFF, 0x8E, 0x0E, 0xBB, 0x90, + 0xAE, 0x13, 0x06, 0xF5, 0x1C, 0x4E, 0x52, 0xF7 + } +}; + +/* Generate and return single random byte */ +static unsigned char randomByte(void){ + unsigned char t; + sqlite3Prng.i++; + t = sqlite3Prng.s[sqlite3Prng.i]; + sqlite3Prng.j += t; + sqlite3Prng.s[sqlite3Prng.i] = sqlite3Prng.s[sqlite3Prng.j]; + sqlite3Prng.s[sqlite3Prng.j] = t; + t += sqlite3Prng.s[sqlite3Prng.i]; + return sqlite3Prng.s[t]; +} + +/* +** Return N random bytes. +*/ +static void randomBlob(int nBuf, unsigned char *zBuf){ + int i; + for(i=0; i>11) & 0x000007FF] ^ + r.aRand3[(iVal>>22) & 0x000003FF] + ; +} + +void testPrngArray(unsigned int iVal, unsigned int *aOut, int nOut){ + int i; + for(i=0; izName; + pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] + ){ + if( zPrev ){ testPrintError("%s, ", zPrev); } + zPrev = pEntry->zName; + } + testPrintError("or %s\n", zPrev); +} + +int testArgSelectX( + void *aData, + const char *zType, + int sz, + const char *zArg, + int *piOut +){ + struct Entry { const char *zName; }; + struct Entry *pEntry; + int nArg = strlen(zArg); + + int i = 0; + int iOut = -1; + int nOut = 0; + + for(pEntry=(struct Entry *)aData; + pEntry->zName; + pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] + ){ + int nName = strlen(pEntry->zName); + if( nArg<=nName && memcmp(pEntry->zName, zArg, nArg)==0 ){ + iOut = i; + if( nName==nArg ){ + nOut = 1; + break; + } + nOut++; + } + i++; + } + + if( nOut!=1 ){ + argError(aData, zType, sz, zArg); + }else{ + *piOut = iOut; + } + return (nOut!=1); +} + +struct timeval zero_time; + +void testTimeInit(void){ + gettimeofday(&zero_time, 0); +} + +int testTimeGet(void){ + struct timeval now; + gettimeofday(&now, 0); + return + (((int)now.tv_sec - (int)zero_time.tv_sec)*1000) + + (((int)now.tv_usec - (int)zero_time.tv_usec)/1000); +} diff --git a/ext/lsm1/lsm-test/lsmtest_win32.c b/ext/lsm1/lsm-test/lsmtest_win32.c new file mode 100644 index 0000000000000000000000000000000000000000..9472723368a72fe1bf87906b5ba5f822147bba4a --- /dev/null +++ b/ext/lsm1/lsm-test/lsmtest_win32.c @@ -0,0 +1,30 @@ + +#include "lsmtest.h" + +#ifdef _WIN32 + +#define TICKS_PER_SECOND (10000000) +#define TICKS_PER_MICROSECOND (10) +#define TICKS_UNIX_EPOCH (116444736000000000LL) + +int win32GetTimeOfDay( + struct timeval *tp, + void *tzp +){ + FILETIME fileTime; + ULONGLONG ticks; + ULONGLONG unixTicks; + + unused_parameter(tzp); + memset(&fileTime, 0, sizeof(FILETIME)); + GetSystemTimeAsFileTime(&fileTime); + ticks = (ULONGLONG)fileTime.dwHighDateTime << 32; + ticks |= (ULONGLONG)fileTime.dwLowDateTime; + unixTicks = ticks - TICKS_UNIX_EPOCH; + tp->tv_sec = (long)(unixTicks / TICKS_PER_SECOND); + unixTicks -= ((ULONGLONG)tp->tv_sec * TICKS_PER_SECOND); + tp->tv_usec = (long)(unixTicks / TICKS_PER_MICROSECOND); + + return 0; +} +#endif diff --git a/ext/lsm1/lsm.h b/ext/lsm1/lsm.h new file mode 100644 index 0000000000000000000000000000000000000000..48701c4c5e45eb1d79e33ec3bd07da9f27d0d1e5 --- /dev/null +++ b/ext/lsm1/lsm.h @@ -0,0 +1,684 @@ +/* +** 2011-08-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file defines the LSM API. +*/ +#ifndef _LSM_H +#define _LSM_H +#include +#ifdef __cplusplus +extern "C" { +#endif + +/* +** Opaque handle types. +*/ +typedef struct lsm_compress lsm_compress; /* Compression library functions */ +typedef struct lsm_compress_factory lsm_compress_factory; +typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ +typedef struct lsm_db lsm_db; /* Database connection handle */ +typedef struct lsm_env lsm_env; /* Runtime environment */ +typedef struct lsm_file lsm_file; /* OS file handle */ +typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ + +/* 64-bit integer type used for file offsets. */ +typedef long long int lsm_i64; /* 64-bit signed integer type */ + +/* Candidate values for the 3rd argument to lsm_env.xLock() */ +#define LSM_LOCK_UNLOCK 0 +#define LSM_LOCK_SHARED 1 +#define LSM_LOCK_EXCL 2 + +/* Flags for lsm_env.xOpen() */ +#define LSM_OPEN_READONLY 0x0001 + +/* +** CAPI: Database Runtime Environment +** +** Run-time environment used by LSM +*/ +struct lsm_env { + int nByte; /* Size of this structure in bytes */ + int iVersion; /* Version number of this structure (1) */ + /****** file i/o ***********************************************/ + void *pVfsCtx; + int (*xFullpath)(lsm_env*, const char *, char *, int *); + int (*xOpen)(lsm_env*, const char *, int flags, lsm_file **); + int (*xRead)(lsm_file *, lsm_i64, void *, int); + int (*xWrite)(lsm_file *, lsm_i64, void *, int); + int (*xTruncate)(lsm_file *, lsm_i64); + int (*xSync)(lsm_file *); + int (*xSectorSize)(lsm_file *); + int (*xRemap)(lsm_file *, lsm_i64, void **, lsm_i64*); + int (*xFileid)(lsm_file *, void *pBuf, int *pnBuf); + int (*xClose)(lsm_file *); + int (*xUnlink)(lsm_env*, const char *); + int (*xLock)(lsm_file*, int, int); + int (*xTestLock)(lsm_file*, int, int, int); + int (*xShmMap)(lsm_file*, int, int, void **); + void (*xShmBarrier)(void); + int (*xShmUnmap)(lsm_file*, int); + /****** memory allocation ****************************************/ + void *pMemCtx; + void *(*xMalloc)(lsm_env*, size_t); /* malloc(3) function */ + void *(*xRealloc)(lsm_env*, void *, size_t); /* realloc(3) function */ + void (*xFree)(lsm_env*, void *); /* free(3) function */ + size_t (*xSize)(lsm_env*, void *); /* xSize function */ + /****** mutexes ****************************************************/ + void *pMutexCtx; + int (*xMutexStatic)(lsm_env*,int,lsm_mutex**); /* Obtain a static mutex */ + int (*xMutexNew)(lsm_env*, lsm_mutex**); /* Get a new dynamic mutex */ + void (*xMutexDel)(lsm_mutex *); /* Delete an allocated mutex */ + void (*xMutexEnter)(lsm_mutex *); /* Grab a mutex */ + int (*xMutexTry)(lsm_mutex *); /* Attempt to obtain a mutex */ + void (*xMutexLeave)(lsm_mutex *); /* Leave a mutex */ + int (*xMutexHeld)(lsm_mutex *); /* Return true if mutex is held */ + int (*xMutexNotHeld)(lsm_mutex *); /* Return true if mutex not held */ + /****** other ****************************************************/ + int (*xSleep)(lsm_env*, int microseconds); + + /* New fields may be added in future releases, in which case the + ** iVersion value will increase. */ +}; + +/* +** Values that may be passed as the second argument to xMutexStatic. +*/ +#define LSM_MUTEX_GLOBAL 1 +#define LSM_MUTEX_HEAP 2 + +/* +** CAPI: LSM Error Codes +*/ +#define LSM_OK 0 +#define LSM_ERROR 1 +#define LSM_BUSY 5 +#define LSM_NOMEM 7 +#define LSM_READONLY 8 +#define LSM_IOERR 10 +#define LSM_CORRUPT 11 +#define LSM_FULL 13 +#define LSM_CANTOPEN 14 +#define LSM_PROTOCOL 15 +#define LSM_MISUSE 21 + +#define LSM_MISMATCH 50 + + +#define LSM_IOERR_NOENT (LSM_IOERR | (1<<8)) + +/* +** CAPI: Creating and Destroying Database Connection Handles +** +** Open and close a database connection handle. +*/ +int lsm_new(lsm_env*, lsm_db **ppDb); +int lsm_close(lsm_db *pDb); + +/* +** CAPI: Connecting to a Database +*/ +int lsm_open(lsm_db *pDb, const char *zFilename); + +/* +** CAPI: Obtaining pointers to database environments +** +** Return a pointer to the environment used by the database connection +** passed as the first argument. Assuming the argument is valid, this +** function always returns a valid environment pointer - it cannot fail. +*/ +lsm_env *lsm_get_env(lsm_db *pDb); + +/* +** The lsm_default_env() function returns a pointer to the default LSM +** environment for the current platform. +*/ +lsm_env *lsm_default_env(void); + + +/* +** CAPI: Configuring a database connection. +** +** The lsm_config() function is used to configure a database connection. +*/ +int lsm_config(lsm_db *, int, ...); + +/* +** The following values may be passed as the second argument to lsm_config(). +** +** LSM_CONFIG_AUTOFLUSH: +** A read/write integer parameter. +** +** This value determines the amount of data allowed to accumulate in a +** live in-memory tree before it is marked as old. After committing a +** transaction, a connection checks if the size of the live in-memory tree, +** including data structure overhead, is greater than the value of this +** option in KB. If it is, and there is not already an old in-memory tree, +** the live in-memory tree is marked as old. +** +** The maximum allowable value is 1048576 (1GB). There is no minimum +** value. If this parameter is set to zero, then an attempt is made to +** mark the live in-memory tree as old after each transaction is committed. +** +** The default value is 1024 (1MB). +** +** LSM_CONFIG_PAGE_SIZE: +** A read/write integer parameter. This parameter may only be set before +** lsm_open() has been called. +** +** LSM_CONFIG_BLOCK_SIZE: +** A read/write integer parameter. +** +** This parameter may only be set before lsm_open() has been called. It +** must be set to a power of two between 64 and 65536, inclusive (block +** sizes between 64KB and 64MB). +** +** If the connection creates a new database, the block size of the new +** database is set to the value of this option in KB. After lsm_open() +** has been called, querying this parameter returns the actual block +** size of the opened database. +** +** The default value is 1024 (1MB blocks). +** +** LSM_CONFIG_SAFETY: +** A read/write integer parameter. Valid values are 0, 1 (the default) +** and 2. This parameter determines how robust the database is in the +** face of a system crash (e.g. a power failure or operating system +** crash). As follows: +** +** 0 (off): No robustness. A system crash may corrupt the database. +** +** 1 (normal): Some robustness. A system crash may not corrupt the +** database file, but recently committed transactions may +** be lost following recovery. +** +** 2 (full): Full robustness. A system crash may not corrupt the +** database file. Following recovery the database file +** contains all successfully committed transactions. +** +** LSM_CONFIG_AUTOWORK: +** A read/write integer parameter. +** +** LSM_CONFIG_AUTOCHECKPOINT: +** A read/write integer parameter. +** +** If this option is set to non-zero value N, then a checkpoint is +** automatically attempted after each N KB of data have been written to +** the database file. +** +** The amount of uncheckpointed data already written to the database file +** is a global parameter. After performing database work (writing to the +** database file), the process checks if the total amount of uncheckpointed +** data exceeds the value of this paramter. If so, a checkpoint is performed. +** This means that this option may cause the connection to perform a +** checkpoint even if the current connection has itself written very little +** data into the database file. +** +** The default value is 2048 (checkpoint every 2MB). +** +** LSM_CONFIG_MMAP: +** A read/write integer parameter. If this value is set to 0, then the +** database file is accessed using ordinary read/write IO functions. Or, +** if it is set to 1, then the database file is memory mapped and accessed +** that way. If this parameter is set to any value N greater than 1, then +** up to the first N KB of the file are memory mapped, and any remainder +** accessed using read/write IO. +** +** The default value is 1 on 64-bit platforms and 32768 on 32-bit platforms. +** +** +** LSM_CONFIG_USE_LOG: +** A read/write boolean parameter. True (the default) to use the log +** file normally. False otherwise. +** +** LSM_CONFIG_AUTOMERGE: +** A read/write integer parameter. The minimum number of segments to +** merge together at a time. Default value 4. +** +** LSM_CONFIG_MAX_FREELIST: +** A read/write integer parameter. The maximum number of free-list +** entries that are stored in a database checkpoint (the others are +** stored elsewhere in the database). +** +** There is no reason for an application to configure or query this +** parameter. It is only present because configuring a small value +** makes certain parts of the lsm code easier to test. +** +** LSM_CONFIG_MULTIPLE_PROCESSES: +** A read/write boolean parameter. This parameter may only be set before +** lsm_open() has been called. If true, the library uses shared-memory +** and posix advisory locks to co-ordinate access by clients from within +** multiple processes. Otherwise, if false, all database clients must be +** located in the same process. The default value is true. +** +** LSM_CONFIG_SET_COMPRESSION: +** Set the compression methods used to compress and decompress database +** content. The argument to this option should be a pointer to a structure +** of type lsm_compress. The lsm_config() method takes a copy of the +** structures contents. +** +** This option may only be used before lsm_open() is called. Invoking it +** after lsm_open() has been called results in an LSM_MISUSE error. +** +** LSM_CONFIG_GET_COMPRESSION: +** Query the compression methods used to compress and decompress database +** content. +** +** LSM_CONFIG_SET_COMPRESSION_FACTORY: +** Configure a factory method to be invoked in case of an LSM_MISMATCH +** error. +** +** LSM_CONFIG_READONLY: +** A read/write boolean parameter. This parameter may only be set before +** lsm_open() is called. +*/ +#define LSM_CONFIG_AUTOFLUSH 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_AUTOMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 +#define LSM_CONFIG_READONLY 16 + +#define LSM_SAFETY_OFF 0 +#define LSM_SAFETY_NORMAL 1 +#define LSM_SAFETY_FULL 2 + +/* +** CAPI: Compression and/or Encryption Hooks +*/ +struct lsm_compress { + void *pCtx; + unsigned int iId; + int (*xBound)(void *, int nSrc); + int (*xCompress)(void *, char *, int *, const char *, int); + int (*xUncompress)(void *, char *, int *, const char *, int); + void (*xFree)(void *pCtx); +}; + +struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *, lsm_db *, unsigned int); + void (*xFree)(void *pCtx); +}; + +#define LSM_COMPRESSION_EMPTY 0 +#define LSM_COMPRESSION_NONE 1 + +/* +** CAPI: Allocating and Freeing Memory +** +** Invoke the memory allocation functions that belong to environment +** pEnv. Or the system defaults if no memory allocation functions have +** been registered. +*/ +void *lsm_malloc(lsm_env*, size_t); +void *lsm_realloc(lsm_env*, void *, size_t); +void lsm_free(lsm_env*, void *); + +/* +** CAPI: Querying a Connection For Operational Data +** +** Query a database connection for operational statistics or data. +*/ +int lsm_info(lsm_db *, int, ...); + +int lsm_get_user_version(lsm_db *, unsigned int *); +int lsm_set_user_version(lsm_db *, unsigned int); + +/* +** The following values may be passed as the second argument to lsm_info(). +** +** LSM_INFO_NWRITE: +** The third parameter should be of type (int *). The location pointed +** to by the third parameter is set to the number of 4KB pages written to +** the database file during the lifetime of this connection. +** +** LSM_INFO_NREAD: +** The third parameter should be of type (int *). The location pointed +** to by the third parameter is set to the number of 4KB pages read from +** the database file during the lifetime of this connection. +** +** LSM_INFO_DB_STRUCTURE: +** The third argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string containing +** the string representation of a Tcl data-structure reflecting the +** current structure of the database file. Specifically, the current state +** of the worker snapshot. The returned string should be eventually freed +** by the caller using lsm_free(). +** +** The returned list contains one element for each level in the database, +** in order from most to least recent. Each element contains a +** single element for each segment comprising the corresponding level, +** starting with the lhs segment, then each of the rhs segments (if any) +** in order from most to least recent. +** +** Each segment element is itself a list of 4 integer values, as follows: +** +**

  1. First page of segment +**
  2. Last page of segment +**
  3. Root page of segment (if applicable) +**
  4. Total number of pages in segment +**
+** +** LSM_INFO_ARRAY_STRUCTURE: +** There should be two arguments passed following this option (i.e. a +** total of four arguments passed to lsm_info()). The first argument +** should be the page number of the first page in a database array +** (perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second +** trailing argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string that must +** be eventually freed using lsm_free() by the caller. +** +** The output string contains the text representation of a Tcl list of +** integers. Each pair of integers represent a range of pages used by +** the identified array. For example, if the array occupies database +** pages 993 to 1024, then pages 2048 to 2777, then the returned string +** will be "993 1024 2048 2777". +** +** If the specified integer argument does not correspond to the first +** page of any database array, LSM_ERROR is returned and the output +** pointer is set to a NULL value. +** +** LSM_INFO_LOG_STRUCTURE: +** The third argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string containing +** the string representation of a Tcl data-structure. The returned +** string should be eventually freed by the caller using lsm_free(). +** +** The Tcl structure returned is a list of six integers that describe +** the current structure of the log file. +** +** LSM_INFO_ARRAY_PAGES: +** +** LSM_INFO_PAGE_ASCII_DUMP: +** As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed +** with calls that specify this option - an integer page number and a +** (char **) used to return a nul-terminated string that must be later +** freed using lsm_free(). In this case the output string is populated +** with a human-readable description of the page content. +** +** If the page cannot be decoded, it is not an error. In this case the +** human-readable output message will report the systems failure to +** interpret the page data. +** +** LSM_INFO_PAGE_HEX_DUMP: +** This argument is similar to PAGE_ASCII_DUMP, except that keys and +** values are represented using hexadecimal notation instead of ascii. +** +** LSM_INFO_FREELIST: +** The third argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string containing +** the string representation of a Tcl data-structure. The returned +** string should be eventually freed by the caller using lsm_free(). +** +** The Tcl structure returned is a list containing one element for each +** free block in the database. The element itself consists of two +** integers - the block number and the id of the snapshot that freed it. +** +** LSM_INFO_CHECKPOINT_SIZE: +** The third argument should be of type (int *). The location pointed to +** by this argument is populated with the number of KB written to the +** database file since the most recent checkpoint. +** +** LSM_INFO_TREE_SIZE: +** If this value is passed as the second argument to an lsm_info() call, it +** should be followed by two arguments of type (int *) (for a total of four +** arguments). +** +** At any time, there are either one or two tree structures held in shared +** memory that new database clients will access (there may also be additional +** tree structures being used by older clients - this API does not provide +** information on them). One tree structure - the current tree - is used to +** accumulate new data written to the database. The other tree structure - +** the old tree - is a read-only tree holding older data and may be flushed +** to disk at any time. +** +** Assuming no error occurs, the location pointed to by the first of the two +** (int *) arguments is set to the size of the old in-memory tree in KB. +** The second is set to the size of the current, or live in-memory tree. +** +** LSM_INFO_COMPRESSION_ID: +** This value should be followed by a single argument of type +** (unsigned int *). If successful, the location pointed to is populated +** with the database compression id before returning. +*/ +#define LSM_INFO_NWRITE 1 +#define LSM_INFO_NREAD 2 +#define LSM_INFO_DB_STRUCTURE 3 +#define LSM_INFO_LOG_STRUCTURE 4 +#define LSM_INFO_ARRAY_STRUCTURE 5 +#define LSM_INFO_PAGE_ASCII_DUMP 6 +#define LSM_INFO_PAGE_HEX_DUMP 7 +#define LSM_INFO_FREELIST 8 +#define LSM_INFO_ARRAY_PAGES 9 +#define LSM_INFO_CHECKPOINT_SIZE 10 +#define LSM_INFO_TREE_SIZE 11 +#define LSM_INFO_FREELIST_SIZE 12 +#define LSM_INFO_COMPRESSION_ID 13 + + +/* +** CAPI: Opening and Closing Write Transactions +** +** These functions are used to open and close transactions and nested +** sub-transactions. +** +** The lsm_begin() function is used to open transactions and sub-transactions. +** A successful call to lsm_begin() ensures that there are at least iLevel +** nested transactions open. To open a top-level transaction, pass iLevel=1. +** To open a sub-transaction within the top-level transaction, iLevel=2. +** Passing iLevel=0 is a no-op. +** +** lsm_commit() is used to commit transactions and sub-transactions. A +** successful call to lsm_commit() ensures that there are at most iLevel +** nested transactions open. To commit a top-level transaction, pass iLevel=0. +** To commit all sub-transactions inside the main transaction, pass iLevel=1. +** +** Function lsm_rollback() is used to roll back transactions and +** sub-transactions. A successful call to lsm_rollback() restores the database +** to the state it was in when the iLevel'th nested sub-transaction (if any) +** was first opened. And then closes transactions to ensure that there are +** at most iLevel nested transactions open. Passing iLevel=0 rolls back and +** closes the top-level transaction. iLevel=1 also rolls back the top-level +** transaction, but leaves it open. iLevel=2 rolls back the sub-transaction +** nested directly inside the top-level transaction (and leaves it open). +*/ +int lsm_begin(lsm_db *pDb, int iLevel); +int lsm_commit(lsm_db *pDb, int iLevel); +int lsm_rollback(lsm_db *pDb, int iLevel); + +/* +** CAPI: Writing to a Database +** +** Write a new value into the database. If a value with a duplicate key +** already exists it is replaced. +*/ +int lsm_insert(lsm_db*, const void *pKey, int nKey, const void *pVal, int nVal); + +/* +** Delete a value from the database. No error is returned if the specified +** key value does not exist in the database. +*/ +int lsm_delete(lsm_db *, const void *pKey, int nKey); + +/* +** Delete all database entries with keys that are greater than (pKey1/nKey1) +** and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and +** (pKey2/nKey2) themselves, if they exist in the database, are not deleted. +** +** Return LSM_OK if successful, or an LSM error code otherwise. +*/ +int lsm_delete_range(lsm_db *, + const void *pKey1, int nKey1, const void *pKey2, int nKey2 +); + +/* +** CAPI: Explicit Database Work and Checkpointing +** +** This function is called by a thread to work on the database structure. +*/ +int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); + +int lsm_flush(lsm_db *pDb); + +/* +** Attempt to checkpoint the current database snapshot. Return an LSM +** error code if an error occurs or LSM_OK otherwise. +** +** If the current snapshot has already been checkpointed, calling this +** function is a no-op. In this case if pnKB is not NULL, *pnKB is +** set to 0. Or, if the current snapshot is successfully checkpointed +** by this function and pbKB is not NULL, *pnKB is set to the number +** of bytes written to the database file since the previous checkpoint +** (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). +*/ +int lsm_checkpoint(lsm_db *pDb, int *pnKB); + +/* +** CAPI: Opening and Closing Database Cursors +** +** Open and close a database cursor. +*/ +int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); +int lsm_csr_close(lsm_cursor *pCsr); + +/* +** CAPI: Positioning Database Cursors +** +** If the fourth parameter is LSM_SEEK_EQ, LSM_SEEK_GE or LSM_SEEK_LE, +** this function searches the database for an entry with key (pKey/nKey). +** If an error occurs, an LSM error code is returned. Otherwise, LSM_OK. +** +** If no error occurs and the requested key is present in the database, the +** cursor is left pointing to the entry with the specified key. Or, if the +** specified key is not present in the database the state of the cursor +** depends on the value passed as the final parameter, as follows: +** +** LSM_SEEK_EQ: +** The cursor is left at EOF (invalidated). A call to lsm_csr_valid() +** returns non-zero. +** +** LSM_SEEK_LE: +** The cursor is left pointing to the largest key in the database that +** is smaller than (pKey/nKey). If the database contains no keys smaller +** than (pKey/nKey), the cursor is left at EOF. +** +** LSM_SEEK_GE: +** The cursor is left pointing to the smallest key in the database that +** is larger than (pKey/nKey). If the database contains no keys larger +** than (pKey/nKey), the cursor is left at EOF. +** +** If the fourth parameter is LSM_SEEK_LEFAST, this function searches the +** database in a similar manner to LSM_SEEK_LE, with two differences: +** +**
  1. Even if a key can be found (the cursor is not left at EOF), the +** lsm_csr_value() function may not be used (attempts to do so return +** LSM_MISUSE). +** +**
  2. The key that the cursor is left pointing to may be one that has +** been recently deleted from the database. In this case it is +** guaranteed that the returned key is larger than any key currently +** in the database that is less than or equal to (pKey/nKey). +**
+** +** LSM_SEEK_LEFAST requests are intended to be used to allocate database +** keys. +*/ +int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek); + +int lsm_csr_first(lsm_cursor *pCsr); +int lsm_csr_last(lsm_cursor *pCsr); + +/* +** Advance the specified cursor to the next or previous key in the database. +** Return LSM_OK if successful, or an LSM error code otherwise. +** +** Functions lsm_csr_seek(), lsm_csr_first() and lsm_csr_last() are "seek" +** functions. Whether or not lsm_csr_next and lsm_csr_prev may be called +** successfully also depends on the most recent seek function called on +** the cursor. Specifically: +** +**
    +**
  • At least one seek function must have been called on the cursor. +**
  • To call lsm_csr_next(), the most recent call to a seek function must +** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying +** LSM_SEEK_GE. +**
  • To call lsm_csr_prev(), the most recent call to a seek function must +** have been either lsm_csr_last() or a call to lsm_csr_seek() specifying +** LSM_SEEK_LE. +**
+** +** Otherwise, if the above conditions are not met when lsm_csr_next or +** lsm_csr_prev is called, LSM_MISUSE is returned and the cursor position +** remains unchanged. +*/ +int lsm_csr_next(lsm_cursor *pCsr); +int lsm_csr_prev(lsm_cursor *pCsr); + +/* +** Values that may be passed as the fourth argument to lsm_csr_seek(). +*/ +#define LSM_SEEK_LEFAST -2 +#define LSM_SEEK_LE -1 +#define LSM_SEEK_EQ 0 +#define LSM_SEEK_GE 1 + +/* +** CAPI: Extracting Data From Database Cursors +** +** Retrieve data from a database cursor. +*/ +int lsm_csr_valid(lsm_cursor *pCsr); +int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey); +int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal); + +/* +** If no error occurs, this function compares the database key passed via +** the pKey/nKey arguments with the key that the cursor passed as the first +** argument currently points to. If the cursors key is less than, equal to +** or greater than pKey/nKey, *piRes is set to less than, equal to or greater +** than zero before returning. LSM_OK is returned in this case. +** +** Or, if an error occurs, an LSM error code is returned and the final +** value of *piRes is undefined. If the cursor does not point to a valid +** key when this function is called, LSM_MISUSE is returned. +*/ +int lsm_csr_cmp(lsm_cursor *pCsr, const void *pKey, int nKey, int *piRes); + +/* +** CAPI: Change these!! +** +** Configure a callback to which debugging and other messages should +** be directed. Only useful for debugging lsm. +*/ +void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); + +/* +** Configure a callback that is invoked if the database connection ever +** writes to the database file. +*/ +void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); + +/* ENDOFAPI */ +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif +#endif /* ifndef _LSM_H */ diff --git a/ext/lsm1/lsmInt.h b/ext/lsm1/lsmInt.h new file mode 100644 index 0000000000000000000000000000000000000000..5060366d0dcb288aba62e5f4a29ff0afc22eb151 --- /dev/null +++ b/ext/lsm1/lsmInt.h @@ -0,0 +1,997 @@ +/* +** 2011-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Internal structure definitions for the LSM module. +*/ +#ifndef _LSM_INT_H +#define _LSM_INT_H + +#include "lsm.h" +#include +#include + +#include +#include +#include +#include + +#ifdef _WIN32 +# ifdef _MSC_VER +# define snprintf _snprintf +# endif +#else +# include +#endif + +#ifdef NDEBUG +# ifdef LSM_DEBUG_EXPENSIVE +# undef LSM_DEBUG_EXPENSIVE +# endif +# ifdef LSM_DEBUG +# undef LSM_DEBUG +# endif +#else +# ifndef LSM_DEBUG +# define LSM_DEBUG +# endif +#endif + +/* #define LSM_DEBUG_EXPENSIVE 1 */ + +/* +** Default values for various data structure parameters. These may be +** overridden by calls to lsm_config(). +*/ +#define LSM_DFLT_PAGE_SIZE (4 * 1024) +#define LSM_DFLT_BLOCK_SIZE (1 * 1024 * 1024) +#define LSM_DFLT_AUTOFLUSH (1 * 1024 * 1024) +#define LSM_DFLT_AUTOCHECKPOINT (i64)(2 * 1024 * 1024) +#define LSM_DFLT_AUTOWORK 1 +#define LSM_DFLT_LOG_SIZE (128*1024) +#define LSM_DFLT_AUTOMERGE 4 +#define LSM_DFLT_SAFETY LSM_SAFETY_NORMAL +#define LSM_DFLT_MMAP (LSM_IS_64_BIT ? 1 : 32768) +#define LSM_DFLT_MULTIPLE_PROCESSES 1 +#define LSM_DFLT_USE_LOG 1 + +/* Initial values for log file checksums. These are only used if the +** database file does not contain a valid checkpoint. */ +#define LSM_CKSUM0_INIT 42 +#define LSM_CKSUM1_INIT 42 + +/* "mmap" mode is currently only used in environments with 64-bit address +** spaces. The following macro is used to test for this. */ +#define LSM_IS_64_BIT (sizeof(void*)==8) + +#define LSM_AUTOWORK_QUANT 32 + +typedef struct Database Database; +typedef struct DbLog DbLog; +typedef struct FileSystem FileSystem; +typedef struct Freelist Freelist; +typedef struct FreelistEntry FreelistEntry; +typedef struct Level Level; +typedef struct LogMark LogMark; +typedef struct LogRegion LogRegion; +typedef struct LogWriter LogWriter; +typedef struct LsmString LsmString; +typedef struct Mempool Mempool; +typedef struct Merge Merge; +typedef struct MergeInput MergeInput; +typedef struct MetaPage MetaPage; +typedef struct MultiCursor MultiCursor; +typedef struct Page Page; +typedef struct Redirect Redirect; +typedef struct Segment Segment; +typedef struct SegmentMerger SegmentMerger; +typedef struct ShmChunk ShmChunk; +typedef struct ShmHeader ShmHeader; +typedef struct ShmReader ShmReader; +typedef struct Snapshot Snapshot; +typedef struct TransMark TransMark; +typedef struct Tree Tree; +typedef struct TreeCursor TreeCursor; +typedef struct TreeHeader TreeHeader; +typedef struct TreeMark TreeMark; +typedef struct TreeRoot TreeRoot; + +#ifndef _SQLITEINT_H_ +typedef unsigned char u8; +typedef unsigned short int u16; +typedef unsigned int u32; +typedef lsm_i64 i64; +typedef unsigned long long int u64; +#endif + +/* A page number is a 64-bit integer. */ +typedef i64 LsmPgno; + +#ifdef LSM_DEBUG +int lsmErrorBkpt(int); +#else +# define lsmErrorBkpt(x) (x) +#endif + +#define LSM_PROTOCOL_BKPT lsmErrorBkpt(LSM_PROTOCOL) +#define LSM_IOERR_BKPT lsmErrorBkpt(LSM_IOERR) +#define LSM_NOMEM_BKPT lsmErrorBkpt(LSM_NOMEM) +#define LSM_CORRUPT_BKPT lsmErrorBkpt(LSM_CORRUPT) +#define LSM_MISUSE_BKPT lsmErrorBkpt(LSM_MISUSE) + +#define unused_parameter(x) (void)(x) +#define array_size(x) (sizeof(x)/sizeof(x[0])) + + +/* The size of each shared-memory chunk */ +#define LSM_SHM_CHUNK_SIZE (32*1024) + +/* The number of bytes reserved at the start of each shm chunk for MM. */ +#define LSM_SHM_CHUNK_HDR (sizeof(ShmChunk)) + +/* The number of available read locks. */ +#define LSM_LOCK_NREADER 6 + +/* The number of available read-write client locks. */ +#define LSM_LOCK_NRWCLIENT 16 + +/* Lock definitions. +*/ +#define LSM_LOCK_DMS1 1 /* Serialize connect/disconnect ops */ +#define LSM_LOCK_DMS2 2 /* Read-write connections */ +#define LSM_LOCK_DMS3 3 /* Read-only connections */ +#define LSM_LOCK_WRITER 4 +#define LSM_LOCK_WORKER 5 +#define LSM_LOCK_CHECKPOINTER 6 +#define LSM_LOCK_ROTRANS 7 +#define LSM_LOCK_READER(i) ((i) + LSM_LOCK_ROTRANS + 1) +#define LSM_LOCK_RWCLIENT(i) ((i) + LSM_LOCK_READER(LSM_LOCK_NREADER)) + +#define LSM_N_LOCK LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT) + +/* +** Meta-page size and usable size. +*/ +#define LSM_META_PAGE_SIZE 4096 + +#define LSM_META_RW_PAGE_SIZE (LSM_META_PAGE_SIZE - LSM_N_LOCK) + +/* +** Hard limit on the number of free-list entries that may be stored in +** a checkpoint (the remainder are stored as a system record in the LSM). +** See also LSM_CONFIG_MAX_FREELIST. +*/ +#define LSM_MAX_FREELIST_ENTRIES 24 + +#define LSM_MAX_BLOCK_REDIRECTS 16 + +#define LSM_ATTEMPTS_BEFORE_PROTOCOL 10000 + + +/* +** Each entry stored in the LSM (or in-memory tree structure) has an +** associated mask of the following flags. +*/ +#define LSM_START_DELETE 0x01 /* Start of open-ended delete range */ +#define LSM_END_DELETE 0x02 /* End of open-ended delete range */ +#define LSM_POINT_DELETE 0x04 /* Delete this key */ +#define LSM_INSERT 0x08 /* Insert this key and value */ +#define LSM_SEPARATOR 0x10 /* True if entry is separator key only */ +#define LSM_SYSTEMKEY 0x20 /* True if entry is a system key (FREELIST) */ + +#define LSM_CONTIGUOUS 0x40 /* Used in lsm_tree.c */ + +/* +** A string that can grow by appending. +*/ +struct LsmString { + lsm_env *pEnv; /* Run-time environment */ + int n; /* Size of string. -1 indicates error */ + int nAlloc; /* Space allocated for z[] */ + char *z; /* The string content */ +}; + +typedef struct LsmFile LsmFile; +struct LsmFile { + lsm_file *pFile; + LsmFile *pNext; +}; + +/* +** An instance of the following type is used to store an ordered list of +** u32 values. +** +** Note: This is a place-holder implementation. It should be replaced by +** a version that avoids making a single large allocation when the array +** contains a large number of values. For this reason, the internals of +** this object should only manipulated by the intArrayXXX() functions in +** lsm_tree.c. +*/ +typedef struct IntArray IntArray; +struct IntArray { + int nAlloc; + int nArray; + u32 *aArray; +}; + +struct Redirect { + int n; /* Number of redirects */ + struct RedirectEntry { + int iFrom; + int iTo; + } *a; +}; + +/* +** An instance of this structure represents a point in the history of the +** tree structure to roll back to. Refer to comments in lsm_tree.c for +** details. +*/ +struct TreeMark { + u32 iRoot; /* Offset of root node in shm file */ + u32 nHeight; /* Current height of tree structure */ + u32 iWrite; /* Write offset in shm file */ + u32 nChunk; /* Number of chunks in shared-memory file */ + u32 iFirst; /* First chunk in linked list */ + u32 iNextShmid; /* Next id to allocate */ + int iRollback; /* Index in lsm->rollback to revert to */ +}; + +/* +** An instance of this structure represents a point in the database log. +*/ +struct LogMark { + i64 iOff; /* Offset into log (see lsm_log.c) */ + int nBuf; /* Size of in-memory buffer here */ + u8 aBuf[8]; /* Bytes of content in aBuf[] */ + u32 cksum0; /* Checksum 0 at offset (iOff-nBuf) */ + u32 cksum1; /* Checksum 1 at offset (iOff-nBuf) */ +}; + +struct TransMark { + TreeMark tree; + LogMark log; +}; + +/* +** A structure that defines the start and end offsets of a region in the +** log file. The size of the region in bytes is (iEnd - iStart), so if +** iEnd==iStart the region is zero bytes in size. +*/ +struct LogRegion { + i64 iStart; /* Start of region in log file */ + i64 iEnd; /* End of region in log file */ +}; + +struct DbLog { + u32 cksum0; /* Checksum 0 at offset iOff */ + u32 cksum1; /* Checksum 1 at offset iOff */ + i64 iSnapshotId; /* Log space has been reclaimed to this ss */ + LogRegion aRegion[3]; /* Log file regions (see docs in lsm_log.c) */ +}; + +struct TreeRoot { + u32 iRoot; + u32 nHeight; + u32 nByte; /* Total size of this tree in bytes */ + u32 iTransId; +}; + +/* +** Tree header structure. +*/ +struct TreeHeader { + u32 iUsedShmid; /* Id of first shm chunk used by this tree */ + u32 iNextShmid; /* Shm-id of next chunk allocated */ + u32 iFirst; /* Chunk number of smallest shm-id */ + u32 nChunk; /* Number of chunks in shared-memory file */ + TreeRoot root; /* Root and height of current tree */ + u32 iWrite; /* Write offset in shm file */ + TreeRoot oldroot; /* Root and height of the previous tree */ + u32 iOldShmid; /* Last shm-id used by previous tree */ + u32 iUsrVersion; /* get/set_user_version() value */ + i64 iOldLog; /* Log offset associated with old tree */ + u32 oldcksum0; + u32 oldcksum1; + DbLog log; /* Current layout of log file */ + u32 aCksum[2]; /* Checksums 1 and 2. */ +}; + +/* +** Database handle structure. +** +** mLock: +** A bitmask representing the locks currently held by the connection. +** An LSM database supports N distinct locks, where N is some number less +** than or equal to 32. Locks are numbered starting from 1 (see the +** definitions for LSM_LOCK_WRITER and co.). +** +** The least significant 32-bits in mLock represent EXCLUSIVE locks. The +** most significant are SHARED locks. So, if a connection holds a SHARED +** lock on lock region iLock, then the following is true: +** +** (mLock & ((iLock+32-1) << 1)) +** +** Or for an EXCLUSIVE lock: +** +** (mLock & ((iLock-1) << 1)) +** +** pCsr: +** Points to the head of a linked list that contains all currently open +** cursors. Once this list becomes empty, the user has no outstanding +** cursors and the database handle can be successfully closed. +** +** pCsrCache: +** This list contains cursor objects that have been closed using +** lsm_csr_close(). Each time a cursor is closed, it is shifted from +** the pCsr list to this list. When a new cursor is opened, this list +** is inspected to see if there exists a cursor object that can be +** reused. This is an optimization only. +*/ +struct lsm_db { + + /* Database handle configuration */ + lsm_env *pEnv; /* runtime environment */ + int (*xCmp)(void *, int, void *, int); /* Compare function */ + + /* Values configured by calls to lsm_config */ + int eSafety; /* LSM_SAFETY_OFF, NORMAL or FULL */ + int bAutowork; /* Configured by LSM_CONFIG_AUTOWORK */ + int nTreeLimit; /* Configured by LSM_CONFIG_AUTOFLUSH */ + int nMerge; /* Configured by LSM_CONFIG_AUTOMERGE */ + int bUseLog; /* Configured by LSM_CONFIG_USE_LOG */ + int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ + int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ + int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ + int iMmap; /* Configured by LSM_CONFIG_MMAP */ + i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ + int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ + int bReadonly; /* Configured by LSM_CONFIG_READONLY */ + lsm_compress compress; /* Compression callbacks */ + lsm_compress_factory factory; /* Compression callback factory */ + + /* Sub-system handles */ + FileSystem *pFS; /* On-disk portion of database */ + Database *pDatabase; /* Database shared data */ + + int iRwclient; /* Read-write client lock held (-1 == none) */ + + /* Client transaction context */ + Snapshot *pClient; /* Client snapshot */ + int iReader; /* Read lock held (-1 == unlocked) */ + int bRoTrans; /* True if a read-only db trans is open */ + MultiCursor *pCsr; /* List of all open cursors */ + LogWriter *pLogWriter; /* Context for writing to the log file */ + int nTransOpen; /* Number of opened write transactions */ + int nTransAlloc; /* Allocated size of aTrans[] array */ + TransMark *aTrans; /* Array of marks for transaction rollback */ + IntArray rollback; /* List of tree-nodes to roll back */ + int bDiscardOld; /* True if lsmTreeDiscardOld() was called */ + + MultiCursor *pCsrCache; /* List of all closed cursors */ + + /* Worker context */ + Snapshot *pWorker; /* Worker snapshot (or NULL) */ + Freelist *pFreelist; /* See sortedNewToplevel() */ + int bUseFreelist; /* True to use pFreelist */ + int bIncrMerge; /* True if currently doing a merge */ + + int bInFactory; /* True if within factory.xFactory() */ + + /* Debugging message callback */ + void (*xLog)(void *, int, const char *); + void *pLogCtx; + + /* Work done notification callback */ + void (*xWork)(lsm_db *, void *); + void *pWorkCtx; + + u64 mLock; /* Mask of current locks. See lsmShmLock(). */ + lsm_db *pNext; /* Next connection to same database */ + + int nShm; /* Size of apShm[] array */ + void **apShm; /* Shared memory chunks */ + ShmHeader *pShmhdr; /* Live shared-memory header */ + TreeHeader treehdr; /* Local copy of tree-header */ + u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)]; +}; + +struct Segment { + LsmPgno iFirst; /* First page of this run */ + LsmPgno iLastPg; /* Last page of this run */ + LsmPgno iRoot; /* Root page number (if any) */ + LsmPgno nSize; /* Size of this run in pages */ + + Redirect *pRedirect; /* Block redirects (or NULL) */ +}; + +/* +** iSplitTopic/pSplitKey/nSplitKey: +** If nRight>0, this buffer contains a copy of the largest key that has +** already been written to the left-hand-side of the level. +*/ +struct Level { + Segment lhs; /* Left-hand (main) segment */ + int nRight; /* Size of apRight[] array */ + Segment *aRhs; /* Old segments being merged into this */ + int iSplitTopic; /* Split key topic (if nRight>0) */ + void *pSplitKey; /* Pointer to split-key (if nRight>0) */ + int nSplitKey; /* Number of bytes in split-key */ + + u16 iAge; /* Number of times data has been written */ + u16 flags; /* Mask of LEVEL_XXX bits */ + Merge *pMerge; /* Merge operation currently underway */ + Level *pNext; /* Next level in tree */ +}; + +/* +** The Level.flags field is set to a combination of the following bits. +** +** LEVEL_FREELIST_ONLY: +** Set if the level consists entirely of free-list entries. +** +** LEVEL_INCOMPLETE: +** This is set while a new toplevel level is being constructed. It is +** never set for any level other than a new toplevel. +*/ +#define LEVEL_FREELIST_ONLY 0x0001 +#define LEVEL_INCOMPLETE 0x0002 + + +/* +** A structure describing an ongoing merge. There is an instance of this +** structure for every Level currently undergoing a merge in the worker +** snapshot. +** +** It is assumed that code that uses an instance of this structure has +** access to the associated Level struct. +** +** iOutputOff: +** The byte offset to write to next within the last page of the +** output segment. +*/ +struct MergeInput { + LsmPgno iPg; /* Page on which next input is stored */ + int iCell; /* Cell containing next input to merge */ +}; +struct Merge { + int nInput; /* Number of input runs being merged */ + MergeInput *aInput; /* Array nInput entries in size */ + MergeInput splitkey; /* Location in file of current splitkey */ + int nSkip; /* Number of separators entries to skip */ + int iOutputOff; /* Write offset on output page */ + LsmPgno iCurrentPtr; /* Current pointer value */ +}; + +/* +** The first argument to this macro is a pointer to a Segment structure. +** Returns true if the structure instance indicates that the separators +** array is valid. +*/ +#define segmentHasSeparators(pSegment) ((pSegment)->sep.iFirst>0) + +/* +** The values that accompany the lock held by a database reader. +*/ +struct ShmReader { + u32 iTreeId; + i64 iLsmId; +}; + +/* +** An instance of this structure is stored in the first shared-memory +** page. The shared-memory header. +** +** bWriter: +** Immediately after opening a write transaction taking the WRITER lock, +** each writer client sets this flag. It is cleared right before the +** WRITER lock is relinquished. If a subsequent writer finds that this +** flag is already set when a write transaction is opened, this indicates +** that a previous writer failed mid-transaction. +** +** iMetaPage: +** If the database file does not contain a valid, synced, checkpoint, this +** value is set to 0. Otherwise, it is set to the meta-page number that +** contains the most recently written checkpoint (either 1 or 2). +** +** hdr1, hdr2: +** The two copies of the in-memory tree header. Two copies are required +** in case a writer fails while updating one of them. +*/ +struct ShmHeader { + u32 aSnap1[LSM_META_PAGE_SIZE / 4]; + u32 aSnap2[LSM_META_PAGE_SIZE / 4]; + u32 bWriter; + u32 iMetaPage; + TreeHeader hdr1; + TreeHeader hdr2; + ShmReader aReader[LSM_LOCK_NREADER]; +}; + +/* +** An instance of this structure is stored at the start of each shared-memory +** chunk except the first (which is the header chunk - see above). +*/ +struct ShmChunk { + u32 iShmid; + u32 iNext; +}; + +/* +** Maximum number of shared-memory chunks allowed in the *-shm file. Since +** each shared-memory chunk is 32KB in size, this is a theoretical limit only. +*/ +#define LSM_MAX_SHMCHUNKS (1<<30) + +/* Return true if shm-sequence "a" is larger than or equal to "b" */ +#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < LSM_MAX_SHMCHUNKS) + +#define LSM_APPLIST_SZ 4 + +/* +** An instance of the following structure stores the in-memory part of +** the current free block list. This structure is to the free block list +** as the in-memory tree is to the users database content. The contents +** of the free block list is found by merging the in-memory components +** with those stored in the LSM, just as the contents of the database is +** found by merging the in-memory tree with the user data entries in the +** LSM. +** +** Each FreelistEntry structure in the array represents either an insert +** or delete operation on the free-list. For deletes, the FreelistEntry.iId +** field is set to -1. For inserts, it is set to zero or greater. +** +** The array of FreelistEntry structures is always sorted in order of +** block number (ascending). +** +** When the in-memory free block list is written into the LSM, each insert +** operation is written separately. The entry key is the bitwise inverse +** of the block number as a 32-bit big-endian integer. This is done so that +** the entries in the LSM are sorted in descending order of block id. +** The associated value is the snapshot id, formated as a varint. +*/ +struct Freelist { + FreelistEntry *aEntry; /* Free list entries */ + int nEntry; /* Number of valid slots in aEntry[] */ + int nAlloc; /* Allocated size of aEntry[] */ +}; +struct FreelistEntry { + u32 iBlk; /* Block number */ + i64 iId; /* Largest snapshot id to use this block */ +}; + +/* +** A snapshot of a database. A snapshot contains all the information required +** to read or write a database file on disk. See the description of struct +** Database below for futher details. +*/ +struct Snapshot { + Database *pDatabase; /* Database this snapshot belongs to */ + u32 iCmpId; /* Id of compression scheme */ + Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ + i64 iId; /* Snapshot id */ + i64 iLogOff; /* Log file offset */ + Redirect redirect; /* Block redirection array */ + + /* Used by worker snapshots only */ + int nBlock; /* Number of blocks in database file */ + LsmPgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ + Freelist freelist; /* Free block list */ + u32 nWrite; /* Total number of pages written to disk */ +}; +#define LSM_INITIAL_SNAPSHOT_ID 11 + +/* +** Functions from file "lsm_ckpt.c". +*/ +int lsmCheckpointWrite(lsm_db *, u32 *); +int lsmCheckpointLevels(lsm_db *, int, void **, int *); +int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal); + +int lsmCheckpointRecover(lsm_db *); +int lsmCheckpointDeserialize(lsm_db *, int, u32 *, Snapshot **); + +int lsmCheckpointLoadWorker(lsm_db *pDb); +int lsmCheckpointStore(lsm_db *pDb, int); + +int lsmCheckpointLoad(lsm_db *pDb, int *); +int lsmCheckpointLoadOk(lsm_db *pDb, int); +int lsmCheckpointClientCacheOk(lsm_db *); + +u32 lsmCheckpointNBlock(u32 *); +i64 lsmCheckpointId(u32 *, int); +u32 lsmCheckpointNWrite(u32 *, int); +i64 lsmCheckpointLogOffset(u32 *); +int lsmCheckpointPgsz(u32 *); +int lsmCheckpointBlksz(u32 *); +void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); +void lsmCheckpointZeroLogoffset(lsm_db *); + +int lsmCheckpointSaveWorker(lsm_db *pDb, int); +int lsmDatabaseFull(lsm_db *pDb); +int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite); + +int lsmCheckpointSize(lsm_db *db, int *pnByte); + +int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId); + +/* +** Functions from file "lsm_tree.c". +*/ +int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); +void lsmTreeRelease(lsm_env *, Tree *); +int lsmTreeInit(lsm_db *); +int lsmTreeRepair(lsm_db *); + +void lsmTreeMakeOld(lsm_db *pDb); +void lsmTreeDiscardOld(lsm_db *pDb); +int lsmTreeHasOld(lsm_db *pDb); + +int lsmTreeSize(lsm_db *); +int lsmTreeEndTransaction(lsm_db *pDb, int bCommit); +int lsmTreeLoadHeader(lsm_db *pDb, int *); +int lsmTreeLoadHeaderOk(lsm_db *, int); + +int lsmTreeInsert(lsm_db *pDb, void *pKey, int nKey, void *pVal, int nVal); +int lsmTreeDelete(lsm_db *db, void *pKey1, int nKey1, void *pKey2, int nKey2); +void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark); +void lsmTreeMark(lsm_db *pDb, TreeMark *pMark); + +int lsmTreeCursorNew(lsm_db *pDb, int, TreeCursor **); +void lsmTreeCursorDestroy(TreeCursor *); + +int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes); +int lsmTreeCursorNext(TreeCursor *pCsr); +int lsmTreeCursorPrev(TreeCursor *pCsr); +int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast); +void lsmTreeCursorReset(TreeCursor *pCsr); +int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey); +int lsmTreeCursorFlags(TreeCursor *pCsr); +int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal); +int lsmTreeCursorValid(TreeCursor *pCsr); +int lsmTreeCursorSave(TreeCursor *pCsr); + +void lsmFlagsToString(int flags, char *zFlags); + +/* +** Functions from file "mem.c". +*/ +void *lsmMalloc(lsm_env*, size_t); +void lsmFree(lsm_env*, void *); +void *lsmRealloc(lsm_env*, void *, size_t); +void *lsmReallocOrFree(lsm_env*, void *, size_t); +void *lsmReallocOrFreeRc(lsm_env *, void *, size_t, int *); + +void *lsmMallocZeroRc(lsm_env*, size_t, int *); +void *lsmMallocRc(lsm_env*, size_t, int *); + +void *lsmMallocZero(lsm_env *pEnv, size_t); +char *lsmMallocStrdup(lsm_env *pEnv, const char *); + +/* +** Functions from file "lsm_mutex.c". +*/ +int lsmMutexStatic(lsm_env*, int, lsm_mutex **); +int lsmMutexNew(lsm_env*, lsm_mutex **); +void lsmMutexDel(lsm_env*, lsm_mutex *); +void lsmMutexEnter(lsm_env*, lsm_mutex *); +int lsmMutexTry(lsm_env*, lsm_mutex *); +void lsmMutexLeave(lsm_env*, lsm_mutex *); + +#ifndef NDEBUG +int lsmMutexHeld(lsm_env *, lsm_mutex *); +int lsmMutexNotHeld(lsm_env *, lsm_mutex *); +#endif + +/************************************************************************** +** Start of functions from "lsm_file.c". +*/ +int lsmFsOpen(lsm_db *, const char *, int); +int lsmFsOpenLog(lsm_db *, int *); +void lsmFsCloseLog(lsm_db *); +void lsmFsClose(FileSystem *); + +int lsmFsUnmap(FileSystem *); + +int lsmFsConfigure(lsm_db *db); + +int lsmFsBlockSize(FileSystem *); +void lsmFsSetBlockSize(FileSystem *, int); +int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom); + +int lsmFsPageSize(FileSystem *); +void lsmFsSetPageSize(FileSystem *, int); + +int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); + +/* Creating, populating, gobbling and deleting sorted runs. */ +void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int); +int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); +int lsmFsSortedFinish(FileSystem *, Segment *); +int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); +int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); + +/* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ +lsm_env *lsmFsEnv(FileSystem *); +lsm_env *lsmPageEnv(Page *); +FileSystem *lsmPageFS(Page *); + +int lsmFsSectorSize(FileSystem *); + +void lsmSortedSplitkey(lsm_db *, Level *, int *); + +/* Reading sorted run content. */ +int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); +int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **); +int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); + +u8 *lsmFsPageData(Page *, int *); +int lsmFsPageRelease(Page *); +int lsmFsPagePersist(Page *); +void lsmFsPageRef(Page *); +LsmPgno lsmFsPageNumber(Page *); + +int lsmFsNRead(FileSystem *); +int lsmFsNWrite(FileSystem *); + +int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **); +int lsmFsMetaPageRelease(MetaPage *); +u8 *lsmFsMetaPageData(MetaPage *, int *); + +#ifdef LSM_DEBUG +int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg); +int lsmFsIntegrityCheck(lsm_db *); +#endif + +LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno); + +int lsmFsPageWritable(Page *); + +/* Functions to read, write and sync the log file. */ +int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); +int lsmFsSyncLog(FileSystem *pFS); +int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); +int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); +int lsmFsTruncateDb(FileSystem *pFS, i64 nByte); +int lsmFsCloseAndDeleteLog(FileSystem *pFS); + +LsmFile *lsmFsDeferClose(FileSystem *pFS); + +/* And to sync the db file */ +int lsmFsSyncDb(FileSystem *, int); + +void lsmFsFlushWaiting(FileSystem *, int *); + +/* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ +int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz); +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut); +int lsmConfigMmap(lsm_db *pDb, int *piParam); + +int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **); +int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); +int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); +int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int); + +int lsmEnvShmMap(lsm_env *, lsm_file *, int, int, void **); +void lsmEnvShmBarrier(lsm_env *); +void lsmEnvShmUnmap(lsm_env *, lsm_file *, int); + +void lsmEnvSleep(lsm_env *, int); + +int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); + +int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *); + +void lsmFsPurgeCache(FileSystem *); + +/* +** End of functions from "lsm_file.c". +**************************************************************************/ + +/* +** Functions from file "lsm_sorted.c". +*/ +int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **); +void lsmSortedCleanup(lsm_db *); +int lsmSortedAutoWork(lsm_db *, int nUnit); + +int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); + +int lsmSaveWorker(lsm_db *, int); + +int lsmFlushTreeToDisk(lsm_db *pDb); + +void lsmSortedRemap(lsm_db *pDb); + +void lsmSortedFreeLevel(lsm_env *pEnv, Level *); + +int lsmSortedAdvanceAll(lsm_db *pDb); + +int lsmSortedLoadMerge(lsm_db *, Level *, u32 *, int *); +int lsmSortedLoadFreelist(lsm_db *pDb, void **, int *); + +void *lsmSortedSplitKey(Level *pLevel, int *pnByte); + +void lsmSortedSaveTreeCursors(lsm_db *); + +int lsmMCursorNew(lsm_db *, MultiCursor **); +void lsmMCursorClose(MultiCursor *, int); +int lsmMCursorSeek(MultiCursor *, int, void *, int , int); +int lsmMCursorFirst(MultiCursor *); +int lsmMCursorPrev(MultiCursor *); +int lsmMCursorLast(MultiCursor *); +int lsmMCursorValid(MultiCursor *); +int lsmMCursorNext(MultiCursor *); +int lsmMCursorKey(MultiCursor *, void **, int *); +int lsmMCursorValue(MultiCursor *, void **, int *); +int lsmMCursorType(MultiCursor *, int *); +lsm_db *lsmMCursorDb(MultiCursor *); +void lsmMCursorFreeCache(lsm_db *); + +int lsmSaveCursors(lsm_db *pDb); +int lsmRestoreCursors(lsm_db *pDb); + +void lsmSortedDumpStructure(lsm_db *pDb, Snapshot *, int, int, const char *); +void lsmFsDumpBlocklists(lsm_db *); + +void lsmSortedExpandBtreePage(Page *pPg, int nOrig); + +void lsmPutU32(u8 *, u32); +u32 lsmGetU32(u8 *); +u64 lsmGetU64(u8 *); + +/* +** Functions from "lsm_varint.c". +*/ +int lsmVarintPut32(u8 *, int); +int lsmVarintGet32(u8 *, int *); +int lsmVarintPut64(u8 *aData, i64 iVal); +int lsmVarintGet64(const u8 *aData, i64 *piVal); + +int lsmVarintLen64(i64); + +int lsmVarintLen32(int); +int lsmVarintSize(u8 c); + +/* +** Functions from file "main.c". +*/ +void lsmLogMessage(lsm_db *, int, const char *, ...); +int lsmInfoFreelist(lsm_db *pDb, char **pzOut); + +/* +** Functions from file "lsm_log.c". +*/ +int lsmLogBegin(lsm_db *pDb); +int lsmLogWrite(lsm_db *, int, void *, int, void *, int); +int lsmLogCommit(lsm_db *); +void lsmLogEnd(lsm_db *pDb, int bCommit); +void lsmLogTell(lsm_db *, LogMark *); +void lsmLogSeek(lsm_db *, LogMark *); +void lsmLogClose(lsm_db *); + +int lsmLogRecover(lsm_db *); +int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); + +/* Valid values for the second argument to lsmLogWrite(). */ +#define LSM_WRITE 0x06 +#define LSM_DELETE 0x08 +#define LSM_DRANGE 0x0A + +/************************************************************************** +** Functions from file "lsm_shared.c". +*/ + +int lsmDbDatabaseConnect(lsm_db*, const char *); +void lsmDbDatabaseRelease(lsm_db *); + +int lsmBeginReadTrans(lsm_db *); +int lsmBeginWriteTrans(lsm_db *); +int lsmBeginFlush(lsm_db *); + +int lsmDetectRoTrans(lsm_db *db, int *); +int lsmBeginRoTrans(lsm_db *db); + +int lsmBeginWork(lsm_db *); +void lsmFinishWork(lsm_db *, int, int *); + +int lsmFinishRecovery(lsm_db *); +void lsmFinishReadTrans(lsm_db *); +int lsmFinishWriteTrans(lsm_db *, int); +int lsmFinishFlush(lsm_db *, int); + +int lsmSnapshotSetFreelist(lsm_db *, int *, int); + +Snapshot *lsmDbSnapshotClient(lsm_db *); +Snapshot *lsmDbSnapshotWorker(lsm_db *); + +void lsmSnapshotSetCkptid(Snapshot *, i64); + +Level *lsmDbSnapshotLevel(Snapshot *); +void lsmDbSnapshotSetLevel(Snapshot *, Level *); + +void lsmDbRecoveryComplete(lsm_db *, int); + +int lsmBlockAllocate(lsm_db *, int, int *); +int lsmBlockFree(lsm_db *, int); +int lsmBlockRefree(lsm_db *, int); + +void lsmFreelistDeltaBegin(lsm_db *); +void lsmFreelistDeltaEnd(lsm_db *); +int lsmFreelistDelta(lsm_db *pDb); + +DbLog *lsmDatabaseLog(lsm_db *pDb); + +#ifdef LSM_DEBUG + int lsmHoldingClientMutex(lsm_db *pDb); + int lsmShmAssertLock(lsm_db *db, int iLock, int eOp); + int lsmShmAssertWorker(lsm_db *db); +#endif + +void lsmFreeSnapshot(lsm_env *, Snapshot *); + + +/* Candidate values for the 3rd argument to lsmShmLock() */ +#define LSM_LOCK_UNLOCK 0 +#define LSM_LOCK_SHARED 1 +#define LSM_LOCK_EXCL 2 + +int lsmShmCacheChunks(lsm_db *db, int nChunk); +int lsmShmLock(lsm_db *db, int iLock, int eOp, int bBlock); +int lsmShmTestLock(lsm_db *db, int iLock, int nLock, int eOp); +void lsmShmBarrier(lsm_db *db); + +#ifdef LSM_DEBUG +void lsmShmHasLock(lsm_db *db, int iLock, int eOp); +#else +# define lsmShmHasLock(x,y,z) +#endif + +int lsmReadlock(lsm_db *, i64 iLsm, u32 iShmMin, u32 iShmMax); + +int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse); +int lsmTreeInUse(lsm_db *db, u32 iLsmId, int *pbInUse); +int lsmFreelistAppend(lsm_env *pEnv, Freelist *p, int iBlk, i64 iId); + +int lsmDbMultiProc(lsm_db *); +void lsmDbDeferredClose(lsm_db *, lsm_file *, LsmFile *); +LsmFile *lsmDbRecycleFd(lsm_db *); + +int lsmWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); + +int lsmCheckCompressionId(lsm_db *, u32); + + +/************************************************************************** +** functions in lsm_str.c +*/ +void lsmStringInit(LsmString*, lsm_env *pEnv); +int lsmStringExtend(LsmString*, int); +int lsmStringAppend(LsmString*, const char *, int); +void lsmStringVAppendf(LsmString*, const char *zFormat, va_list, va_list); +void lsmStringAppendf(LsmString*, const char *zFormat, ...); +void lsmStringClear(LsmString*); +char *lsmMallocPrintf(lsm_env*, const char*, ...); +int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n); + +int lsmStrlen(const char *zName); + + + +/* +** Round up a number to the next larger multiple of 8. This is used +** to force 8-byte alignment on 64-bit architectures. +*/ +#define ROUND8(x) (((x)+7)&~7) + +#define LSM_MIN(x,y) ((x)>(y) ? (y) : (x)) +#define LSM_MAX(x,y) ((x)>(y) ? (x) : (y)) + +#endif diff --git a/ext/lsm1/lsm_ckpt.c b/ext/lsm1/lsm_ckpt.c new file mode 100644 index 0000000000000000000000000000000000000000..1c4f788ad654e32d3d071f4587a64d08f283f7b5 --- /dev/null +++ b/ext/lsm1/lsm_ckpt.c @@ -0,0 +1,1239 @@ +/* +** 2011-09-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code to read and write checkpoints. +** +** A checkpoint represents the database layout at a single point in time. +** It includes a log offset. When an existing database is opened, the +** current state is determined by reading the newest checkpoint and updating +** it with all committed transactions from the log that follow the specified +** offset. +*/ +#include "lsmInt.h" + +/* +** CHECKPOINT BLOB FORMAT: +** +** A checkpoint blob is a series of unsigned 32-bit integers stored in +** big-endian byte order. As follows: +** +** Checkpoint header (see the CKPT_HDR_XXX #defines): +** +** 1. The checkpoint id MSW. +** 2. The checkpoint id LSW. +** 3. The number of integer values in the entire checkpoint, including +** the two checksum values. +** 4. The compression scheme id. +** 5. The total number of blocks in the database. +** 6. The block size. +** 7. The number of levels. +** 8. The nominal database page size. +** 9. The number of pages (in total) written to the database file. +** +** Log pointer: +** +** 1. The log offset MSW. +** 2. The log offset LSW. +** 3. Log checksum 0. +** 4. Log checksum 1. +** +** Note that the "log offset" is not the literal byte offset. Instead, +** it is the byte offset multiplied by 2, with least significant bit +** toggled each time the log pointer value is changed. This is to make +** sure that this field changes each time the log pointer is updated, +** even if the log file itself is disabled. See lsmTreeMakeOld(). +** +** See ckptExportLog() and ckptImportLog(). +** +** Append points: +** +** 8 integers (4 * 64-bit page numbers). See ckptExportAppendlist(). +** +** For each level in the database, a level record. Formatted as follows: +** +** 0. Age of the level (least significant 16-bits). And flags mask (most +** significant 16-bits). +** 1. The number of right-hand segments (nRight, possibly 0), +** 2. Segment record for left-hand segment (8 integers defined below), +** 3. Segment record for each right-hand segment (8 integers defined below), +** 4. If nRight>0, The number of segments involved in the merge +** 5. if nRight>0, Current nSkip value (see Merge structure defn.), +** 6. For each segment in the merge: +** 5a. Page number of next cell to read during merge (this field +** is 64-bits - 2 integers) +** 5b. Cell number of next cell to read during merge +** 7. Page containing current split-key (64-bits - 2 integers). +** 8. Cell within page containing current split-key. +** 9. Current pointer value (64-bits - 2 integers). +** +** The block redirect array: +** +** 1. Number of redirections (maximum LSM_MAX_BLOCK_REDIRECTS). +** 2. For each redirection: +** a. "from" block number +** b. "to" block number +** +** The in-memory freelist entries. Each entry is either an insert or a +** delete. The in-memory freelist is to the free-block-list as the +** in-memory tree is to the users database content. +** +** 1. Number of free-list entries stored in checkpoint header. +** 2. Number of free blocks (in total). +** 3. Total number of blocks freed during database lifetime. +** 4. For each entry: +** 2a. Block number of free block. +** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, +** or the associated checkpoint id for an insert. +** +** The checksum: +** +** 1. Checksum value 1. +** 2. Checksum value 2. +** +** In the above, a segment record consists of the following four 64-bit +** fields (converted to 2 * u32 by storing the MSW followed by LSW): +** +** 1. First page of array, +** 2. Last page of array, +** 3. Root page of array (or 0), +** 4. Size of array in pages. +*/ + +/* +** LARGE NUMBERS OF LEVEL RECORDS: +** +** A limit on the number of rhs segments that may be present in the database +** file. Defining this limit ensures that all level records fit within +** the 4096 byte limit for checkpoint blobs. +** +** The number of right-hand-side segments in a database is counted as +** follows: +** +** * For each level in the database not undergoing a merge, add 1. +** +** * For each level in the database that is undergoing a merge, add +** the number of segments on the rhs of the level. +** +** A level record not undergoing a merge is 10 integers. A level record +** with nRhs rhs segments and (nRhs+1) input segments (i.e. including the +** separators from the next level) is (11*nRhs+20) integers. The maximum +** per right-hand-side level is therefore 21 integers. So the maximum +** size of all level records in a checkpoint is 21*40=820 integers. +** +** TODO: Before pointer values were changed from 32 to 64 bits, the above +** used to come to 420 bytes - leaving significant space for a free-list +** prefix. No more. To fix this, reduce the size of the level records in +** a db snapshot, and improve management of the free-list tail in +** lsm_sorted.c. +*/ +#define LSM_MAX_RHS_SEGMENTS 40 + +/* +** LARGE NUMBERS OF FREELIST ENTRIES: +** +** There is also a limit (LSM_MAX_FREELIST_ENTRIES - defined in lsmInt.h) +** on the number of free-list entries stored in a checkpoint. Since each +** free-list entry consists of 3 integers, the maximum free-list size is +** 3*100=300 integers. Combined with the limit on rhs segments defined +** above, this ensures that a checkpoint always fits within a 4096 byte +** meta page. +** +** If the database contains more than 100 free blocks, the "overflow" flag +** in the checkpoint header is set and the remainder are stored in the +** system FREELIST entry in the LSM (along with user data). The value +** accompanying the FREELIST key in the LSM is, like a checkpoint, an array +** of 32-bit big-endian integers. As follows: +** +** For each entry: +** a. Block number of free block. +** b. MSW of associated checkpoint id. +** c. LSW of associated checkpoint id. +** +** The number of entries is not required - it is implied by the size of the +** value blob containing the integer array. +** +** Note that the limit defined by LSM_MAX_FREELIST_ENTRIES is a hard limit. +** The actual value used may be configured using LSM_CONFIG_MAX_FREELIST. +*/ + +/* +** The argument to this macro must be of type u32. On a little-endian +** architecture, it returns the u32 value that results from interpreting +** the 4 bytes as a big-endian value. On a big-endian architecture, it +** returns the value that would be produced by intepreting the 4 bytes +** of the input value as a little-endian integer. +*/ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +static const int one = 1; +#define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) + +/* Sizes, in integers, of various parts of the checkpoint. */ +#define CKPT_HDR_SIZE 9 +#define CKPT_LOGPTR_SIZE 4 +#define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) + +/* A #define to describe each integer in the checkpoint header. */ +#define CKPT_HDR_ID_MSW 0 +#define CKPT_HDR_ID_LSW 1 +#define CKPT_HDR_NCKPT 2 +#define CKPT_HDR_CMPID 3 +#define CKPT_HDR_NBLOCK 4 +#define CKPT_HDR_BLKSZ 5 +#define CKPT_HDR_NLEVEL 6 +#define CKPT_HDR_PGSZ 7 +#define CKPT_HDR_NWRITE 8 + +#define CKPT_HDR_LO_MSW 9 +#define CKPT_HDR_LO_LSW 10 +#define CKPT_HDR_LO_CKSUM1 11 +#define CKPT_HDR_LO_CKSUM2 12 + +typedef struct CkptBuffer CkptBuffer; + +/* +** Dynamic buffer used to accumulate data for a checkpoint. +*/ +struct CkptBuffer { + lsm_env *pEnv; + int nAlloc; + u32 *aCkpt; +}; + +/* +** Calculate the checksum of the checkpoint specified by arguments aCkpt and +** nCkpt. Store the checksum in *piCksum1 and *piCksum2 before returning. +** +** The value of the nCkpt parameter includes the two checksum values at +** the end of the checkpoint. They are not used as inputs to the checksum +** calculation. The checksum is based on the array of (nCkpt-2) integers +** at aCkpt[]. +*/ +static void ckptChecksum(u32 *aCkpt, u32 nCkpt, u32 *piCksum1, u32 *piCksum2){ + u32 i; + u32 cksum1 = 1; + u32 cksum2 = 2; + + if( nCkpt % 2 ){ + cksum1 += aCkpt[nCkpt-3] & 0x0000FFFF; + cksum2 += aCkpt[nCkpt-3] & 0xFFFF0000; + } + + for(i=0; (i+3)=p->nAlloc ){ + int nNew = LSM_MAX(8, iIdx*2); + p->aCkpt = (u32 *)lsmReallocOrFree(p->pEnv, p->aCkpt, nNew*sizeof(u32)); + if( !p->aCkpt ){ + *pRc = LSM_NOMEM_BKPT; + return; + } + p->nAlloc = nNew; + } + p->aCkpt[iIdx] = iVal; +} + +/* +** Argument aInt points to an array nInt elements in size. Switch the +** endian-ness of each element of the array. +*/ +static void ckptChangeEndianness(u32 *aInt, int nInt){ + if( LSM_LITTLE_ENDIAN ){ + int i; + for(i=0; iaCkpt, nCkpt+2, &aCksum[0], &aCksum[1]); + ckptSetValue(p, nCkpt, aCksum[0], pRc); + ckptSetValue(p, nCkpt+1, aCksum[1], pRc); + } +} + +static void ckptAppend64(CkptBuffer *p, int *piOut, i64 iVal, int *pRc){ + int iOut = *piOut; + ckptSetValue(p, iOut++, (iVal >> 32) & 0xFFFFFFFF, pRc); + ckptSetValue(p, iOut++, (iVal & 0xFFFFFFFF), pRc); + *piOut = iOut; +} + +static i64 ckptRead64(u32 *a){ + return (((i64)a[0]) << 32) + (i64)a[1]; +} + +static i64 ckptGobble64(u32 *a, int *piIn){ + int iIn = *piIn; + *piIn += 2; + return ckptRead64(&a[iIn]); +} + + +/* +** Append a 6-value segment record corresponding to pSeg to the checkpoint +** buffer passed as the third argument. +*/ +static void ckptExportSegment( + Segment *pSeg, + CkptBuffer *p, + int *piOut, + int *pRc +){ + ckptAppend64(p, piOut, pSeg->iFirst, pRc); + ckptAppend64(p, piOut, pSeg->iLastPg, pRc); + ckptAppend64(p, piOut, pSeg->iRoot, pRc); + ckptAppend64(p, piOut, pSeg->nSize, pRc); +} + +static void ckptExportLevel( + Level *pLevel, /* Level object to serialize */ + CkptBuffer *p, /* Append new level record to this ckpt */ + int *piOut, /* IN/OUT: Size of checkpoint so far */ + int *pRc /* IN/OUT: Error code */ +){ + int iOut = *piOut; + Merge *pMerge; + + pMerge = pLevel->pMerge; + ckptSetValue(p, iOut++, (u32)pLevel->iAge + (u32)(pLevel->flags<<16), pRc); + ckptSetValue(p, iOut++, pLevel->nRight, pRc); + ckptExportSegment(&pLevel->lhs, p, &iOut, pRc); + + assert( (pLevel->nRight>0)==(pMerge!=0) ); + if( pMerge ){ + int i; + for(i=0; inRight; i++){ + ckptExportSegment(&pLevel->aRhs[i], p, &iOut, pRc); + } + assert( pMerge->nInput==pLevel->nRight + || pMerge->nInput==pLevel->nRight+1 + ); + ckptSetValue(p, iOut++, pMerge->nInput, pRc); + ckptSetValue(p, iOut++, pMerge->nSkip, pRc); + for(i=0; inInput; i++){ + ckptAppend64(p, &iOut, pMerge->aInput[i].iPg, pRc); + ckptSetValue(p, iOut++, pMerge->aInput[i].iCell, pRc); + } + ckptAppend64(p, &iOut, pMerge->splitkey.iPg, pRc); + ckptSetValue(p, iOut++, pMerge->splitkey.iCell, pRc); + ckptAppend64(p, &iOut, pMerge->iCurrentPtr, pRc); + } + + *piOut = iOut; +} + +/* +** Populate the log offset fields of the checkpoint buffer. 4 values. +*/ +static void ckptExportLog( + lsm_db *pDb, + int bFlush, + CkptBuffer *p, + int *piOut, + int *pRc +){ + int iOut = *piOut; + + assert( iOut==CKPT_HDR_LO_MSW ); + + if( bFlush ){ + i64 iOff = pDb->treehdr.iOldLog; + ckptAppend64(p, &iOut, iOff, pRc); + ckptSetValue(p, iOut++, pDb->treehdr.oldcksum0, pRc); + ckptSetValue(p, iOut++, pDb->treehdr.oldcksum1, pRc); + }else{ + for(; iOut<=CKPT_HDR_LO_CKSUM2; iOut++){ + ckptSetValue(p, iOut, pDb->pShmhdr->aSnap2[iOut], pRc); + } + } + + assert( *pRc || iOut==CKPT_HDR_LO_CKSUM2+1 ); + *piOut = iOut; +} + +static void ckptExportAppendlist( + lsm_db *db, /* Database connection */ + CkptBuffer *p, /* Checkpoint buffer to write to */ + int *piOut, /* IN/OUT: Offset within checkpoint buffer */ + int *pRc /* IN/OUT: Error code */ +){ + int i; + LsmPgno *aiAppend = db->pWorker->aiAppend; + + for(i=0; ipFS; /* File system object */ + Snapshot *pSnap = pDb->pWorker; /* Worker snapshot */ + int nLevel = 0; /* Number of levels in checkpoint */ + int iLevel; /* Used to count out nLevel levels */ + int iOut = 0; /* Current offset in aCkpt[] */ + Level *pLevel; /* Level iterator */ + int i; /* Iterator used while serializing freelist */ + CkptBuffer ckpt; + + /* Initialize the output buffer */ + memset(&ckpt, 0, sizeof(CkptBuffer)); + ckpt.pEnv = pDb->pEnv; + iOut = CKPT_HDR_SIZE; + + /* Write the log offset into the checkpoint. */ + ckptExportLog(pDb, bLog, &ckpt, &iOut, &rc); + + /* Write the append-point list */ + ckptExportAppendlist(pDb, &ckpt, &iOut, &rc); + + /* Figure out how many levels will be written to the checkpoint. */ + for(pLevel=lsmDbSnapshotLevel(pSnap); pLevel; pLevel=pLevel->pNext) nLevel++; + + /* Serialize nLevel levels. */ + iLevel = 0; + for(pLevel=lsmDbSnapshotLevel(pSnap); iLevelpNext){ + ckptExportLevel(pLevel, &ckpt, &iOut, &rc); + iLevel++; + } + + /* Write the block-redirect list */ + ckptSetValue(&ckpt, iOut++, pSnap->redirect.n, &rc); + for(i=0; iredirect.n; i++){ + ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iFrom, &rc); + ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iTo, &rc); + } + + /* Write the freelist */ + assert( pSnap->freelist.nEntry<=pDb->nMaxFreelist ); + if( rc==LSM_OK ){ + int nFree = pSnap->freelist.nEntry; + ckptSetValue(&ckpt, iOut++, nFree, &rc); + for(i=0; ifreelist.aEntry[i]; + ckptSetValue(&ckpt, iOut++, p->iBlk, &rc); + ckptSetValue(&ckpt, iOut++, (p->iId >> 32) & 0xFFFFFFFF, &rc); + ckptSetValue(&ckpt, iOut++, p->iId & 0xFFFFFFFF, &rc); + } + } + + /* Write the checkpoint header */ + assert( iId>=0 ); + assert( pSnap->iCmpId==pDb->compress.iId + || pSnap->iCmpId==LSM_COMPRESSION_EMPTY + ); + ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); + ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); + ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); + ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); + ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); + ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); + ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); + ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); + ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); + + if( bCksum ){ + ckptAddChecksum(&ckpt, iOut, &rc); + }else{ + ckptSetValue(&ckpt, iOut, 0, &rc); + ckptSetValue(&ckpt, iOut+1, 0, &rc); + } + iOut += 2; + assert( iOut<=1024 ); + +#ifdef LSM_LOG_FREELIST + lsmLogMessage(pDb, rc, + "ckptExportSnapshot(): id=%lld freelist: %d", iId, pSnap->freelist.nEntry + ); + for(i=0; ifreelist.nEntry; i++){ + lsmLogMessage(pDb, rc, + "ckptExportSnapshot(): iBlk=%d id=%lld", + pSnap->freelist.aEntry[i].iBlk, + pSnap->freelist.aEntry[i].iId + ); + } +#endif + + *ppCkpt = (void *)ckpt.aCkpt; + if( pnCkpt ) *pnCkpt = sizeof(u32)*iOut; + return rc; +} + + +/* +** Helper function for ckptImport(). +*/ +static void ckptNewSegment( + u32 *aIn, + int *piIn, + Segment *pSegment /* Populate this structure */ +){ + assert( pSegment->iFirst==0 && pSegment->iLastPg==0 ); + assert( pSegment->nSize==0 && pSegment->iRoot==0 ); + pSegment->iFirst = ckptGobble64(aIn, piIn); + pSegment->iLastPg = ckptGobble64(aIn, piIn); + pSegment->iRoot = ckptGobble64(aIn, piIn); + pSegment->nSize = ckptGobble64(aIn, piIn); + assert( pSegment->iFirst ); +} + +static int ckptSetupMerge(lsm_db *pDb, u32 *aInt, int *piIn, Level *pLevel){ + Merge *pMerge; /* Allocated Merge object */ + int nInput; /* Number of input segments in merge */ + int iIn = *piIn; /* Next value to read from aInt[] */ + int i; /* Iterator variable */ + int nByte; /* Number of bytes to allocate */ + + /* Allocate the Merge object. If malloc() fails, return LSM_NOMEM. */ + nInput = (int)aInt[iIn++]; + nByte = sizeof(Merge) + sizeof(MergeInput) * nInput; + pMerge = (Merge *)lsmMallocZero(pDb->pEnv, nByte); + if( !pMerge ) return LSM_NOMEM_BKPT; + pLevel->pMerge = pMerge; + + /* Populate the Merge object. */ + pMerge->aInput = (MergeInput *)&pMerge[1]; + pMerge->nInput = nInput; + pMerge->iOutputOff = -1; + pMerge->nSkip = (int)aInt[iIn++]; + for(i=0; iaInput[i].iPg = ckptGobble64(aInt, &iIn); + pMerge->aInput[i].iCell = (int)aInt[iIn++]; + } + pMerge->splitkey.iPg = ckptGobble64(aInt, &iIn); + pMerge->splitkey.iCell = (int)aInt[iIn++]; + pMerge->iCurrentPtr = ckptGobble64(aInt, &iIn); + + /* Set *piIn and return LSM_OK. */ + *piIn = iIn; + return LSM_OK; +} + + +static int ckptLoadLevels( + lsm_db *pDb, + u32 *aIn, + int *piIn, + int nLevel, + Level **ppLevel +){ + int i; + int rc = LSM_OK; + Level *pRet = 0; + Level **ppNext; + int iIn = *piIn; + + ppNext = &pRet; + for(i=0; rc==LSM_OK && ipEnv, sizeof(Level), &rc); + if( rc==LSM_OK ){ + pLevel->iAge = (u16)(aIn[iIn] & 0x0000FFFF); + pLevel->flags = (u16)((aIn[iIn]>>16) & 0x0000FFFF); + iIn++; + pLevel->nRight = aIn[iIn++]; + if( pLevel->nRight ){ + int nByte = sizeof(Segment) * pLevel->nRight; + pLevel->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); + } + if( rc==LSM_OK ){ + *ppNext = pLevel; + ppNext = &pLevel->pNext; + + /* Allocate the main segment */ + ckptNewSegment(aIn, &iIn, &pLevel->lhs); + + /* Allocate each of the right-hand segments, if any */ + for(iRight=0; iRightnRight; iRight++){ + ckptNewSegment(aIn, &iIn, &pLevel->aRhs[iRight]); + } + + /* Set up the Merge object, if required */ + if( pLevel->nRight>0 ){ + rc = ckptSetupMerge(pDb, aIn, &iIn, pLevel); + } + } + } + } + + if( rc!=LSM_OK ){ + /* An OOM must have occurred. Free any level structures allocated and + ** return the error to the caller. */ + lsmSortedFreeLevel(pDb->pEnv, pRet); + pRet = 0; + } + + *ppLevel = pRet; + *piIn = iIn; + return rc; +} + + +int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal){ + int rc = LSM_OK; + if( nVal>0 ){ + u32 *aIn; + + aIn = lsmMallocRc(pDb->pEnv, nVal, &rc); + if( aIn ){ + Level *pLevel = 0; + Level *pParent; + + int nIn; + int nLevel; + int iIn = 1; + memcpy(aIn, pVal, nVal); + nIn = nVal / sizeof(u32); + + ckptChangeEndianness(aIn, nIn); + nLevel = aIn[0]; + rc = ckptLoadLevels(pDb, aIn, &iIn, nLevel, &pLevel); + lsmFree(pDb->pEnv, aIn); + assert( rc==LSM_OK || pLevel==0 ); + if( rc==LSM_OK ){ + pParent = lsmDbSnapshotLevel(pDb->pWorker); + assert( pParent ); + while( pParent->pNext ) pParent = pParent->pNext; + pParent->pNext = pLevel; + } + } + } + + return rc; +} + +/* +** Return the data for the LEVELS record. +** +** The size of the checkpoint that can be stored in the database header +** must not exceed 1024 32-bit integers. Normally, it does not. However, +** if it does, part of the checkpoint must be stored in the LSM. This +** routine returns that part. +*/ +int lsmCheckpointLevels( + lsm_db *pDb, /* Database handle */ + int nLevel, /* Number of levels to write to blob */ + void **paVal, /* OUT: Pointer to LEVELS blob */ + int *pnVal /* OUT: Size of LEVELS blob in bytes */ +){ + Level *p; /* Used to iterate through levels */ + int nAll= 0; + int rc; + int i; + int iOut; + CkptBuffer ckpt; + assert( nLevel>0 ); + + for(p=lsmDbSnapshotLevel(pDb->pWorker); p; p=p->pNext) nAll++; + + assert( nAll>nLevel ); + nAll -= nLevel; + for(p=lsmDbSnapshotLevel(pDb->pWorker); p && nAll>0; p=p->pNext) nAll--; + + memset(&ckpt, 0, sizeof(CkptBuffer)); + ckpt.pEnv = pDb->pEnv; + + ckptSetValue(&ckpt, 0, nLevel, &rc); + iOut = 1; + for(i=0; rc==LSM_OK && ipNext; + } + assert( rc!=LSM_OK || p==0 ); + + if( rc==LSM_OK ){ + ckptChangeEndianness(ckpt.aCkpt, iOut); + *paVal = (void *)ckpt.aCkpt; + *pnVal = iOut * sizeof(u32); + }else{ + *pnVal = 0; + *paVal = 0; + } + + return rc; +} + +/* +** Read the checkpoint id from meta-page pPg. +*/ +static i64 ckptLoadId(MetaPage *pPg){ + i64 ret = 0; + if( pPg ){ + int nData; + u8 *aData = lsmFsMetaPageData(pPg, &nData); + ret = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32) + + ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); + } + return ret; +} + +/* +** Return true if the buffer passed as an argument contains a valid +** checkpoint. +*/ +static int ckptChecksumOk(u32 *aCkpt){ + u32 nCkpt = aCkpt[CKPT_HDR_NCKPT]; + u32 cksum1; + u32 cksum2; + + if( nCkpt(LSM_META_RW_PAGE_SIZE)/sizeof(u32) ){ + return 0; + } + ckptChecksum(aCkpt, nCkpt, &cksum1, &cksum2); + return (cksum1==aCkpt[nCkpt-2] && cksum2==aCkpt[nCkpt-1]); +} + +/* +** Attempt to load a checkpoint from meta page iMeta. +** +** This function is a no-op if *pRc is set to any value other than LSM_OK +** when it is called. If an error occurs, *pRc is set to an LSM error code +** before returning. +** +** If no error occurs and the checkpoint is successfully loaded, copy it to +** ShmHeader.aSnap1[] and ShmHeader.aSnap2[], and set ShmHeader.iMetaPage +** to indicate its origin. In this case return 1. Or, if the checkpoint +** cannot be loaded (because the checksum does not compute), return 0. +*/ +static int ckptTryLoad(lsm_db *pDb, MetaPage *pPg, u32 iMeta, int *pRc){ + int bLoaded = 0; /* Return value */ + if( *pRc==LSM_OK ){ + int rc = LSM_OK; /* Error code */ + u32 *aCkpt = 0; /* Pointer to buffer containing checkpoint */ + u32 nCkpt; /* Number of elements in aCkpt[] */ + int nData; /* Bytes of data in aData[] */ + u8 *aData; /* Meta page data */ + + aData = lsmFsMetaPageData(pPg, &nData); + nCkpt = (u32)lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); + if( nCkpt<=nData/sizeof(u32) && nCkpt>CKPT_HDR_NCKPT ){ + aCkpt = (u32 *)lsmMallocRc(pDb->pEnv, nCkpt*sizeof(u32), &rc); + } + if( aCkpt ){ + memcpy(aCkpt, aData, nCkpt*sizeof(u32)); + ckptChangeEndianness(aCkpt, nCkpt); + if( ckptChecksumOk(aCkpt) ){ + ShmHeader *pShm = pDb->pShmhdr; + memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); + memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); + memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); + pShm->iMetaPage = iMeta; + bLoaded = 1; + } + } + + lsmFree(pDb->pEnv, aCkpt); + *pRc = rc; + } + return bLoaded; +} + +/* +** Initialize the shared-memory header with an empty snapshot. This function +** is called when no valid snapshot can be found in the database header. +*/ +static void ckptLoadEmpty(lsm_db *pDb){ + u32 aCkpt[] = { + 0, /* CKPT_HDR_ID_MSW */ + 10, /* CKPT_HDR_ID_LSW */ + 0, /* CKPT_HDR_NCKPT */ + LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ + 0, /* CKPT_HDR_NBLOCK */ + 0, /* CKPT_HDR_BLKSZ */ + 0, /* CKPT_HDR_NLEVEL */ + 0, /* CKPT_HDR_PGSZ */ + 0, /* CKPT_HDR_NWRITE */ + 0, 0, 1234, 5678, /* The log pointer and initial checksum */ + 0,0,0,0, 0,0,0,0, /* The append list */ + 0, /* The redirected block list */ + 0, /* The free block list */ + 0, 0 /* Space for checksum values */ + }; + u32 nCkpt = array_size(aCkpt); + ShmHeader *pShm = pDb->pShmhdr; + + aCkpt[CKPT_HDR_NCKPT] = nCkpt; + aCkpt[CKPT_HDR_BLKSZ] = pDb->nDfltBlksz; + aCkpt[CKPT_HDR_PGSZ] = pDb->nDfltPgsz; + ckptChecksum(aCkpt, array_size(aCkpt), &aCkpt[nCkpt-2], &aCkpt[nCkpt-1]); + + memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); + memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); + memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); +} + +/* +** This function is called as part of database recovery to initialize the +** ShmHeader.aSnap1[] and ShmHeader.aSnap2[] snapshots. +*/ +int lsmCheckpointRecover(lsm_db *pDb){ + int rc = LSM_OK; /* Return Code */ + i64 iId1; /* Id of checkpoint on meta-page 1 */ + i64 iId2; /* Id of checkpoint on meta-page 2 */ + int bLoaded = 0; /* True once checkpoint has been loaded */ + int cmp; /* True if (iId2>iId1) */ + MetaPage *apPg[2] = {0, 0}; /* Meta-pages 1 and 2 */ + + rc = lsmFsMetaPageGet(pDb->pFS, 0, 1, &apPg[0]); + if( rc==LSM_OK ) rc = lsmFsMetaPageGet(pDb->pFS, 0, 2, &apPg[1]); + + iId1 = ckptLoadId(apPg[0]); + iId2 = ckptLoadId(apPg[1]); + cmp = (iId2 > iId1); + bLoaded = ckptTryLoad(pDb, apPg[cmp?1:0], (cmp?2:1), &rc); + if( bLoaded==0 ){ + bLoaded = ckptTryLoad(pDb, apPg[cmp?0:1], (cmp?1:2), &rc); + } + + /* The database does not contain a valid checkpoint. Initialize the shared + ** memory header with an empty checkpoint. */ + if( bLoaded==0 ){ + ckptLoadEmpty(pDb); + } + + lsmFsMetaPageRelease(apPg[0]); + lsmFsMetaPageRelease(apPg[1]); + + return rc; +} + +/* +** Store the snapshot in pDb->aSnapshot[] in meta-page iMeta. +*/ +int lsmCheckpointStore(lsm_db *pDb, int iMeta){ + MetaPage *pPg = 0; + int rc; + + assert( iMeta==1 || iMeta==2 ); + rc = lsmFsMetaPageGet(pDb->pFS, 1, iMeta, &pPg); + if( rc==LSM_OK ){ + u8 *aData; + int nData; + int nCkpt; + + nCkpt = (int)pDb->aSnapshot[CKPT_HDR_NCKPT]; + aData = lsmFsMetaPageData(pPg, &nData); + memcpy(aData, pDb->aSnapshot, nCkpt*sizeof(u32)); + ckptChangeEndianness((u32 *)aData, nCkpt); + rc = lsmFsMetaPageRelease(pPg); + } + + return rc; +} + +/* +** Copy the current client snapshot from shared-memory to pDb->aSnapshot[]. +*/ +int lsmCheckpointLoad(lsm_db *pDb, int *piRead){ + int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; + ShmHeader *pShm = pDb->pShmhdr; + while( (nRem--)>0 ){ + int nInt; + + nInt = pShm->aSnap1[CKPT_HDR_NCKPT]; + if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ + memcpy(pDb->aSnapshot, pShm->aSnap1, nInt*sizeof(u32)); + if( ckptChecksumOk(pDb->aSnapshot) ){ + if( piRead ) *piRead = 1; + return LSM_OK; + } + } + + nInt = pShm->aSnap2[CKPT_HDR_NCKPT]; + if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ + memcpy(pDb->aSnapshot, pShm->aSnap2, nInt*sizeof(u32)); + if( ckptChecksumOk(pDb->aSnapshot) ){ + if( piRead ) *piRead = 2; + return LSM_OK; + } + } + + lsmShmBarrier(pDb); + } + return LSM_PROTOCOL_BKPT; +} + +int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ + int rc; + + assert( db->pClient==0 && db->pWorker==0 ); + rc = lsmCheckpointLoad(db, 0); + if( rc==LSM_OK ){ + *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; + } + + return rc; +} + +int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ + u32 *aShm; + assert( iSnap==1 || iSnap==2 ); + aShm = (iSnap==1) ? pDb->pShmhdr->aSnap1 : pDb->pShmhdr->aSnap2; + return (lsmCheckpointId(pDb->aSnapshot, 0)==lsmCheckpointId(aShm, 0) ); +} + +int lsmCheckpointClientCacheOk(lsm_db *pDb){ + return ( pDb->pClient + && pDb->pClient->iId==lsmCheckpointId(pDb->aSnapshot, 0) + && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap1, 0) + && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap2, 0) + ); +} + +int lsmCheckpointLoadWorker(lsm_db *pDb){ + int rc; + ShmHeader *pShm = pDb->pShmhdr; + int nInt1; + int nInt2; + + /* Must be holding the WORKER lock to do this. Or DMS2. */ + assert( + lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) + || lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) + ); + + /* Check that the two snapshots match. If not, repair them. */ + nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; + nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; + if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ + if( ckptChecksumOk(pShm->aSnap1) ){ + memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); + }else if( ckptChecksumOk(pShm->aSnap2) ){ + memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); + }else{ + return LSM_PROTOCOL_BKPT; + } + } + + rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); + if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; + + if( rc==LSM_OK ){ + rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); + } + +#if 0 + assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); +#endif + return rc; +} + +int lsmCheckpointDeserialize( + lsm_db *pDb, + int bInclFreelist, /* If true, deserialize free-list */ + u32 *aCkpt, + Snapshot **ppSnap +){ + int rc = LSM_OK; + Snapshot *pNew; + + pNew = (Snapshot *)lsmMallocZeroRc(pDb->pEnv, sizeof(Snapshot), &rc); + if( rc==LSM_OK ){ + Level *pLvl; + int nFree; + int i; + int nLevel = (int)aCkpt[CKPT_HDR_NLEVEL]; + int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; + + pNew->iId = lsmCheckpointId(aCkpt, 0); + pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; + pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; + rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); + pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); + pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; + + /* Make a copy of the append-list */ + for(i=0; iaiAppend[i] = ckptRead64(a); + } + + /* Read the block-redirect list */ + pNew->redirect.n = aCkpt[iIn++]; + if( pNew->redirect.n ){ + pNew->redirect.a = lsmMallocZeroRc(pDb->pEnv, + (sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS), &rc + ); + if( rc==LSM_OK ){ + for(i=0; iredirect.n; i++){ + pNew->redirect.a[i].iFrom = aCkpt[iIn++]; + pNew->redirect.a[i].iTo = aCkpt[iIn++]; + } + } + for(pLvl=pNew->pLevel; pLvl->pNext; pLvl=pLvl->pNext); + if( pLvl->nRight ){ + pLvl->aRhs[pLvl->nRight-1].pRedirect = &pNew->redirect; + }else{ + pLvl->lhs.pRedirect = &pNew->redirect; + } + } + + /* Copy the free-list */ + if( rc==LSM_OK && bInclFreelist ){ + nFree = aCkpt[iIn++]; + if( nFree ){ + pNew->freelist.aEntry = (FreelistEntry *)lsmMallocZeroRc( + pDb->pEnv, sizeof(FreelistEntry)*nFree, &rc + ); + if( rc==LSM_OK ){ + int j; + for(j=0; jfreelist.aEntry[j]; + p->iBlk = aCkpt[iIn++]; + p->iId = ((i64)(aCkpt[iIn])<<32) + aCkpt[iIn+1]; + iIn += 2; + } + pNew->freelist.nEntry = pNew->freelist.nAlloc = nFree; + } + } + } + } + + if( rc!=LSM_OK ){ + lsmFreeSnapshot(pDb->pEnv, pNew); + pNew = 0; + } + + *ppSnap = pNew; + return rc; +} + +/* +** Connection pDb must be the worker connection in order to call this +** function. It returns true if the database already contains the maximum +** number of levels or false otherwise. +** +** This is used when flushing the in-memory tree to disk. If the database +** is already full, then the caller should invoke lsm_work() or similar +** until it is not full before creating a new level by flushing the in-memory +** tree to disk. Limiting the number of levels in the database ensures that +** the records describing them always fit within the checkpoint blob. +*/ +int lsmDatabaseFull(lsm_db *pDb){ + Level *p; + int nRhs = 0; + + assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); + assert( pDb->pWorker ); + + for(p=pDb->pWorker->pLevel; p; p=p->pNext){ + nRhs += (p->nRight ? p->nRight : 1); + } + + return (nRhs >= LSM_MAX_RHS_SEGMENTS); +} + +/* +** The connection passed as the only argument is currently the worker +** connection. Some work has been performed on the database by the connection, +** but no new snapshot has been written into shared memory. +** +** This function updates the shared-memory worker and client snapshots with +** the new snapshot produced by the work performed by pDb. +** +** If successful, LSM_OK is returned. Otherwise, if an error occurs, an LSM +** error code is returned. +*/ +int lsmCheckpointSaveWorker(lsm_db *pDb, int bFlush){ + Snapshot *pSnap = pDb->pWorker; + ShmHeader *pShm = pDb->pShmhdr; + void *p = 0; + int n = 0; + int rc; + + pSnap->iId++; + rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId, 1, &p, &n); + if( rc!=LSM_OK ) return rc; + assert( ckptChecksumOk((u32 *)p) ); + + assert( n<=LSM_META_RW_PAGE_SIZE ); + memcpy(pShm->aSnap2, p, n); + lsmShmBarrier(pDb); + memcpy(pShm->aSnap1, p, n); + lsmFree(pDb->pEnv, p); + + /* assert( lsmFsIntegrityCheck(pDb) ); */ + return LSM_OK; +} + +/* +** This function is used to determine the snapshot-id of the most recently +** checkpointed snapshot. Variable ShmHeader.iMetaPage indicates which of +** the two meta-pages said snapshot resides on (if any). +** +** If successful, this function loads the snapshot from the meta-page, +** verifies its checksum and sets *piId to the snapshot-id before returning +** LSM_OK. Or, if the checksum attempt fails, *piId is set to zero and +** LSM_OK returned. If an error occurs, an LSM error code is returned and +** the final value of *piId is undefined. +*/ +int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite){ + int rc = LSM_OK; + MetaPage *pPg; + u32 iMeta; + + iMeta = pDb->pShmhdr->iMetaPage; + if( iMeta==1 || iMeta==2 ){ + rc = lsmFsMetaPageGet(pDb->pFS, 0, iMeta, &pPg); + if( rc==LSM_OK ){ + int nCkpt; + int nData; + u8 *aData; + + aData = lsmFsMetaPageData(pPg, &nData); + assert( nData==LSM_META_RW_PAGE_SIZE ); + nCkpt = lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); + if( nCkpt<(LSM_META_RW_PAGE_SIZE/sizeof(u32)) ){ + u32 *aCopy = lsmMallocRc(pDb->pEnv, sizeof(u32) * nCkpt, &rc); + if( aCopy ){ + memcpy(aCopy, aData, nCkpt*sizeof(u32)); + ckptChangeEndianness(aCopy, nCkpt); + if( ckptChecksumOk(aCopy) ){ + if( piId ) *piId = lsmCheckpointId(aCopy, 0); + if( piLog ) *piLog = (lsmCheckpointLogOffset(aCopy) >> 1); + if( pnWrite ) *pnWrite = aCopy[CKPT_HDR_NWRITE]; + } + lsmFree(pDb->pEnv, aCopy); + } + } + lsmFsMetaPageRelease(pPg); + } + } + + if( (iMeta!=1 && iMeta!=2) || rc!=LSM_OK || pDb->pShmhdr->iMetaPage!=iMeta ){ + if( piId ) *piId = 0; + if( piLog ) *piLog = 0; + if( pnWrite ) *pnWrite = 0; + } + return rc; +} + +/* +** Return the checkpoint-id of the checkpoint array passed as the first +** argument to this function. If the second argument is true, then assume +** that the checkpoint is made up of 32-bit big-endian integers. If it +** is false, assume that the integers are in machine byte order. +*/ +i64 lsmCheckpointId(u32 *aCkpt, int bDisk){ + i64 iId; + if( bDisk ){ + u8 *aData = (u8 *)aCkpt; + iId = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32); + iId += ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); + }else{ + iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; + } + return iId; +} + +u32 lsmCheckpointNBlock(u32 *aCkpt){ + return aCkpt[CKPT_HDR_NBLOCK]; +} + +u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ + if( bDisk ){ + return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); + }else{ + return aCkpt[CKPT_HDR_NWRITE]; + } +} + +i64 lsmCheckpointLogOffset(u32 *aCkpt){ + return ((i64)aCkpt[CKPT_HDR_LO_MSW] << 32) + (i64)aCkpt[CKPT_HDR_LO_LSW]; +} + +int lsmCheckpointPgsz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_PGSZ]; } + +int lsmCheckpointBlksz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_BLKSZ]; } + +void lsmCheckpointLogoffset( + u32 *aCkpt, + DbLog *pLog +){ + pLog->aRegion[2].iStart = (lsmCheckpointLogOffset(aCkpt) >> 1); + + pLog->cksum0 = aCkpt[CKPT_HDR_LO_CKSUM1]; + pLog->cksum1 = aCkpt[CKPT_HDR_LO_CKSUM2]; + pLog->iSnapshotId = lsmCheckpointId(aCkpt, 0); +} + +void lsmCheckpointZeroLogoffset(lsm_db *pDb){ + u32 nCkpt; + + nCkpt = pDb->aSnapshot[CKPT_HDR_NCKPT]; + assert( nCkpt>CKPT_HDR_NCKPT ); + assert( nCkpt==pDb->pShmhdr->aSnap1[CKPT_HDR_NCKPT] ); + assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap1, nCkpt*sizeof(u32)) ); + assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap2, nCkpt*sizeof(u32)) ); + + pDb->aSnapshot[CKPT_HDR_LO_MSW] = 0; + pDb->aSnapshot[CKPT_HDR_LO_LSW] = 0; + ckptChecksum(pDb->aSnapshot, nCkpt, + &pDb->aSnapshot[nCkpt-2], &pDb->aSnapshot[nCkpt-1] + ); + + memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); + memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); +} + +/* +** Set the output variable to the number of KB of data written into the +** database file since the most recent checkpoint. +*/ +int lsmCheckpointSize(lsm_db *db, int *pnKB){ + int rc = LSM_OK; + u32 nSynced; + + /* Set nSynced to the number of pages that had been written when the + ** database was last checkpointed. */ + rc = lsmCheckpointSynced(db, 0, 0, &nSynced); + + if( rc==LSM_OK ){ + u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; + u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; + *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); + } + + return rc; +} diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c new file mode 100644 index 0000000000000000000000000000000000000000..fd78835bbb0be978a6d5460540847583d486f866 --- /dev/null +++ b/ext/lsm1/lsm_file.c @@ -0,0 +1,3311 @@ +/* +** 2011-08-26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** NORMAL DATABASE FILE FORMAT +** +** The following database file format concepts are used by the code in +** this file to read and write the database file. +** +** Pages: +** +** A database file is divided into pages. The first 8KB of the file consists +** of two 4KB meta-pages. The meta-page size is not configurable. The +** remainder of the file is made up of database pages. The default database +** page size is 4KB. Database pages are aligned to page-size boundaries, +** so if the database page size is larger than 8KB there is a gap between +** the end of the meta pages and the start of the database pages. +** +** Database pages are numbered based on their position in the file. Page N +** begins at byte offset ((N-1)*pgsz). This means that page 1 does not +** exist - since it would always overlap with the meta pages. If the +** page-size is (say) 512 bytes, then the first usable page in the database +** is page 33. +** +** It is assumed that the first two meta pages and the data that follows +** them are located on different disk sectors. So that if a power failure +** while writing to a meta page there is no risk of damage to the other +** meta page or any other part of the database file. TODO: This may need +** to be revisited. +** +** Blocks: +** +** The database file is also divided into blocks. The default block size is +** 1MB. When writing to the database file, an attempt is made to write data +** in contiguous block-sized chunks. +** +** The first and last page on each block are special in that they are 4 +** bytes smaller than all other pages. This is because the last four bytes +** of space on the first and last pages of each block are reserved for +** pointers to other blocks (i.e. a 32-bit block number). +** +** Runs: +** +** A run is a sequence of pages that the upper layer uses to store a +** sorted array of database keys (and accompanying data - values, FC +** pointers and so on). Given a page within a run, it is possible to +** navigate to the next page in the run as follows: +** +** a) if the current page is not the last in a block, the next page +** in the run is located immediately after the current page, OR +** +** b) if the current page is the last page in a block, the next page +** in the run is the first page on the block identified by the +** block pointer stored in the last 4 bytes of the current block. +** +** It is possible to navigate to the previous page in a similar fashion, +** using the block pointer embedded in the last 4 bytes of the first page +** of each block as required. +** +** The upper layer is responsible for identifying by page number the +** first and last page of any run that it needs to navigate - there are +** no "end-of-run" markers stored or identified by this layer. This is +** necessary as clients reading different database snapshots may access +** different subsets of a run. +** +** THE LOG FILE +** +** This file opens and closes the log file. But it does not contain any +** logic related to the log file format. Instead, it exports the following +** functions that are used by the code in lsm_log.c to read and write the +** log file: +** +** lsmFsOpenLog +** lsmFsWriteLog +** lsmFsSyncLog +** lsmFsReadLog +** lsmFsTruncateLog +** lsmFsCloseAndDeleteLog +** +** COMPRESSED DATABASE FILE FORMAT +** +** The compressed database file format is very similar to the normal format. +** The file still begins with two 4KB meta-pages (which are never compressed). +** It is still divided into blocks. +** +** The first and last four bytes of each block are reserved for 32-bit +** pointer values. Similar to the way four bytes are carved from the end of +** the first and last page of each block in uncompressed databases. From +** the point of view of the upper layer, all pages are the same size - this +** is different from the uncompressed format where the first and last pages +** on each block are 4 bytes smaller than the others. +** +** Pages are stored in variable length compressed form, as follows: +** +** * 3-byte size field containing the size of the compressed page image +** in bytes. The most significant bit of each byte of the size field +** is always set. The remaining 7 bits are used to store a 21-bit +** integer value (in big-endian order - the first byte in the field +** contains the most significant 7 bits). Since the maximum allowed +** size of a compressed page image is (2^17 - 1) bytes, there are +** actually 4 unused bits in the size field. +** +** In other words, if the size of the compressed page image is nSz, +** the header can be serialized as follows: +** +** u8 aHdr[3] +** aHdr[0] = 0x80 | (u8)(nSz >> 14); +** aHdr[1] = 0x80 | (u8)(nSz >> 7); +** aHdr[2] = 0x80 | (u8)(nSz >> 0); +** +** * Compressed page image. +** +** * A second copy of the 3-byte record header. +** +** A page number is a byte offset into the database file. So the smallest +** possible page number is 8192 (immediately after the two meta-pages). +** The first and root page of a segment are identified by a page number +** corresponding to the byte offset of the first byte in the corresponding +** page record. The last page of a segment is identified by the byte offset +** of the last byte in its record. +** +** Unlike uncompressed pages, compressed page records may span blocks. +** +** Sometimes, in order to avoid touching sectors that contain synced data +** when writing, it is necessary to insert unused space between compressed +** page records. This can be done as follows: +** +** * For less than 6 bytes of empty space, the first and last byte +** of the free space contain the total number of free bytes. For +** example: +** +** Block of 4 free bytes: 0x04 0x?? 0x?? 0x04 +** Block of 2 free bytes: 0x02 0x02 +** A single free byte: 0x01 +** +** * For 6 or more bytes of empty space, a record similar to a +** compressed page record is added to the segment. A padding record +** is distinguished from a compressed page record by the most +** significant bit of the second byte of the size field, which is +** cleared instead of set. +*/ +#include "lsmInt.h" + +#include +#include +#include + +/* +** File-system object. Each database connection allocates a single instance +** of the following structure. It is used for all access to the database and +** log files. +** +** The database file may be accessed via two methods - using mmap() or using +** read() and write() calls. In the general case both methods are used - a +** prefix of the file is mapped into memory and the remainder accessed using +** read() and write(). This is helpful when accessing very large files (or +** files that may grow very large during the lifetime of a database +** connection) on systems with 32-bit address spaces. However, it also requires +** that this object manage two distinct types of Page objects simultaneously - +** those that carry pointers to the mapped file and those that carry arrays +** populated by read() calls. +** +** pFree: +** The head of a singly-linked list that containing currently unused Page +** structures suitable for use as mmap-page handles. Connected by the +** Page.pFreeNext pointers. +** +** pMapped: +** The head of a singly-linked list that contains all pages that currently +** carry pointers to the mapped region. This is used if the region is +** every remapped - the pointers carried by existing pages can be adjusted +** to account for the remapping. Connected by the Page.pMappedNext pointers. +** +** pWaiting: +** When the upper layer wishes to append a new b-tree page to a segment, +** it allocates a Page object that carries a malloc'd block of memory - +** regardless of the mmap-related configuration. The page is not assigned +** a page number at first. When the upper layer has finished constructing +** the page contents, it calls lsmFsPagePersist() to assign a page number +** to it. At this point it is likely that N pages have been written to the +** segment, the (N+1)th page is still outstanding and the b-tree page is +** assigned page number (N+2). To avoid writing page (N+2) before page +** (N+1), the recently completed b-tree page is held in the singly linked +** list headed by pWaiting until page (N+1) has been written. +** +** Function lsmFsFlushWaiting() is responsible for eventually writing +** waiting pages to disk. +** +** apHash/nHash: +** Hash table used to store all Page objects that carry malloc'd arrays, +** except those b-tree pages that have not yet been assigned page numbers. +** Once they have been assigned page numbers - they are added to this +** hash table. +** +** Hash table overflow chains are connected using the Page.pHashNext +** pointers. +** +** pLruFirst, pLruLast: +** The first and last entries in a doubly-linked list of pages. This +** list contains all pages with malloc'd data that are present in the +** hash table and have a ref-count of zero. +*/ +struct FileSystem { + lsm_db *pDb; /* Database handle that owns this object */ + lsm_env *pEnv; /* Environment pointer */ + char *zDb; /* Database file name */ + char *zLog; /* Database file name */ + int nMetasize; /* Size of meta pages in bytes */ + int nMetaRwSize; /* Read/written size of meta pages in bytes */ + i64 nPagesize; /* Database page-size in bytes */ + i64 nBlocksize; /* Database block-size in bytes */ + + /* r/w file descriptors for both files. */ + LsmFile *pLsmFile; /* Used after lsm_close() to link into list */ + lsm_file *fdDb; /* Database file */ + lsm_file *fdLog; /* Log file */ + int szSector; /* Database file sector size */ + + /* If this is a compressed database, a pointer to the compression methods. + ** For an uncompressed database, a NULL pointer. */ + lsm_compress *pCompress; + u8 *aIBuffer; /* Buffer to compress to */ + u8 *aOBuffer; /* Buffer to uncompress from */ + int nBuffer; /* Allocated size of above buffers in bytes */ + + /* mmap() page related things */ + i64 nMapLimit; /* Maximum bytes of file to map */ + void *pMap; /* Current mapping of database file */ + i64 nMap; /* Bytes mapped at pMap */ + Page *pFree; /* Unused Page structures */ + Page *pMapped; /* List of Page structs that point to pMap */ + + /* Page cache parameters for non-mmap() pages */ + int nCacheMax; /* Configured cache size (in pages) */ + int nCacheAlloc; /* Current cache size (in pages) */ + Page *pLruFirst; /* Head of the LRU list */ + Page *pLruLast; /* Tail of the LRU list */ + int nHash; /* Number of hash slots in hash table */ + Page **apHash; /* nHash Hash slots */ + Page *pWaiting; /* b-tree pages waiting to be written */ + + /* Statistics */ + int nOut; /* Number of outstanding pages */ + int nWrite; /* Total number of pages written */ + int nRead; /* Total number of pages read */ +}; + +/* +** Database page handle. +** +** pSeg: +** When lsmFsSortedAppend() is called on a compressed database, the new +** page is not assigned a page number or location in the database file +** immediately. Instead, these are assigned by the lsmFsPagePersist() call +** right before it writes the compressed page image to disk. +** +** The lsmFsSortedAppend() function sets the pSeg pointer to point to the +** segment that the new page will be a part of. It is unset by +** lsmFsPagePersist() after the page is written to disk. +*/ +struct Page { + u8 *aData; /* Buffer containing page data */ + int nData; /* Bytes of usable data at aData[] */ + LsmPgno iPg; /* Page number */ + int nRef; /* Number of outstanding references */ + int flags; /* Combination of PAGE_XXX flags */ + Page *pHashNext; /* Next page in hash table slot */ + Page *pLruNext; /* Next page in LRU list */ + Page *pLruPrev; /* Previous page in LRU list */ + FileSystem *pFS; /* File system that owns this page */ + + /* Only used in compressed database mode: */ + int nCompress; /* Compressed size (or 0 for uncomp. db) */ + int nCompressPrev; /* Compressed size of prev page */ + Segment *pSeg; /* Segment this page will be written to */ + + /* Pointers for singly linked lists */ + Page *pWaitingNext; /* Next page in FileSystem.pWaiting list */ + Page *pFreeNext; /* Next page in FileSystem.pFree list */ + Page *pMappedNext; /* Next page in FileSystem.pMapped list */ +}; + +/* +** Meta-data page handle. There are two meta-data pages at the start of +** the database file, each FileSystem.nMetasize bytes in size. +*/ +struct MetaPage { + int iPg; /* Either 1 or 2 */ + int bWrite; /* Write back to db file on release */ + u8 *aData; /* Pointer to buffer */ + FileSystem *pFS; /* FileSystem that owns this page */ +}; + +/* +** Values for LsmPage.flags +*/ +#define PAGE_DIRTY 0x00000001 /* Set if page is dirty */ +#define PAGE_FREE 0x00000002 /* Set if Page.aData requires lsmFree() */ +#define PAGE_HASPREV 0x00000004 /* Set if page is first on uncomp. block */ + +/* +** Number of pgsz byte pages omitted from the start of block 1. The start +** of block 1 contains two 4096 byte meta pages (8192 bytes in total). +*/ +#define BLOCK1_HDR_SIZE(pgsz) LSM_MAX(1, 8192/(pgsz)) + +/* +** If NDEBUG is not defined, set a breakpoint in function lsmIoerrBkpt() +** to catch IO errors (any error returned by a VFS method). +*/ +#ifndef NDEBUG +static void lsmIoerrBkpt(void){ + static int nErr = 0; + nErr++; +} +static int IOERR_WRAPPER(int rc){ + if( rc!=LSM_OK ) lsmIoerrBkpt(); + return rc; +} +#else +# define IOERR_WRAPPER(rc) (rc) +#endif + +#ifdef NDEBUG +# define assert_lists_are_ok(x) +#else +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash); + +static void assert_lists_are_ok(FileSystem *pFS){ +#if 0 + Page *p; + + assert( pFS->nMapLimit>=0 ); + + /* Check that all pages in the LRU list have nRef==0, pointers to buffers + ** in heap memory, and corresponding entries in the hash table. */ + for(p=pFS->pLruFirst; p; p=p->pLruNext){ + assert( p==pFS->pLruFirst || p->pLruPrev!=0 ); + assert( p==pFS->pLruLast || p->pLruNext!=0 ); + assert( p->pLruPrev==0 || p->pLruPrev->pLruNext==p ); + assert( p->pLruNext==0 || p->pLruNext->pLruPrev==p ); + assert( p->nRef==0 ); + assert( p->flags & PAGE_FREE ); + assert( p==fsPageFindInHash(pFS, p->iPg, 0) ); + } +#endif +} +#endif + +/* +** Wrappers around the VFS methods of the lsm_env object: +** +** lsmEnvOpen() +** lsmEnvRead() +** lsmEnvWrite() +** lsmEnvSync() +** lsmEnvSectorSize() +** lsmEnvClose() +** lsmEnvTruncate() +** lsmEnvUnlink() +** lsmEnvRemap() +*/ +int lsmEnvOpen(lsm_env *pEnv, const char *zFile, int flags, lsm_file **ppNew){ + return pEnv->xOpen(pEnv, zFile, flags, ppNew); +} + +static int lsmEnvRead( + lsm_env *pEnv, + lsm_file *pFile, + lsm_i64 iOff, + void *pRead, + int nRead +){ + return IOERR_WRAPPER( pEnv->xRead(pFile, iOff, pRead, nRead) ); +} + +static int lsmEnvWrite( + lsm_env *pEnv, + lsm_file *pFile, + lsm_i64 iOff, + const void *pWrite, + int nWrite +){ + return IOERR_WRAPPER( pEnv->xWrite(pFile, iOff, (void *)pWrite, nWrite) ); +} + +static int lsmEnvSync(lsm_env *pEnv, lsm_file *pFile){ + return IOERR_WRAPPER( pEnv->xSync(pFile) ); +} + +static int lsmEnvSectorSize(lsm_env *pEnv, lsm_file *pFile){ + return pEnv->xSectorSize(pFile); +} + +int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile){ + return IOERR_WRAPPER( pEnv->xClose(pFile) ); +} + +static int lsmEnvTruncate(lsm_env *pEnv, lsm_file *pFile, lsm_i64 nByte){ + return IOERR_WRAPPER( pEnv->xTruncate(pFile, nByte) ); +} + +static int lsmEnvUnlink(lsm_env *pEnv, const char *zDel){ + return IOERR_WRAPPER( pEnv->xUnlink(pEnv, zDel) ); +} + +static int lsmEnvRemap( + lsm_env *pEnv, + lsm_file *pFile, + i64 szMin, + void **ppMap, + i64 *pszMap +){ + return pEnv->xRemap(pFile, szMin, ppMap, pszMap); +} + +int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock){ + if( pFile==0 ) return LSM_OK; + return pEnv->xLock(pFile, iLock, eLock); +} + +int lsmEnvTestLock( + lsm_env *pEnv, + lsm_file *pFile, + int iLock, + int nLock, + int eLock +){ + return pEnv->xTestLock(pFile, iLock, nLock, eLock); +} + +int lsmEnvShmMap( + lsm_env *pEnv, + lsm_file *pFile, + int iChunk, + int sz, + void **ppOut +){ + return pEnv->xShmMap(pFile, iChunk, sz, ppOut); +} + +void lsmEnvShmBarrier(lsm_env *pEnv){ + pEnv->xShmBarrier(); +} + +void lsmEnvShmUnmap(lsm_env *pEnv, lsm_file *pFile, int bDel){ + pEnv->xShmUnmap(pFile, bDel); +} + +void lsmEnvSleep(lsm_env *pEnv, int nUs){ + pEnv->xSleep(pEnv, nUs); +} + + +/* +** Write the contents of string buffer pStr into the log file, starting at +** offset iOff. +*/ +int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr){ + assert( pFS->fdLog ); + return lsmEnvWrite(pFS->pEnv, pFS->fdLog, iOff, pStr->z, pStr->n); +} + +/* +** fsync() the log file. +*/ +int lsmFsSyncLog(FileSystem *pFS){ + assert( pFS->fdLog ); + return lsmEnvSync(pFS->pEnv, pFS->fdLog); +} + +/* +** Read nRead bytes of data starting at offset iOff of the log file. Append +** the results to string buffer pStr. +*/ +int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr){ + int rc; /* Return code */ + assert( pFS->fdLog ); + rc = lsmStringExtend(pStr, nRead); + if( rc==LSM_OK ){ + rc = lsmEnvRead(pFS->pEnv, pFS->fdLog, iOff, &pStr->z[pStr->n], nRead); + pStr->n += nRead; + } + return rc; +} + +/* +** Truncate the log file to nByte bytes in size. +*/ +int lsmFsTruncateLog(FileSystem *pFS, i64 nByte){ + if( pFS->fdLog==0 ) return LSM_OK; + return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); +} + +/* +** Truncate the db file to nByte bytes in size. +*/ +int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ + if( pFS->fdDb==0 ) return LSM_OK; + return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); +} + +/* +** Close the log file. Then delete it from the file-system. This function +** is called during database shutdown only. +*/ +int lsmFsCloseAndDeleteLog(FileSystem *pFS){ + char *zDel; + + if( pFS->fdLog ){ + lsmEnvClose(pFS->pEnv, pFS->fdLog ); + pFS->fdLog = 0; + } + + zDel = lsmMallocPrintf(pFS->pEnv, "%s-log", pFS->zDb); + if( zDel ){ + lsmEnvUnlink(pFS->pEnv, zDel); + lsmFree(pFS->pEnv, zDel); + } + return LSM_OK; +} + +/* +** Return true if page iReal of the database should be accessed using mmap. +** False otherwise. +*/ +static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){ + return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit); +} + +/* +** Given that there are currently nHash slots in the hash table, return +** the hash key for file iFile, page iPg. +*/ +static int fsHashKey(int nHash, LsmPgno iPg){ + return (iPg % nHash); +} + +/* +** This is a helper function for lsmFsOpen(). It opens a single file on +** disk (either the database or log file). +*/ +static lsm_file *fsOpenFile( + FileSystem *pFS, /* File system object */ + int bReadonly, /* True to open this file read-only */ + int bLog, /* True for log, false for db */ + int *pRc /* IN/OUT: Error code */ +){ + lsm_file *pFile = 0; + if( *pRc==LSM_OK ){ + int flags = (bReadonly ? LSM_OPEN_READONLY : 0); + const char *zPath = (bLog ? pFS->zLog : pFS->zDb); + + *pRc = lsmEnvOpen(pFS->pEnv, zPath, flags, &pFile); + } + return pFile; +} + +/* +** If it is not already open, this function opens the log file. It returns +** LSM_OK if successful (or if the log file was already open) or an LSM +** error code otherwise. +** +** The log file must be opened before any of the following may be called: +** +** lsmFsWriteLog +** lsmFsSyncLog +** lsmFsReadLog +*/ +int lsmFsOpenLog(lsm_db *db, int *pbOpen){ + int rc = LSM_OK; + FileSystem *pFS = db->pFS; + + if( 0==pFS->fdLog ){ + pFS->fdLog = fsOpenFile(pFS, db->bReadonly, 1, &rc); + + if( rc==LSM_IOERR_NOENT && db->bReadonly ){ + rc = LSM_OK; + } + } + + if( pbOpen ) *pbOpen = (pFS->fdLog!=0); + return rc; +} + +/* +** Close the log file, if it is open. +*/ +void lsmFsCloseLog(lsm_db *db){ + FileSystem *pFS = db->pFS; + if( pFS->fdLog ){ + lsmEnvClose(pFS->pEnv, pFS->fdLog); + pFS->fdLog = 0; + } +} + +/* +** Open a connection to a database stored within the file-system. +** +** If parameter bReadonly is true, then open a read-only file-descriptor +** on the database file. It is possible that bReadonly will be false even +** if the user requested that pDb be opened read-only. This is because the +** file-descriptor may later on be recycled by a read-write connection. +** If the db file can be opened for read-write access, it always is. Parameter +** bReadonly is only ever true if it has already been determined that the +** db can only be opened for read-only access. +** +** Return LSM_OK if successful or an lsm error code otherwise. +*/ +int lsmFsOpen( + lsm_db *pDb, /* Database connection to open fd for */ + const char *zDb, /* Full path to database file */ + int bReadonly /* True to open db file read-only */ +){ + FileSystem *pFS; + int rc = LSM_OK; + int nDb = strlen(zDb); + int nByte; + + assert( pDb->pFS==0 ); + assert( pDb->pWorker==0 && pDb->pClient==0 ); + + nByte = sizeof(FileSystem) + nDb+1 + nDb+4+1; + pFS = (FileSystem *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); + if( pFS ){ + LsmFile *pLsmFile; + pFS->zDb = (char *)&pFS[1]; + pFS->zLog = &pFS->zDb[nDb+1]; + pFS->nPagesize = LSM_DFLT_PAGE_SIZE; + pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; + pFS->nMetasize = LSM_META_PAGE_SIZE; + pFS->nMetaRwSize = LSM_META_RW_PAGE_SIZE; + pFS->pDb = pDb; + pFS->pEnv = pDb->pEnv; + + /* Make a copy of the database and log file names. */ + memcpy(pFS->zDb, zDb, nDb+1); + memcpy(pFS->zLog, zDb, nDb); + memcpy(&pFS->zLog[nDb], "-log", 5); + + /* Allocate the hash-table here. At some point, it should be changed + ** so that it can grow dynamicly. */ + pFS->nCacheMax = 2048*1024 / pFS->nPagesize; + pFS->nHash = 4096; + pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc); + + /* Open the database file */ + pLsmFile = lsmDbRecycleFd(pDb); + if( pLsmFile ){ + pFS->pLsmFile = pLsmFile; + pFS->fdDb = pLsmFile->pFile; + memset(pLsmFile, 0, sizeof(LsmFile)); + }else{ + pFS->pLsmFile = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmFile), &rc); + if( rc==LSM_OK ){ + pFS->fdDb = fsOpenFile(pFS, bReadonly, 0, &rc); + } + } + + if( rc!=LSM_OK ){ + lsmFsClose(pFS); + pFS = 0; + }else{ + pFS->szSector = lsmEnvSectorSize(pFS->pEnv, pFS->fdDb); + } + } + + pDb->pFS = pFS; + return rc; +} + +/* +** Configure the file-system object according to the current values of +** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. +*/ +int lsmFsConfigure(lsm_db *db){ + FileSystem *pFS = db->pFS; + if( pFS ){ + lsm_env *pEnv = pFS->pEnv; + Page *pPg; + + assert( pFS->nOut==0 ); + assert( pFS->pWaiting==0 ); + assert( pFS->pMapped==0 ); + + /* Reset any compression/decompression buffers already allocated */ + lsmFree(pEnv, pFS->aIBuffer); + lsmFree(pEnv, pFS->aOBuffer); + pFS->nBuffer = 0; + + /* Unmap the file, if it is currently mapped */ + if( pFS->pMap ){ + lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); + pFS->nMapLimit = 0; + } + + /* Free all allocated page structures */ + pPg = pFS->pLruFirst; + while( pPg ){ + Page *pNext = pPg->pLruNext; + assert( pPg->flags & PAGE_FREE ); + lsmFree(pEnv, pPg->aData); + lsmFree(pEnv, pPg); + pPg = pNext; + } + + pPg = pFS->pFree; + while( pPg ){ + Page *pNext = pPg->pFreeNext; + lsmFree(pEnv, pPg); + pPg = pNext; + } + + /* Zero pointers that point to deleted page objects */ + pFS->nCacheAlloc = 0; + pFS->pLruFirst = 0; + pFS->pLruLast = 0; + pFS->pFree = 0; + if( pFS->apHash ){ + memset(pFS->apHash, 0, pFS->nHash*sizeof(pFS->apHash[0])); + } + + /* Configure the FileSystem object */ + if( db->compress.xCompress ){ + pFS->pCompress = &db->compress; + pFS->nMapLimit = 0; + }else{ + pFS->pCompress = 0; + if( db->iMmap==1 ){ + /* Unlimited */ + pFS->nMapLimit = (i64)1 << 60; + }else{ + /* iMmap is a limit in KB. Set nMapLimit to the same value in bytes. */ + pFS->nMapLimit = (i64)db->iMmap * 1024; + } + } + } + + return LSM_OK; +} + +/* +** Close and destroy a FileSystem object. +*/ +void lsmFsClose(FileSystem *pFS){ + if( pFS ){ + Page *pPg; + lsm_env *pEnv = pFS->pEnv; + + assert( pFS->nOut==0 ); + pPg = pFS->pLruFirst; + while( pPg ){ + Page *pNext = pPg->pLruNext; + if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); + lsmFree(pEnv, pPg); + pPg = pNext; + } + + pPg = pFS->pFree; + while( pPg ){ + Page *pNext = pPg->pFreeNext; + if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); + lsmFree(pEnv, pPg); + pPg = pNext; + } + + if( pFS->fdDb ) lsmEnvClose(pFS->pEnv, pFS->fdDb ); + if( pFS->fdLog ) lsmEnvClose(pFS->pEnv, pFS->fdLog ); + lsmFree(pEnv, pFS->pLsmFile); + lsmFree(pEnv, pFS->apHash); + lsmFree(pEnv, pFS->aIBuffer); + lsmFree(pEnv, pFS->aOBuffer); + lsmFree(pEnv, pFS); + } +} + +/* +** This function is called when closing a database handle (i.e. lsm_close()) +** if there exist other connections to the same database within this process. +** In that case the file-descriptor open on the database file is not closed +** when the FileSystem object is destroyed, as this would cause any POSIX +** locks held by the other connections to be silently dropped (see "man close" +** for details). Instead, the file-descriptor is stored in a list by the +** lsm_shared.c module until it is either closed or reused. +** +** This function returns a pointer to an object that can be linked into +** the list described above. The returned object now 'owns' the database +** file descriptr, so that when the FileSystem object is destroyed, it +** will not be closed. +** +** This function may be called at most once in the life-time of a +** FileSystem object. The results of any operations involving the database +** file descriptor are undefined once this function has been called. +** +** None of this is necessary on non-POSIX systems. But we do it anyway in +** the name of using as similar code as possible on all platforms. +*/ +LsmFile *lsmFsDeferClose(FileSystem *pFS){ + LsmFile *p = pFS->pLsmFile; + assert( p->pNext==0 ); + p->pFile = pFS->fdDb; + pFS->fdDb = 0; + pFS->pLsmFile = 0; + return p; +} + +/* +** Allocate a buffer and populate it with the output of the xFileid() +** method of the database file handle. If successful, set *ppId to point +** to the buffer and *pnId to the number of bytes in the buffer and return +** LSM_OK. Otherwise, set *ppId and *pnId to zero and return an LSM +** error code. +*/ +int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId){ + lsm_env *pEnv = pDb->pEnv; + FileSystem *pFS = pDb->pFS; + int rc; + int nId = 0; + void *pId; + + rc = pEnv->xFileid(pFS->fdDb, 0, &nId); + pId = lsmMallocZeroRc(pEnv, nId, &rc); + if( rc==LSM_OK ) rc = pEnv->xFileid(pFS->fdDb, pId, &nId); + + if( rc!=LSM_OK ){ + lsmFree(pEnv, pId); + pId = 0; + nId = 0; + } + + *ppId = pId; + *pnId = nId; + return rc; +} + +/* +** Return the nominal page-size used by this file-system. Actual pages +** may be smaller or larger than this value. +*/ +int lsmFsPageSize(FileSystem *pFS){ + return pFS->nPagesize; +} + +/* +** Return the block-size used by this file-system. +*/ +int lsmFsBlockSize(FileSystem *pFS){ + return pFS->nBlocksize; +} + +/* +** Configure the nominal page-size used by this file-system. Actual +** pages may be smaller or larger than this value. +*/ +void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){ + pFS->nPagesize = nPgsz; + pFS->nCacheMax = 2048*1024 / pFS->nPagesize; +} + +/* +** Configure the block-size used by this file-system. +*/ +void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){ + pFS->nBlocksize = nBlocksize; +} + +/* +** Return the page number of the first page on block iBlock. Blocks are +** numbered starting from 1. +** +** For a compressed database, page numbers are byte offsets. The first +** page on each block is the byte offset immediately following the 4-byte +** "previous block" pointer at the start of each block. +*/ +static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){ + LsmPgno iPg; + if( pFS->pCompress ){ + if( iBlock==1 ){ + iPg = pFS->nMetasize * 2 + 4; + }else{ + iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4; + } + }else{ + const i64 nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + if( iBlock==1 ){ + iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize); + }else{ + iPg = 1 + (iBlock-1) * nPagePerBlock; + } + } + return iPg; +} + +/* +** Return the page number of the last page on block iBlock. Blocks are +** numbered starting from 1. +** +** For a compressed database, page numbers are byte offsets. The first +** page on each block is the byte offset of the byte immediately before +** the 4-byte "next block" pointer at the end of each block. +*/ +static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){ + if( pFS->pCompress ){ + return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4; + }else{ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + return iBlock * nPagePerBlock; + } +} + +/* +** Return the block number of the block that page iPg is located on. +** Blocks are numbered starting from 1. +*/ +static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){ + if( pFS->pCompress ){ + return (int)((iPg / pFS->nBlocksize) + 1); + }else{ + return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize))); + } +} + +/* +** Return true if page iPg is the last page on its block. +** +** This function is only called in non-compressed database mode. +*/ +static int fsIsLast(FileSystem *pFS, LsmPgno iPg){ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + assert( !pFS->pCompress ); + return ( iPg && (iPg % nPagePerBlock)==0 ); +} + +/* +** Return true if page iPg is the first page on its block. +** +** This function is only called in non-compressed database mode. +*/ +static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + assert( !pFS->pCompress ); + return ( (iPg % nPagePerBlock)==1 + || (iPgnData; + } + return pPage->aData; +} + +/* +** Return the page number of a page. +*/ +LsmPgno lsmFsPageNumber(Page *pPage){ + /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */ + return pPage ? pPage->iPg : 0; +} + +/* +** Page pPg is currently part of the LRU list belonging to pFS. Remove +** it from the list. pPg->pLruNext and pPg->pLruPrev are cleared by this +** operation. +*/ +static void fsPageRemoveFromLru(FileSystem *pFS, Page *pPg){ + assert( pPg->pLruNext || pPg==pFS->pLruLast ); + assert( pPg->pLruPrev || pPg==pFS->pLruFirst ); + if( pPg->pLruNext ){ + pPg->pLruNext->pLruPrev = pPg->pLruPrev; + }else{ + pFS->pLruLast = pPg->pLruPrev; + } + if( pPg->pLruPrev ){ + pPg->pLruPrev->pLruNext = pPg->pLruNext; + }else{ + pFS->pLruFirst = pPg->pLruNext; + } + pPg->pLruPrev = 0; + pPg->pLruNext = 0; +} + +/* +** Page pPg is not currently part of the LRU list belonging to pFS. Add it. +*/ +static void fsPageAddToLru(FileSystem *pFS, Page *pPg){ + assert( pPg->pLruNext==0 && pPg->pLruPrev==0 ); + pPg->pLruPrev = pFS->pLruLast; + if( pPg->pLruPrev ){ + pPg->pLruPrev->pLruNext = pPg; + }else{ + pFS->pLruFirst = pPg; + } + pFS->pLruLast = pPg; +} + +/* +** Page pPg is currently stored in the apHash/nHash hash table. Remove it. +*/ +static void fsPageRemoveFromHash(FileSystem *pFS, Page *pPg){ + int iHash; + Page **pp; + + iHash = fsHashKey(pFS->nHash, pPg->iPg); + for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext); + *pp = pPg->pHashNext; + pPg->pHashNext = 0; +} + +/* +** Free a Page object allocated by fsPageBuffer(). +*/ +static void fsPageBufferFree(Page *pPg){ + pPg->pFS->nCacheAlloc--; + lsmFree(pPg->pFS->pEnv, pPg->aData); + lsmFree(pPg->pFS->pEnv, pPg); +} + + +/* +** Purge the cache of all non-mmap pages with nRef==0. +*/ +void lsmFsPurgeCache(FileSystem *pFS){ + Page *pPg; + + pPg = pFS->pLruFirst; + while( pPg ){ + Page *pNext = pPg->pLruNext; + assert( pPg->flags & PAGE_FREE ); + fsPageRemoveFromHash(pFS, pPg); + fsPageBufferFree(pPg); + pPg = pNext; + } + pFS->pLruFirst = 0; + pFS->pLruLast = 0; + + assert( pFS->nCacheAlloc<=pFS->nOut && pFS->nCacheAlloc>=0 ); +} + +/* +** Search the hash-table for page iPg. If an entry is round, return a pointer +** to it. Otherwise, return NULL. +** +** Either way, if argument piHash is not NULL set *piHash to the hash slot +** number that page iPg would be stored in before returning. +*/ +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){ + Page *p; /* Return value */ + int iHash = fsHashKey(pFS->nHash, iPg); + + if( piHash ) *piHash = iHash; + for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ + if( p->iPg==iPg) break; + } + return p; +} + +/* +** Allocate and return a non-mmap Page object. If there are already +** nCacheMax such Page objects outstanding, try to recycle an existing +** Page instead. +*/ +static int fsPageBuffer( + FileSystem *pFS, + Page **ppOut +){ + int rc = LSM_OK; + Page *pPage = 0; + if( pFS->pLruFirst==0 || pFS->nCacheAllocnCacheMax ){ + /* Allocate a new Page object */ + pPage = lsmMallocZero(pFS->pEnv, sizeof(Page)); + if( !pPage ){ + rc = LSM_NOMEM_BKPT; + }else{ + pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize); + if( !pPage->aData ){ + lsmFree(pFS->pEnv, pPage); + rc = LSM_NOMEM_BKPT; + pPage = 0; + }else{ + pFS->nCacheAlloc++; + } + } + }else{ + /* Reuse an existing Page object */ + u8 *aData; + pPage = pFS->pLruFirst; + aData = pPage->aData; + fsPageRemoveFromLru(pFS, pPage); + fsPageRemoveFromHash(pFS, pPage); + + memset(pPage, 0, sizeof(Page)); + pPage->aData = aData; + } + + if( pPage ){ + pPage->flags = PAGE_FREE; + } + *ppOut = pPage; + return rc; +} + +/* +** Assuming *pRc is initially LSM_OK, attempt to ensure that the +** memory-mapped region is at least iSz bytes in size. If it is not already, +** iSz bytes in size, extend it and update the pointers associated with any +** outstanding Page objects. +** +** If *pRc is not LSM_OK when this function is called, it is a no-op. +** Otherwise, *pRc is set to an lsm error code if an error occurs, or +** left unmodified otherwise. +** +** This function is never called in compressed database mode. +*/ +static void fsGrowMapping( + FileSystem *pFS, /* File system object */ + i64 iSz, /* Minimum size to extend mapping to */ + int *pRc /* IN/OUT: Error code */ +){ + assert( PAGE_HASPREV==4 ); + + if( *pRc==LSM_OK && iSz>pFS->nMap ){ + int rc; + u8 *aOld = pFS->pMap; + rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, iSz, &pFS->pMap, &pFS->nMap); + if( rc==LSM_OK && pFS->pMap!=aOld ){ + Page *pFix; + i64 iOff = (u8 *)pFS->pMap - aOld; + for(pFix=pFS->pMapped; pFix; pFix=pFix->pMappedNext){ + pFix->aData += iOff; + } + lsmSortedRemap(pFS->pDb); + } + *pRc = rc; + } +} + +/* +** If it is mapped, unmap the database file. +*/ +int lsmFsUnmap(FileSystem *pFS){ + int rc = LSM_OK; + if( pFS ){ + rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); + } + return rc; +} + +/* +** fsync() the database file. +*/ +int lsmFsSyncDb(FileSystem *pFS, int nBlock){ + return lsmEnvSync(pFS->pEnv, pFS->fdDb); +} + +/* +** If block iBlk has been redirected according to the redirections in the +** object passed as the first argument, return the destination block to +** which it is redirected. Otherwise, return a copy of iBlk. +*/ +static int fsRedirectBlock(Redirect *p, int iBlk){ + if( p ){ + int i; + for(i=0; in; i++){ + if( iBlk==p->a[i].iFrom ) return p->a[i].iTo; + } + } + assert( iBlk!=0 ); + return iBlk; +} + +/* +** If page iPg has been redirected according to the redirections in the +** object passed as the second argument, return the destination page to +** which it is redirected. Otherwise, return a copy of iPg. +*/ +LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){ + LsmPgno iReal = iPg; + + if( pRedir ){ + const int nPagePerBlock = ( + pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) + ); + int iBlk = fsPageToBlock(pFS, iPg); + int i; + for(i=0; in; i++){ + int iFrom = pRedir->a[i].iFrom; + if( iFrom>iBlk ) break; + if( iFrom==iBlk ){ + int iTo = pRedir->a[i].iTo; + iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; + if( iTo==1 ){ + iReal += (fsFirstPageOnBlock(pFS, 1)-1); + } + break; + } + } + } + + assert( iReal!=0 ); + return iReal; +} + +/* Required by the circular fsBlockNext<->fsPageGet dependency. */ +static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *); + +/* +** Parameter iBlock is a database file block. This function reads the value +** stored in the blocks "next block" pointer and stores it in *piNext. +** LSM_OK is returned if everything is successful, or an LSM error code +** otherwise. +*/ +static int fsBlockNext( + FileSystem *pFS, /* File-system object handle */ + Segment *pSeg, /* Use this segment for block redirects */ + int iBlock, /* Read field from this block */ + int *piNext /* OUT: Next block in linked list */ +){ + int rc; + int iRead; /* Read block from here */ + + if( pSeg ){ + iRead = fsRedirectBlock(pSeg->pRedirect, iBlock); + }else{ + iRead = iBlock; + } + + assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); + if( pFS->pCompress ){ + i64 iOff; /* File offset to read data from */ + u8 aNext[4]; /* 4-byte pointer read from db file */ + + iOff = (i64)iRead * pFS->nBlocksize - sizeof(aNext); + rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aNext, sizeof(aNext)); + if( rc==LSM_OK ){ + *piNext = (int)lsmGetU32(aNext); + } + }else{ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + Page *pLast; + rc = fsPageGet(pFS, 0, iRead*nPagePerBlock, 0, &pLast, 0); + if( rc==LSM_OK ){ + *piNext = lsmGetU32(&pLast->aData[pFS->nPagesize-4]); + lsmFsPageRelease(pLast); + } + } + + if( pSeg ){ + *piNext = fsRedirectBlock(pSeg->pRedirect, *piNext); + } + return rc; +} + +/* +** Return the page number of the last page on the same block as page iPg. +*/ +LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){ + return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg)); +} + +/* +** Read nData bytes of data from offset iOff of the database file into +** buffer aData. If this means reading past the end of a block, follow +** the block pointer to the next block and continue reading. +** +** Offset iOff is an absolute offset - not subject to any block redirection. +** However any block pointer followed is. Use pSeg->pRedirect in this case. +** +** This function is only called in compressed database mode. +*/ +static int fsReadData( + FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* Block redirection */ + i64 iOff, /* Read data from this offset */ + u8 *aData, /* Buffer to read data into */ + int nData /* Number of bytes to read */ +){ + i64 iEob; /* End of block */ + int nRead; + int rc; + + assert( pFS->pCompress ); + + iEob = fsLastPageOnPagesBlock(pFS, iOff) + 1; + nRead = (int)LSM_MIN(iEob - iOff, nData); + + rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nRead); + if( rc==LSM_OK && nRead!=nData ){ + int iBlk; + + rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); + if( rc==LSM_OK ){ + i64 iOff2 = fsFirstPageOnBlock(pFS, iBlk); + rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff2, &aData[nRead], nData-nRead); + } + } + + return rc; +} + +/* +** Parameter iBlock is a database file block. This function reads the value +** stored in the blocks "previous block" pointer and stores it in *piPrev. +** LSM_OK is returned if everything is successful, or an LSM error code +** otherwise. +*/ +static int fsBlockPrev( + FileSystem *pFS, /* File-system object handle */ + Segment *pSeg, /* Use this segment for block redirects */ + int iBlock, /* Read field from this block */ + int *piPrev /* OUT: Previous block in linked list */ +){ + int rc = LSM_OK; /* Return code */ + + assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); + assert( iBlock>0 ); + + if( pFS->pCompress ){ + i64 iOff = fsFirstPageOnBlock(pFS, iBlock) - 4; + u8 aPrev[4]; /* 4-byte pointer read from db file */ + rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aPrev, sizeof(aPrev)); + if( rc==LSM_OK ){ + Redirect *pRedir = (pSeg ? pSeg->pRedirect : 0); + *piPrev = fsRedirectBlock(pRedir, (int)lsmGetU32(aPrev)); + } + }else{ + assert( 0 ); + } + return rc; +} + +/* +** Encode and decode routines for record size fields. +*/ +static void putRecordSize(u8 *aBuf, int nByte, int bFree){ + aBuf[0] = (u8)(nByte >> 14) | 0x80; + aBuf[1] = ((u8)(nByte >> 7) & 0x7F) | (bFree ? 0x00 : 0x80); + aBuf[2] = (u8)nByte | 0x80; +} +static int getRecordSize(u8 *aBuf, int *pbFree){ + int nByte; + nByte = (aBuf[0] & 0x7F) << 14; + nByte += (aBuf[1] & 0x7F) << 7; + nByte += (aBuf[2] & 0x7F); + *pbFree = !(aBuf[1] & 0x80); + return nByte; +} + +/* +** Subtract iSub from database file offset iOff and set *piRes to the +** result. If doing so means passing the start of a block, follow the +** block pointer stored in the first 4 bytes of the block. +** +** Offset iOff is an absolute offset - not subject to any block redirection. +** However any block pointer followed is. Use pSeg->pRedirect in this case. +** +** Return LSM_OK if successful or an lsm error code if an error occurs. +*/ +static int fsSubtractOffset( + FileSystem *pFS, + Segment *pSeg, + i64 iOff, + int iSub, + i64 *piRes +){ + i64 iStart; + int iBlk = 0; + int rc; + + assert( pFS->pCompress ); + + iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); + if( (iOff-iSub)>=iStart ){ + *piRes = (iOff-iSub); + return LSM_OK; + } + + rc = fsBlockPrev(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); + *piRes = fsLastPageOnBlock(pFS, iBlk) - iSub + (iOff - iStart + 1); + return rc; +} + +/* +** Add iAdd to database file offset iOff and set *piRes to the +** result. If doing so means passing the end of a block, follow the +** block pointer stored in the last 4 bytes of the block. +** +** Offset iOff is an absolute offset - not subject to any block redirection. +** However any block pointer followed is. Use pSeg->pRedirect in this case. +** +** Return LSM_OK if successful or an lsm error code if an error occurs. +*/ +static int fsAddOffset( + FileSystem *pFS, + Segment *pSeg, + i64 iOff, + int iAdd, + i64 *piRes +){ + i64 iEob; + int iBlk; + int rc; + + assert( pFS->pCompress ); + + iEob = fsLastPageOnPagesBlock(pFS, iOff); + if( (iOff+iAdd)<=iEob ){ + *piRes = (iOff+iAdd); + return LSM_OK; + } + + rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); + *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); + return rc; +} + +/* +** If it is not already allocated, allocate either the FileSystem.aOBuffer (if +** bWrite is true) or the FileSystem.aIBuffer (if bWrite is false). Return +** LSM_OK if successful if the attempt to allocate memory fails. +*/ +static int fsAllocateBuffer(FileSystem *pFS, int bWrite){ + u8 **pp; /* Pointer to either aIBuffer or aOBuffer */ + + assert( pFS->pCompress ); + + /* If neither buffer has been allocated, figure out how large they + ** should be. Store this value in FileSystem.nBuffer. */ + if( pFS->nBuffer==0 ){ + assert( pFS->aIBuffer==0 && pFS->aOBuffer==0 ); + pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize); + if( pFS->nBuffer<(pFS->szSector+6) ){ + pFS->nBuffer = pFS->szSector+6; + } + } + + pp = (bWrite ? &pFS->aOBuffer : &pFS->aIBuffer); + if( *pp==0 ){ + *pp = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); + if( *pp==0 ) return LSM_NOMEM_BKPT; + } + + return LSM_OK; +} + +/* +** This function is only called in compressed database mode. It reads and +** uncompresses the compressed data for page pPg from the database and +** populates the pPg->aData[] buffer and pPg->nCompress field. +** +** It is possible that instead of a page record, there is free space +** at offset pPg->iPgno. In this case no data is read from the file, but +** output variable *pnSpace is set to the total number of free bytes. +** +** LSM_OK is returned if successful, or an LSM error code otherwise. +*/ +static int fsReadPagedata( + FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* pPg is part of this segment */ + Page *pPg, /* Page to read and uncompress data for */ + int *pnSpace /* OUT: Total bytes of free space */ +){ + lsm_compress *p = pFS->pCompress; + i64 iOff = pPg->iPg; + u8 aSz[3]; + int rc; + + assert( p && pPg->nCompress==0 ); + + if( fsAllocateBuffer(pFS, 0) ) return LSM_NOMEM; + + rc = fsReadData(pFS, pSeg, iOff, aSz, sizeof(aSz)); + + if( rc==LSM_OK ){ + int bFree; + if( aSz[0] & 0x80 ){ + pPg->nCompress = (int)getRecordSize(aSz, &bFree); + }else{ + pPg->nCompress = (int)aSz[0] - sizeof(aSz)*2; + bFree = 1; + } + if( bFree ){ + if( pnSpace ){ + *pnSpace = pPg->nCompress + sizeof(aSz)*2; + }else{ + rc = LSM_CORRUPT_BKPT; + } + }else{ + rc = fsAddOffset(pFS, pSeg, iOff, 3, &iOff); + if( rc==LSM_OK ){ + if( pPg->nCompress>pFS->nBuffer ){ + rc = LSM_CORRUPT_BKPT; + }else{ + rc = fsReadData(pFS, pSeg, iOff, pFS->aIBuffer, pPg->nCompress); + } + if( rc==LSM_OK ){ + int n = pFS->nPagesize; + rc = p->xUncompress(p->pCtx, + (char *)pPg->aData, &n, + (const char *)pFS->aIBuffer, pPg->nCompress + ); + if( rc==LSM_OK && n!=pPg->pFS->nPagesize ){ + rc = LSM_CORRUPT_BKPT; + } + } + } + } + } + return rc; +} + +/* +** Return a handle for a database page. +** +** If this file-system object is accessing a compressed database it may be +** that there is no page record at database file offset iPg. Instead, there +** may be a free space record. In this case, set *ppPg to NULL and *pnSpace +** to the total number of free bytes before returning. +** +** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code. +*/ +static int fsPageGet( + FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* Block redirection to use (or NULL) */ + LsmPgno iPg, /* Page id */ + int noContent, /* True to not load content from disk */ + Page **ppPg, /* OUT: New page handle */ + int *pnSpace /* OUT: Bytes of free space */ +){ + Page *p; + int iHash; + int rc = LSM_OK; + + /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is + ** not NULL, and the block containing iPg has been redirected, then iReal + ** is the page number after redirection. */ + LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); + + assert_lists_are_ok(pFS); + assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); + assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); + *ppPg = 0; + + /* Search the hash-table for the page */ + p = fsPageFindInHash(pFS, iReal, &iHash); + + if( p ){ + assert( p->flags & PAGE_FREE ); + if( p->nRef==0 ) fsPageRemoveFromLru(pFS, p); + }else{ + + if( fsMmapPage(pFS, iReal) ){ + i64 iEnd = (i64)iReal * pFS->nPagesize; + fsGrowMapping(pFS, iEnd, &rc); + if( rc!=LSM_OK ) return rc; + + if( pFS->pFree ){ + p = pFS->pFree; + pFS->pFree = p->pFreeNext; + assert( p->nRef==0 ); + }else{ + p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); + if( rc ) return rc; + p->pFS = pFS; + } + p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (iReal-1)]; + p->iPg = iReal; + + /* This page now carries a pointer to the mapping. Link it in to + ** the FileSystem.pMapped list. */ + assert( p->pMappedNext==0 ); + p->pMappedNext = pFS->pMapped; + pFS->pMapped = p; + + assert( pFS->pCompress==0 ); + assert( (p->flags & PAGE_FREE)==0 ); + }else{ + rc = fsPageBuffer(pFS, &p); + if( rc==LSM_OK ){ + int nSpace = 0; + p->iPg = iReal; + p->nRef = 0; + p->pFS = pFS; + assert( p->flags==0 || p->flags==PAGE_FREE ); + +#ifdef LSM_DEBUG + memset(p->aData, 0x56, pFS->nPagesize); +#endif + assert( p->pLruNext==0 && p->pLruPrev==0 ); + if( noContent==0 ){ + if( pFS->pCompress ){ + rc = fsReadPagedata(pFS, pSeg, p, &nSpace); + }else{ + int nByte = pFS->nPagesize; + i64 iOff = (i64)(iReal-1) * pFS->nPagesize; + rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, p->aData, nByte); + } + pFS->nRead++; + } + + /* If the xRead() call was successful (or not attempted), link the + ** page into the page-cache hash-table. Otherwise, if it failed, + ** free the buffer. */ + if( rc==LSM_OK && nSpace==0 ){ + p->pHashNext = pFS->apHash[iHash]; + pFS->apHash[iHash] = p; + }else{ + fsPageBufferFree(p); + p = 0; + if( pnSpace ) *pnSpace = nSpace; + } + } + } + + assert( (rc==LSM_OK && (p || (pnSpace && *pnSpace))) + || (rc!=LSM_OK && p==0) + ); + } + + if( rc==LSM_OK && p ){ + if( pFS->pCompress==0 && (fsIsLast(pFS, iReal) || fsIsFirst(pFS, iReal)) ){ + p->nData = pFS->nPagesize - 4; + if( fsIsFirst(pFS, iReal) && p->nRef==0 ){ + p->aData += 4; + p->flags |= PAGE_HASPREV; + } + }else{ + p->nData = pFS->nPagesize; + } + pFS->nOut += (p->nRef==0); + p->nRef++; + } + *ppPg = p; + return rc; +} + +/* +** Read the 64-bit checkpoint id of the checkpoint currently stored on meta +** page iMeta of the database file. If no error occurs, store the id value +** in *piVal and return LSM_OK. Otherwise, return an LSM error code and leave +** *piVal unmodified. +** +** If a checkpointer connection is currently updating meta-page iMeta, or an +** earlier checkpointer crashed while doing so, the value read into *piVal +** may be garbage. It is the callers responsibility to deal with this. +*/ +int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){ + FileSystem *pFS = db->pFS; + int rc = LSM_OK; + + assert( iMeta==1 || iMeta==2 ); + if( pFS->nMapLimit>0 ){ + fsGrowMapping(pFS, iMeta*LSM_META_PAGE_SIZE, &rc); + if( rc==LSM_OK ){ + *piVal = (i64)lsmGetU64(&((u8 *)pFS->pMap)[(iMeta-1)*LSM_META_PAGE_SIZE]); + } + }else{ + MetaPage *pMeta = 0; + rc = lsmFsMetaPageGet(pFS, 0, iMeta, &pMeta); + if( rc==LSM_OK ){ + *piVal = (i64)lsmGetU64(pMeta->aData); + lsmFsMetaPageRelease(pMeta); + } + } + + return rc; +} + + +/* +** Return true if the first or last page of segment pRun falls between iFirst +** and iLast, inclusive, and pRun is not equal to pIgnore. +*/ +static int fsRunEndsBetween( + Segment *pRun, + Segment *pIgnore, + LsmPgno iFirst, + LsmPgno iLast +){ + return (pRun!=pIgnore && ( + (pRun->iFirst>=iFirst && pRun->iFirst<=iLast) + || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast) + )); +} + +/* +** Return true if level pLevel contains a segment other than pIgnore for +** which the first or last page is between iFirst and iLast, inclusive. +*/ +static int fsLevelEndsBetween( + Level *pLevel, + Segment *pIgnore, + LsmPgno iFirst, + LsmPgno iLast +){ + int i; + + if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){ + return 1; + } + for(i=0; inRight; i++){ + if( fsRunEndsBetween(&pLevel->aRhs[i], pIgnore, iFirst, iLast) ){ + return 1; + } + } + + return 0; +} + +/* +** Block iBlk is no longer in use by segment pIgnore. If it is not in use +** by any other segment, move it to the free block list. +*/ +static int fsFreeBlock( + FileSystem *pFS, /* File system object */ + Snapshot *pSnapshot, /* Worker snapshot */ + Segment *pIgnore, /* Ignore this run when searching */ + int iBlk /* Block number of block to free */ +){ + int rc = LSM_OK; /* Return code */ + LsmPgno iFirst; /* First page on block iBlk */ + LsmPgno iLast; /* Last page on block iBlk */ + Level *pLevel; /* Used to iterate through levels */ + + int iIn; /* Used to iterate through append points */ + int iOut = 0; /* Used to output append points */ + LsmPgno *aApp = pSnapshot->aiAppend; + + iFirst = fsFirstPageOnBlock(pFS, iBlk); + iLast = fsLastPageOnBlock(pFS, iBlk); + + /* Check if any other run in the snapshot has a start or end page + ** within this block. If there is such a run, return early. */ + for(pLevel=lsmDbSnapshotLevel(pSnapshot); pLevel; pLevel=pLevel->pNext){ + if( fsLevelEndsBetween(pLevel, pIgnore, iFirst, iLast) ){ + return LSM_OK; + } + } + + /* Remove any entries that lie on this block from the append-list. */ + for(iIn=0; iIniLast ){ + aApp[iOut++] = aApp[iIn]; + } + } + while( iOutpDb, iBlk); + } + return rc; +} + +/* +** Delete or otherwise recycle the blocks currently occupied by run pDel. +*/ +int lsmFsSortedDelete( + FileSystem *pFS, + Snapshot *pSnapshot, + int bZero, /* True to zero the Segment structure */ + Segment *pDel +){ + if( pDel->iFirst ){ + int rc = LSM_OK; + + int iBlk; + int iLastBlk; + + iBlk = fsPageToBlock(pFS, pDel->iFirst); + iLastBlk = fsPageToBlock(pFS, pDel->iLastPg); + + /* Mark all blocks currently used by this sorted run as free */ + while( iBlk && rc==LSM_OK ){ + int iNext = 0; + if( iBlk!=iLastBlk ){ + rc = fsBlockNext(pFS, pDel, iBlk, &iNext); + }else if( bZero==0 && pDel->iLastPg!=fsLastPageOnBlock(pFS, iLastBlk) ){ + break; + } + rc = fsFreeBlock(pFS, pSnapshot, pDel, iBlk); + iBlk = iNext; + } + + if( pDel->pRedirect ){ + assert( pDel->pRedirect==&pSnapshot->redirect ); + pSnapshot->redirect.n = 0; + } + + if( bZero ) memset(pDel, 0, sizeof(Segment)); + } + return LSM_OK; +} + +/* +** aPgno is an array containing nPgno page numbers. Return the smallest page +** number from the array that falls on block iBlk. Or, if none of the pages +** in aPgno[] fall on block iBlk, return 0. +*/ +static LsmPgno firstOnBlock( + FileSystem *pFS, + int iBlk, + LsmPgno *aPgno, + int nPgno +){ + LsmPgno iRet = 0; + int i; + for(i=0; ipRedirect, iPg)); +} + +/* +** Return true if the second argument is not NULL and any of the first +** last or root pages lie on a redirected block. +*/ +static int fsSegmentRedirects(FileSystem *pFS, Segment *p){ + return (p && ( + fsPageRedirects(pFS, p, p->iFirst) + || fsPageRedirects(pFS, p, p->iRoot) + || fsPageRedirects(pFS, p, p->iLastPg) + )); +} +#endif + +/* +** Argument aPgno is an array of nPgno page numbers. All pages belong to +** the segment pRun. This function gobbles from the start of the run to the +** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is +** the new first page of the run). +*/ +void lsmFsGobble( + lsm_db *pDb, + Segment *pRun, + LsmPgno *aPgno, + int nPgno +){ + int rc = LSM_OK; + FileSystem *pFS = pDb->pFS; + Snapshot *pSnapshot = pDb->pWorker; + int iBlk; + + assert( pRun->nSize>0 ); + assert( 0==fsSegmentRedirects(pFS, pRun) ); + assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) ); + + iBlk = fsPageToBlock(pFS, pRun->iFirst); + pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); + + while( rc==LSM_OK ){ + int iNext = 0; + LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); + if( iFirst ){ + pRun->iFirst = iFirst; + break; + } + rc = fsBlockNext(pFS, pRun, iBlk, &iNext); + if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk); + pRun->nSize -= ( + 1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk) + ); + iBlk = iNext; + } + + pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); + assert( pRun->nSize>0 ); +} + +/* +** This function is only used in compressed database mode. +** +** Argument iPg is the page number (byte offset) of a page within segment +** pSeg. The page record, including all headers, is nByte bytes in size. +** Before returning, set *piNext to the page number of the next page in +** the segment, or to zero if iPg is the last. +** +** In other words, do: +** +** *piNext = iPg + nByte; +** +** But take block overflow and redirection into account. +*/ +static int fsNextPageOffset( + FileSystem *pFS, /* File system object */ + Segment *pSeg, /* Segment to move within */ + LsmPgno iPg, /* Offset of current page */ + int nByte, /* Size of current page including headers */ + LsmPgno *piNext /* OUT: Offset of next page. Or zero (EOF) */ +){ + LsmPgno iNext; + int rc; + + assert( pFS->pCompress ); + + rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); + if( pSeg && iNext==pSeg->iLastPg ){ + iNext = 0; + }else if( rc==LSM_OK ){ + rc = fsAddOffset(pFS, pSeg, iNext, 1, &iNext); + } + + *piNext = iNext; + return rc; +} + +/* +** This function is only used in compressed database mode. +** +** Argument iPg is the page number of a pagethat appears in segment pSeg. +** This function determines the page number of the previous page in the +** same run. *piPrev is set to the previous page number before returning. +** +** LSM_OK is returned if no error occurs. Otherwise, an lsm error code. +** If any value other than LSM_OK is returned, then the final value of +** *piPrev is undefined. +*/ +static int fsGetPageBefore( + FileSystem *pFS, + Segment *pSeg, + LsmPgno iPg, + LsmPgno *piPrev +){ + u8 aSz[3]; + int rc; + i64 iRead; + + assert( pFS->pCompress ); + + rc = fsSubtractOffset(pFS, pSeg, iPg, sizeof(aSz), &iRead); + if( rc==LSM_OK ) rc = fsReadData(pFS, pSeg, iRead, aSz, sizeof(aSz)); + + if( rc==LSM_OK ){ + int bFree; + int nSz; + if( aSz[2] & 0x80 ){ + nSz = getRecordSize(aSz, &bFree) + sizeof(aSz)*2; + }else{ + nSz = (int)(aSz[2] & 0x7F); + bFree = 1; + } + rc = fsSubtractOffset(pFS, pSeg, iPg, nSz, piPrev); + } + + return rc; +} + +/* +** The first argument to this function is a valid reference to a database +** file page that is part of a sorted run. If parameter eDir is -1, this +** function attempts to locate and load the previous page in the same run. +** Or, if eDir is +1, it attempts to find the next page in the same run. +** The results of passing an eDir value other than positive or negative one +** are undefined. +** +** If parameter pRun is not NULL then it must point to the run that page +** pPg belongs to. In this case, if pPg is the first or last page of the +** run, and the request is for the previous or next page, respectively, +** *ppNext is set to NULL before returning LSM_OK. If pRun is NULL, then it +** is assumed that the next or previous page, as requested, exists. +** +** If the previous/next page does exist and is successfully loaded, *ppNext +** is set to point to it and LSM_OK is returned. Otherwise, if an error +** occurs, *ppNext is set to NULL and and lsm error code returned. +** +** Page references returned by this function should be released by the +** caller using lsmFsPageRelease(). +*/ +int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ + int rc = LSM_OK; + FileSystem *pFS = pPg->pFS; + LsmPgno iPg = pPg->iPg; + + assert( 0==fsSegmentRedirects(pFS, pRun) ); + if( pFS->pCompress ){ + int nSpace = pPg->nCompress + 2*3; + + do { + if( eDir>0 ){ + rc = fsNextPageOffset(pFS, pRun, iPg, nSpace, &iPg); + }else{ + if( iPg==pRun->iFirst ){ + iPg = 0; + }else{ + rc = fsGetPageBefore(pFS, pRun, iPg, &iPg); + } + } + + nSpace = 0; + if( iPg!=0 ){ + rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, &nSpace); + assert( (*ppNext==0)==(rc!=LSM_OK || nSpace>0) ); + }else{ + *ppNext = 0; + } + }while( nSpace>0 && rc==LSM_OK ); + + }else{ + Redirect *pRedir = pRun ? pRun->pRedirect : 0; + assert( eDir==1 || eDir==-1 ); + if( eDir<0 ){ + if( pRun && iPg==pRun->iFirst ){ + *ppNext = 0; + return LSM_OK; + }else if( fsIsFirst(pFS, iPg) ){ + assert( pPg->flags & PAGE_HASPREV ); + iPg = fsLastPageOnBlock(pFS, lsmGetU32(&pPg->aData[-4])); + }else{ + iPg--; + } + }else{ + if( pRun ){ + if( iPg==pRun->iLastPg ){ + *ppNext = 0; + return LSM_OK; + } + } + + if( fsIsLast(pFS, iPg) ){ + int iBlk = fsRedirectBlock( + pRedir, lsmGetU32(&pPg->aData[pFS->nPagesize-4]) + ); + iPg = fsFirstPageOnBlock(pFS, iBlk); + }else{ + iPg++; + } + } + rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, 0); + } + + return rc; +} + +/* +** This function is called when creating a new segment to determine if the +** first part of it can be written following an existing segment on an +** already allocated block. If it is possible, the page number of the first +** page to use for the new segment is returned. Otherwise zero. +** +** If argument pLvl is not NULL, then this function will not attempt to +** start the new segment immediately following any segment that is part +** of the right-hand-side of pLvl. +*/ +static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){ + int i; + LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; + LsmPgno iRet = 0; + + for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ + if( (iRet = aiAppend[i]) ){ + if( pLvl ){ + int iBlk = fsPageToBlock(pFS, iRet); + int j; + for(j=0; iRet && jnRight; j++){ + if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ + iRet = 0; + } + } + } + if( iRet ) aiAppend[i] = 0; + } + } + return iRet; +} + +/* +** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and +** return a pointer to it. The page is writable until either +** lsmFsPagePersist() is called on it or the ref-count drops to zero. +*/ +int lsmFsSortedAppend( + FileSystem *pFS, + Snapshot *pSnapshot, + Level *pLvl, + int bDefer, + Page **ppOut +){ + int rc = LSM_OK; + Page *pPg = 0; + LsmPgno iApp = 0; + LsmPgno iNext = 0; + Segment *p = &pLvl->lhs; + LsmPgno iPrev = p->iLastPg; + + *ppOut = 0; + assert( p->pRedirect==0 ); + + if( pFS->pCompress || bDefer ){ + /* In compressed database mode the page is not assigned a page number + ** or location in the database file at this point. This will be done + ** by the lsmFsPagePersist() call. */ + rc = fsPageBuffer(pFS, &pPg); + if( rc==LSM_OK ){ + pPg->pFS = pFS; + pPg->pSeg = p; + pPg->iPg = 0; + pPg->flags |= PAGE_DIRTY; + pPg->nData = pFS->nPagesize; + assert( pPg->aData ); + if( pFS->pCompress==0 ) pPg->nData -= 4; + + pPg->nRef = 1; + pFS->nOut++; + } + }else{ + if( iPrev==0 ){ + iApp = findAppendPoint(pFS, pLvl); + }else if( fsIsLast(pFS, iPrev) ){ + int iNext2; + rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iPrev), &iNext2); + if( rc!=LSM_OK ) return rc; + iApp = fsFirstPageOnBlock(pFS, iNext2); + }else{ + iApp = iPrev + 1; + } + + /* If this is the first page allocated, or if the page allocated is the + ** last in the block, also allocate the next block here. */ + if( iApp==0 || fsIsLast(pFS, iApp) ){ + int iNew; /* New block number */ + + rc = lsmBlockAllocate(pFS->pDb, 0, &iNew); + if( rc!=LSM_OK ) return rc; + if( iApp==0 ){ + iApp = fsFirstPageOnBlock(pFS, iNew); + }else{ + iNext = fsFirstPageOnBlock(pFS, iNew); + } + } + + /* Grab the new page. */ + pPg = 0; + rc = fsPageGet(pFS, 0, iApp, 1, &pPg, 0); + assert( rc==LSM_OK || pPg==0 ); + + /* If this is the first or last page of a block, fill in the pointer + ** value at the end of the new page. */ + if( rc==LSM_OK ){ + p->nSize++; + p->iLastPg = iApp; + if( p->iFirst==0 ) p->iFirst = iApp; + pPg->flags |= PAGE_DIRTY; + + if( fsIsLast(pFS, iApp) ){ + lsmPutU32(&pPg->aData[pFS->nPagesize-4], fsPageToBlock(pFS, iNext)); + }else if( fsIsFirst(pFS, iApp) ){ + lsmPutU32(&pPg->aData[-4], fsPageToBlock(pFS, iPrev)); + } + } + } + + *ppOut = pPg; + return rc; +} + +/* +** Mark the segment passed as the second argument as finished. Once a segment +** is marked as finished it is not possible to append any further pages to +** it. +** +** Return LSM_OK if successful or an lsm error code if an error occurs. +*/ +int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ + int rc = LSM_OK; + if( p && p->iLastPg ){ + assert( p->pRedirect==0 ); + + /* Check if the last page of this run happens to be the last of a block. + ** If it is, then an extra block has already been allocated for this run. + ** Shift this extra block back to the free-block list. + ** + ** Otherwise, add the first free page in the last block used by the run + ** to the lAppend list. + */ + if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ + int i; + LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; + for(i=0; iiLastPg+1; + break; + } + } + }else if( pFS->pCompress==0 ){ + Page *pLast; + rc = fsPageGet(pFS, 0, p->iLastPg, 0, &pLast, 0); + if( rc==LSM_OK ){ + int iBlk = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); + lsmBlockRefree(pFS->pDb, iBlk); + lsmFsPageRelease(pLast); + } + }else{ + int iBlk = 0; + rc = fsBlockNext(pFS, p, fsPageToBlock(pFS, p->iLastPg), &iBlk); + if( rc==LSM_OK ){ + lsmBlockRefree(pFS->pDb, iBlk); + } + } + } + return rc; +} + +/* +** Obtain a reference to page number iPg. +** +** Return LSM_OK if successful, or an lsm error code if an error occurs. +*/ +int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){ + return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); +} + +/* +** Obtain a reference to the last page in the segment passed as the +** second argument. +** +** Return LSM_OK if successful, or an lsm error code if an error occurs. +*/ +int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){ + int rc; + LsmPgno iPg = pSeg->iLastPg; + if( pFS->pCompress ){ + int nSpace; + iPg++; + do { + nSpace = 0; + rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg); + if( rc==LSM_OK ){ + rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, &nSpace); + } + }while( rc==LSM_OK && nSpace>0 ); + + }else{ + rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); + } + return rc; +} + +/* +** Return a reference to meta-page iPg. If successful, LSM_OK is returned +** and *ppPg populated with the new page reference. The reference should +** be released by the caller using lsmFsPageRelease(). +** +** Otherwise, if an error occurs, *ppPg is set to NULL and an LSM error +** code is returned. +*/ +int lsmFsMetaPageGet( + FileSystem *pFS, /* File-system connection */ + int bWrite, /* True for write access, false for read */ + int iPg, /* Either 1 or 2 */ + MetaPage **ppPg /* OUT: Pointer to MetaPage object */ +){ + int rc = LSM_OK; + MetaPage *pPg; + assert( iPg==1 || iPg==2 ); + + pPg = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); + + if( pPg ){ + i64 iOff = (iPg-1) * pFS->nMetasize; + if( pFS->nMapLimit>0 ){ + fsGrowMapping(pFS, 2*pFS->nMetasize, &rc); + pPg->aData = (u8 *)(pFS->pMap) + iOff; + }else{ + pPg->aData = lsmMallocRc(pFS->pEnv, pFS->nMetasize, &rc); + if( rc==LSM_OK && bWrite==0 ){ + rc = lsmEnvRead( + pFS->pEnv, pFS->fdDb, iOff, pPg->aData, pFS->nMetaRwSize + ); + } +#ifndef NDEBUG + /* pPg->aData causes an uninitialized access via a downstreadm write(). + After discussion on this list, this memory should not, for performance + reasons, be memset. However, tracking down "real" misuse is more + difficult with this "false" positive, so it is set when NDEBUG. + */ + else if( rc==LSM_OK ){ + memset( pPg->aData, 0x77, pFS->nMetasize ); + } +#endif + } + + if( rc!=LSM_OK ){ + if( pFS->nMapLimit==0 ) lsmFree(pFS->pEnv, pPg->aData); + lsmFree(pFS->pEnv, pPg); + pPg = 0; + }else{ + pPg->iPg = iPg; + pPg->bWrite = bWrite; + pPg->pFS = pFS; + } + } + + *ppPg = pPg; + return rc; +} + +/* +** Release a meta-page reference obtained via a call to lsmFsMetaPageGet(). +*/ +int lsmFsMetaPageRelease(MetaPage *pPg){ + int rc = LSM_OK; + if( pPg ){ + FileSystem *pFS = pPg->pFS; + + if( pFS->nMapLimit==0 ){ + if( pPg->bWrite ){ + i64 iOff = (pPg->iPg==2 ? pFS->nMetasize : 0); + int nWrite = pFS->nMetaRwSize; + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, pPg->aData, nWrite); + } + lsmFree(pFS->pEnv, pPg->aData); + } + + lsmFree(pFS->pEnv, pPg); + } + return rc; +} + +/* +** Return a pointer to a buffer containing the data associated with the +** meta-page passed as the first argument. If parameter pnData is not NULL, +** set *pnData to the size of the meta-page in bytes before returning. +*/ +u8 *lsmFsMetaPageData(MetaPage *pPg, int *pnData){ + if( pnData ) *pnData = pPg->pFS->nMetaRwSize; + return pPg->aData; +} + +/* +** Return true if page is currently writable. This is used in assert() +** statements only. +*/ +#ifndef NDEBUG +int lsmFsPageWritable(Page *pPg){ + return (pPg->flags & PAGE_DIRTY) ? 1 : 0; +} +#endif + +/* +** This is called when block iFrom is being redirected to iTo. If page +** number (*piPg) lies on block iFrom, then calculate the equivalent +** page on block iTo and set *piPg to this value before returning. +*/ +static void fsMovePage( + FileSystem *pFS, /* File system object */ + int iTo, /* Destination block */ + int iFrom, /* Source block */ + LsmPgno *piPg /* IN/OUT: Page number */ +){ + LsmPgno iPg = *piPg; + if( iFrom==fsPageToBlock(pFS, iPg) ){ + const int nPagePerBlock = ( + pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) + ); + *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; + } +} + +/* +** Copy the contents of block iFrom to block iTo. +** +** It is safe to assume that there are no outstanding references to pages +** on block iTo. And that block iFrom is not currently being written. In +** other words, the data can be read and written directly. +*/ +int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){ + Snapshot *p = pFS->pDb->pWorker; + int rc = LSM_OK; + int i; + i64 nMap; + + i64 iFromOff = (i64)(iFrom-1) * pFS->nBlocksize; + i64 iToOff = (i64)(iTo-1) * pFS->nBlocksize; + + assert( iTo!=1 ); + assert( iFrom>iTo ); + + /* Grow the mapping as required. */ + nMap = LSM_MIN(pFS->nMapLimit, (i64)iFrom * pFS->nBlocksize); + fsGrowMapping(pFS, nMap, &rc); + + if( rc==LSM_OK ){ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + int nSz = pFS->nPagesize; + u8 *aBuf = 0; + u8 *aData = 0; + + for(i=0; rc==LSM_OK && inMapLimit ){ + u8 *aMap = (u8 *)(pFS->pMap); + aData = &aMap[iOff]; + }else{ + if( aBuf==0 ){ + aBuf = (u8 *)lsmMallocRc(pFS->pEnv, nSz, &rc); + if( aBuf==0 ) break; + } + aData = aBuf; + rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); + } + + /* Copy aData to the to page */ + if( rc==LSM_OK ){ + iOff = iToOff + i*nSz; + if( (iOff+nSz)<=pFS->nMapLimit ){ + u8 *aMap = (u8 *)(pFS->pMap); + memcpy(&aMap[iOff], aData, nSz); + }else{ + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); + } + } + } + lsmFree(pFS->pEnv, aBuf); + lsmFsPurgeCache(pFS); + } + + /* Update append-point list if necessary */ + for(i=0; iaiAppend[i]); + } + + /* Update the Segment structure itself */ + fsMovePage(pFS, iTo, iFrom, &pSeg->iFirst); + fsMovePage(pFS, iTo, iFrom, &pSeg->iLastPg); + fsMovePage(pFS, iTo, iFrom, &pSeg->iRoot); + + return rc; +} + +/* +** Append raw data to a segment. Return the database file offset that the +** data is written to (this may be used as the page number if the data +** being appended is a new page record). +** +** This function is only used in compressed database mode. +*/ +static LsmPgno fsAppendData( + FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* Segment to append to */ + const u8 *aData, /* Buffer containing data to write */ + int nData, /* Size of buffer aData[] in bytes */ + int *pRc /* IN/OUT: Error code */ +){ + LsmPgno iRet = 0; + int rc = *pRc; + assert( pFS->pCompress ); + if( rc==LSM_OK ){ + int nRem = 0; + int nWrite = 0; + LsmPgno iLastOnBlock; + LsmPgno iApp = pSeg->iLastPg+1; + + /* If this is the first data written into the segment, find an append-point + ** or allocate a new block. */ + if( iApp==1 ){ + pSeg->iFirst = iApp = findAppendPoint(pFS, 0); + if( iApp==0 ){ + int iBlk; + rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); + pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); + } + } + iRet = iApp; + + /* Write as much data as is possible at iApp (usually all of it). */ + iLastOnBlock = fsLastPageOnPagesBlock(pFS, iApp); + if( rc==LSM_OK ){ + int nSpace = (int)(iLastOnBlock - iApp + 1); + nWrite = LSM_MIN(nData, nSpace); + nRem = nData - nWrite; + assert( nWrite>=0 ); + if( nWrite!=0 ){ + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite); + } + iApp += nWrite; + } + + /* If required, allocate a new block and write the rest of the data + ** into it. Set the next and previous block pointers to link the new + ** block to the old. */ + assert( nRem<=0 || (iApp-1)==iLastOnBlock ); + if( rc==LSM_OK && (iApp-1)==iLastOnBlock ){ + u8 aPtr[4]; /* Space to serialize a u32 */ + int iBlk; /* New block number */ + + if( nWrite>0 ){ + /* Allocate a new block. */ + rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); + + /* Set the "next" pointer on the old block */ + if( rc==LSM_OK ){ + assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 ); + lsmPutU32(aPtr, iBlk); + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr)); + } + + /* Set the "prev" pointer on the new block */ + if( rc==LSM_OK ){ + LsmPgno iWrite; + lsmPutU32(aPtr, fsPageToBlock(pFS, iApp)); + iWrite = fsFirstPageOnBlock(pFS, iBlk); + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr)); + if( nRem>0 ) iApp = iWrite; + } + }else{ + /* The next block is already allocated. */ + assert( nRem>0 ); + assert( pSeg->pRedirect==0 ); + rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iApp), &iBlk); + iRet = iApp = fsFirstPageOnBlock(pFS, iBlk); + } + + /* Write the remaining data into the new block */ + if( rc==LSM_OK && nRem>0 ){ + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, &aData[nWrite], nRem); + iApp += nRem; + } + } + + pSeg->iLastPg = iApp-1; + *pRc = rc; + } + + return iRet; +} + +/* +** This function is only called in compressed database mode. It +** compresses the contents of page pPg and writes the result to the +** buffer at pFS->aOBuffer. The size of the compressed data is stored in +** pPg->nCompress. +** +** If buffer pFS->aOBuffer[] has not been allocated then this function +** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. +*/ +static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ + lsm_compress *p = pFS->pCompress; + + if( fsAllocateBuffer(pFS, 1) ) return LSM_NOMEM; + assert( pPg->nData==pFS->nPagesize ); + + pPg->nCompress = pFS->nBuffer; + return p->xCompress(p->pCtx, + (char *)pFS->aOBuffer, &pPg->nCompress, + (const char *)pPg->aData, pPg->nData + ); +} + +/* +** Append a new page to segment pSeg. Set output variable *piNew to the +** page number of the new page before returning. +** +** If the new page is the last on its block, then the 'next' block that +** will be used by the segment is allocated here too. In this case output +** variable *piNext is set to the block number of the next block. +** +** If the new page is the first on its block but not the first in the +** entire segment, set output variable *piPrev to the block number of +** the previous block in the segment. +** +** LSM_OK is returned if successful, or an lsm error code otherwise. If +** any value other than LSM_OK is returned, then the final value of all +** output variables is undefined. +*/ +static int fsAppendPage( + FileSystem *pFS, + Segment *pSeg, + LsmPgno *piNew, + int *piPrev, + int *piNext +){ + LsmPgno iPrev = pSeg->iLastPg; + int rc; + assert( iPrev!=0 ); + + *piPrev = 0; + *piNext = 0; + + if( fsIsLast(pFS, iPrev) ){ + /* Grab the first page on the next block (which has already be + ** allocated). In this case set *piPrev to tell the caller to set + ** the "previous block" pointer in the first 4 bytes of the page. + */ + int iNext; + int iBlk = fsPageToBlock(pFS, iPrev); + assert( pSeg->pRedirect==0 ); + rc = fsBlockNext(pFS, 0, iBlk, &iNext); + if( rc!=LSM_OK ) return rc; + *piNew = fsFirstPageOnBlock(pFS, iNext); + *piPrev = iBlk; + }else{ + *piNew = iPrev+1; + if( fsIsLast(pFS, *piNew) ){ + /* Allocate the next block here. */ + int iBlk; + rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); + if( rc!=LSM_OK ) return rc; + *piNext = iBlk; + } + } + + pSeg->nSize++; + pSeg->iLastPg = *piNew; + return LSM_OK; +} + +/* +** Flush all pages in the FileSystem.pWaiting list to disk. +*/ +void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){ + int rc = *pRc; + Page *pPg; + + pPg = pFS->pWaiting; + pFS->pWaiting = 0; + + while( pPg ){ + Page *pNext = pPg->pWaitingNext; + if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg); + assert( pPg->nRef==1 ); + lsmFsPageRelease(pPg); + pPg = pNext; + } + *pRc = rc; +} + +/* +** If there exists a hash-table entry associated with page iPg, remove it. +*/ +static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){ + Page *p; + int iHash = fsHashKey(pFS->nHash, iPg); + + for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext); + + if( p ){ + assert( p->nRef==0 || (p->flags & PAGE_FREE)==0 ); + fsPageRemoveFromHash(pFS, p); + p->iPg = 0; + iHash = fsHashKey(pFS->nHash, 0); + p->pHashNext = pFS->apHash[iHash]; + pFS->apHash[iHash] = p; + } +} + +/* +** If the page passed as an argument is dirty, update the database file +** (or mapping of the database file) with its current contents and mark +** the page as clean. +** +** Return LSM_OK if the operation is a success, or an LSM error code +** otherwise. +*/ +int lsmFsPagePersist(Page *pPg){ + int rc = LSM_OK; + if( pPg && (pPg->flags & PAGE_DIRTY) ){ + FileSystem *pFS = pPg->pFS; + + if( pFS->pCompress ){ + int iHash; /* Hash key of assigned page number */ + u8 aSz[3]; /* pPg->nCompress as a 24-bit big-endian */ + assert( pPg->pSeg && pPg->iPg==0 && pPg->nCompress==0 ); + + /* Compress the page image. */ + rc = fsCompressIntoBuffer(pFS, pPg); + + /* Serialize the compressed size into buffer aSz[] */ + putRecordSize(aSz, pPg->nCompress, 0); + + /* Write the serialized page record into the database file. */ + pPg->iPg = fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); + fsAppendData(pFS, pPg->pSeg, pFS->aOBuffer, pPg->nCompress, &rc); + fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); + + /* Now that it has a page number, insert the page into the hash table */ + iHash = fsHashKey(pFS->nHash, pPg->iPg); + pPg->pHashNext = pFS->apHash[iHash]; + pFS->apHash[iHash] = pPg; + + pPg->pSeg->nSize += (sizeof(aSz) * 2) + pPg->nCompress; + + pPg->flags &= ~PAGE_DIRTY; + pFS->nWrite++; + }else{ + + if( pPg->iPg==0 ){ + /* No page number has been assigned yet. This occurs with pages used + ** in the b-tree hierarchy. They were not assigned page numbers when + ** they were created as doing so would cause this call to + ** lsmFsPagePersist() to write an out-of-order page. Instead a page + ** number is assigned here so that the page data will be appended + ** to the current segment. + */ + Page **pp; + int iPrev = 0; + int iNext = 0; + int iHash; + + assert( pPg->pSeg->iFirst ); + assert( pPg->flags & PAGE_FREE ); + assert( (pPg->flags & PAGE_HASPREV)==0 ); + assert( pPg->nData==pFS->nPagesize-4 ); + + rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); + if( rc!=LSM_OK ) return rc; + + assert( pPg->flags & PAGE_FREE ); + iHash = fsHashKey(pFS->nHash, pPg->iPg); + fsRemoveHashEntry(pFS, pPg->iPg); + pPg->pHashNext = pFS->apHash[iHash]; + pFS->apHash[iHash] = pPg; + assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); + + if( iPrev ){ + assert( iNext==0 ); + memmove(&pPg->aData[4], pPg->aData, pPg->nData); + lsmPutU32(pPg->aData, iPrev); + pPg->flags |= PAGE_HASPREV; + pPg->aData += 4; + }else if( iNext ){ + assert( iPrev==0 ); + lsmPutU32(&pPg->aData[pPg->nData], iNext); + }else{ + int nData = pPg->nData; + pPg->nData += 4; + lsmSortedExpandBtreePage(pPg, nData); + } + + pPg->nRef++; + for(pp=&pFS->pWaiting; *pp; pp=&(*pp)->pWaitingNext); + *pp = pPg; + assert( pPg->pWaitingNext==0 ); + + }else{ + i64 iOff; /* Offset to write within database file */ + + iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); + if( fsMmapPage(pFS, pPg->iPg)==0 ){ + u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); + }else if( pPg->flags & PAGE_FREE ){ + fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); + if( rc==LSM_OK ){ + u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; + u8 *aFrom = pPg->aData - (pPg->flags & PAGE_HASPREV); + memcpy(aTo, aFrom, pFS->nPagesize); + lsmFree(pFS->pEnv, aFrom); + pFS->nCacheAlloc--; + pPg->aData = aTo + (pPg->flags & PAGE_HASPREV); + pPg->flags &= ~PAGE_FREE; + fsPageRemoveFromHash(pFS, pPg); + pPg->pMappedNext = pFS->pMapped; + pFS->pMapped = pPg; + } + } + + lsmFsFlushWaiting(pFS, &rc); + pPg->flags &= ~PAGE_DIRTY; + pFS->nWrite++; + } + } + } + + return rc; +} + +/* +** For non-compressed databases, this function is a no-op. For compressed +** databases, it adds a padding record to the segment passed as the third +** argument. +** +** The size of the padding records is selected so that the last byte +** written is the last byte of a disk sector. This means that if a +** snapshot is taken and checkpointed, subsequent worker processes will +** not write to any sector that contains checkpointed data. +*/ +int lsmFsSortedPadding( + FileSystem *pFS, + Snapshot *pSnapshot, + Segment *pSeg +){ + int rc = LSM_OK; + if( pFS->pCompress && pSeg->iFirst ){ + LsmPgno iLast2; + LsmPgno iLast = pSeg->iLastPg; /* Current last page of segment */ + int nPad; /* Bytes of padding required */ + u8 aSz[3]; + + iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1; + assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) ); + nPad = (int)(iLast2 - iLast); + + if( iLast2>fsLastPageOnPagesBlock(pFS, iLast) ){ + nPad -= 4; + } + assert( nPad>=0 ); + + if( nPad>=6 ){ + pSeg->nSize += nPad; + nPad -= 6; + putRecordSize(aSz, nPad, 1); + fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); + memset(pFS->aOBuffer, 0, nPad); + fsAppendData(pFS, pSeg, pFS->aOBuffer, nPad, &rc); + fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); + }else if( nPad>0 ){ + u8 aBuf[5] = {0,0,0,0,0}; + aBuf[0] = (u8)nPad; + aBuf[nPad-1] = (u8)nPad; + fsAppendData(pFS, pSeg, aBuf, nPad, &rc); + } + + assert( rc!=LSM_OK + || pSeg->iLastPg==fsLastPageOnPagesBlock(pFS, pSeg->iLastPg) + || ((pSeg->iLastPg + 1) % pFS->szSector)==0 + ); + } + + return rc; +} + + +/* +** Increment the reference count on the page object passed as the first +** argument. +*/ +void lsmFsPageRef(Page *pPg){ + if( pPg ){ + pPg->nRef++; + } +} + +/* +** Release a page-reference obtained using fsPageGet(). +*/ +int lsmFsPageRelease(Page *pPg){ + int rc = LSM_OK; + if( pPg ){ + assert( pPg->nRef>0 ); + pPg->nRef--; + if( pPg->nRef==0 ){ + FileSystem *pFS = pPg->pFS; + rc = lsmFsPagePersist(pPg); + pFS->nOut--; + + assert( pPg->pFS->pCompress + || fsIsFirst(pPg->pFS, pPg->iPg)==0 + || (pPg->flags & PAGE_HASPREV) + ); + pPg->aData -= (pPg->flags & PAGE_HASPREV); + pPg->flags &= ~PAGE_HASPREV; + + if( (pPg->flags & PAGE_FREE)==0 ){ + /* Removed from mapped list */ + Page **pp; + for(pp=&pFS->pMapped; (*pp)!=pPg; pp=&(*pp)->pMappedNext); + *pp = pPg->pMappedNext; + pPg->pMappedNext = 0; + + /* Add to free list */ + pPg->pFreeNext = pFS->pFree; + pFS->pFree = pPg; + }else{ + fsPageAddToLru(pFS, pPg); + } + } + } + + return rc; +} + +/* +** Return the total number of pages read from the database file. +*/ +int lsmFsNRead(FileSystem *pFS){ return pFS->nRead; } + +/* +** Return the total number of pages written to the database file. +*/ +int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } + +/* +** Return a copy of the environment pointer used by the file-system object. +*/ +lsm_env *lsmFsEnv(FileSystem *pFS){ + return pFS->pEnv; +} + +/* +** Return a copy of the environment pointer used by the file-system object +** to which this page belongs. +*/ +lsm_env *lsmPageEnv(Page *pPg) { + return pPg->pFS->pEnv; +} + +/* +** Return a pointer to the file-system object associated with the Page +** passed as the only argument. +*/ +FileSystem *lsmPageFS(Page *pPg){ + return pPg->pFS; +} + +/* +** Return the sector-size as reported by the log file handle. +*/ +int lsmFsSectorSize(FileSystem *pFS){ + return pFS->szSector; +} + +/* +** Helper function for lsmInfoArrayStructure(). +*/ +static Segment *startsWith(Segment *pRun, LsmPgno iFirst){ + return (iFirst==pRun->iFirst) ? pRun : 0; +} + +/* +** Return the segment that starts with page iFirst, if any. If no such segment +** can be found, return NULL. +*/ +static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){ + Level *pLvl; /* Used to iterate through db levels */ + Segment *pSeg = 0; /* Pointer to segment to return */ + + for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){ + if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){ + int i; + for(i=0; inRight; i++){ + if( (pSeg = startsWith(&pLvl->aRhs[i], iFirst)) ) break; + } + } + } + + return pSeg; +} + +/* +** This function implements the lsm_info(LSM_INFO_ARRAY_STRUCTURE) request. +** If successful, *pzOut is set to point to a nul-terminated string +** containing the array structure and LSM_OK is returned. The caller should +** eventually free the string using lsmFree(). +** +** If an error occurs, *pzOut is set to NULL and an LSM error code returned. +*/ +int lsmInfoArrayStructure( + lsm_db *pDb, + int bBlock, /* True for block numbers only */ + LsmPgno iFirst, + char **pzOut +){ + int rc = LSM_OK; + Snapshot *pWorker; /* Worker snapshot */ + Segment *pArray = 0; /* Array to report on */ + int bUnlock = 0; + + *pzOut = 0; + if( iFirst==0 ) return LSM_ERROR; + + /* Obtain the worker snapshot */ + pWorker = pDb->pWorker; + if( !pWorker ){ + rc = lsmBeginWork(pDb); + if( rc!=LSM_OK ) return rc; + pWorker = pDb->pWorker; + bUnlock = 1; + } + + /* Search for the array that starts on page iFirst */ + pArray = findSegment(pWorker, iFirst); + + if( pArray==0 ){ + /* Could not find the requested array. This is an error. */ + rc = LSM_ERROR; + }else{ + FileSystem *pFS = pDb->pFS; + LsmString str; + int iBlk; + int iLastBlk; + + iBlk = fsPageToBlock(pFS, pArray->iFirst); + iLastBlk = fsPageToBlock(pFS, pArray->iLastPg); + + lsmStringInit(&str, pDb->pEnv); + if( bBlock ){ + lsmStringAppendf(&str, "%d", iBlk); + while( iBlk!=iLastBlk ){ + fsBlockNext(pFS, pArray, iBlk, &iBlk); + lsmStringAppendf(&str, " %d", iBlk); + } + }else{ + lsmStringAppendf(&str, "%d", pArray->iFirst); + while( iBlk!=iLastBlk ){ + lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); + fsBlockNext(pFS, pArray, iBlk, &iBlk); + lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); + } + lsmStringAppendf(&str, " %d", pArray->iLastPg); + } + + *pzOut = str.z; + } + + if( bUnlock ){ + int rcwork = LSM_BUSY; + lsmFinishWork(pDb, 0, &rcwork); + } + return rc; +} + +int lsmFsSegmentContainsPg( + FileSystem *pFS, + Segment *pSeg, + LsmPgno iPg, + int *pbRes +){ + Redirect *pRedir = pSeg->pRedirect; + int rc = LSM_OK; + int iBlk; + int iLastBlk; + int iPgBlock; /* Block containing page iPg */ + + iPgBlock = fsPageToBlock(pFS, pSeg->iFirst); + iBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iFirst)); + iLastBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iLastPg)); + + while( iBlk!=iLastBlk && iBlk!=iPgBlock && rc==LSM_OK ){ + rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); + } + + *pbRes = (iBlk==iPgBlock); + return rc; +} + +/* +** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request. +** If successful, *pzOut is set to point to a nul-terminated string +** containing the array structure and LSM_OK is returned. The caller should +** eventually free the string using lsmFree(). +** +** If an error occurs, *pzOut is set to NULL and an LSM error code returned. +*/ +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){ + int rc = LSM_OK; + Snapshot *pWorker; /* Worker snapshot */ + Segment *pSeg = 0; /* Array to report on */ + int bUnlock = 0; + + *pzOut = 0; + if( iFirst==0 ) return LSM_ERROR; + + /* Obtain the worker snapshot */ + pWorker = pDb->pWorker; + if( !pWorker ){ + rc = lsmBeginWork(pDb); + if( rc!=LSM_OK ) return rc; + pWorker = pDb->pWorker; + bUnlock = 1; + } + + /* Search for the array that starts on page iFirst */ + pSeg = findSegment(pWorker, iFirst); + + if( pSeg==0 ){ + /* Could not find the requested array. This is an error. */ + rc = LSM_ERROR; + }else{ + Page *pPg = 0; + FileSystem *pFS = pDb->pFS; + LsmString str; + + lsmStringInit(&str, pDb->pEnv); + rc = lsmFsDbPageGet(pFS, pSeg, iFirst, &pPg); + while( rc==LSM_OK && pPg ){ + Page *pNext = 0; + lsmStringAppendf(&str, " %lld", lsmFsPageNumber(pPg)); + rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); + lsmFsPageRelease(pPg); + pPg = pNext; + } + + if( rc!=LSM_OK ){ + lsmFree(pDb->pEnv, str.z); + }else{ + *pzOut = str.z; + } + } + + if( bUnlock ){ + int rcwork = LSM_BUSY; + lsmFinishWork(pDb, 0, &rcwork); + } + return rc; +} + +/* +** The following macros are used by the integrity-check code. Associated with +** each block in the database is an 8-bit bit mask (the entry in the aUsed[] +** array). As the integrity-check meanders through the database, it sets the +** following bits to indicate how each block is used. +** +** INTEGRITY_CHECK_FIRST_PG: +** First page of block is in use by sorted run. +** +** INTEGRITY_CHECK_LAST_PG: +** Last page of block is in use by sorted run. +** +** INTEGRITY_CHECK_USED: +** At least one page of the block is in use by a sorted run. +** +** INTEGRITY_CHECK_FREE: +** The free block list contains an entry corresponding to this block. +*/ +#define INTEGRITY_CHECK_FIRST_PG 0x01 +#define INTEGRITY_CHECK_LAST_PG 0x02 +#define INTEGRITY_CHECK_USED 0x04 +#define INTEGRITY_CHECK_FREE 0x08 + +/* +** Helper function for lsmFsIntegrityCheck() +*/ +static void checkBlocks( + FileSystem *pFS, + Segment *pSeg, + int bExtra, /* If true, count the "next" block if any */ + int nUsed, + u8 *aUsed +){ + if( pSeg ){ + if( pSeg && pSeg->nSize>0 ){ + int rc; + int iBlk; /* Current block (during iteration) */ + int iLastBlk; /* Last block of segment */ + int iFirstBlk; /* First block of segment */ + int bLastIsLastOnBlock; /* True iLast is the last on its block */ + + assert( 0==fsSegmentRedirects(pFS, pSeg) ); + iBlk = iFirstBlk = fsPageToBlock(pFS, pSeg->iFirst); + iLastBlk = fsPageToBlock(pFS, pSeg->iLastPg); + + bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==pSeg->iLastPg); + assert( iBlk>0 ); + + do { + /* iBlk is a part of this sorted run. */ + aUsed[iBlk-1] |= INTEGRITY_CHECK_USED; + + /* If the first page of this block is also part of the segment, + ** set the flag to indicate that the first page of iBlk is in use. + */ + if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst || iBlk!=iFirstBlk ){ + assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); + aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; + } + + /* Unless the sorted run finishes before the last page on this block, + ** the last page of this block is also in use. */ + if( iBlk!=iLastBlk || bLastIsLastOnBlock ){ + assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_LAST_PG)==0 ); + aUsed[iBlk-1] |= INTEGRITY_CHECK_LAST_PG; + } + + /* Special case. The sorted run being scanned is the output run of + ** a level currently undergoing an incremental merge. The sorted + ** run ends on the last page of iBlk, but the next block has already + ** been allocated. So mark it as in use as well. */ + if( iBlk==iLastBlk && bLastIsLastOnBlock && bExtra ){ + int iExtra = 0; + rc = fsBlockNext(pFS, pSeg, iBlk, &iExtra); + assert( rc==LSM_OK ); + + assert( aUsed[iExtra-1]==0 ); + aUsed[iExtra-1] |= INTEGRITY_CHECK_USED; + aUsed[iExtra-1] |= INTEGRITY_CHECK_FIRST_PG; + aUsed[iExtra-1] |= INTEGRITY_CHECK_LAST_PG; + } + + /* Move on to the next block in the sorted run. Or set iBlk to zero + ** in order to break out of the loop if this was the last block in + ** the run. */ + if( iBlk==iLastBlk ){ + iBlk = 0; + }else{ + rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); + assert( rc==LSM_OK ); + } + }while( iBlk ); + } + } +} + +typedef struct CheckFreelistCtx CheckFreelistCtx; +struct CheckFreelistCtx { + u8 *aUsed; + int nBlock; +}; +static int checkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ + CheckFreelistCtx *p = (CheckFreelistCtx *)pCtx; + + assert( iBlk>=1 ); + assert( iBlk<=p->nBlock ); + assert( p->aUsed[iBlk-1]==0 ); + p->aUsed[iBlk-1] = INTEGRITY_CHECK_FREE; + return 0; +} + +/* +** This function checks that all blocks in the database file are accounted +** for. For each block, exactly one of the following must be true: +** +** + the block is part of a sorted run, or +** + the block is on the free-block list +** +** This function also checks that there are no references to blocks with +** out-of-range block numbers. +** +** If no errors are found, non-zero is returned. If an error is found, an +** assert() fails. +*/ +int lsmFsIntegrityCheck(lsm_db *pDb){ + CheckFreelistCtx ctx; + FileSystem *pFS = pDb->pFS; + int i; + int rc; + Freelist freelist = {0, 0, 0}; + u8 *aUsed; + Level *pLevel; + Snapshot *pWorker = pDb->pWorker; + int nBlock = pWorker->nBlock; + +#if 0 + static int nCall = 0; + nCall++; + printf("%d calls\n", nCall); +#endif + + aUsed = lsmMallocZero(pDb->pEnv, nBlock); + if( aUsed==0 ){ + /* Malloc has failed. Since this function is only called within debug + ** builds, this probably means the user is running an OOM injection test. + ** Regardless, it will not be possible to run the integrity-check at this + ** time, so assume the database is Ok and return non-zero. */ + return 1; + } + + for(pLevel=pWorker->pLevel; pLevel; pLevel=pLevel->pNext){ + int j; + checkBlocks(pFS, &pLevel->lhs, (pLevel->nRight!=0), nBlock, aUsed); + for(j=0; jnRight; j++){ + checkBlocks(pFS, &pLevel->aRhs[j], 0, nBlock, aUsed); + } + } + + /* Mark all blocks in the free-list as used */ + ctx.aUsed = aUsed; + ctx.nBlock = nBlock; + rc = lsmWalkFreelist(pDb, 0, checkFreelistCb, (void *)&ctx); + + if( rc==LSM_OK ){ + for(i=0; ipEnv, aUsed); + lsmFree(pDb->pEnv, freelist.aEntry); + + return 1; +} + +#ifndef NDEBUG +/* +** Return true if pPg happens to be the last page in segment pSeg. Or false +** otherwise. This function is only invoked as part of assert() conditions. +*/ +int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){ + if( pPg->pFS->pCompress ){ + LsmPgno iNext = 0; + int rc; + rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext); + return (rc!=LSM_OK || iNext==0); + } + return (pPg->iPg==pSeg->iLastPg); +} +#endif diff --git a/ext/lsm1/lsm_log.c b/ext/lsm1/lsm_log.c new file mode 100644 index 0000000000000000000000000000000000000000..a66e40bccd58bb2a61ef6f4be6ab1fee7d76d770 --- /dev/null +++ b/ext/lsm1/lsm_log.c @@ -0,0 +1,1156 @@ +/* +** 2011-08-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of LSM database logging. Logging +** has one purpose in LSM - to make transactions durable. +** +** When data is written to an LSM database, it is initially stored in an +** in-memory tree structure. Since this structure is in volatile memory, +** if a power failure or application crash occurs it may be lost. To +** prevent loss of data in this case, each time a record is written to the +** in-memory tree an equivalent record is appended to the log on disk. +** If a power failure or application crash does occur, data can be recovered +** by reading the log. +** +** A log file consists of the following types of records representing data +** written into the database: +** +** LOG_WRITE: A key-value pair written to the database. +** LOG_DELETE: A delete key issued to the database. +** LOG_COMMIT: A transaction commit. +** +** And the following types of records for ancillary purposes.. +** +** LOG_EOF: A record indicating the end of a log file. +** LOG_PAD1: A single byte padding record. +** LOG_PAD2: An N byte padding record (N>1). +** LOG_JUMP: A pointer to another offset within the log file. +** +** Each transaction written to the log contains one or more LOG_WRITE and/or +** LOG_DELETE records, followed by a LOG_COMMIT record. The LOG_COMMIT record +** contains an 8-byte checksum based on all previous data written to the +** log file. +** +** LOG CHECKSUMS & RECOVERY +** +** Checksums are found in two types of log records: LOG_COMMIT and +** LOG_CKSUM records. In order to recover content from a log, a client +** reads each record from the start of the log, calculating a checksum as +** it does. Each time a LOG_COMMIT or LOG_CKSUM is encountered, the +** recovery process verifies that the checksum stored in the log +** matches the calculated checksum. If it does not, the recovery process +** can stop reading the log. +** +** If a recovery process reads records (other than COMMIT or CKSUM) +** consisting of at least LSM_CKSUM_MAXDATA bytes, then the next record in +** the log must be either a LOG_CKSUM or LOG_COMMIT record. If it is +** not, the recovery process also stops reading the log. +** +** To recover the log file, it must be read twice. The first time to +** determine the location of the last valid commit record. And the second +** time to load data into the in-memory tree. +** +** Todo: Surely there is a better way... +** +** LOG WRAPPING +** +** If the log file were never deleted or wrapped, it would be possible to +** read it from start to end each time is required recovery (i.e each time +** the number of database clients changes from 0 to 1). Effectively reading +** the entire history of the database each time. This would quickly become +** inefficient. Additionally, since the log file would grow without bound, +** it wastes storage space. +** +** Instead, part of each checkpoint written into the database file contains +** a log offset (and other information required to read the log starting at +** at this offset) at which to begin recovery. Offset $O. +** +** Once a checkpoint has been written and synced into the database file, it +** is guaranteed that no recovery process will need to read any data before +** offset $O of the log file. It is therefore safe to begin overwriting +** any data that occurs before offset $O. +** +** This implementation separates the log into three regions mapped into +** the log file - regions 0, 1 and 2. During recovery, regions are read +** in ascending order (i.e. 0, then 1, then 2). Each region is zero or +** more bytes in size. +** +** |---1---|..|--0--|.|--2--|.... +** +** New records are always appended to the end of region 2. +** +** Initially (when it is empty), all three regions are zero bytes in size. +** Each of them are located at the beginning of the file. As records are +** added to the log, region 2 grows, so that the log consists of a zero +** byte region 1, followed by a zero byte region 0, followed by an N byte +** region 2. After one or more checkpoints have been written to disk, +** the start point of region 2 is moved to $O. For example: +** +** A) ||.........|--2--|.... +** +** (both regions 0 and 1 are 0 bytes in size at offset 0). +** +** Eventually, the log wraps around to write new records into the start. +** At this point, region 2 is renamed to region 0. Region 0 is renamed +** to region 2. After appending a few records to the new region 2, the +** log file looks like this: +** +** B) ||--2--|...|--0--|.... +** +** (region 1 is still 0 bytes in size, located at offset 0). +** +** Any checkpoints made at this point may reduce the size of region 0. +** However, if they do not, and region 2 expands so that it is about to +** overwrite the start of region 0, then region 2 is renamed to region 1, +** and a new region 2 created at the end of the file following the existing +** region 0. +** +** C) |---1---|..|--0--|.|-2-| +** +** In this state records are appended to region 2 until checkpoints have +** contracted regions 0 AND 1 UNTil they are both zero bytes in size. They +** are then shifted to the start of the log file, leaving the system in +** the equivalent of state A above. +** +** Alternatively, state B may transition directly to state A if the size +** of region 0 is reduced to zero bytes before region 2 threatens to +** encroach upon it. +** +** LOG_PAD1 & LOG_PAD2 RECORDS +** +** PAD1 and PAD2 records may appear in a log file at any point. They allow +** a process writing the log file align the beginning of transactions with +** the beginning of disk sectors, which increases robustness. +** +** RECORD FORMATS: +** +** LOG_EOF: * A single 0x00 byte. +** +** LOG_PAD1: * A single 0x01 byte. +** +** LOG_PAD2: * A single 0x02 byte, followed by +** * The number of unused bytes (N) as a varint, +** * An N byte block of unused space. +** +** LOG_COMMIT: * A single 0x03 byte. +** * An 8-byte checksum. +** +** LOG_JUMP: * A single 0x04 byte. +** * Absolute file offset to jump to, encoded as a varint. +** +** LOG_WRITE: * A single 0x06 or 0x07 byte, +** * The number of bytes in the key, encoded as a varint, +** * The number of bytes in the value, encoded as a varint, +** * If the first byte was 0x07, an 8 byte checksum. +** * The key data, +** * The value data. +** +** LOG_DELETE: * A single 0x08 or 0x09 byte, +** * The number of bytes in the key, encoded as a varint, +** * If the first byte was 0x09, an 8 byte checksum. +** * The key data. +** +** Varints are as described in lsm_varint.c (SQLite 4 format). +** +** CHECKSUMS: +** +** The checksum is calculated using two 32-bit unsigned integers, s0 and +** s1. The initial value for both is 42. It is updated each time a record +** is written into the log file by treating the encoded (binary) record as +** an array of 32-bit little-endian integers. Then, if x[] is the integer +** array, updating the checksum accumulators as follows: +** +** for i from 0 to n-1 step 2: +** s0 += x[i] + s1; +** s1 += x[i+1] + s0; +** endfor +** +** If the record is not an even multiple of 8-bytes in size it is padded +** with zeroes to make it so before the checksum is updated. +** +** The checksum stored in a COMMIT, WRITE or DELETE is based on all bytes +** up to the start of the 8-byte checksum itself, including the COMMIT, +** WRITE or DELETE fields that appear before the checksum in the record. +** +** VARINT FORMAT +** +** See lsm_varint.c. +*/ + +#ifndef _LSM_INT_H +# include "lsmInt.h" +#endif + +/* Log record types */ +#define LSM_LOG_EOF 0x00 +#define LSM_LOG_PAD1 0x01 +#define LSM_LOG_PAD2 0x02 +#define LSM_LOG_COMMIT 0x03 +#define LSM_LOG_JUMP 0x04 + +#define LSM_LOG_WRITE 0x06 +#define LSM_LOG_WRITE_CKSUM 0x07 + +#define LSM_LOG_DELETE 0x08 +#define LSM_LOG_DELETE_CKSUM 0x09 + +#define LSM_LOG_DRANGE 0x0A +#define LSM_LOG_DRANGE_CKSUM 0x0B + +/* Require a checksum every 32KB. */ +#define LSM_CKSUM_MAXDATA (32*1024) + +/* Do not wrap a log file smaller than this in bytes. */ +#define LSM_MIN_LOGWRAP (128*1024) + +/* +** szSector: +** Commit records must be aligned to end on szSector boundaries. If +** the safety-mode is set to NORMAL or OFF, this value is 1. Otherwise, +** if the safety-mode is set to FULL, it is the size of the file-system +** sectors as reported by lsmFsSectorSize(). +*/ +struct LogWriter { + u32 cksum0; /* Checksum 0 at offset iOff */ + u32 cksum1; /* Checksum 1 at offset iOff */ + int iCksumBuf; /* Bytes of buf that have been checksummed */ + i64 iOff; /* Offset at start of buffer buf */ + int szSector; /* Sector size for this transaction */ + LogRegion jump; /* Avoid writing to this region */ + i64 iRegion1End; /* End of first region written by trans */ + i64 iRegion2Start; /* Start of second regions written by trans */ + LsmString buf; /* Buffer containing data not yet written */ +}; + +/* +** Return the result of interpreting the first 4 bytes in buffer aIn as +** a 32-bit unsigned little-endian integer. +*/ +static u32 getU32le(u8 *aIn){ + return ((u32)aIn[3] << 24) + + ((u32)aIn[2] << 16) + + ((u32)aIn[1] << 8) + + ((u32)aIn[0]); +} + + +/* +** This function is the same as logCksum(), except that pointer "a" need +** not be aligned to an 8-byte boundary or padded with zero bytes. This +** version is slower, but sometimes more convenient to use. +*/ +static void logCksumUnaligned( + char *z, /* Input buffer */ + int n, /* Size of input buffer in bytes */ + u32 *pCksum0, /* IN/OUT: Checksum value 1 */ + u32 *pCksum1 /* IN/OUT: Checksum value 2 */ +){ + u8 *a = (u8 *)z; + u32 cksum0 = *pCksum0; + u32 cksum1 = *pCksum1; + int nIn = (n/8) * 8; + int i; + + assert( n>0 ); + for(i=0; inIn ); + memcpy(aBuf, &a[nIn], n-nIn); + cksum0 += getU32le(aBuf) + cksum1; + cksum1 += getU32le(&aBuf[4]) + cksum0; + } + + *pCksum0 = cksum0; + *pCksum1 = cksum1; +} + +/* +** Update pLog->cksum0 and pLog->cksum1 so that the first nBuf bytes in the +** write buffer (pLog->buf) are included in the checksum. +*/ +static void logUpdateCksum(LogWriter *pLog, int nBuf){ + assert( (pLog->iCksumBuf % 8)==0 ); + assert( pLog->iCksumBuf<=nBuf ); + assert( (nBuf % 8)==0 || nBuf==pLog->buf.n ); + if( nBuf>pLog->iCksumBuf ){ + logCksumUnaligned( + &pLog->buf.z[pLog->iCksumBuf], nBuf-pLog->iCksumBuf, + &pLog->cksum0, &pLog->cksum1 + ); + } + pLog->iCksumBuf = nBuf; +} + +static i64 firstByteOnSector(LogWriter *pLog, i64 iOff){ + return (iOff / pLog->szSector) * pLog->szSector; +} +static i64 lastByteOnSector(LogWriter *pLog, i64 iOff){ + return firstByteOnSector(pLog, iOff) + pLog->szSector - 1; +} + +/* +** If possible, reclaim log file space. Log file space is reclaimed after +** a snapshot that points to the same data in the database file is synced +** into the db header. +*/ +static int logReclaimSpace(lsm_db *pDb){ + int rc; + int iMeta; + int bRotrans; /* True if there exists some ro-trans */ + + /* Test if there exists some other connection with a read-only transaction + ** open. If there does, then log file space may not be reclaimed. */ + rc = lsmDetectRoTrans(pDb, &bRotrans); + if( rc!=LSM_OK || bRotrans ) return rc; + + iMeta = (int)pDb->pShmhdr->iMetaPage; + if( iMeta==1 || iMeta==2 ){ + DbLog *pLog = &pDb->treehdr.log; + i64 iSyncedId; + + /* Read the snapshot-id of the snapshot stored on meta-page iMeta. Note + ** that in theory, the value read is untrustworthy (due to a race + ** condition - see comments above lsmFsReadSyncedId()). So it is only + ** ever used to conclude that no log space can be reclaimed. If it seems + ** to indicate that it may be possible to reclaim log space, a + ** second call to lsmCheckpointSynced() (which does return trustworthy + ** values) is made below to confirm. */ + rc = lsmFsReadSyncedId(pDb, iMeta, &iSyncedId); + + if( rc==LSM_OK && pLog->iSnapshotId!=iSyncedId ){ + i64 iSnapshotId = 0; + i64 iOff = 0; + rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); + if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; + if( iOff>=p->iStart && iOff<=p->iEnd ) break; + p->iStart = 0; + p->iEnd = 0; + } + assert( iRegion<3 ); + pLog->aRegion[iRegion].iStart = iOff; + pLog->iSnapshotId = iSnapshotId; + } + } + } + return rc; +} + +/* +** This function is called when a write-transaction is first opened. It +** is assumed that the caller is holding the client-mutex when it is +** called. +** +** Before returning, this function allocates the LogWriter object that +** will be used to write to the log file during the write transaction. +** LSM_OK is returned if no error occurs, otherwise an LSM error code. +*/ +int lsmLogBegin(lsm_db *pDb){ + int rc = LSM_OK; + LogWriter *pNew; + LogRegion *aReg; + + if( pDb->bUseLog==0 ) return LSM_OK; + + /* If the log file has not yet been opened, open it now. Also allocate + ** the LogWriter structure, if it has not already been allocated. */ + rc = lsmFsOpenLog(pDb, 0); + if( pDb->pLogWriter==0 ){ + pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); + if( pNew ){ + lsmStringInit(&pNew->buf, pDb->pEnv); + rc = lsmStringExtend(&pNew->buf, 2); + } + pDb->pLogWriter = pNew; + }else{ + pNew = pDb->pLogWriter; + assert( (u8 *)(&pNew[1])==(u8 *)(&((&pNew->buf)[1])) ); + memset(pNew, 0, ((u8 *)&pNew->buf) - (u8 *)pNew); + pNew->buf.n = 0; + } + + if( rc==LSM_OK ){ + /* The following call detects whether or not a new snapshot has been + ** synced into the database file. If so, it updates the contents of + ** the pDb->treehdr.log structure to reclaim any space in the log + ** file that is no longer required. + ** + ** TODO: Calling this every transaction is overkill. And since the + ** call has to read and checksum a snapshot from the database file, + ** it is expensive. It would be better to figure out a way so that + ** this is only called occasionally - say for every 32KB written to + ** the log file. + */ + rc = logReclaimSpace(pDb); + } + if( rc!=LSM_OK ){ + lsmLogClose(pDb); + return rc; + } + + /* Set the effective sector-size for this transaction. Sectors are assumed + ** to be one byte in size if the safety-mode is OFF or NORMAL, or as + ** reported by lsmFsSectorSize if it is FULL. */ + if( pDb->eSafety==LSM_SAFETY_FULL ){ + pNew->szSector = lsmFsSectorSize(pDb->pFS); + assert( pNew->szSector>0 ); + }else{ + pNew->szSector = 1; + } + + /* There are now three scenarios: + ** + ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins + ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap + ** around to the start and write data into the start of the log file. + ** + ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the + ** file than region 0. In this case, append data to region 2, but + ** remember to jump over region 1 if required. + ** + ** 3) Region 2 is the last in the file. Append to it. + */ + aReg = &pDb->treehdr.log.aRegion[0]; + + assert( aReg[0].iEnd==0 || aReg[0].iEnd>aReg[0].iStart ); + assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); + + pNew->cksum0 = pDb->treehdr.log.cksum0; + pNew->cksum1 = pDb->treehdr.log.cksum1; + + if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ + /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP + ** into the log file in this case. Pad it out to 8 bytes using a PAD2 + ** record so that the checksums can be updated immediately. */ + u8 aJump[] = { + LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 + }; + + lsmStringBinAppend(&pNew->buf, aJump, sizeof(aJump)); + logUpdateCksum(pNew, pNew->buf.n); + rc = lsmFsWriteLog(pDb->pFS, aReg[2].iEnd, &pNew->buf); + pNew->iCksumBuf = pNew->buf.n = 0; + + aReg[2].iEnd += 8; + pNew->jump = aReg[0] = aReg[2]; + aReg[2].iStart = aReg[2].iEnd = 0; + }else if( aReg[1].iEnd==0 && aReg[2].iEndiOff = aReg[2].iEnd; + pNew->jump = aReg[0]; + }else{ + /* Case 3. */ + assert( aReg[2].iStart>=aReg[0].iEnd && aReg[2].iStart>=aReg[1].iEnd ); + pNew->iOff = aReg[2].iEnd; + } + + if( pNew->jump.iStart ){ + i64 iRound; + assert( pNew->jump.iStart>pNew->iOff ); + + iRound = firstByteOnSector(pNew, pNew->jump.iStart); + if( iRound>pNew->iOff ) pNew->jump.iStart = iRound; + pNew->jump.iEnd = lastByteOnSector(pNew, pNew->jump.iEnd); + } + + assert( pDb->pLogWriter==pNew ); + return rc; +} + +/* +** This function is called when a write-transaction is being closed. +** Parameter bCommit is true if the transaction is being committed, +** or false otherwise. The caller must hold the client-mutex to call +** this function. +** +** A call to this function deletes the LogWriter object allocated by +** lsmLogBegin(). If the transaction is being committed, the shared state +** in *pLog is updated before returning. +*/ +void lsmLogEnd(lsm_db *pDb, int bCommit){ + DbLog *pLog; + LogWriter *p; + p = pDb->pLogWriter; + + if( p==0 ) return; + pLog = &pDb->treehdr.log; + + if( bCommit ){ + pLog->aRegion[2].iEnd = p->iOff; + pLog->cksum0 = p->cksum0; + pLog->cksum1 = p->cksum1; + if( p->iRegion1End ){ + /* This happens when the transaction had to jump over some other + ** part of the log. */ + assert( pLog->aRegion[1].iEnd==0 ); + assert( pLog->aRegion[2].iStartiRegion1End ); + pLog->aRegion[1].iStart = pLog->aRegion[2].iStart; + pLog->aRegion[1].iEnd = p->iRegion1End; + pLog->aRegion[2].iStart = p->iRegion2Start; + } + } +} + +static int jumpIfRequired( + lsm_db *pDb, + LogWriter *pLog, + int nReq, + int *pbJump +){ + /* Determine if it is necessary to add an LSM_LOG_JUMP to jump over the + ** jump region before writing the LSM_LOG_WRITE or DELETE record. This + ** is necessary if there is insufficient room between the current offset + ** and the jump region to fit the new WRITE/DELETE record and the largest + ** possible JUMP record with up to 7 bytes of padding (a total of 17 + ** bytes). */ + if( (pLog->jump.iStart > (pLog->iOff + pLog->buf.n)) + && (pLog->jump.iStart < (pLog->iOff + pLog->buf.n + (nReq + 17))) + ){ + int rc; /* Return code */ + i64 iJump; /* Offset to jump to */ + u8 aJump[10]; /* Encoded jump record */ + int nJump; /* Valid bytes in aJump[] */ + int nPad; /* Bytes of padding required */ + + /* Serialize the JUMP record */ + iJump = pLog->jump.iEnd+1; + aJump[0] = LSM_LOG_JUMP; + nJump = 1 + lsmVarintPut64(&aJump[1], iJump); + + /* Adding padding to the contents of the buffer so that it will be a + ** multiple of 8 bytes in size after the JUMP record is appended. This + ** is not strictly required, it just makes the keeping the running + ** checksum up to date in this file a little simpler. */ + nPad = (pLog->buf.n + nJump) % 8; + if( nPad ){ + u8 aPad[7] = {0,0,0,0,0,0,0}; + nPad = 8-nPad; + if( nPad==1 ){ + aPad[0] = LSM_LOG_PAD1; + }else{ + aPad[0] = LSM_LOG_PAD2; + aPad[1] = (u8)(nPad-2); + } + rc = lsmStringBinAppend(&pLog->buf, aPad, nPad); + if( rc!=LSM_OK ) return rc; + } + + /* Append the JUMP record to the buffer. Then flush the buffer to disk + ** and update the checksums. The next write to the log file (assuming + ** there is no transaction rollback) will be to offset iJump (just past + ** the jump region). */ + rc = lsmStringBinAppend(&pLog->buf, aJump, nJump); + if( rc!=LSM_OK ) return rc; + assert( (pLog->buf.n % 8)==0 ); + rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); + if( rc!=LSM_OK ) return rc; + logUpdateCksum(pLog, pLog->buf.n); + pLog->iRegion1End = (pLog->iOff + pLog->buf.n); + pLog->iRegion2Start = iJump; + pLog->iOff = iJump; + pLog->iCksumBuf = pLog->buf.n = 0; + if( pbJump ) *pbJump = 1; + } + + return LSM_OK; +} + +static int logCksumAndFlush(lsm_db *pDb){ + int rc; /* Return code */ + LogWriter *pLog = pDb->pLogWriter; + + /* Calculate the checksum value. Append it to the buffer. */ + logUpdateCksum(pLog, pLog->buf.n); + lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum0); + pLog->buf.n += 4; + lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum1); + pLog->buf.n += 4; + + /* Write the contents of the buffer to disk. */ + rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); + pLog->iOff += pLog->buf.n; + pLog->iCksumBuf = pLog->buf.n = 0; + + return rc; +} + +/* +** Write the contents of the log-buffer to disk. Then write either a CKSUM +** or COMMIT record, depending on the value of parameter eType. +*/ +static int logFlush(lsm_db *pDb, int eType){ + int rc; + int nReq; + LogWriter *pLog = pDb->pLogWriter; + + assert( eType==LSM_LOG_COMMIT ); + assert( pLog ); + + /* Commit record is always 9 bytes in size. */ + nReq = 9; + if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ) nReq += pLog->szSector + 17; + rc = jumpIfRequired(pDb, pLog, nReq, 0); + + /* If this is a COMMIT, add padding to the log so that the COMMIT record + ** is aligned against the end of a disk sector. In other words, add padding + ** so that the first byte following the COMMIT record lies on a different + ** sector. */ + if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ){ + int nPad; /* Bytes of padding to add */ + + /* Determine the value of nPad. */ + nPad = ((pLog->iOff + pLog->buf.n + 9) % pLog->szSector); + if( nPad ) nPad = pLog->szSector - nPad; + rc = lsmStringExtend(&pLog->buf, nPad); + if( rc!=LSM_OK ) return rc; + + while( nPad ){ + if( nPad==1 ){ + pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD1; + nPad = 0; + }else{ + int n = LSM_MIN(200, nPad-2); + pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD2; + pLog->buf.z[pLog->buf.n++] = (char)n; + nPad -= 2; + memset(&pLog->buf.z[pLog->buf.n], 0x2B, n); + pLog->buf.n += n; + nPad -= n; + } + } + } + + /* Make sure there is room in the log-buffer to add the CKSUM or COMMIT + ** record. Then add the first byte of it. */ + rc = lsmStringExtend(&pLog->buf, 9); + if( rc!=LSM_OK ) return rc; + pLog->buf.z[pLog->buf.n++] = (char)eType; + memset(&pLog->buf.z[pLog->buf.n], 0, 8); + + rc = logCksumAndFlush(pDb); + + /* If this is a commit and synchronous=full, sync the log to disk. */ + if( rc==LSM_OK && eType==LSM_LOG_COMMIT && pDb->eSafety==LSM_SAFETY_FULL ){ + rc = lsmFsSyncLog(pDb->pFS); + } + return rc; +} + +/* +** Append an LSM_LOG_WRITE (if nVal>=0) or LSM_LOG_DELETE (if nVal<0) +** record to the database log. +*/ +int lsmLogWrite( + lsm_db *pDb, /* Database handle */ + int eType, + void *pKey, int nKey, /* Database key to write to log */ + void *pVal, int nVal /* Database value (or nVal<0) to write */ +){ + int rc = LSM_OK; + LogWriter *pLog; /* Log object to write to */ + int nReq; /* Bytes of space required in log */ + int bCksum = 0; /* True to embed a checksum in this record */ + + assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE ); + assert( LSM_LOG_WRITE==LSM_WRITE ); + assert( LSM_LOG_DELETE==LSM_DELETE ); + assert( LSM_LOG_DRANGE==LSM_DRANGE ); + assert( (eType==LSM_LOG_DELETE)==(nVal<0) ); + + if( pDb->bUseLog==0 ) return LSM_OK; + pLog = pDb->pLogWriter; + + /* Determine how many bytes of space are required, assuming that a checksum + ** will be embedded in this record (even though it may not be). */ + nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey; + if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal; + + /* Jump over the jump region if required. Set bCksum to true to tell the + ** code below to include a checksum in the record if either (a) writing + ** this record would mean that more than LSM_CKSUM_MAXDATA bytes of data + ** have been written to the log since the last checksum, or (b) the jump + ** is taken. */ + rc = jumpIfRequired(pDb, pLog, nReq, &bCksum); + if( (pLog->buf.n+nReq) > LSM_CKSUM_MAXDATA ) bCksum = 1; + + if( rc==LSM_OK ){ + rc = lsmStringExtend(&pLog->buf, nReq); + } + if( rc==LSM_OK ){ + u8 *a = (u8 *)&pLog->buf.z[pLog->buf.n]; + + /* Write the record header - the type byte followed by either 1 (for + ** DELETE) or 2 (for WRITE) varints. */ + assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) ); + assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) ); + assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) ); + *(a++) = (u8)eType | (u8)bCksum; + a += lsmVarintPut32(a, nKey); + if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal); + + if( bCksum ){ + pLog->buf.n = (a - (u8 *)pLog->buf.z); + rc = logCksumAndFlush(pDb); + a = (u8 *)&pLog->buf.z[pLog->buf.n]; + } + + memcpy(a, pKey, nKey); + a += nKey; + if( eType!=LSM_LOG_DELETE ){ + memcpy(a, pVal, nVal); + a += nVal; + } + pLog->buf.n = a - (u8 *)pLog->buf.z; + assert( pLog->buf.n<=pLog->buf.nAlloc ); + } + + return rc; +} + +/* +** Append an LSM_LOG_COMMIT record to the database log. +*/ +int lsmLogCommit(lsm_db *pDb){ + if( pDb->bUseLog==0 ) return LSM_OK; + return logFlush(pDb, LSM_LOG_COMMIT); +} + +/* +** Store the current offset and other checksum related information in the +** structure *pMark. Later, *pMark can be passed to lsmLogSeek() to "rewind" +** the LogWriter object to the current log file offset. This is used when +** rolling back savepoint transactions. +*/ +void lsmLogTell( + lsm_db *pDb, /* Database handle */ + LogMark *pMark /* Populate this object with current offset */ +){ + LogWriter *pLog; + int nCksum; + + if( pDb->bUseLog==0 ) return; + pLog = pDb->pLogWriter; + nCksum = pLog->buf.n & 0xFFFFFFF8; + logUpdateCksum(pLog, nCksum); + assert( pLog->iCksumBuf==nCksum ); + pMark->nBuf = pLog->buf.n - nCksum; + memcpy(pMark->aBuf, &pLog->buf.z[nCksum], pMark->nBuf); + + pMark->iOff = pLog->iOff + pLog->buf.n; + pMark->cksum0 = pLog->cksum0; + pMark->cksum1 = pLog->cksum1; +} + +/* +** Seek (rewind) back to the log file offset stored by an ealier call to +** lsmLogTell() in *pMark. +*/ +void lsmLogSeek( + lsm_db *pDb, /* Database handle */ + LogMark *pMark /* Object containing log offset to seek to */ +){ + LogWriter *pLog; + + if( pDb->bUseLog==0 ) return; + pLog = pDb->pLogWriter; + + assert( pMark->iOff<=pLog->iOff+pLog->buf.n ); + if( (pMark->iOff & 0xFFFFFFF8)>=pLog->iOff ){ + pLog->buf.n = (int)(pMark->iOff - pLog->iOff); + pLog->iCksumBuf = (pLog->buf.n & 0xFFFFFFF8); + }else{ + pLog->buf.n = pMark->nBuf; + memcpy(pLog->buf.z, pMark->aBuf, pMark->nBuf); + pLog->iCksumBuf = 0; + pLog->iOff = pMark->iOff - pMark->nBuf; + } + pLog->cksum0 = pMark->cksum0; + pLog->cksum1 = pMark->cksum1; + + if( pMark->iOff > pLog->iRegion1End ) pLog->iRegion1End = 0; + if( pMark->iOff > pLog->iRegion2Start ) pLog->iRegion2Start = 0; +} + +/* +** This function does the work for an lsm_info(LOG_STRUCTURE) request. +*/ +int lsmInfoLogStructure(lsm_db *pDb, char **pzVal){ + int rc = LSM_OK; + char *zVal = 0; + + /* If there is no read or write transaction open, read the latest + ** tree-header from shared-memory to report on. If necessary, update + ** it based on the contents of the database header. + ** + ** No locks are taken here - these are passive read operations only. + */ + if( pDb->pCsr==0 && pDb->nTransOpen==0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + if( rc==LSM_OK ) rc = logReclaimSpace(pDb); + } + + if( rc==LSM_OK ){ + DbLog *pLog = &pDb->treehdr.log; + zVal = lsmMallocPrintf(pDb->pEnv, + "%d %d %d %d %d %d", + (int)pLog->aRegion[0].iStart, (int)pLog->aRegion[0].iEnd, + (int)pLog->aRegion[1].iStart, (int)pLog->aRegion[1].iEnd, + (int)pLog->aRegion[2].iStart, (int)pLog->aRegion[2].iEnd + ); + if( !zVal ) rc = LSM_NOMEM_BKPT; + } + + *pzVal = zVal; + return rc; +} + +/************************************************************************* +** Begin code for log recovery. +*/ + +typedef struct LogReader LogReader; +struct LogReader { + FileSystem *pFS; /* File system to read from */ + i64 iOff; /* File offset at end of buf content */ + int iBuf; /* Current read offset in buf */ + LsmString buf; /* Buffer containing file content */ + + int iCksumBuf; /* Offset in buf corresponding to cksum[01] */ + u32 cksum0; /* Checksum 0 at offset iCksumBuf */ + u32 cksum1; /* Checksum 1 at offset iCksumBuf */ +}; + +static void logReaderBlob( + LogReader *p, /* Log reader object */ + LsmString *pBuf, /* Dynamic storage, if required */ + int nBlob, /* Number of bytes to read */ + u8 **ppBlob, /* OUT: Pointer to blob read */ + int *pRc /* IN/OUT: Error code */ +){ + static const int LOG_READ_SIZE = 512; + int rc = *pRc; /* Return code */ + int nReq = nBlob; /* Bytes required */ + + while( rc==LSM_OK && nReq>0 ){ + int nAvail; /* Bytes of data available in p->buf */ + if( p->buf.n==p->iBuf ){ + int nCksum; /* Total bytes requiring checksum */ + int nCarry = 0; /* Total bytes requiring checksum */ + + nCksum = p->iBuf - p->iCksumBuf; + if( nCksum>0 ){ + nCarry = nCksum % 8; + nCksum = ((nCksum / 8) * 8); + if( nCksum>0 ){ + logCksumUnaligned( + &p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1 + ); + } + } + if( nCarry>0 ) memcpy(p->buf.z, &p->buf.z[p->iBuf-nCarry], nCarry); + p->buf.n = nCarry; + p->iBuf = nCarry; + + rc = lsmFsReadLog(p->pFS, p->iOff, LOG_READ_SIZE, &p->buf); + if( rc!=LSM_OK ) break; + p->iCksumBuf = 0; + p->iOff += LOG_READ_SIZE; + } + + nAvail = p->buf.n - p->iBuf; + if( ppBlob && nReq==nBlob && nBlob<=nAvail ){ + *ppBlob = (u8 *)&p->buf.z[p->iBuf]; + p->iBuf += nBlob; + nReq = 0; + }else{ + int nCopy = LSM_MIN(nAvail, nReq); + if( nBlob==nReq ){ + pBuf->n = 0; + } + rc = lsmStringBinAppend(pBuf, (u8 *)&p->buf.z[p->iBuf], nCopy); + nReq -= nCopy; + p->iBuf += nCopy; + if( nReq==0 && ppBlob ){ + *ppBlob = (u8*)pBuf->z; + } + } + } + + *pRc = rc; +} + +static void logReaderVarint( + LogReader *p, + LsmString *pBuf, + int *piVal, /* OUT: Value read from log */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==LSM_OK ){ + u8 *aVarint; + if( p->buf.n==p->iBuf ){ + logReaderBlob(p, 0, 10, &aVarint, pRc); + if( LSM_OK==*pRc ) p->iBuf -= (10 - lsmVarintGet32(aVarint, piVal)); + }else{ + logReaderBlob(p, pBuf, lsmVarintSize(p->buf.z[p->iBuf]), &aVarint, pRc); + if( LSM_OK==*pRc ) lsmVarintGet32(aVarint, piVal); + } + } +} + +static void logReaderByte(LogReader *p, u8 *pByte, int *pRc){ + u8 *pPtr = 0; + logReaderBlob(p, 0, 1, &pPtr, pRc); + if( pPtr ) *pByte = *pPtr; +} + +static void logReaderCksum(LogReader *p, LsmString *pBuf, int *pbEof, int *pRc){ + if( *pRc==LSM_OK ){ + u8 *pPtr = 0; + u32 cksum0, cksum1; + int nCksum = p->iBuf - p->iCksumBuf; + + /* Update in-memory (expected) checksums */ + assert( nCksum>=0 ); + logCksumUnaligned(&p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1); + p->iCksumBuf = p->iBuf + 8; + logReaderBlob(p, pBuf, 8, &pPtr, pRc); + assert( pPtr || *pRc ); + + /* Read the checksums from the log file. Set *pbEof if they do not match. */ + if( pPtr ){ + cksum0 = lsmGetU32(pPtr); + cksum1 = lsmGetU32(&pPtr[4]); + *pbEof = (cksum0!=p->cksum0 || cksum1!=p->cksum1); + p->iCksumBuf = p->iBuf; + } + } +} + +static void logReaderInit( + lsm_db *pDb, /* Database handle */ + DbLog *pLog, /* Log object associated with pDb */ + int bInitBuf, /* True if p->buf is uninitialized */ + LogReader *p /* Initialize this LogReader object */ +){ + p->pFS = pDb->pFS; + p->iOff = pLog->aRegion[2].iStart; + p->cksum0 = pLog->cksum0; + p->cksum1 = pLog->cksum1; + if( bInitBuf ){ lsmStringInit(&p->buf, pDb->pEnv); } + p->buf.n = 0; + p->iCksumBuf = 0; + p->iBuf = 0; +} + +/* +** This function is called after reading the header of a LOG_DELETE or +** LOG_WRITE record. Parameter nByte is the total size of the key and +** value that follow the header just read. Return true if the size and +** position of the record indicate that it should contain a checksum. +*/ +static int logRequireCksum(LogReader *p, int nByte){ + return ((p->iBuf + nByte - p->iCksumBuf) > LSM_CKSUM_MAXDATA); +} + +/* +** Recover the contents of the log file. +*/ +int lsmLogRecover(lsm_db *pDb){ + LsmString buf1; /* Key buffer */ + LsmString buf2; /* Value buffer */ + LogReader reader; /* Log reader object */ + int rc = LSM_OK; /* Return code */ + int nCommit = 0; /* Number of transactions to recover */ + int iPass; + int nJump = 0; /* Number of LSM_LOG_JUMP records in pass 0 */ + DbLog *pLog; + int bOpen; + + rc = lsmFsOpenLog(pDb, &bOpen); + if( rc!=LSM_OK ) return rc; + + rc = lsmTreeInit(pDb); + if( rc!=LSM_OK ) return rc; + + pLog = &pDb->treehdr.log; + lsmCheckpointLogoffset(pDb->pShmhdr->aSnap2, pLog); + + logReaderInit(pDb, pLog, 1, &reader); + lsmStringInit(&buf1, pDb->pEnv); + lsmStringInit(&buf2, pDb->pEnv); + + /* The outer for() loop runs at most twice. The first iteration is to + ** count the number of committed transactions in the log. The second + ** iterates through those transactions and updates the in-memory tree + ** structure with their contents. */ + if( bOpen ){ + for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ + int bEof = 0; + + while( rc==LSM_OK && !bEof ){ + u8 eType = 0; + logReaderByte(&reader, &eType, &rc); + + switch( eType ){ + case LSM_LOG_PAD1: + break; + + case LSM_LOG_PAD2: { + int nPad; + logReaderVarint(&reader, &buf1, &nPad, &rc); + logReaderBlob(&reader, &buf1, nPad, 0, &rc); + break; + } + + case LSM_LOG_DRANGE: + case LSM_LOG_DRANGE_CKSUM: + case LSM_LOG_WRITE: + case LSM_LOG_WRITE_CKSUM: { + int nKey; + int nVal; + u8 *aVal; + logReaderVarint(&reader, &buf1, &nKey, &rc); + logReaderVarint(&reader, &buf2, &nVal, &rc); + + if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){ + logReaderCksum(&reader, &buf1, &bEof, &rc); + }else{ + bEof = logRequireCksum(&reader, nKey+nVal); + } + if( bEof ) break; + + logReaderBlob(&reader, &buf1, nKey, 0, &rc); + logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); + if( iPass==1 && rc==LSM_OK ){ + if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){ + rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); + }else{ + rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal); + } + } + break; + } + + case LSM_LOG_DELETE: + case LSM_LOG_DELETE_CKSUM: { + int nKey; u8 *aKey; + logReaderVarint(&reader, &buf1, &nKey, &rc); + + if( eType==LSM_LOG_DELETE_CKSUM ){ + logReaderCksum(&reader, &buf1, &bEof, &rc); + }else{ + bEof = logRequireCksum(&reader, nKey); + } + if( bEof ) break; + + logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); + if( iPass==1 && rc==LSM_OK ){ + rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); + } + break; + } + + case LSM_LOG_COMMIT: + logReaderCksum(&reader, &buf1, &bEof, &rc); + if( bEof==0 ){ + nCommit++; + assert( nCommit>0 || iPass==1 ); + if( nCommit==0 ) bEof = 1; + } + break; + + case LSM_LOG_JUMP: { + int iOff = 0; + logReaderVarint(&reader, &buf1, &iOff, &rc); + if( rc==LSM_OK ){ + if( iPass==1 ){ + if( pLog->aRegion[2].iStart==0 ){ + assert( pLog->aRegion[1].iStart==0 ); + pLog->aRegion[1].iEnd = reader.iOff; + }else{ + assert( pLog->aRegion[0].iStart==0 ); + pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; + pLog->aRegion[0].iEnd = reader.iOff-reader.buf.n+reader.iBuf; + } + pLog->aRegion[2].iStart = iOff; + }else{ + if( (nJump++)==2 ){ + bEof = 1; + } + } + + reader.iOff = iOff; + reader.buf.n = reader.iBuf; + } + break; + } + + default: + /* Including LSM_LOG_EOF */ + bEof = 1; + break; + } + } + + if( rc==LSM_OK && iPass==0 ){ + if( nCommit==0 ){ + if( pLog->aRegion[2].iStart==0 ){ + iPass = 1; + }else{ + pLog->aRegion[2].iStart = 0; + iPass = -1; + lsmCheckpointZeroLogoffset(pDb); + } + } + logReaderInit(pDb, pLog, 0, &reader); + nCommit = nCommit * -1; + } + } + } + + /* Initialize DbLog object */ + if( rc==LSM_OK ){ + pLog->aRegion[2].iEnd = reader.iOff - reader.buf.n + reader.iBuf; + pLog->cksum0 = reader.cksum0; + pLog->cksum1 = reader.cksum1; + } + + if( rc==LSM_OK ){ + rc = lsmFinishRecovery(pDb); + }else{ + lsmFinishRecovery(pDb); + } + + if( pDb->bRoTrans ){ + lsmFsCloseLog(pDb); + } + + lsmStringClear(&buf1); + lsmStringClear(&buf2); + lsmStringClear(&reader.buf); + return rc; +} + +void lsmLogClose(lsm_db *db){ + if( db->pLogWriter ){ + lsmFree(db->pEnv, db->pLogWriter->buf.z); + lsmFree(db->pEnv, db->pLogWriter); + db->pLogWriter = 0; + } +} diff --git a/ext/lsm1/lsm_main.c b/ext/lsm1/lsm_main.c new file mode 100644 index 0000000000000000000000000000000000000000..f2b353105a508b291b3d952adbaabb600a7473eb --- /dev/null +++ b/ext/lsm1/lsm_main.c @@ -0,0 +1,1008 @@ +/* +** 2011-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** The main interface to the LSM module. +*/ +#include "lsmInt.h" + + +#ifdef LSM_DEBUG +/* +** This function returns a copy of its only argument. +** +** When the library is built with LSM_DEBUG defined, this function is called +** whenever an error code is generated (not propagated - generated). So +** if the library is mysteriously returning (say) LSM_IOERR, a breakpoint +** may be set in this function to determine why. +*/ +int lsmErrorBkpt(int rc){ + /* Set breakpoint here! */ + return rc; +} + +/* +** This function contains various assert() statements that test that the +** lsm_db structure passed as an argument is internally consistent. +*/ +static void assert_db_state(lsm_db *pDb){ + + /* If there is at least one cursor or a write transaction open, the database + ** handle must be holding a pointer to a client snapshot. And the reverse + ** - if there are no open cursors and no write transactions then there must + ** not be a client snapshot. */ + + assert( (pDb->pCsr!=0||pDb->nTransOpen>0)==(pDb->iReader>=0||pDb->bRoTrans) ); + + assert( (pDb->iReader<0 && pDb->bRoTrans==0) || pDb->pClient!=0 ); + + assert( pDb->nTransOpen>=0 ); +} +#else +# define assert_db_state(x) +#endif + +/* +** The default key-compare function. +*/ +static int xCmp(void *p1, int n1, void *p2, int n2){ + int res; + res = memcmp(p1, p2, LSM_MIN(n1, n2)); + if( res==0 ) res = (n1-n2); + return res; +} + +static void xLog(void *pCtx, int rc, const char *z){ + (void)(rc); + (void)(pCtx); + fprintf(stderr, "%s\n", z); + fflush(stderr); +} + +/* +** Allocate a new db handle. +*/ +int lsm_new(lsm_env *pEnv, lsm_db **ppDb){ + lsm_db *pDb; + + /* If the user did not provide an environment, use the default. */ + if( pEnv==0 ) pEnv = lsm_default_env(); + assert( pEnv ); + + /* Allocate the new database handle */ + *ppDb = pDb = (lsm_db *)lsmMallocZero(pEnv, sizeof(lsm_db)); + if( pDb==0 ) return LSM_NOMEM_BKPT; + + /* Initialize the new object */ + pDb->pEnv = pEnv; + pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; + pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; + pDb->bAutowork = LSM_DFLT_AUTOWORK; + pDb->eSafety = LSM_DFLT_SAFETY; + pDb->xCmp = xCmp; + pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; + pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; + pDb->nMerge = LSM_DFLT_AUTOMERGE; + pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; + pDb->bUseLog = LSM_DFLT_USE_LOG; + pDb->iReader = -1; + pDb->iRwclient = -1; + pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; + pDb->iMmap = LSM_DFLT_MMAP; + pDb->xLog = xLog; + pDb->compress.iId = LSM_COMPRESSION_NONE; + return LSM_OK; +} + +lsm_env *lsm_get_env(lsm_db *pDb){ + assert( pDb->pEnv ); + return pDb->pEnv; +} + +/* +** If database handle pDb is currently holding a client snapshot, but does +** not have any open cursors or write transactions, release it. +*/ +static void dbReleaseClientSnapshot(lsm_db *pDb){ + if( pDb->nTransOpen==0 && pDb->pCsr==0 ){ + lsmFinishReadTrans(pDb); + } +} + +static int getFullpathname( + lsm_env *pEnv, + const char *zRel, + char **pzAbs +){ + int nAlloc = 0; + char *zAlloc = 0; + int nReq = 0; + int rc; + + do{ + nAlloc = nReq; + rc = pEnv->xFullpath(pEnv, zRel, zAlloc, &nReq); + if( nReq>nAlloc ){ + zAlloc = lsmReallocOrFreeRc(pEnv, zAlloc, nReq, &rc); + } + }while( nReq>nAlloc && rc==LSM_OK ); + + if( rc!=LSM_OK ){ + lsmFree(pEnv, zAlloc); + zAlloc = 0; + } + *pzAbs = zAlloc; + return rc; +} + +/* +** Check that the bits in the db->mLock mask are consistent with the +** value stored in db->iRwclient. An assert shall fail otherwise. +*/ +static void assertRwclientLockValue(lsm_db *db){ +#ifndef NDEBUG + u64 msk; /* Mask of mLock bits for RWCLIENT locks */ + u64 rwclient = 0; /* Bit corresponding to db->iRwclient */ + + if( db->iRwclient>=0 ){ + rwclient = ((u64)1 << (LSM_LOCK_RWCLIENT(db->iRwclient)-1)); + } + msk = ((u64)1 << (LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT)-1)) - 1; + msk -= (((u64)1 << (LSM_LOCK_RWCLIENT(0)-1)) - 1); + + assert( (db->mLock & msk)==rwclient ); +#endif +} + +/* +** Open a new connection to database zFilename. +*/ +int lsm_open(lsm_db *pDb, const char *zFilename){ + int rc; + + if( pDb->pDatabase ){ + rc = LSM_MISUSE; + }else{ + char *zFull; + + /* Translate the possibly relative pathname supplied by the user into + ** an absolute pathname. This is required because the supplied path + ** is used (either directly or with "-log" appended to it) for more + ** than one purpose - to open both the database and log files, and + ** perhaps to unlink the log file during disconnection. An absolute + ** path is required to ensure that the correct files are operated + ** on even if the application changes the cwd. */ + rc = getFullpathname(pDb->pEnv, zFilename, &zFull); + assert( rc==LSM_OK || zFull==0 ); + + /* Connect to the database. */ + if( rc==LSM_OK ){ + rc = lsmDbDatabaseConnect(pDb, zFull); + } + + if( pDb->bReadonly==0 ){ + /* Configure the file-system connection with the page-size and block-size + ** of this database. Even if the database file is zero bytes in size + ** on disk, these values have been set in shared-memory by now, and so + ** are guaranteed not to change during the lifetime of this connection. + */ + if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ + lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); + lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); + } + } + + lsmFree(pDb->pEnv, zFull); + assertRwclientLockValue(pDb); + } + + assert( pDb->bReadonly==0 || pDb->bReadonly==1 ); + assert( rc!=LSM_OK || (pDb->pShmhdr==0)==(pDb->bReadonly==1) ); + + return rc; +} + +int lsm_close(lsm_db *pDb){ + int rc = LSM_OK; + if( pDb ){ + assert_db_state(pDb); + if( pDb->pCsr || pDb->nTransOpen ){ + rc = LSM_MISUSE_BKPT; + }else{ + lsmMCursorFreeCache(pDb); + lsmFreeSnapshot(pDb->pEnv, pDb->pClient); + pDb->pClient = 0; + + assertRwclientLockValue(pDb); + + lsmDbDatabaseRelease(pDb); + lsmLogClose(pDb); + lsmFsClose(pDb->pFS); + /* assert( pDb->mLock==0 ); */ + + /* Invoke any destructors registered for the compression or + ** compression factory callbacks. */ + if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); + if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); + + lsmFree(pDb->pEnv, pDb->rollback.aArray); + lsmFree(pDb->pEnv, pDb->aTrans); + lsmFree(pDb->pEnv, pDb->apShm); + lsmFree(pDb->pEnv, pDb); + } + } + return rc; +} + +int lsm_config(lsm_db *pDb, int eParam, ...){ + int rc = LSM_OK; + va_list ap; + va_start(ap, eParam); + + switch( eParam ){ + case LSM_CONFIG_AUTOFLUSH: { + /* This parameter is read and written in KB. But all internal + ** processing is done in bytes. */ + int *piVal = va_arg(ap, int *); + int iVal = *piVal; + if( iVal>=0 && iVal<=(1024*1024) ){ + pDb->nTreeLimit = iVal*1024; + } + *piVal = (pDb->nTreeLimit / 1024); + break; + } + + case LSM_CONFIG_AUTOWORK: { + int *piVal = va_arg(ap, int *); + if( *piVal>=0 ){ + pDb->bAutowork = *piVal; + } + *piVal = pDb->bAutowork; + break; + } + + case LSM_CONFIG_AUTOCHECKPOINT: { + /* This parameter is read and written in KB. But all internal processing + ** (including the lsm_db.nAutockpt variable) is done in bytes. */ + int *piVal = va_arg(ap, int *); + if( *piVal>=0 ){ + int iVal = *piVal; + pDb->nAutockpt = (i64)iVal * 1024; + } + *piVal = (int)(pDb->nAutockpt / 1024); + break; + } + + case LSM_CONFIG_PAGE_SIZE: { + int *piVal = va_arg(ap, int *); + if( pDb->pDatabase ){ + /* If lsm_open() has been called, this is a read-only parameter. + ** Set the output variable to the page-size according to the + ** FileSystem object. */ + *piVal = lsmFsPageSize(pDb->pFS); + }else{ + if( *piVal>=256 && *piVal<=65536 && ((*piVal-1) & *piVal)==0 ){ + pDb->nDfltPgsz = *piVal; + }else{ + *piVal = pDb->nDfltPgsz; + } + } + break; + } + + case LSM_CONFIG_BLOCK_SIZE: { + /* This parameter is read and written in KB. But all internal + ** processing is done in bytes. */ + int *piVal = va_arg(ap, int *); + if( pDb->pDatabase ){ + /* If lsm_open() has been called, this is a read-only parameter. + ** Set the output variable to the block-size in KB according to the + ** FileSystem object. */ + *piVal = lsmFsBlockSize(pDb->pFS) / 1024; + }else{ + int iVal = *piVal; + if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ + pDb->nDfltBlksz = iVal * 1024; + }else{ + *piVal = pDb->nDfltBlksz / 1024; + } + } + break; + } + + case LSM_CONFIG_SAFETY: { + int *piVal = va_arg(ap, int *); + if( *piVal>=0 && *piVal<=2 ){ + pDb->eSafety = *piVal; + } + *piVal = pDb->eSafety; + break; + } + + case LSM_CONFIG_MMAP: { + int *piVal = va_arg(ap, int *); + if( pDb->iReader<0 && *piVal>=0 ){ + pDb->iMmap = *piVal; + rc = lsmFsConfigure(pDb); + } + *piVal = pDb->iMmap; + break; + } + + case LSM_CONFIG_USE_LOG: { + int *piVal = va_arg(ap, int *); + if( pDb->nTransOpen==0 && (*piVal==0 || *piVal==1) ){ + pDb->bUseLog = *piVal; + } + *piVal = pDb->bUseLog; + break; + } + + case LSM_CONFIG_AUTOMERGE: { + int *piVal = va_arg(ap, int *); + if( *piVal>1 ) pDb->nMerge = *piVal; + *piVal = pDb->nMerge; + break; + } + + case LSM_CONFIG_MAX_FREELIST: { + int *piVal = va_arg(ap, int *); + if( *piVal>=2 && *piVal<=LSM_MAX_FREELIST_ENTRIES ){ + pDb->nMaxFreelist = *piVal; + } + *piVal = pDb->nMaxFreelist; + break; + } + + case LSM_CONFIG_MULTIPLE_PROCESSES: { + int *piVal = va_arg(ap, int *); + if( pDb->pDatabase ){ + /* If lsm_open() has been called, this is a read-only parameter. + ** Set the output variable to true if this connection is currently + ** in multi-process mode. */ + *piVal = lsmDbMultiProc(pDb); + }else{ + pDb->bMultiProc = *piVal = (*piVal!=0); + } + break; + } + + case LSM_CONFIG_READONLY: { + int *piVal = va_arg(ap, int *); + /* If lsm_open() has been called, this is a read-only parameter. */ + if( pDb->pDatabase==0 && *piVal>=0 ){ + pDb->bReadonly = *piVal = (*piVal!=0); + } + *piVal = pDb->bReadonly; + break; + } + + case LSM_CONFIG_SET_COMPRESSION: { + lsm_compress *p = va_arg(ap, lsm_compress *); + if( pDb->iReader>=0 && pDb->bInFactory==0 ){ + /* May not change compression schemes with an open transaction */ + rc = LSM_MISUSE_BKPT; + }else{ + if( pDb->compress.xFree ){ + /* Invoke any destructor belonging to the current compression. */ + pDb->compress.xFree(pDb->compress.pCtx); + } + if( p->xBound==0 ){ + memset(&pDb->compress, 0, sizeof(lsm_compress)); + pDb->compress.iId = LSM_COMPRESSION_NONE; + }else{ + memcpy(&pDb->compress, p, sizeof(lsm_compress)); + } + rc = lsmFsConfigure(pDb); + } + break; + } + + case LSM_CONFIG_SET_COMPRESSION_FACTORY: { + lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); + if( pDb->factory.xFree ){ + /* Invoke any destructor belonging to the current factory. */ + pDb->factory.xFree(pDb->factory.pCtx); + } + memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); + break; + } + + case LSM_CONFIG_GET_COMPRESSION: { + lsm_compress *p = va_arg(ap, lsm_compress *); + memcpy(p, &pDb->compress, sizeof(lsm_compress)); + break; + } + + default: + rc = LSM_MISUSE; + break; + } + + va_end(ap); + return rc; +} + +void lsmAppendSegmentList(LsmString *pStr, char *zPre, Segment *pSeg){ + lsmStringAppendf(pStr, "%s{%lld %lld %lld %lld}", zPre, + pSeg->iFirst, pSeg->iLastPg, pSeg->iRoot, pSeg->nSize + ); +} + +static int infoGetWorker(lsm_db *pDb, Snapshot **pp, int *pbUnlock){ + int rc = LSM_OK; + + assert( *pbUnlock==0 ); + if( !pDb->pWorker ){ + rc = lsmBeginWork(pDb); + if( rc!=LSM_OK ) return rc; + *pbUnlock = 1; + } + if( pp ) *pp = pDb->pWorker; + return rc; +} + +static void infoFreeWorker(lsm_db *pDb, int bUnlock){ + if( bUnlock ){ + int rcdummy = LSM_BUSY; + lsmFinishWork(pDb, 0, &rcdummy); + } +} + +int lsmStructList( + lsm_db *pDb, /* Database handle */ + char **pzOut /* OUT: Nul-terminated string (tcl list) */ +){ + Level *pTopLevel = 0; /* Top level of snapshot to report on */ + int rc = LSM_OK; + Level *p; + LsmString s; + Snapshot *pWorker; /* Worker snapshot */ + int bUnlock = 0; + + /* Obtain the worker snapshot */ + rc = infoGetWorker(pDb, &pWorker, &bUnlock); + if( rc!=LSM_OK ) return rc; + + /* Format the contents of the snapshot as text */ + pTopLevel = lsmDbSnapshotLevel(pWorker); + lsmStringInit(&s, pDb->pEnv); + for(p=pTopLevel; rc==LSM_OK && p; p=p->pNext){ + int i; + lsmStringAppendf(&s, "%s{%d", (s.n ? " " : ""), (int)p->iAge); + lsmAppendSegmentList(&s, " ", &p->lhs); + for(i=0; rc==LSM_OK && inRight; i++){ + lsmAppendSegmentList(&s, " ", &p->aRhs[i]); + } + lsmStringAppend(&s, "}", 1); + } + rc = s.n>=0 ? LSM_OK : LSM_NOMEM; + + /* Release the snapshot and return */ + infoFreeWorker(pDb, bUnlock); + *pzOut = s.z; + return rc; +} + +static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ + LsmString *pStr = (LsmString *)pCtx; + lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); + return 0; +} + +int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ + Snapshot *pWorker; /* Worker snapshot */ + int bUnlock = 0; + LsmString s; + int rc; + + /* Obtain the worker snapshot */ + rc = infoGetWorker(pDb, &pWorker, &bUnlock); + if( rc!=LSM_OK ) return rc; + + lsmStringInit(&s, pDb->pEnv); + rc = lsmWalkFreelist(pDb, 0, infoFreelistCb, &s); + if( rc!=LSM_OK ){ + lsmFree(pDb->pEnv, s.z); + }else{ + *pzOut = s.z; + } + + /* Release the snapshot and return */ + infoFreeWorker(pDb, bUnlock); + return rc; +} + +static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ + ShmHeader *pShm = db->pShmhdr; + TreeHeader *p = &pShm->hdr1; + + /* The following code suffers from two race conditions, as it accesses and + ** trusts the contents of shared memory without verifying checksums: + ** + ** * The two values read - TreeHeader.root.nByte and oldroot.nByte - are + ** 32-bit fields. It is assumed that reading from one of these + ** is atomic - that it is not possible to read a partially written + ** garbage value. However the two values may be mutually inconsistent. + ** + ** * TreeHeader.iLogOff is a 64-bit value. And lsmCheckpointLogOffset() + ** reads a 64-bit value from a snapshot stored in shared memory. It + ** is assumed that in each case it is possible to read a partially + ** written garbage value. If this occurs, then the value returned + ** for the size of the "old" tree may reflect the size of an "old" + ** tree that was recently flushed to disk. + ** + ** Given the context in which this function is called (as a result of an + ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to + ** be problems. + */ + *pnNewKB = ((int)p->root.nByte + 1023) / 1024; + if( p->iOldShmid ){ + if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ + *pnOldKB = 0; + }else{ + *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; + } + }else{ + *pnOldKB = 0; + } + + return LSM_OK; +} + +int lsm_info(lsm_db *pDb, int eParam, ...){ + int rc = LSM_OK; + va_list ap; + va_start(ap, eParam); + + switch( eParam ){ + case LSM_INFO_NWRITE: { + int *piVal = va_arg(ap, int *); + *piVal = lsmFsNWrite(pDb->pFS); + break; + } + + case LSM_INFO_NREAD: { + int *piVal = va_arg(ap, int *); + *piVal = lsmFsNRead(pDb->pFS); + break; + } + + case LSM_INFO_DB_STRUCTURE: { + char **pzVal = va_arg(ap, char **); + rc = lsmStructList(pDb, pzVal); + break; + } + + case LSM_INFO_ARRAY_STRUCTURE: { + LsmPgno pgno = va_arg(ap, LsmPgno); + char **pzVal = va_arg(ap, char **); + rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); + break; + } + + case LSM_INFO_ARRAY_PAGES: { + LsmPgno pgno = va_arg(ap, LsmPgno); + char **pzVal = va_arg(ap, char **); + rc = lsmInfoArrayPages(pDb, pgno, pzVal); + break; + } + + case LSM_INFO_PAGE_HEX_DUMP: + case LSM_INFO_PAGE_ASCII_DUMP: { + LsmPgno pgno = va_arg(ap, LsmPgno); + char **pzVal = va_arg(ap, char **); + int bUnlock = 0; + rc = infoGetWorker(pDb, 0, &bUnlock); + if( rc==LSM_OK ){ + int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); + rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal); + } + infoFreeWorker(pDb, bUnlock); + break; + } + + case LSM_INFO_LOG_STRUCTURE: { + char **pzVal = va_arg(ap, char **); + rc = lsmInfoLogStructure(pDb, pzVal); + break; + } + + case LSM_INFO_FREELIST: { + char **pzVal = va_arg(ap, char **); + rc = lsmInfoFreelist(pDb, pzVal); + break; + } + + case LSM_INFO_CHECKPOINT_SIZE: { + int *pnKB = va_arg(ap, int *); + rc = lsmCheckpointSize(pDb, pnKB); + break; + } + + case LSM_INFO_TREE_SIZE: { + int *pnOld = va_arg(ap, int *); + int *pnNew = va_arg(ap, int *); + rc = infoTreeSize(pDb, pnOld, pnNew); + break; + } + + case LSM_INFO_COMPRESSION_ID: { + unsigned int *piOut = va_arg(ap, unsigned int *); + if( pDb->pClient ){ + *piOut = pDb->pClient->iCmpId; + }else{ + rc = lsmInfoCompressionId(pDb, piOut); + } + break; + } + + default: + rc = LSM_MISUSE; + break; + } + + va_end(ap); + return rc; +} + +static int doWriteOp( + lsm_db *pDb, + int bDeleteRange, + const void *pKey, int nKey, /* Key to write or delete */ + const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ +){ + int rc = LSM_OK; /* Return code */ + int bCommit = 0; /* True to commit before returning */ + + if( pDb->nTransOpen==0 ){ + bCommit = 1; + rc = lsm_begin(pDb, 1); + } + + if( rc==LSM_OK ){ + int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE)); + rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal); + } + + lsmSortedSaveTreeCursors(pDb); + + if( rc==LSM_OK ){ + int pgsz = lsmFsPageSize(pDb->pFS); + int nQuant = LSM_AUTOWORK_QUANT * pgsz; + int nBefore; + int nAfter; + int nDiff; + + if( nQuant>pDb->nTreeLimit ){ + nQuant = LSM_MAX(pDb->nTreeLimit, pgsz); + } + + nBefore = lsmTreeSize(pDb); + if( bDeleteRange ){ + rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal); + }else{ + rc = lsmTreeInsert(pDb, (void *)pKey, nKey, (void *)pVal, nVal); + } + + nAfter = lsmTreeSize(pDb); + nDiff = (nAfter/nQuant) - (nBefore/nQuant); + if( rc==LSM_OK && pDb->bAutowork && nDiff!=0 ){ + rc = lsmSortedAutoWork(pDb, nDiff * LSM_AUTOWORK_QUANT); + } + } + + /* If a transaction was opened at the start of this function, commit it. + ** Or, if an error has occurred, roll it back. */ + if( bCommit ){ + if( rc==LSM_OK ){ + rc = lsm_commit(pDb, 0); + }else{ + lsm_rollback(pDb, 0); + } + } + + return rc; +} + +/* +** Write a new value into the database. +*/ +int lsm_insert( + lsm_db *db, /* Database connection */ + const void *pKey, int nKey, /* Key to write or delete */ + const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ +){ + return doWriteOp(db, 0, pKey, nKey, pVal, nVal); +} + +/* +** Delete a value from the database. +*/ +int lsm_delete(lsm_db *db, const void *pKey, int nKey){ + return doWriteOp(db, 0, pKey, nKey, 0, -1); +} + +/* +** Delete a range of database keys. +*/ +int lsm_delete_range( + lsm_db *db, /* Database handle */ + const void *pKey1, int nKey1, /* Lower bound of range to delete */ + const void *pKey2, int nKey2 /* Upper bound of range to delete */ +){ + int rc = LSM_OK; + if( db->xCmp((void *)pKey1, nKey1, (void *)pKey2, nKey2)<0 ){ + rc = doWriteOp(db, 1, pKey1, nKey1, pKey2, nKey2); + } + return rc; +} + +/* +** Open a new cursor handle. +** +** If there are currently no other open cursor handles, and no open write +** transaction, open a read transaction here. +*/ +int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr){ + int rc = LSM_OK; /* Return code */ + MultiCursor *pCsr = 0; /* New cursor object */ + + /* Open a read transaction if one is not already open. */ + assert_db_state(pDb); + + if( pDb->pShmhdr==0 ){ + assert( pDb->bReadonly ); + rc = lsmBeginRoTrans(pDb); + }else if( pDb->iReader<0 ){ + rc = lsmBeginReadTrans(pDb); + } + + /* Allocate the multi-cursor. */ + if( rc==LSM_OK ){ + rc = lsmMCursorNew(pDb, &pCsr); + } + + /* If an error has occured, set the output to NULL and delete any partially + ** allocated cursor. If this means there are no open cursors, release the + ** client snapshot. */ + if( rc!=LSM_OK ){ + lsmMCursorClose(pCsr, 0); + dbReleaseClientSnapshot(pDb); + } + + assert_db_state(pDb); + *ppCsr = (lsm_cursor *)pCsr; + return rc; +} + +/* +** Close a cursor opened using lsm_csr_open(). +*/ +int lsm_csr_close(lsm_cursor *p){ + if( p ){ + lsm_db *pDb = lsmMCursorDb((MultiCursor *)p); + assert_db_state(pDb); + lsmMCursorClose((MultiCursor *)p, 1); + dbReleaseClientSnapshot(pDb); + assert_db_state(pDb); + } + return LSM_OK; +} + +/* +** Attempt to seek the cursor to the database entry specified by pKey/nKey. +** If an error occurs (e.g. an OOM or IO error), return an LSM error code. +** Otherwise, return LSM_OK. +*/ +int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek){ + return lsmMCursorSeek((MultiCursor *)pCsr, 0, (void *)pKey, nKey, eSeek); +} + +int lsm_csr_next(lsm_cursor *pCsr){ + return lsmMCursorNext((MultiCursor *)pCsr); +} + +int lsm_csr_prev(lsm_cursor *pCsr){ + return lsmMCursorPrev((MultiCursor *)pCsr); +} + +int lsm_csr_first(lsm_cursor *pCsr){ + return lsmMCursorFirst((MultiCursor *)pCsr); +} + +int lsm_csr_last(lsm_cursor *pCsr){ + return lsmMCursorLast((MultiCursor *)pCsr); +} + +int lsm_csr_valid(lsm_cursor *pCsr){ + return lsmMCursorValid((MultiCursor *)pCsr); +} + +int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey){ + return lsmMCursorKey((MultiCursor *)pCsr, (void **)ppKey, pnKey); +} + +int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal){ + return lsmMCursorValue((MultiCursor *)pCsr, (void **)ppVal, pnVal); +} + +void lsm_config_log( + lsm_db *pDb, + void (*xLog)(void *, int, const char *), + void *pCtx +){ + pDb->xLog = xLog; + pDb->pLogCtx = pCtx; +} + +void lsm_config_work_hook( + lsm_db *pDb, + void (*xWork)(lsm_db *, void *), + void *pCtx +){ + pDb->xWork = xWork; + pDb->pWorkCtx = pCtx; +} + +void lsmLogMessage(lsm_db *pDb, int rc, const char *zFormat, ...){ + if( pDb->xLog ){ + LsmString s; + va_list ap, ap2; + lsmStringInit(&s, pDb->pEnv); + va_start(ap, zFormat); + va_start(ap2, zFormat); + lsmStringVAppendf(&s, zFormat, ap, ap2); + va_end(ap); + va_end(ap2); + pDb->xLog(pDb->pLogCtx, rc, s.z); + lsmStringClear(&s); + } +} + +int lsm_begin(lsm_db *pDb, int iLevel){ + int rc; + + assert_db_state( pDb ); + rc = (pDb->bReadonly ? LSM_READONLY : LSM_OK); + + /* A value less than zero means open one more transaction. */ + if( iLevel<0 ) iLevel = pDb->nTransOpen + 1; + if( iLevel>pDb->nTransOpen ){ + int i; + + /* Extend the pDb->aTrans[] array if required. */ + if( rc==LSM_OK && pDb->nTransAllocpEnv, pDb->aTrans, nByte); + if( !aNew ){ + rc = LSM_NOMEM; + }else{ + nByte = sizeof(TransMark) * (iLevel+1 - pDb->nTransAlloc); + memset(&aNew[pDb->nTransAlloc], 0, nByte); + pDb->nTransAlloc = iLevel+1; + pDb->aTrans = aNew; + } + } + + if( rc==LSM_OK && pDb->nTransOpen==0 ){ + rc = lsmBeginWriteTrans(pDb); + } + + if( rc==LSM_OK ){ + for(i=pDb->nTransOpen; iaTrans[i].tree); + lsmLogTell(pDb, &pDb->aTrans[i].log); + } + pDb->nTransOpen = iLevel; + } + } + + return rc; +} + +int lsm_commit(lsm_db *pDb, int iLevel){ + int rc = LSM_OK; + + assert_db_state( pDb ); + + /* A value less than zero means close the innermost nested transaction. */ + if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); + + if( iLevelnTransOpen ){ + if( iLevel==0 ){ + int rc2; + /* Commit the transaction to disk. */ + if( rc==LSM_OK ) rc = lsmLogCommit(pDb); + if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ + rc = lsmFsSyncLog(pDb->pFS); + } + rc2 = lsmFinishWriteTrans(pDb, (rc==LSM_OK)); + if( rc==LSM_OK ) rc = rc2; + } + pDb->nTransOpen = iLevel; + } + dbReleaseClientSnapshot(pDb); + return rc; +} + +int lsm_rollback(lsm_db *pDb, int iLevel){ + int rc = LSM_OK; + assert_db_state( pDb ); + + if( pDb->nTransOpen ){ + /* A value less than zero means close the innermost nested transaction. */ + if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); + + if( iLevel<=pDb->nTransOpen ){ + TransMark *pMark = &pDb->aTrans[(iLevel==0 ? 0 : iLevel-1)]; + lsmTreeRollback(pDb, &pMark->tree); + if( iLevel ) lsmLogSeek(pDb, &pMark->log); + pDb->nTransOpen = iLevel; + } + + if( pDb->nTransOpen==0 ){ + lsmFinishWriteTrans(pDb, 0); + } + dbReleaseClientSnapshot(pDb); + } + + return rc; +} + +int lsm_get_user_version(lsm_db *pDb, unsigned int *piUsr){ + int rc = LSM_OK; /* Return code */ + + /* Open a read transaction if one is not already open. */ + assert_db_state(pDb); + if( pDb->pShmhdr==0 ){ + assert( pDb->bReadonly ); + rc = lsmBeginRoTrans(pDb); + }else if( pDb->iReader<0 ){ + rc = lsmBeginReadTrans(pDb); + } + + /* Allocate the multi-cursor. */ + if( rc==LSM_OK ){ + *piUsr = pDb->treehdr.iUsrVersion; + } + + dbReleaseClientSnapshot(pDb); + assert_db_state(pDb); + return rc; +} + +int lsm_set_user_version(lsm_db *pDb, unsigned int iUsr){ + int rc = LSM_OK; /* Return code */ + int bCommit = 0; /* True to commit before returning */ + + if( pDb->nTransOpen==0 ){ + bCommit = 1; + rc = lsm_begin(pDb, 1); + } + + if( rc==LSM_OK ){ + pDb->treehdr.iUsrVersion = iUsr; + } + + /* If a transaction was opened at the start of this function, commit it. + ** Or, if an error has occurred, roll it back. */ + if( bCommit ){ + if( rc==LSM_OK ){ + rc = lsm_commit(pDb, 0); + }else{ + lsm_rollback(pDb, 0); + } + } + + return rc; +} diff --git a/ext/lsm1/lsm_mem.c b/ext/lsm1/lsm_mem.c new file mode 100644 index 0000000000000000000000000000000000000000..13dd9fe312c8f5882c71f250847a63b59b9291fb --- /dev/null +++ b/ext/lsm1/lsm_mem.c @@ -0,0 +1,104 @@ +/* +** 2011-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Helper routines for memory allocation. +*/ +#include "lsmInt.h" + +/* +** The following routines are called internally by LSM sub-routines. In +** this case a valid environment pointer must be supplied. +*/ +void *lsmMalloc(lsm_env *pEnv, size_t N){ + assert( pEnv ); + return pEnv->xMalloc(pEnv, N); +} +void lsmFree(lsm_env *pEnv, void *p){ + assert( pEnv ); + pEnv->xFree(pEnv, p); +} +void *lsmRealloc(lsm_env *pEnv, void *p, size_t N){ + assert( pEnv ); + return pEnv->xRealloc(pEnv, p, N); +} + +/* +** Core memory allocation routines for LSM. +*/ +void *lsm_malloc(lsm_env *pEnv, size_t N){ + return lsmMalloc(pEnv ? pEnv : lsm_default_env(), N); +} +void lsm_free(lsm_env *pEnv, void *p){ + lsmFree(pEnv ? pEnv : lsm_default_env(), p); +} +void *lsm_realloc(lsm_env *pEnv, void *p, size_t N){ + return lsmRealloc(pEnv ? pEnv : lsm_default_env(), p, N); +} + +void *lsmMallocZero(lsm_env *pEnv, size_t N){ + void *pRet; + assert( pEnv ); + pRet = lsmMalloc(pEnv, N); + if( pRet ) memset(pRet, 0, N); + return pRet; +} + +void *lsmMallocRc(lsm_env *pEnv, size_t N, int *pRc){ + void *pRet = 0; + if( *pRc==LSM_OK ){ + pRet = lsmMalloc(pEnv, N); + if( pRet==0 ){ + *pRc = LSM_NOMEM_BKPT; + } + } + return pRet; +} + +void *lsmMallocZeroRc(lsm_env *pEnv, size_t N, int *pRc){ + void *pRet = 0; + if( *pRc==LSM_OK ){ + pRet = lsmMallocZero(pEnv, N); + if( pRet==0 ){ + *pRc = LSM_NOMEM_BKPT; + } + } + return pRet; +} + +void *lsmReallocOrFree(lsm_env *pEnv, void *p, size_t N){ + void *pNew; + pNew = lsm_realloc(pEnv, p, N); + if( !pNew ) lsm_free(pEnv, p); + return pNew; +} + +void *lsmReallocOrFreeRc(lsm_env *pEnv, void *p, size_t N, int *pRc){ + void *pRet = 0; + if( *pRc ){ + lsmFree(pEnv, p); + }else{ + pRet = lsmReallocOrFree(pEnv, p, N); + if( !pRet ) *pRc = LSM_NOMEM_BKPT; + } + return pRet; +} + +char *lsmMallocStrdup(lsm_env *pEnv, const char *zIn){ + int nByte; + char *zRet; + nByte = strlen(zIn); + zRet = lsmMalloc(pEnv, nByte+1); + if( zRet ){ + memcpy(zRet, zIn, nByte+1); + } + return zRet; +} diff --git a/ext/lsm1/lsm_mutex.c b/ext/lsm1/lsm_mutex.c new file mode 100644 index 0000000000000000000000000000000000000000..cb99b2a61e23a10d7c3c627f8bb0994ff5f757fe --- /dev/null +++ b/ext/lsm1/lsm_mutex.c @@ -0,0 +1,88 @@ +/* +** 2012-01-30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Mutex functions for LSM. +*/ +#include "lsmInt.h" + +/* +** Allocate a new mutex. +*/ +int lsmMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ + return pEnv->xMutexNew(pEnv, ppNew); +} + +/* +** Return a handle for one of the static mutexes. +*/ +int lsmMutexStatic(lsm_env *pEnv, int iMutex, lsm_mutex **ppStatic){ + return pEnv->xMutexStatic(pEnv, iMutex, ppStatic); +} + +/* +** Free a mutex allocated by lsmMutexNew(). +*/ +void lsmMutexDel(lsm_env *pEnv, lsm_mutex *pMutex){ + if( pMutex ) pEnv->xMutexDel(pMutex); +} + +/* +** Enter a mutex. +*/ +void lsmMutexEnter(lsm_env *pEnv, lsm_mutex *pMutex){ + pEnv->xMutexEnter(pMutex); +} + +/* +** Attempt to enter a mutex, but do not block. If successful, return zero. +** Otherwise, if the mutex is already held by some other thread and is not +** entered, return non zero. +** +** Each successful call to this function must be matched by a call to +** lsmMutexLeave(). +*/ +int lsmMutexTry(lsm_env *pEnv, lsm_mutex *pMutex){ + return pEnv->xMutexTry(pMutex); +} + +/* +** Leave a mutex. +*/ +void lsmMutexLeave(lsm_env *pEnv, lsm_mutex *pMutex){ + pEnv->xMutexLeave(pMutex); +} + +#ifndef NDEBUG +/* +** Return non-zero if the mutex passed as the second argument is held +** by the calling thread, or zero otherwise. If the implementation is not +** able to tell if the mutex is held by the caller, it should return +** non-zero. +** +** This function is only used as part of assert() statements. +*/ +int lsmMutexHeld(lsm_env *pEnv, lsm_mutex *pMutex){ + return pEnv->xMutexHeld ? pEnv->xMutexHeld(pMutex) : 1; +} + +/* +** Return non-zero if the mutex passed as the second argument is not +** held by the calling thread, or zero otherwise. If the implementation +** is not able to tell if the mutex is held by the caller, it should +** return non-zero. +** +** This function is only used as part of assert() statements. +*/ +int lsmMutexNotHeld(lsm_env *pEnv, lsm_mutex *pMutex){ + return pEnv->xMutexNotHeld ? pEnv->xMutexNotHeld(pMutex) : 1; +} +#endif diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c new file mode 100644 index 0000000000000000000000000000000000000000..09f9338488ca92bd2fa86eef8b14b039a19c78d8 --- /dev/null +++ b/ext/lsm1/lsm_shared.c @@ -0,0 +1,1994 @@ +/* +** 2012-01-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Utilities used to help multiple LSM clients to coexist within the +** same process space. +*/ +#include "lsmInt.h" + +/* +** Global data. All global variables used by code in this file are grouped +** into the following structure instance. +** +** pDatabase: +** Linked list of all Database objects allocated within this process. +** This list may not be traversed without holding the global mutex (see +** functions enterGlobalMutex() and leaveGlobalMutex()). +*/ +static struct SharedData { + Database *pDatabase; /* Linked list of all Database objects */ +} gShared; + +/* +** Database structure. There is one such structure for each distinct +** database accessed by this process. They are stored in the singly linked +** list starting at global variable gShared.pDatabase. Database objects are +** reference counted. Once the number of connections to the associated +** database drops to zero, they are removed from the linked list and deleted. +** +** pFile: +** In multi-process mode, this file descriptor is used to obtain locks +** and to access shared-memory. In single process mode, its only job is +** to hold the exclusive lock on the file. +** +*/ +struct Database { + /* Protected by the global mutex (enterGlobalMutex/leaveGlobalMutex): */ + char *zName; /* Canonical path to database file */ + int nName; /* strlen(zName) */ + int nDbRef; /* Number of associated lsm_db handles */ + Database *pDbNext; /* Next Database structure in global list */ + + /* Protected by the local mutex (pClientMutex) */ + int bReadonly; /* True if Database.pFile is read-only */ + int bMultiProc; /* True if running in multi-process mode */ + lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ + LsmFile *pLsmFile; /* List of deferred closes */ + lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ + int nShmChunk; /* Number of entries in apShmChunk[] array */ + void **apShmChunk; /* Array of "shared" memory regions */ + lsm_db *pConn; /* List of connections to this db. */ +}; + +/* +** Functions to enter and leave the global mutex. This mutex is used +** to protect the global linked-list headed at gShared.pDatabase. +*/ +static int enterGlobalMutex(lsm_env *pEnv){ + lsm_mutex *p; + int rc = lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); + if( rc==LSM_OK ) lsmMutexEnter(pEnv, p); + return rc; +} +static void leaveGlobalMutex(lsm_env *pEnv){ + lsm_mutex *p; + lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); + lsmMutexLeave(pEnv, p); +} + +#ifdef LSM_DEBUG +static int holdingGlobalMutex(lsm_env *pEnv){ + lsm_mutex *p; + lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); + return lsmMutexHeld(pEnv, p); +} +#endif + +#if 0 +static void assertNotInFreelist(Freelist *p, int iBlk){ + int i; + for(i=0; inEntry; i++){ + assert( p->aEntry[i].iBlk!=iBlk ); + } +} +#else +# define assertNotInFreelist(x,y) +#endif + +/* +** Append an entry to the free-list. If (iId==-1), this is a delete. +*/ +int freelistAppend(lsm_db *db, u32 iBlk, i64 iId){ + lsm_env *pEnv = db->pEnv; + Freelist *p; + int i; + + assert( iId==-1 || iId>=0 ); + p = db->bUseFreelist ? db->pFreelist : &db->pWorker->freelist; + + /* Extend the space allocated for the freelist, if required */ + assert( p->nAlloc>=p->nEntry ); + if( p->nAlloc==p->nEntry ){ + int nNew; + int nByte; + FreelistEntry *aNew; + + nNew = (p->nAlloc==0 ? 4 : p->nAlloc*2); + nByte = sizeof(FreelistEntry) * nNew; + aNew = (FreelistEntry *)lsmRealloc(pEnv, p->aEntry, nByte); + if( !aNew ) return LSM_NOMEM_BKPT; + p->nAlloc = nNew; + p->aEntry = aNew; + } + + for(i=0; inEntry; i++){ + assert( i==0 || p->aEntry[i].iBlk > p->aEntry[i-1].iBlk ); + if( p->aEntry[i].iBlk>=iBlk ) break; + } + + if( inEntry && p->aEntry[i].iBlk==iBlk ){ + /* Clobber an existing entry */ + p->aEntry[i].iId = iId; + }else{ + /* Insert a new entry into the list */ + int nByte = sizeof(FreelistEntry)*(p->nEntry-i); + memmove(&p->aEntry[i+1], &p->aEntry[i], nByte); + p->aEntry[i].iBlk = iBlk; + p->aEntry[i].iId = iId; + p->nEntry++; + } + + return LSM_OK; +} + +/* +** This function frees all resources held by the Database structure passed +** as the only argument. +*/ +static void freeDatabase(lsm_env *pEnv, Database *p){ + assert( holdingGlobalMutex(pEnv) ); + if( p ){ + /* Free the mutexes */ + lsmMutexDel(pEnv, p->pClientMutex); + + if( p->pFile ){ + lsmEnvClose(pEnv, p->pFile); + } + + /* Free the array of shm pointers */ + lsmFree(pEnv, p->apShmChunk); + + /* Free the memory allocated for the Database struct itself */ + lsmFree(pEnv, p); + } +} + +typedef struct DbTruncateCtx DbTruncateCtx; +struct DbTruncateCtx { + int nBlock; + i64 iInUse; +}; + +static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ + DbTruncateCtx *p = (DbTruncateCtx *)pCtx; + if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; + p->nBlock--; + return 0; +} + +static int dbTruncate(lsm_db *pDb, i64 iInUse){ + int rc = LSM_OK; +#if 0 + int i; + DbTruncateCtx ctx; + + assert( pDb->pWorker ); + ctx.nBlock = pDb->pWorker->nBlock; + ctx.iInUse = iInUse; + + rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); + for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ + rc = freelistAppend(pDb, i, -1); + } + + if( rc==LSM_OK ){ +#ifdef LSM_LOG_FREELIST + if( ctx.nBlock!=pDb->pWorker->nBlock ){ + lsmLogMessage(pDb, 0, + "dbTruncate(): truncated db to %d blocks",ctx.nBlock + ); + } +#endif + pDb->pWorker->nBlock = ctx.nBlock; + } +#endif + return rc; +} + + +/* +** This function is called during database shutdown (when the number of +** connections drops from one to zero). It truncates the database file +** to as small a size as possible without truncating away any blocks that +** contain data. +*/ +static int dbTruncateFile(lsm_db *pDb){ + int rc; + + assert( pDb->pWorker==0 ); + assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) ); + rc = lsmCheckpointLoadWorker(pDb); + + if( rc==LSM_OK ){ + DbTruncateCtx ctx; + + /* Walk the database free-block-list in reverse order. Set ctx.nBlock + ** to the block number of the last block in the database that actually + ** contains data. */ + ctx.nBlock = pDb->pWorker->nBlock; + ctx.iInUse = -1; + rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); + + /* If the last block that contains data is not already the last block in + ** the database file, truncate the database file so that it is. */ + if( rc==LSM_OK ){ + rc = lsmFsTruncateDb( + pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) + ); + } + } + + lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); + pDb->pWorker = 0; + return rc; +} + +static void doDbDisconnect(lsm_db *pDb){ + int rc; + + if( pDb->bReadonly ){ + lsmShmLock(pDb, LSM_LOCK_DMS3, LSM_LOCK_UNLOCK, 0); + }else{ + /* Block for an exclusive lock on DMS1. This lock serializes all calls + ** to doDbConnect() and doDbDisconnect() across all processes. */ + rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); + if( rc==LSM_OK ){ + + lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); + + /* Try an exclusive lock on DMS2. If successful, this is the last + ** connection to the database. In this case flush the contents of the + ** in-memory tree to disk and write a checkpoint. */ + rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 1, LSM_LOCK_EXCL); + if( rc==LSM_OK ){ + rc = lsmShmTestLock(pDb, LSM_LOCK_CHECKPOINTER, 1, LSM_LOCK_EXCL); + } + if( rc==LSM_OK ){ + int bReadonly = 0; /* True if there exist read-only conns. */ + + /* Flush the in-memory tree, if required. If there is data to flush, + ** this will create a new client snapshot in Database.pClient. The + ** checkpoint (serialization) of this snapshot may be written to disk + ** by the following block. + ** + ** There is no need to take a WRITER lock here. That there are no + ** other locks on DMS2 guarantees that there are no other read-write + ** connections at this time (and the lock on DMS1 guarantees that + ** no new ones may appear). + */ + rc = lsmTreeLoadHeader(pDb, 0); + if( rc==LSM_OK && (lsmTreeHasOld(pDb) || lsmTreeSize(pDb)>0) ){ + rc = lsmFlushTreeToDisk(pDb); + } + + /* Now check if there are any read-only connections. If there are, + ** then do not truncate the db file or unlink the shared-memory + ** region. */ + if( rc==LSM_OK ){ + rc = lsmShmTestLock(pDb, LSM_LOCK_DMS3, 1, LSM_LOCK_EXCL); + if( rc==LSM_BUSY ){ + bReadonly = 1; + rc = LSM_OK; + } + } + + /* Write a checkpoint to disk. */ + if( rc==LSM_OK ){ + rc = lsmCheckpointWrite(pDb, 0); + } + + /* If the checkpoint was written successfully, delete the log file + ** and, if possible, truncate the database file. */ + if( rc==LSM_OK ){ + int bRotrans = 0; + Database *p = pDb->pDatabase; + + /* The log file may only be deleted if there are no clients + ** read-only clients running rotrans transactions. */ + rc = lsmDetectRoTrans(pDb, &bRotrans); + if( rc==LSM_OK && bRotrans==0 ){ + lsmFsCloseAndDeleteLog(pDb->pFS); + } + + /* The database may only be truncated if there exist no read-only + ** clients - either connected or running rotrans transactions. */ + if( bReadonly==0 && bRotrans==0 ){ + lsmFsUnmap(pDb->pFS); + dbTruncateFile(pDb); + if( p->pFile && p->bMultiProc ){ + lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); + } + } + } + } + } + + if( pDb->iRwclient>=0 ){ + lsmShmLock(pDb, LSM_LOCK_RWCLIENT(pDb->iRwclient), LSM_LOCK_UNLOCK, 0); + pDb->iRwclient = -1; + } + + lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + } + pDb->pShmhdr = 0; +} + +static int doDbConnect(lsm_db *pDb){ + const int nUsMax = 100000; /* Max value for nUs */ + int nUs = 1000; /* us to wait between DMS1 attempts */ + int rc; + + /* Obtain a pointer to the shared-memory header */ + assert( pDb->pShmhdr==0 ); + assert( pDb->bReadonly==0 ); + + /* Block for an exclusive lock on DMS1. This lock serializes all calls + ** to doDbConnect() and doDbDisconnect() across all processes. */ + while( 1 ){ + rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); + if( rc!=LSM_BUSY ) break; + lsmEnvSleep(pDb->pEnv, nUs); + nUs = nUs * 2; + if( nUs>nUsMax ) nUs = nUsMax; + } + if( rc==LSM_OK ){ + rc = lsmShmCacheChunks(pDb, 1); + } + if( rc!=LSM_OK ) return rc; + pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; + + /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first + ** and only connection to the database. In this case initialize the + ** shared-memory and run log file recovery. */ + assert( LSM_LOCK_DMS3==1+LSM_LOCK_DMS2 ); + rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 2, LSM_LOCK_EXCL); + if( rc==LSM_OK ){ + memset(pDb->pShmhdr, 0, sizeof(ShmHeader)); + rc = lsmCheckpointRecover(pDb); + if( rc==LSM_OK ){ + rc = lsmLogRecover(pDb); + } + if( rc==LSM_OK ){ + ShmHeader *pShm = pDb->pShmhdr; + pShm->aReader[0].iLsmId = lsmCheckpointId(pShm->aSnap1, 0); + pShm->aReader[0].iTreeId = pDb->treehdr.iUsedShmid; + } + }else if( rc==LSM_BUSY ){ + rc = LSM_OK; + } + + /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" + ** fail, as connections may only hold an exclusive lock on DMS2 if they + ** first hold an exclusive lock on DMS1. And this connection is currently + ** holding the exclusive lock on DSM1. + ** + ** However, if some other connection has the database open in single-process + ** mode, this operation will fail. In this case, return the error to the + ** caller - the attempt to connect to the db has failed. + */ + if( rc==LSM_OK ){ + rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); + } + + /* If anything went wrong, unlock DMS2. Otherwise, try to take an exclusive + ** lock on one of the LSM_LOCK_RWCLIENT() locks. Unlock DMS1 in any case. */ + if( rc!=LSM_OK ){ + pDb->pShmhdr = 0; + }else{ + int i; + for(i=0; iiRwclient = i; + if( rc2!=LSM_BUSY ){ + rc = rc2; + break; + } + } + } + lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + + return rc; +} + +static int dbOpenSharedFd(lsm_env *pEnv, Database *p, int bRoOk){ + int rc; + + rc = lsmEnvOpen(pEnv, p->zName, 0, &p->pFile); + if( rc==LSM_IOERR && bRoOk ){ + rc = lsmEnvOpen(pEnv, p->zName, LSM_OPEN_READONLY, &p->pFile); + p->bReadonly = 1; + } + + return rc; +} + +/* +** Return a reference to the shared Database handle for the database +** identified by canonical path zName. If this is the first connection to +** the named database, a new Database object is allocated. Otherwise, a +** pointer to an existing object is returned. +** +** If successful, *ppDatabase is set to point to the shared Database +** structure and LSM_OK returned. Otherwise, *ppDatabase is set to NULL +** and and LSM error code returned. +** +** Each successful call to this function should be (eventually) matched +** by a call to lsmDbDatabaseRelease(). +*/ +int lsmDbDatabaseConnect( + lsm_db *pDb, /* Database handle */ + const char *zName /* Full-path to db file */ +){ + lsm_env *pEnv = pDb->pEnv; + int rc; /* Return code */ + Database *p = 0; /* Pointer returned via *ppDatabase */ + int nName = lsmStrlen(zName); + + assert( pDb->pDatabase==0 ); + rc = enterGlobalMutex(pEnv); + if( rc==LSM_OK ){ + + /* Search the global list for an existing object. TODO: Need something + ** better than the memcmp() below to figure out if a given Database + ** object represents the requested file. */ + for(p=gShared.pDatabase; p; p=p->pDbNext){ + if( nName==p->nName && 0==memcmp(zName, p->zName, nName) ) break; + } + + /* If no suitable Database object was found, allocate a new one. */ + if( p==0 ){ + p = (Database *)lsmMallocZeroRc(pEnv, sizeof(Database)+nName+1, &rc); + + /* If the allocation was successful, fill in other fields and + ** allocate the client mutex. */ + if( rc==LSM_OK ){ + p->bMultiProc = pDb->bMultiProc; + p->zName = (char *)&p[1]; + p->nName = nName; + memcpy((void *)p->zName, zName, nName+1); + rc = lsmMutexNew(pEnv, &p->pClientMutex); + } + + /* If nothing has gone wrong so far, open the shared fd. And if that + ** succeeds and this connection requested single-process mode, + ** attempt to take the exclusive lock on DMS2. */ + if( rc==LSM_OK ){ + int bReadonly = (pDb->bReadonly && pDb->bMultiProc); + rc = dbOpenSharedFd(pDb->pEnv, p, bReadonly); + } + + if( rc==LSM_OK && p->bMultiProc==0 ){ + /* Hold an exclusive lock DMS1 while grabbing DMS2. This ensures + ** that any ongoing call to doDbDisconnect() (even one in another + ** process) is finished before proceeding. */ + assert( p->bReadonly==0 ); + rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_EXCL); + if( rc==LSM_OK ){ + rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); + lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK); + } + } + + if( rc==LSM_OK ){ + p->pDbNext = gShared.pDatabase; + gShared.pDatabase = p; + }else{ + freeDatabase(pEnv, p); + p = 0; + } + } + + if( p ){ + p->nDbRef++; + } + leaveGlobalMutex(pEnv); + + if( p ){ + lsmMutexEnter(pDb->pEnv, p->pClientMutex); + pDb->pNext = p->pConn; + p->pConn = pDb; + lsmMutexLeave(pDb->pEnv, p->pClientMutex); + } + } + + pDb->pDatabase = p; + if( rc==LSM_OK ){ + assert( p ); + rc = lsmFsOpen(pDb, zName, p->bReadonly); + } + + /* If the db handle is read-write, then connect to the system now. Run + ** recovery as necessary. Or, if this is a read-only database handle, + ** defer attempting to connect to the system until a read-transaction + ** is opened. */ + if( rc==LSM_OK ){ + rc = lsmFsConfigure(pDb); + } + if( rc==LSM_OK && pDb->bReadonly==0 ){ + rc = doDbConnect(pDb); + } + + return rc; +} + +static void dbDeferClose(lsm_db *pDb){ + if( pDb->pFS ){ + LsmFile *pLsmFile; + Database *p = pDb->pDatabase; + pLsmFile = lsmFsDeferClose(pDb->pFS); + pLsmFile->pNext = p->pLsmFile; + p->pLsmFile = pLsmFile; + } +} + +LsmFile *lsmDbRecycleFd(lsm_db *db){ + LsmFile *pRet; + Database *p = db->pDatabase; + lsmMutexEnter(db->pEnv, p->pClientMutex); + if( (pRet = p->pLsmFile)!=0 ){ + p->pLsmFile = pRet->pNext; + } + lsmMutexLeave(db->pEnv, p->pClientMutex); + return pRet; +} + +/* +** Release a reference to a Database object obtained from +** lsmDbDatabaseConnect(). There should be exactly one call to this function +** for each successful call to Find(). +*/ +void lsmDbDatabaseRelease(lsm_db *pDb){ + Database *p = pDb->pDatabase; + if( p ){ + lsm_db **ppDb; + + if( pDb->pShmhdr ){ + doDbDisconnect(pDb); + } + + lsmFsUnmap(pDb->pFS); + lsmMutexEnter(pDb->pEnv, p->pClientMutex); + for(ppDb=&p->pConn; *ppDb!=pDb; ppDb=&((*ppDb)->pNext)); + *ppDb = pDb->pNext; + dbDeferClose(pDb); + lsmMutexLeave(pDb->pEnv, p->pClientMutex); + + enterGlobalMutex(pDb->pEnv); + p->nDbRef--; + if( p->nDbRef==0 ){ + LsmFile *pIter; + LsmFile *pNext; + Database **pp; + + /* Remove the Database structure from the linked list. */ + for(pp=&gShared.pDatabase; *pp!=p; pp=&((*pp)->pDbNext)); + *pp = p->pDbNext; + + /* If they were allocated from the heap, free the shared memory chunks */ + if( p->bMultiProc==0 ){ + int i; + for(i=0; inShmChunk; i++){ + lsmFree(pDb->pEnv, p->apShmChunk[i]); + } + } + + /* Close any outstanding file descriptors */ + for(pIter=p->pLsmFile; pIter; pIter=pNext){ + pNext = pIter->pNext; + lsmEnvClose(pDb->pEnv, pIter->pFile); + lsmFree(pDb->pEnv, pIter); + } + freeDatabase(pDb->pEnv, p); + } + leaveGlobalMutex(pDb->pEnv); + } +} + +Level *lsmDbSnapshotLevel(Snapshot *pSnapshot){ + return pSnapshot->pLevel; +} + +void lsmDbSnapshotSetLevel(Snapshot *pSnap, Level *pLevel){ + pSnap->pLevel = pLevel; +} + +/* TODO: Shuffle things around to get rid of this */ +static int firstSnapshotInUse(lsm_db *, i64 *); + +/* +** Context object used by the lsmWalkFreelist() utility. +*/ +typedef struct WalkFreelistCtx WalkFreelistCtx; +struct WalkFreelistCtx { + lsm_db *pDb; + int bReverse; + Freelist *pFreelist; + int iFree; + int (*xUsr)(void *, int, i64); /* User callback function */ + void *pUsrctx; /* User callback context */ + int bDone; /* Set to true after xUsr() returns true */ +}; + +/* +** Callback used by lsmWalkFreelist(). +*/ +static int walkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ + WalkFreelistCtx *p = (WalkFreelistCtx *)pCtx; + const int iDir = (p->bReverse ? -1 : 1); + Freelist *pFree = p->pFreelist; + + assert( p->bDone==0 ); + assert( iBlk>=0 ); + if( pFree ){ + while( (p->iFree < pFree->nEntry) && p->iFree>=0 ){ + FreelistEntry *pEntry = &pFree->aEntry[p->iFree]; + if( (p->bReverse==0 && pEntry->iBlk>(u32)iBlk) + || (p->bReverse!=0 && pEntry->iBlk<(u32)iBlk) + ){ + break; + }else{ + p->iFree += iDir; + if( pEntry->iId>=0 + && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) + ){ + p->bDone = 1; + return 1; + } + if( pEntry->iBlk==(u32)iBlk ) return 0; + } + } + } + + if( p->xUsr(p->pUsrctx, iBlk, iSnapshot) ){ + p->bDone = 1; + return 1; + } + return 0; +} + +/* +** The database handle passed as the first argument must be the worker +** connection. This function iterates through the contents of the current +** free block list, invoking the supplied callback once for each list +** element. +** +** The difference between this function and lsmSortedWalkFreelist() is +** that lsmSortedWalkFreelist() only considers those free-list elements +** stored within the LSM. This function also merges in any in-memory +** elements. +*/ +int lsmWalkFreelist( + lsm_db *pDb, /* Database handle (must be worker) */ + int bReverse, /* True to iterate from largest to smallest */ + int (*x)(void *, int, i64), /* Callback function */ + void *pCtx /* First argument to pass to callback */ +){ + const int iDir = (bReverse ? -1 : 1); + int rc; + int iCtx; + + WalkFreelistCtx ctx[2]; + + ctx[0].pDb = pDb; + ctx[0].bReverse = bReverse; + ctx[0].pFreelist = &pDb->pWorker->freelist; + if( ctx[0].pFreelist && bReverse ){ + ctx[0].iFree = ctx[0].pFreelist->nEntry-1; + }else{ + ctx[0].iFree = 0; + } + ctx[0].xUsr = walkFreelistCb; + ctx[0].pUsrctx = (void *)&ctx[1]; + ctx[0].bDone = 0; + + ctx[1].pDb = pDb; + ctx[1].bReverse = bReverse; + ctx[1].pFreelist = pDb->pFreelist; + if( ctx[1].pFreelist && bReverse ){ + ctx[1].iFree = ctx[1].pFreelist->nEntry-1; + }else{ + ctx[1].iFree = 0; + } + ctx[1].xUsr = x; + ctx[1].pUsrctx = pCtx; + ctx[1].bDone = 0; + + rc = lsmSortedWalkFreelist(pDb, bReverse, walkFreelistCb, (void *)&ctx[0]); + + if( ctx[0].bDone==0 ){ + for(iCtx=0; iCtx<2; iCtx++){ + int i; + WalkFreelistCtx *p = &ctx[iCtx]; + for(i=p->iFree; + p->pFreelist && rc==LSM_OK && ipFreelist->nEntry && i>=0; + i += iDir + ){ + FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; + if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ + return LSM_OK; + } + } + } + } + + return rc; +} + + +typedef struct FindFreeblockCtx FindFreeblockCtx; +struct FindFreeblockCtx { + i64 iInUse; + int iRet; + int bNotOne; +}; + +static int findFreeblockCb(void *pCtx, int iBlk, i64 iSnapshot){ + FindFreeblockCtx *p = (FindFreeblockCtx *)pCtx; + if( iSnapshotiInUse && (iBlk!=1 || p->bNotOne==0) ){ + p->iRet = iBlk; + return 1; + } + return 0; +} + +static int findFreeblock(lsm_db *pDb, i64 iInUse, int bNotOne, int *piRet){ + int rc; /* Return code */ + FindFreeblockCtx ctx; /* Context object */ + + ctx.iInUse = iInUse; + ctx.iRet = 0; + ctx.bNotOne = bNotOne; + rc = lsmWalkFreelist(pDb, 0, findFreeblockCb, (void *)&ctx); + *piRet = ctx.iRet; + + return rc; +} + +/* +** Allocate a new database file block to write data to, either by extending +** the database file or by recycling a free-list entry. The worker snapshot +** must be held in order to call this function. +** +** If successful, *piBlk is set to the block number allocated and LSM_OK is +** returned. Otherwise, *piBlk is zeroed and an lsm error code returned. +*/ +int lsmBlockAllocate(lsm_db *pDb, int iBefore, int *piBlk){ + Snapshot *p = pDb->pWorker; + int iRet = 0; /* Block number of allocated block */ + int rc = LSM_OK; + i64 iInUse = 0; /* Snapshot id still in use */ + i64 iSynced = 0; /* Snapshot id synced to disk */ + + assert( p ); + +#ifdef LSM_LOG_FREELIST + { + static int nCall = 0; + char *zFree = 0; + nCall++; + rc = lsmInfoFreelist(pDb, &zFree); + if( rc!=LSM_OK ) return rc; + lsmLogMessage(pDb, 0, "lsmBlockAllocate(): %d freelist: %s", nCall, zFree); + lsmFree(pDb->pEnv, zFree); + } +#endif + + /* Set iInUse to the smallest snapshot id that is either: + ** + ** * Currently in use by a database client, + ** * May be used by a database client in the future, or + ** * Is the most recently checkpointed snapshot (i.e. the one that will + ** be used following recovery if a failure occurs at this point). + */ + rc = lsmCheckpointSynced(pDb, &iSynced, 0, 0); + if( rc==LSM_OK && iSynced==0 ) iSynced = p->iId; + iInUse = iSynced; + if( rc==LSM_OK && pDb->iReader>=0 ){ + assert( pDb->pClient ); + iInUse = LSM_MIN(iInUse, pDb->pClient->iId); + } + if( rc==LSM_OK ) rc = firstSnapshotInUse(pDb, &iInUse); + +#ifdef LSM_LOG_FREELIST + { + lsmLogMessage(pDb, 0, "lsmBlockAllocate(): " + "snapshot-in-use: %lld (iSynced=%lld) (client-id=%lld)", + iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) + ); + } +#endif + + + /* Unless there exists a read-only transaction (which prevents us from + ** recycling any blocks regardless, query the free block list for a + ** suitable block to reuse. + ** + ** It might seem more natural to check for a read-only transaction at + ** the start of this function. However, it is better do wait until after + ** the call to lsmCheckpointSynced() to do so. + */ + if( rc==LSM_OK ){ + int bRotrans; + rc = lsmDetectRoTrans(pDb, &bRotrans); + + if( rc==LSM_OK && bRotrans==0 ){ + rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); + } + } + + if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ + iRet = 0; + + }else if( rc==LSM_OK ){ + /* If a block was found in the free block list, use it and remove it from + ** the list. Otherwise, if no suitable block was found, allocate one from + ** the end of the file. */ + if( iRet>0 ){ +#ifdef LSM_LOG_FREELIST + lsmLogMessage(pDb, 0, + "reusing block %d (snapshot-in-use=%lld)", iRet, iInUse); +#endif + rc = freelistAppend(pDb, iRet, -1); + if( rc==LSM_OK ){ + rc = dbTruncate(pDb, iInUse); + } + }else{ + iRet = ++(p->nBlock); +#ifdef LSM_LOG_FREELIST + lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); +#endif + } + } + + assert( iBefore>0 || iRet>0 || rc!=LSM_OK ); + *piBlk = iRet; + return rc; +} + +/* +** Free a database block. The worker snapshot must be held in order to call +** this function. +** +** If successful, LSM_OK is returned. Otherwise, an lsm error code (e.g. +** LSM_NOMEM). +*/ +int lsmBlockFree(lsm_db *pDb, int iBlk){ + Snapshot *p = pDb->pWorker; + assert( lsmShmAssertWorker(pDb) ); + +#ifdef LSM_LOG_FREELIST + lsmLogMessage(pDb, LSM_OK, "lsmBlockFree(): Free block %d", iBlk); +#endif + + return freelistAppend(pDb, iBlk, p->iId); +} + +/* +** Refree a database block. The worker snapshot must be held in order to call +** this function. +** +** Refreeing is required when a block is allocated using lsmBlockAllocate() +** but then not used. This function is used to push the block back onto +** the freelist. Refreeing a block is different from freeing is, as a refreed +** block may be reused immediately. Whereas a freed block can not be reused +** until (at least) after the next checkpoint. +*/ +int lsmBlockRefree(lsm_db *pDb, int iBlk){ + int rc = LSM_OK; /* Return code */ + +#ifdef LSM_LOG_FREELIST + lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); +#endif + + rc = freelistAppend(pDb, iBlk, 0); + return rc; +} + +/* +** If required, copy a database checkpoint from shared memory into the +** database itself. +** +** The WORKER lock must not be held when this is called. This is because +** this function may indirectly call fsync(). And the WORKER lock should +** not be held that long (in case it is required by a client flushing an +** in-memory tree to disk). +*/ +int lsmCheckpointWrite(lsm_db *pDb, u32 *pnWrite){ + int rc; /* Return Code */ + u32 nWrite = 0; + + assert( pDb->pWorker==0 ); + assert( 1 || pDb->pClient==0 ); + assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK) ); + + rc = lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_EXCL, 0); + if( rc!=LSM_OK ) return rc; + + rc = lsmCheckpointLoad(pDb, 0); + if( rc==LSM_OK ){ + int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); + ShmHeader *pShm = pDb->pShmhdr; + int bDone = 0; /* True if checkpoint is already stored */ + + /* Check if this checkpoint has already been written to the database + ** file. If so, set variable bDone to true. */ + if( pShm->iMetaPage ){ + MetaPage *pPg; /* Meta page */ + u8 *aData; /* Meta-page data buffer */ + int nData; /* Size of aData[] in bytes */ + i64 iCkpt; /* Id of checkpoint just loaded */ + i64 iDisk = 0; /* Id of checkpoint already stored in db */ + iCkpt = lsmCheckpointId(pDb->aSnapshot, 0); + rc = lsmFsMetaPageGet(pDb->pFS, 0, pShm->iMetaPage, &pPg); + if( rc==LSM_OK ){ + aData = lsmFsMetaPageData(pPg, &nData); + iDisk = lsmCheckpointId((u32 *)aData, 1); + nWrite = lsmCheckpointNWrite((u32 *)aData, 1); + lsmFsMetaPageRelease(pPg); + } + bDone = (iDisk>=iCkpt); + } + + if( rc==LSM_OK && bDone==0 ){ + int iMeta = (pShm->iMetaPage % 2) + 1; + if( pDb->eSafety!=LSM_SAFETY_OFF ){ + rc = lsmFsSyncDb(pDb->pFS, nBlock); + } + if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); + if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ + rc = lsmFsSyncDb(pDb->pFS, 0); + } + if( rc==LSM_OK ){ + pShm->iMetaPage = iMeta; + nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; + } +#ifdef LSM_LOG_WORK + lsmLogMessage(pDb, 0, "finish checkpoint %d", + (int)lsmCheckpointId(pDb->aSnapshot, 0) + ); +#endif + } + } + + lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); + if( pnWrite && rc==LSM_OK ) *pnWrite = nWrite; + return rc; +} + +int lsmBeginWork(lsm_db *pDb){ + int rc; + + /* Attempt to take the WORKER lock */ + rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); + + /* Deserialize the current worker snapshot */ + if( rc==LSM_OK ){ + rc = lsmCheckpointLoadWorker(pDb); + } + return rc; +} + +void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ + if( p ){ + lsmSortedFreeLevel(pEnv, p->pLevel); + lsmFree(pEnv, p->freelist.aEntry); + lsmFree(pEnv, p->redirect.a); + lsmFree(pEnv, p); + } +} + +/* +** Attempt to populate one of the read-lock slots to contain lock values +** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. +** +** It is not an error if no slot can be populated because the write-lock +** cannot be obtained. If any other error occurs, return an LSM error code. +** Otherwise, LSM_OK. +** +** This function is called at various points to try to ensure that there +** always exists at least one read-lock slot that can be used by a read-only +** client. And so that, in the usual case, there is an "exact match" available +** whenever a read transaction is opened by any client. At present this +** function is called when: +** +** * A write transaction that called lsmTreeDiscardOld() is committed, and +** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). +*/ +static int dbSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ + int rc = LSM_OK; + ShmHeader *pShm = db->pShmhdr; + int i; + + /* Check if there is already a slot containing the required values. */ + for(i=0; iaReader[i]; + if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; + } + + /* Iterate through all read-lock slots, attempting to take a write-lock + ** on each of them. If a write-lock succeeds, populate the locked slot + ** with the required values and break out of the loop. */ + for(i=0; rc==LSM_OK && iaReader[i]; + p->iLsmId = iLsm; + p->iTreeId = iShm; + lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); + break; + } + } + + return rc; +} + +/* +** Release the read-lock currently held by connection db. +*/ +int dbReleaseReadlock(lsm_db *db){ + int rc = LSM_OK; + if( db->iReader>=0 ){ + rc = lsmShmLock(db, LSM_LOCK_READER(db->iReader), LSM_LOCK_UNLOCK, 0); + db->iReader = -1; + } + db->bRoTrans = 0; + return rc; +} + + +/* +** Argument bFlush is true if the contents of the in-memory tree has just +** been flushed to disk. The significance of this is that once the snapshot +** created to hold the updated state of the database is synced to disk, log +** file space can be recycled. +*/ +void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ + int rc = *pRc; + assert( rc!=0 || pDb->pWorker ); + if( pDb->pWorker ){ + /* If no error has occurred, serialize the worker snapshot and write + ** it to shared memory. */ + if( rc==LSM_OK ){ + rc = lsmSaveWorker(pDb, bFlush); + } + + /* Assuming no error has occurred, update a read lock slot with the + ** new snapshot id (see comments above function dbSetReadLock()). */ + if( rc==LSM_OK ){ + if( pDb->iReader<0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + } + if( rc==LSM_OK ){ + rc = dbSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); + } + } + + /* Free the snapshot object. */ + lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); + pDb->pWorker = 0; + } + + lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); + *pRc = rc; +} + +/* +** Called when recovery is finished. +*/ +int lsmFinishRecovery(lsm_db *pDb){ + lsmTreeEndTransaction(pDb, 1); + return LSM_OK; +} + +/* +** Check if the currently configured compression functions +** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its +** compression id set to iReq. Compression routines are compatible if iReq +** is zero (indicating the database is empty), or if it is equal to the +** compression id of the configured compression routines. +** +** If the check shows that the current compression are incompatible and there +** is a compression factory registered, give it a chance to install new +** compression routines. +** +** If, after any registered factory is invoked, the compression functions +** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. +*/ +int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ + if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ + if( pDb->factory.xFactory ){ + pDb->bInFactory = 1; + pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); + pDb->bInFactory = 0; + } + if( pDb->compress.iId!=iReq ){ + /* Incompatible */ + return LSM_MISMATCH; + } + } + /* Compatible */ + return LSM_OK; +} + +/* +** Begin a read transaction. This function is a no-op if the connection +** passed as the only argument already has an open read transaction. +*/ +int lsmBeginReadTrans(lsm_db *pDb){ + const int MAX_READLOCK_ATTEMPTS = 10; + const int nMaxAttempt = (pDb->bRoTrans ? 1 : MAX_READLOCK_ATTEMPTS); + + int rc = LSM_OK; /* Return code */ + int iAttempt = 0; + + assert( pDb->pWorker==0 ); + + while( rc==LSM_OK && pDb->iReader<0 && (iAttempt++)pCsr==0 && pDb->nTransOpen==0 ); + + /* Load the in-memory tree header. */ + rc = lsmTreeLoadHeader(pDb, &iTreehdr); + + /* Load the database snapshot */ + if( rc==LSM_OK ){ + if( lsmCheckpointClientCacheOk(pDb)==0 ){ + lsmFreeSnapshot(pDb->pEnv, pDb->pClient); + pDb->pClient = 0; + lsmMCursorFreeCache(pDb); + lsmFsPurgeCache(pDb->pFS); + rc = lsmCheckpointLoad(pDb, &iSnap); + }else{ + iSnap = 1; + } + } + + /* Take a read-lock on the tree and snapshot just loaded. Then check + ** that the shared-memory still contains the same values. If so, proceed. + ** Otherwise, relinquish the read-lock and retry the whole procedure + ** (starting with loading the in-memory tree header). */ + if( rc==LSM_OK ){ + u32 iShmMax = pDb->treehdr.iUsedShmid; + u32 iShmMin = pDb->treehdr.iNextShmid+1-LSM_MAX_SHMCHUNKS; + rc = lsmReadlock( + pDb, lsmCheckpointId(pDb->aSnapshot, 0), iShmMin, iShmMax + ); + if( rc==LSM_OK ){ + if( lsmTreeLoadHeaderOk(pDb, iTreehdr) + && lsmCheckpointLoadOk(pDb, iSnap) + ){ + /* Read lock has been successfully obtained. Deserialize the + ** checkpoint just loaded. TODO: This will be removed after + ** lsm_sorted.c is changed to work directly from the serialized + ** version of the snapshot. */ + if( pDb->pClient==0 ){ + rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); + } + assert( (rc==LSM_OK)==(pDb->pClient!=0) ); + assert( pDb->iReader>=0 ); + + /* Check that the client has the right compression hooks loaded. + ** If not, set rc to LSM_MISMATCH. */ + if( rc==LSM_OK ){ + rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); + } + }else{ + rc = dbReleaseReadlock(pDb); + } + } + + if( rc==LSM_BUSY ){ + rc = LSM_OK; + } + } +#if 0 +if( rc==LSM_OK && pDb->pClient ){ + fprintf(stderr, + "reading %p: snapshot:%d used-shmid:%d trans-id:%d iOldShmid=%d\n", + (void *)pDb, + (int)pDb->pClient->iId, (int)pDb->treehdr.iUsedShmid, + (int)pDb->treehdr.root.iTransId, + (int)pDb->treehdr.iOldShmid + ); +} +#endif + } + + if( rc==LSM_OK ){ + rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); + } + if( rc!=LSM_OK ){ + dbReleaseReadlock(pDb); + } + if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; + return rc; +} + +/* +** This function is used by a read-write connection to determine if there +** are currently one or more read-only transactions open on the database +** (in this context a read-only transaction is one opened by a read-only +** connection on a non-live database). +** +** If no error occurs, LSM_OK is returned and *pbExists is set to true if +** some other connection has a read-only transaction open, or false +** otherwise. If an error occurs an LSM error code is returned and the final +** value of *pbExist is undefined. +*/ +int lsmDetectRoTrans(lsm_db *db, int *pbExist){ + int rc; + + /* Only a read-write connection may use this function. */ + assert( db->bReadonly==0 ); + + rc = lsmShmTestLock(db, LSM_LOCK_ROTRANS, 1, LSM_LOCK_EXCL); + if( rc==LSM_BUSY ){ + *pbExist = 1; + rc = LSM_OK; + }else{ + *pbExist = 0; + } + + return rc; +} + +/* +** db is a read-only database handle in the disconnected state. This function +** attempts to open a read-transaction on the database. This may involve +** connecting to the database system (opening shared memory etc.). +*/ +int lsmBeginRoTrans(lsm_db *db){ + int rc = LSM_OK; + + assert( db->bReadonly && db->pShmhdr==0 ); + assert( db->iReader<0 ); + + if( db->bRoTrans==0 ){ + + /* Attempt a shared-lock on DMS1. */ + rc = lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_SHARED, 0); + if( rc!=LSM_OK ) return rc; + + rc = lsmShmTestLock( + db, LSM_LOCK_RWCLIENT(0), LSM_LOCK_NREADER, LSM_LOCK_SHARED + ); + if( rc==LSM_OK ){ + /* System is not live. Take a SHARED lock on the ROTRANS byte and + ** release DMS1. Locking ROTRANS tells all read-write clients that they + ** may not recycle any disk space from within the database or log files, + ** as a read-only client may be using it. */ + rc = lsmShmLock(db, LSM_LOCK_ROTRANS, LSM_LOCK_SHARED, 0); + lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + + if( rc==LSM_OK ){ + db->bRoTrans = 1; + rc = lsmShmCacheChunks(db, 1); + if( rc==LSM_OK ){ + db->pShmhdr = (ShmHeader *)db->apShm[0]; + memset(db->pShmhdr, 0, sizeof(ShmHeader)); + rc = lsmCheckpointRecover(db); + if( rc==LSM_OK ){ + rc = lsmLogRecover(db); + } + } + } + }else if( rc==LSM_BUSY ){ + /* System is live! */ + rc = lsmShmLock(db, LSM_LOCK_DMS3, LSM_LOCK_SHARED, 0); + lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + if( rc==LSM_OK ){ + rc = lsmShmCacheChunks(db, 1); + if( rc==LSM_OK ){ + db->pShmhdr = (ShmHeader *)db->apShm[0]; + } + } + } + + /* In 'lsm_open()' we don't update the page and block sizes in the + ** Filesystem for 'readonly' connection. Because member 'db->pShmhdr' is a + ** nullpointer, this prevents loading a checkpoint. Now that the system is + ** live this member should be set. So we can update both values in + ** the Filesystem. + ** + ** Configure the file-system connection with the page-size and block-size + ** of this database. Even if the database file is zero bytes in size + ** on disk, these values have been set in shared-memory by now, and so + ** are guaranteed not to change during the lifetime of this connection. */ + if( LSM_OK==rc + && 0==lsmCheckpointClientCacheOk(db) + && LSM_OK==(rc=lsmCheckpointLoad(db, 0)) + ){ + lsmFsSetPageSize(db->pFS, lsmCheckpointPgsz(db->aSnapshot)); + lsmFsSetBlockSize(db->pFS, lsmCheckpointBlksz(db->aSnapshot)); + } + + if( rc==LSM_OK ){ + rc = lsmBeginReadTrans(db); + } + } + + return rc; +} + +/* +** Close the currently open read transaction. +*/ +void lsmFinishReadTrans(lsm_db *pDb){ + + /* Worker connections should not be closing read transactions. And + ** read transactions should only be closed after all cursors and write + ** transactions have been closed. Finally pClient should be non-NULL + ** only iff pDb->iReader>=0. */ + assert( pDb->pWorker==0 ); + assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); + + if( pDb->bRoTrans ){ + int i; + for(i=0; inShm; i++){ + lsmFree(pDb->pEnv, pDb->apShm[i]); + } + lsmFree(pDb->pEnv, pDb->apShm); + pDb->apShm = 0; + pDb->nShm = 0; + pDb->pShmhdr = 0; + + lsmShmLock(pDb, LSM_LOCK_ROTRANS, LSM_LOCK_UNLOCK, 0); + } + dbReleaseReadlock(pDb); +} + +/* +** Open a write transaction. +*/ +int lsmBeginWriteTrans(lsm_db *pDb){ + int rc = LSM_OK; /* Return code */ + ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ + + assert( pDb->nTransOpen==0 ); + assert( pDb->bDiscardOld==0 ); + assert( pDb->bReadonly==0 ); + + /* If there is no read-transaction open, open one now. */ + if( pDb->iReader<0 ){ + rc = lsmBeginReadTrans(pDb); + } + + /* Attempt to take the WRITER lock */ + if( rc==LSM_OK ){ + rc = lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL, 0); + } + + /* If the previous writer failed mid-transaction, run emergency rollback. */ + if( rc==LSM_OK && pShm->bWriter ){ + rc = lsmTreeRepair(pDb); + if( rc==LSM_OK ) pShm->bWriter = 0; + } + + /* Check that this connection is currently reading from the most recent + ** version of the database. If not, return LSM_BUSY. */ + if( rc==LSM_OK && memcmp(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)) ){ + rc = LSM_BUSY; + } + + if( rc==LSM_OK ){ + rc = lsmLogBegin(pDb); + } + + /* If everything was successful, set the "transaction-in-progress" flag + ** and return LSM_OK. Otherwise, if some error occurred, relinquish the + ** WRITER lock and return an error code. */ + if( rc==LSM_OK ){ + TreeHeader *p = &pDb->treehdr; + pShm->bWriter = 1; + p->root.iTransId++; + if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ + lsmTreeDiscardOld(pDb); + pDb->bDiscardOld = 1; + } + }else{ + lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); + if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); + } + return rc; +} + +/* +** End the current write transaction. The connection is left with an open +** read transaction. It is an error to call this if there is no open write +** transaction. +** +** If the transaction was committed, then a commit record has already been +** written into the log file when this function is called. Or, if the +** transaction was rolled back, both the log file and in-memory tree +** structure have already been restored. In either case, this function +** merely releases locks and other resources held by the write-transaction. +** +** LSM_OK is returned if successful, or an LSM error code otherwise. +*/ +int lsmFinishWriteTrans(lsm_db *pDb, int bCommit){ + int rc = LSM_OK; + int bFlush = 0; + + lsmLogEnd(pDb, bCommit); + if( rc==LSM_OK && bCommit && lsmTreeSize(pDb)>pDb->nTreeLimit ){ + bFlush = 1; + lsmTreeMakeOld(pDb); + } + lsmTreeEndTransaction(pDb, bCommit); + + if( rc==LSM_OK ){ + if( bFlush && pDb->bAutowork ){ + rc = lsmSortedAutoWork(pDb, 1); + }else if( bCommit && pDb->bDiscardOld ){ + rc = dbSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); + } + } + pDb->bDiscardOld = 0; + lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); + + if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ + pDb->xWork(pDb, pDb->pWorkCtx); + } + return rc; +} + + +/* +** Return non-zero if the caller is holding the client mutex. +*/ +#ifdef LSM_DEBUG +int lsmHoldingClientMutex(lsm_db *pDb){ + return lsmMutexHeld(pDb->pEnv, pDb->pDatabase->pClientMutex); +} +#endif + +static int slotIsUsable(ShmReader *p, i64 iLsm, u32 iShmMin, u32 iShmMax){ + return( + p->iLsmId && p->iLsmId<=iLsm + && shm_sequence_ge(iShmMax, p->iTreeId) + && shm_sequence_ge(p->iTreeId, iShmMin) + ); +} + +/* +** Obtain a read-lock on database version identified by the combination +** of snapshot iLsm and tree iTree. Return LSM_OK if successful, or +** an LSM error code otherwise. +*/ +int lsmReadlock(lsm_db *db, i64 iLsm, u32 iShmMin, u32 iShmMax){ + int rc = LSM_OK; + ShmHeader *pShm = db->pShmhdr; + int i; + + assert( db->iReader<0 ); + assert( shm_sequence_ge(iShmMax, iShmMin) ); + + /* This is a no-op if the read-only transaction flag is set. */ + if( db->bRoTrans ){ + db->iReader = 0; + return LSM_OK; + } + + /* Search for an exact match. */ + for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; + if( p->iLsmId==iLsm && p->iTreeId==iShmMax ){ + rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); + if( rc==LSM_OK && p->iLsmId==iLsm && p->iTreeId==iShmMax ){ + db->iReader = i; + }else if( rc==LSM_BUSY ){ + rc = LSM_OK; + } + } + } + + /* Try to obtain a write-lock on each slot, in order. If successful, set + ** the slot values to iLsm/iTree. */ + for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; + p->iLsmId = iLsm; + p->iTreeId = iShmMax; + rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); + assert( rc!=LSM_BUSY ); + if( rc==LSM_OK ) db->iReader = i; + } + } + + /* Search for any usable slot */ + for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; + if( slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ + rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); + if( rc==LSM_OK && slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ + db->iReader = i; + }else if( rc==LSM_BUSY ){ + rc = LSM_OK; + } + } + } + + if( rc==LSM_OK && db->iReader<0 ){ + rc = LSM_BUSY; + } + return rc; +} + +/* +** This is used to check if there exists a read-lock locking a particular +** version of either the in-memory tree or database file. +** +** If iLsmId is non-zero, then it is a snapshot id. If there exists a +** read-lock using this snapshot or newer, set *pbInUse to true. Or, +** if there is no such read-lock, set it to false. +** +** Or, if iLsmId is zero, then iShmid is a shared-memory sequence id. +** Search for a read-lock using this sequence id or newer. etc. +*/ +static int isInUse(lsm_db *db, i64 iLsmId, u32 iShmid, int *pbInUse){ + ShmHeader *pShm = db->pShmhdr; + int i; + int rc = LSM_OK; + + for(i=0; rc==LSM_OK && iaReader[i]; + if( p->iLsmId ){ + if( (iLsmId!=0 && p->iLsmId!=0 && iLsmId>=p->iLsmId) + || (iLsmId==0 && shm_sequence_ge(p->iTreeId, iShmid)) + ){ + rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); + if( rc==LSM_OK ){ + p->iLsmId = 0; + lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); + } + } + } + } + + if( rc==LSM_BUSY ){ + *pbInUse = 1; + return LSM_OK; + } + *pbInUse = 0; + return rc; +} + +/* +** This function is called by worker connections to determine the smallest +** snapshot id that is currently in use by a database client. The worker +** connection uses this result to determine whether or not it is safe to +** recycle a database block. +*/ +static int firstSnapshotInUse( + lsm_db *db, /* Database handle */ + i64 *piInUse /* IN/OUT: Smallest snapshot id in use */ +){ + ShmHeader *pShm = db->pShmhdr; + i64 iInUse = *piInUse; + int i; + + assert( iInUse>0 ); + for(i=0; iaReader[i]; + if( p->iLsmId ){ + i64 iThis = p->iLsmId; + if( iThis!=0 && iInUse>iThis ){ + int rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); + if( rc==LSM_OK ){ + p->iLsmId = 0; + lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); + }else if( rc==LSM_BUSY ){ + iInUse = iThis; + }else{ + /* Some error other than LSM_BUSY. Return the error code to + ** the caller in this case. */ + return rc; + } + } + } + } + + *piInUse = iInUse; + return LSM_OK; +} + +int lsmTreeInUse(lsm_db *db, u32 iShmid, int *pbInUse){ + if( db->treehdr.iUsedShmid==iShmid ){ + *pbInUse = 1; + return LSM_OK; + } + return isInUse(db, 0, iShmid, pbInUse); +} + +int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse){ + if( db->pClient && db->pClient->iId<=iLsmId ){ + *pbInUse = 1; + return LSM_OK; + } + return isInUse(db, iLsmId, 0, pbInUse); +} + +/* +** This function may only be called after a successful call to +** lsmDbDatabaseConnect(). It returns true if the connection is in +** multi-process mode, or false otherwise. +*/ +int lsmDbMultiProc(lsm_db *pDb){ + return pDb->pDatabase && pDb->pDatabase->bMultiProc; +} + + +/************************************************************************* +************************************************************************** +************************************************************************** +************************************************************************** +************************************************************************** +*************************************************************************/ + +/* +** Ensure that database connection db has cached pointers to at least the +** first nChunk chunks of shared memory. +*/ +int lsmShmCacheChunks(lsm_db *db, int nChunk){ + int rc = LSM_OK; + if( nChunk>db->nShm ){ + static const int NINCR = 16; + Database *p = db->pDatabase; + lsm_env *pEnv = db->pEnv; + int nAlloc; + int i; + + /* Ensure that the db->apShm[] array is large enough. If an attempt to + ** allocate memory fails, return LSM_NOMEM immediately. The apShm[] array + ** is always extended in multiples of 16 entries - so the actual allocated + ** size can be inferred from nShm. */ + nAlloc = ((db->nShm + NINCR - 1) / NINCR) * NINCR; + while( nChunk>=nAlloc ){ + void **apShm; + nAlloc += NINCR; + apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); + if( !apShm ) return LSM_NOMEM_BKPT; + db->apShm = apShm; + } + + if( db->bRoTrans ){ + for(i=db->nShm; rc==LSM_OK && iapShm[i] = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + db->nShm++; + } + + }else{ + + /* Enter the client mutex */ + lsmMutexEnter(pEnv, p->pClientMutex); + + /* Extend the Database objects apShmChunk[] array if necessary. Using the + ** same pattern as for the lsm_db.apShm[] array above. */ + nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; + while( nChunk>=nAlloc ){ + void **apShm; + nAlloc += NINCR; + apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); + if( !apShm ){ + rc = LSM_NOMEM_BKPT; + break; + } + p->apShmChunk = apShm; + } + + for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ + void *pChunk = 0; + if( p->bMultiProc==0 ){ + /* Single process mode */ + pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + }else{ + /* Multi-process mode */ + rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); + } + if( rc==LSM_OK ){ + p->apShmChunk[i] = pChunk; + p->nShmChunk++; + } + } + if( rc==LSM_OK ){ + db->apShm[i] = p->apShmChunk[i]; + db->nShm++; + } + } + + /* Release the client mutex */ + lsmMutexLeave(pEnv, p->pClientMutex); + } + } + + return rc; +} + +static int lockSharedFile(lsm_env *pEnv, Database *p, int iLock, int eOp){ + int rc = LSM_OK; + if( p->bMultiProc ){ + rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); + } + return rc; +} + +/* +** Test if it would be possible for connection db to obtain a lock of type +** eType on the nLock locks starting at iLock. If so, return LSM_OK. If it +** would not be possible to obtain the lock due to a lock held by another +** connection, return LSM_BUSY. If an IO or other error occurs (i.e. in the +** lsm_env.xTestLock function), return some other LSM error code. +** +** Note that this function never actually locks the database - it merely +** queries the system to see if there exists a lock that would prevent +** it from doing so. +*/ +int lsmShmTestLock( + lsm_db *db, + int iLock, + int nLock, + int eOp +){ + int rc = LSM_OK; + lsm_db *pIter; + Database *p = db->pDatabase; + int i; + u64 mask = 0; + + for(i=iLock; i<(iLock+nLock); i++){ + mask |= ((u64)1 << (iLock-1)); + if( eOp==LSM_LOCK_EXCL ) mask |= ((u64)1 << (iLock+32-1)); + } + + lsmMutexEnter(db->pEnv, p->pClientMutex); + for(pIter=p->pConn; pIter; pIter=pIter->pNext){ + if( pIter!=db && (pIter->mLock & mask) ){ + assert( pIter!=db ); + break; + } + } + + if( pIter ){ + rc = LSM_BUSY; + }else if( p->bMultiProc ){ + rc = lsmEnvTestLock(db->pEnv, p->pFile, iLock, nLock, eOp); + } + + lsmMutexLeave(db->pEnv, p->pClientMutex); + return rc; +} + +/* +** Attempt to obtain the lock identified by the iLock and bExcl parameters. +** If successful, return LSM_OK. If the lock cannot be obtained because +** there exists some other conflicting lock, return LSM_BUSY. If some other +** error occurs, return an LSM error code. +** +** Parameter iLock must be one of LSM_LOCK_WRITER, WORKER or CHECKPOINTER, +** or else a value returned by the LSM_LOCK_READER macro. +*/ +int lsmShmLock( + lsm_db *db, + int iLock, + int eOp, /* One of LSM_LOCK_UNLOCK, SHARED or EXCL */ + int bBlock /* True for a blocking lock */ +){ + lsm_db *pIter; + const u64 me = ((u64)1 << (iLock-1)); + const u64 ms = ((u64)1 << (iLock+32-1)); + int rc = LSM_OK; + Database *p = db->pDatabase; + + assert( eOp!=LSM_LOCK_EXCL || p->bReadonly==0 ); + assert( iLock>=1 && iLock<=LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1) ); + assert( LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1)<=32 ); + assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); + + /* Check for a no-op. Proceed only if this is not one of those. */ + if( (eOp==LSM_LOCK_UNLOCK && (db->mLock & (me|ms))!=0) + || (eOp==LSM_LOCK_SHARED && (db->mLock & (me|ms))!=ms) + || (eOp==LSM_LOCK_EXCL && (db->mLock & me)==0) + ){ + int nExcl = 0; /* Number of connections holding EXCLUSIVE */ + int nShared = 0; /* Number of connections holding SHARED */ + lsmMutexEnter(db->pEnv, p->pClientMutex); + + /* Figure out the locks currently held by this process on iLock, not + ** including any held by connection db. */ + for(pIter=p->pConn; pIter; pIter=pIter->pNext){ + assert( (pIter->mLock & me)==0 || (pIter->mLock & ms)!=0 ); + if( pIter!=db ){ + if( pIter->mLock & me ){ + nExcl++; + }else if( pIter->mLock & ms ){ + nShared++; + } + } + } + assert( nExcl==0 || nExcl==1 ); + assert( nExcl==0 || nShared==0 ); + assert( nExcl==0 || (db->mLock & (me|ms))==0 ); + + switch( eOp ){ + case LSM_LOCK_UNLOCK: + if( nShared==0 ){ + lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_UNLOCK); + } + db->mLock &= ~(me|ms); + break; + + case LSM_LOCK_SHARED: + if( nExcl ){ + rc = LSM_BUSY; + }else{ + if( nShared==0 ){ + rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); + } + if( rc==LSM_OK ){ + db->mLock |= ms; + db->mLock &= ~me; + } + } + break; + + default: + assert( eOp==LSM_LOCK_EXCL ); + if( nExcl || nShared ){ + rc = LSM_BUSY; + }else{ + rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); + if( rc==LSM_OK ){ + db->mLock |= (me|ms); + } + } + break; + } + + lsmMutexLeave(db->pEnv, p->pClientMutex); + } + + return rc; +} + +#ifdef LSM_DEBUG + +int shmLockType(lsm_db *db, int iLock){ + const u64 me = ((u64)1 << (iLock-1)); + const u64 ms = ((u64)1 << (iLock+32-1)); + + if( db->mLock & me ) return LSM_LOCK_EXCL; + if( db->mLock & ms ) return LSM_LOCK_SHARED; + return LSM_LOCK_UNLOCK; +} + +/* +** The arguments passed to this function are similar to those passed to +** the lsmShmLock() function. However, instead of obtaining a new lock +** this function returns true if the specified connection already holds +** (or does not hold) such a lock, depending on the value of eOp. As +** follows: +** +** (eOp==LSM_LOCK_UNLOCK) -> true if db has no lock on iLock +** (eOp==LSM_LOCK_SHARED) -> true if db has at least a SHARED lock on iLock. +** (eOp==LSM_LOCK_EXCL) -> true if db has an EXCLUSIVE lock on iLock. +*/ +int lsmShmAssertLock(lsm_db *db, int iLock, int eOp){ + int ret = 0; + int eHave; + + assert( iLock>=1 && iLock<=LSM_LOCK_READER(LSM_LOCK_NREADER-1) ); + assert( iLock<=16 ); + assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); + + eHave = shmLockType(db, iLock); + + switch( eOp ){ + case LSM_LOCK_UNLOCK: + ret = (eHave==LSM_LOCK_UNLOCK); + break; + case LSM_LOCK_SHARED: + ret = (eHave!=LSM_LOCK_UNLOCK); + break; + case LSM_LOCK_EXCL: + ret = (eHave==LSM_LOCK_EXCL); + break; + default: + assert( !"bad eOp value passed to lsmShmAssertLock()" ); + break; + } + + return ret; +} + +int lsmShmAssertWorker(lsm_db *db){ + return lsmShmAssertLock(db, LSM_LOCK_WORKER, LSM_LOCK_EXCL) && db->pWorker; +} + +/* +** This function does not contribute to library functionality, and is not +** included in release builds. It is intended to be called from within +** an interactive debugger. +** +** When called, this function prints a single line of human readable output +** to stdout describing the locks currently held by the connection. For +** example: +** +** (gdb) call print_db_locks(pDb) +** (shared on dms2) (exclusive on writer) +*/ +void print_db_locks(lsm_db *db){ + int iLock; + for(iLock=0; iLock<16; iLock++){ + int bOne = 0; + const char *azLock[] = {0, "shared", "exclusive"}; + const char *azName[] = { + 0, "dms1", "dms2", "writer", "worker", "checkpointer", + "reader0", "reader1", "reader2", "reader3", "reader4", "reader5" + }; + int eHave = shmLockType(db, iLock); + if( azLock[eHave] ){ + printf("%s(%s on %s)", (bOne?" ":""), azLock[eHave], azName[iLock]); + bOne = 1; + } + } + printf("\n"); +} +void print_all_db_locks(lsm_db *db){ + lsm_db *p; + for(p=db->pDatabase->pConn; p; p=p->pNext){ + printf("%s connection %p ", ((p==db)?"*":""), p); + print_db_locks(p); + } +} +#endif + +void lsmShmBarrier(lsm_db *db){ + lsmEnvShmBarrier(db->pEnv); +} + +int lsm_checkpoint(lsm_db *pDb, int *pnKB){ + int rc; /* Return code */ + u32 nWrite = 0; /* Number of pages checkpointed */ + + /* Attempt the checkpoint. If successful, nWrite is set to the number of + ** pages written between this and the previous checkpoint. */ + rc = lsmCheckpointWrite(pDb, &nWrite); + + /* If required, calculate the output variable (KB of data checkpointed). + ** Set it to zero if an error occured. */ + if( pnKB ){ + int nKB = 0; + if( rc==LSM_OK && nWrite ){ + nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; + } + *pnKB = nKB; + } + + return rc; +} diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c new file mode 100644 index 0000000000000000000000000000000000000000..6e5243e850666ec7a2a2cca9a61314af8f344f97 --- /dev/null +++ b/ext/lsm1/lsm_sorted.c @@ -0,0 +1,6195 @@ +/* +** 2011-08-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** PAGE FORMAT: +** +** The maximum page size is 65536 bytes. +** +** Since all records are equal to or larger than 2 bytes in size, and +** some space within the page is consumed by the page footer, there must +** be less than 2^15 records on each page. +** +** Each page ends with a footer that describes the pages contents. This +** footer serves as similar purpose to the page header in an SQLite database. +** A footer is used instead of a header because it makes it easier to +** populate a new page based on a sorted list of key/value pairs. +** +** The footer consists of the following values (starting at the end of +** the page and continuing backwards towards the start). All values are +** stored as unsigned big-endian integers. +** +** * Number of records on page (2 bytes). +** * Flags field (2 bytes). +** * Left-hand pointer value (8 bytes). +** * The starting offset of each record (2 bytes per record). +** +** Records may span pages. Unless it happens to be an exact fit, the part +** of the final record that starts on page X that does not fit on page X +** is stored at the start of page (X+1). This means there may be pages where +** (N==0). And on most pages the first record that starts on the page will +** not start at byte offset 0. For example: +** +** aaaaa bbbbb ccc
cc eeeee fffff g
gggg.... +** +** RECORD FORMAT: +** +** The first byte of the record is a flags byte. It is a combination +** of the following flags (defined in lsmInt.h): +** +** LSM_START_DELETE +** LSM_END_DELETE +** LSM_POINT_DELETE +** LSM_INSERT +** LSM_SEPARATOR +** LSM_SYSTEMKEY +** +** Immediately following the type byte is a pointer to the smallest key +** in the next file that is larger than the key in the current record. The +** pointer is encoded as a varint. When added to the 32-bit page number +** stored in the footer, it is the page number of the page that contains the +** smallest key in the next sorted file that is larger than this key. +** +** Next is the number of bytes in the key, encoded as a varint. +** +** If the LSM_INSERT flag is set, the number of bytes in the value, as +** a varint, is next. +** +** Finally, the blob of data containing the key, and for LSM_INSERT +** records, the value as well. +*/ + +#ifndef _LSM_INT_H +# include "lsmInt.h" +#endif + +#define LSM_LOG_STRUCTURE 0 +#define LSM_LOG_DATA 0 + +/* +** Macros to help decode record types. +*/ +#define rtTopic(eType) ((eType) & LSM_SYSTEMKEY) +#define rtIsDelete(eType) (((eType) & 0x0F)==LSM_POINT_DELETE) + +#define rtIsSeparator(eType) (((eType) & LSM_SEPARATOR)!=0) +#define rtIsWrite(eType) (((eType) & LSM_INSERT)!=0) +#define rtIsSystem(eType) (((eType) & LSM_SYSTEMKEY)!=0) + +/* +** The following macros are used to access a page footer. +*/ +#define SEGMENT_NRECORD_OFFSET(pgsz) ((pgsz) - 2) +#define SEGMENT_FLAGS_OFFSET(pgsz) ((pgsz) - 2 - 2) +#define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8) +#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2) + +#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1) + +#define SEGMENT_BTREE_FLAG 0x0001 +#define PGFTR_SKIP_NEXT_FLAG 0x0002 +#define PGFTR_SKIP_THIS_FLAG 0x0004 + + +#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD +# define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 +#endif + +typedef struct SegmentPtr SegmentPtr; +typedef struct LsmBlob LsmBlob; + +struct LsmBlob { + lsm_env *pEnv; + void *pData; + int nData; + int nAlloc; +}; + +/* +** A SegmentPtr object may be used for one of two purposes: +** +** * To iterate and/or seek within a single Segment (the combination of a +** main run and an optional sorted run). +** +** * To iterate through the separators array of a segment. +*/ +struct SegmentPtr { + Level *pLevel; /* Level object segment is part of */ + Segment *pSeg; /* Segment to access */ + + /* Current page. See segmentPtrLoadPage(). */ + Page *pPg; /* Current page */ + u16 flags; /* Copy of page flags field */ + int nCell; /* Number of cells on pPg */ + LsmPgno iPtr; /* Base cascade pointer */ + + /* Current cell. See segmentPtrLoadCell() */ + int iCell; /* Current record within page pPg */ + int eType; /* Type of current record */ + LsmPgno iPgPtr; /* Cascade pointer offset */ + void *pKey; int nKey; /* Key associated with current record */ + void *pVal; int nVal; /* Current record value (eType==WRITE only) */ + + /* Blobs used to allocate buffers for pKey and pVal as required */ + LsmBlob blob1; + LsmBlob blob2; +}; + +/* +** Used to iterate through the keys stored in a b-tree hierarchy from start +** to finish. Only First() and Next() operations are required. +** +** btreeCursorNew() +** btreeCursorFirst() +** btreeCursorNext() +** btreeCursorFree() +** btreeCursorPosition() +** btreeCursorRestore() +*/ +typedef struct BtreePg BtreePg; +typedef struct BtreeCursor BtreeCursor; +struct BtreePg { + Page *pPage; + int iCell; +}; +struct BtreeCursor { + Segment *pSeg; /* Iterate through this segments btree */ + FileSystem *pFS; /* File system to read pages from */ + int nDepth; /* Allocated size of aPg[] */ + int iPg; /* Current entry in aPg[]. -1 -> EOF. */ + BtreePg *aPg; /* Pages from root to current location */ + + /* Cache of current entry. pKey==0 for EOF. */ + void *pKey; + int nKey; + int eType; + LsmPgno iPtr; + + /* Storage for key, if not local */ + LsmBlob blob; +}; + + +/* +** A cursor used for merged searches or iterations through up to one +** Tree structure and any number of sorted files. +** +** lsmMCursorNew() +** lsmMCursorSeek() +** lsmMCursorNext() +** lsmMCursorPrev() +** lsmMCursorFirst() +** lsmMCursorLast() +** lsmMCursorKey() +** lsmMCursorValue() +** lsmMCursorValid() +** +** iFree: +** This variable is only used by cursors providing input data for a +** new top-level segment. Such cursors only ever iterate forwards, not +** backwards. +*/ +struct MultiCursor { + lsm_db *pDb; /* Connection that owns this cursor */ + MultiCursor *pNext; /* Next cursor owned by connection pDb */ + int flags; /* Mask of CURSOR_XXX flags */ + + int eType; /* Cache of current key type */ + LsmBlob key; /* Cache of current key (or NULL) */ + LsmBlob val; /* Cache of current value */ + + /* All the component cursors: */ + TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */ + int iFree; /* Next element of free-list (-ve for eof) */ + SegmentPtr *aPtr; /* Array of segment pointers */ + int nPtr; /* Size of array aPtr[] */ + BtreeCursor *pBtCsr; /* b-tree cursor (db writes only) */ + + /* Comparison results */ + int nTree; /* Size of aTree[] array */ + int *aTree; /* Array of comparison results */ + + /* Used by cursors flushing the in-memory tree only */ + void *pSystemVal; /* Pointer to buffer to free */ + + /* Used by worker cursors only */ + LsmPgno *pPrevMergePtr; +}; + +/* +** The following constants are used to assign integers to each component +** cursor of a multi-cursor. +*/ +#define CURSOR_DATA_TREE0 0 /* Current tree cursor (apTreeCsr[0]) */ +#define CURSOR_DATA_TREE1 1 /* The "old" tree, if any (apTreeCsr[1]) */ +#define CURSOR_DATA_SYSTEM 2 /* Free-list entries (new-toplevel only) */ +#define CURSOR_DATA_SEGMENT 3 /* First segment pointer (aPtr[0]) */ + +/* +** CURSOR_IGNORE_DELETE +** If set, this cursor will not visit SORTED_DELETE keys. +** +** CURSOR_FLUSH_FREELIST +** This cursor is being used to create a new toplevel. It should also +** iterate through the contents of the in-memory free block list. +** +** CURSOR_IGNORE_SYSTEM +** If set, this cursor ignores system keys. +** +** CURSOR_NEXT_OK +** Set if it is Ok to call lsm_csr_next(). +** +** CURSOR_PREV_OK +** Set if it is Ok to call lsm_csr_prev(). +** +** CURSOR_READ_SEPARATORS +** Set if this cursor should visit the separator keys in segment +** aPtr[nPtr-1]. +** +** CURSOR_SEEK_EQ +** Cursor has undergone a successful lsm_csr_seek(LSM_SEEK_EQ) operation. +** The key and value are stored in MultiCursor.key and MultiCursor.val +** respectively. +*/ +#define CURSOR_IGNORE_DELETE 0x00000001 +#define CURSOR_FLUSH_FREELIST 0x00000002 +#define CURSOR_IGNORE_SYSTEM 0x00000010 +#define CURSOR_NEXT_OK 0x00000020 +#define CURSOR_PREV_OK 0x00000040 +#define CURSOR_READ_SEPARATORS 0x00000080 +#define CURSOR_SEEK_EQ 0x00000100 + +typedef struct MergeWorker MergeWorker; +typedef struct Hierarchy Hierarchy; + +struct Hierarchy { + Page **apHier; + int nHier; +}; + +/* +** aSave: +** When mergeWorkerNextPage() is called to advance to the next page in +** the output segment, if the bStore flag for an element of aSave[] is +** true, it is cleared and the corresponding iPgno value is set to the +** page number of the page just completed. +** +** aSave[0] is used to record the pointer value to be pushed into the +** b-tree hierarchy. aSave[1] is used to save the page number of the +** page containing the indirect key most recently written to the b-tree. +** see mergeWorkerPushHierarchy() for details. +*/ +struct MergeWorker { + lsm_db *pDb; /* Database handle */ + Level *pLevel; /* Worker snapshot Level being merged */ + MultiCursor *pCsr; /* Cursor to read new segment contents from */ + int bFlush; /* True if this is an in-memory tree flush */ + Hierarchy hier; /* B-tree hierarchy under construction */ + Page *pPage; /* Current output page */ + int nWork; /* Number of calls to mergeWorkerNextPage() */ + LsmPgno *aGobble; /* Gobble point for each input segment */ + + LsmPgno iIndirect; + struct SavedPgno { + LsmPgno iPgno; + int bStore; + } aSave[2]; +}; + +#ifdef LSM_DEBUG_EXPENSIVE +static int assertPointersOk(lsm_db *, Segment *, Segment *, int); +static int assertBtreeOk(lsm_db *, Segment *); +static void assertRunInOrder(lsm_db *pDb, Segment *pSeg); +#else +#define assertRunInOrder(x,y) +#define assertBtreeOk(x,y) +#endif + + +struct FilePage { u8 *aData; int nData; }; +static u8 *fsPageData(Page *pPg, int *pnData){ + *pnData = ((struct FilePage *)(pPg))->nData; + return ((struct FilePage *)(pPg))->aData; +} +/*UNUSED static u8 *fsPageDataPtr(Page *pPg){ + return ((struct FilePage *)(pPg))->aData; +}*/ + +/* +** Write nVal as a 16-bit unsigned big-endian integer into buffer aOut. +*/ +void lsmPutU16(u8 *aOut, u16 nVal){ + aOut[0] = (u8)((nVal>>8) & 0xFF); + aOut[1] = (u8)(nVal & 0xFF); +} + +void lsmPutU32(u8 *aOut, u32 nVal){ + aOut[0] = (u8)((nVal>>24) & 0xFF); + aOut[1] = (u8)((nVal>>16) & 0xFF); + aOut[2] = (u8)((nVal>> 8) & 0xFF); + aOut[3] = (u8)((nVal ) & 0xFF); +} + +int lsmGetU16(u8 *aOut){ + return (aOut[0] << 8) + aOut[1]; +} + +u32 lsmGetU32(u8 *aOut){ + return ((u32)aOut[0] << 24) + + ((u32)aOut[1] << 16) + + ((u32)aOut[2] << 8) + + ((u32)aOut[3]); +} + +u64 lsmGetU64(u8 *aOut){ + return ((u64)aOut[0] << 56) + + ((u64)aOut[1] << 48) + + ((u64)aOut[2] << 40) + + ((u64)aOut[3] << 32) + + ((u64)aOut[4] << 24) + + ((u32)aOut[5] << 16) + + ((u32)aOut[6] << 8) + + ((u32)aOut[7]); +} + +void lsmPutU64(u8 *aOut, u64 nVal){ + aOut[0] = (u8)((nVal>>56) & 0xFF); + aOut[1] = (u8)((nVal>>48) & 0xFF); + aOut[2] = (u8)((nVal>>40) & 0xFF); + aOut[3] = (u8)((nVal>>32) & 0xFF); + aOut[4] = (u8)((nVal>>24) & 0xFF); + aOut[5] = (u8)((nVal>>16) & 0xFF); + aOut[6] = (u8)((nVal>> 8) & 0xFF); + aOut[7] = (u8)((nVal ) & 0xFF); +} + +static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){ + assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) ); + if( pBlob->nAllocpData = lsmReallocOrFree(pEnv, pBlob->pData, nData); + if( !pBlob->pData ) return LSM_NOMEM_BKPT; + pBlob->nAlloc = nData; + pBlob->pEnv = pEnv; + } + return LSM_OK; +} + +static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){ + if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM; + memcpy(pBlob->pData, pData, nData); + pBlob->nData = nData; + return LSM_OK; +} + +#if 0 +static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){ + return sortedBlobSet(pDest, pSrc->pData, pSrc->nData); +} +#endif + +static void sortedBlobFree(LsmBlob *pBlob){ + assert( pBlob->pEnv || pBlob->pData==0 ); + if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); + memset(pBlob, 0, sizeof(LsmBlob)); +} + +static int sortedReadData( + Segment *pSeg, + Page *pPg, + int iOff, + int nByte, + void **ppData, + LsmBlob *pBlob +){ + int rc = LSM_OK; + int iEnd; + int nData; + int nCell; + u8 *aData; + + aData = fsPageData(pPg, &nData); + nCell = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); + iEnd = SEGMENT_EOF(nData, nCell); + assert( iEnd>0 && iEndnData = nByte; + aDest = (u8 *)pBlob->pData; + *ppData = pBlob->pData; + + /* Increment the pointer pages ref-count. */ + lsmFsPageRef(pPg); + + while( rc==LSM_OK ){ + Page *pNext; + int flags; + + /* Copy data from pPg into the output buffer. */ + int nCopy = LSM_MIN(nRem, iEnd-i); + if( nCopy>0 ){ + memcpy(&aDest[nByte-nRem], &aData[i], nCopy); + nRem -= nCopy; + i += nCopy; + assert( nRem==0 || i==iEnd ); + } + assert( nRem>=0 ); + if( nRem==0 ) break; + i -= iEnd; + + /* Grab the next page in the segment */ + + do { + rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); + if( rc==LSM_OK && pNext==0 ){ + rc = LSM_CORRUPT_BKPT; + } + if( rc ) break; + lsmFsPageRelease(pPg); + pPg = pNext; + aData = fsPageData(pPg, &nData); + flags = lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]); + }while( flags&SEGMENT_BTREE_FLAG ); + + iEnd = SEGMENT_EOF(nData, lsmGetU16(&aData[nData-2])); + assert( iEnd>0 && iEnd=0 ); + aCell = pageGetCell(aData, nData, iCell); + lsmVarintGet64(&aCell[1], &iRet); + return iRet; +} + +static u8 *pageGetKey( + Segment *pSeg, /* Segment pPg belongs to */ + Page *pPg, /* Page to read from */ + int iCell, /* Index of cell on page to read */ + int *piTopic, /* OUT: Topic associated with this key */ + int *pnKey, /* OUT: Size of key in bytes */ + LsmBlob *pBlob /* If required, use this for dynamic memory */ +){ + u8 *pKey; + i64 nDummy; + int eType; + u8 *aData; + int nData; + + aData = fsPageData(pPg, &nData); + + assert( !(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); + assert( iCellpData || nKey==pBlob->nData ); + if( (void *)aKey!=pBlob->pData ){ + rc = sortedBlobSet(pEnv, pBlob, aKey, nKey); + } + + return rc; +} + +static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){ + LsmPgno iRef; + u8 *aData; + int nData; + u8 *aCell; + + aData = fsPageData(pPg, &nData); + aCell = pageGetCell(aData, nData, iKey); + assert( aCell[0]==0 ); + aCell++; + aCell += lsmVarintGet64(aCell, &iRef); + lsmVarintGet64(aCell, &iRef); + assert( iRef>0 ); + return iRef; +} + +#define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i))) +#define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i))) + +static int pageGetBtreeKey( + Segment *pSeg, /* Segment page pPg belongs to */ + Page *pPg, + int iKey, + LsmPgno *piPtr, + int *piTopic, + void **ppKey, + int *pnKey, + LsmBlob *pBlob +){ + u8 *aData; + int nData; + u8 *aCell; + int eType; + + aData = fsPageData(pPg, &nData); + assert( SEGMENT_BTREE_FLAG & pageGetFlags(aData, nData) ); + assert( iKey>=0 && iKeypData; + *pnKey = pBlob->nData; + }else{ + aCell += GETVARINT32(aCell, *pnKey); + *ppKey = aCell; + } + if( piTopic ) *piTopic = rtTopic(eType); + + return LSM_OK; +} + +static int btreeCursorLoadKey(BtreeCursor *pCsr){ + int rc = LSM_OK; + if( pCsr->iPg<0 ){ + pCsr->pKey = 0; + pCsr->nKey = 0; + pCsr->eType = 0; + }else{ + LsmPgno dummy; + int iPg = pCsr->iPg; + int iCell = pCsr->aPg[iPg].iCell; + while( iCell<0 && (--iPg)>=0 ){ + iCell = pCsr->aPg[iPg].iCell-1; + } + if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT; + + rc = pageGetBtreeKey( + pCsr->pSeg, + pCsr->aPg[iPg].pPage, iCell, + &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob + ); + pCsr->eType |= LSM_SEPARATOR; + } + + return rc; +} + +static LsmPgno btreeCursorPtr(u8 *aData, int nData, int iCell){ + int nCell; + + nCell = pageGetNRec(aData, nData); + if( iCell>=nCell ){ + return pageGetPtr(aData, nData); + } + return pageGetRecordPtr(aData, nData, iCell); +} + +static int btreeCursorNext(BtreeCursor *pCsr){ + int rc = LSM_OK; + + BtreePg *pPg = &pCsr->aPg[pCsr->iPg]; + int nCell; + u8 *aData; + int nData; + + assert( pCsr->iPg>=0 ); + assert( pCsr->iPg==pCsr->nDepth-1 ); + + aData = fsPageData(pPg->pPage, &nData); + nCell = pageGetNRec(aData, nData); + assert( pPg->iCell<=nCell ); + pPg->iCell++; + if( pPg->iCell==nCell ){ + LsmPgno iLoad; + + /* Up to parent. */ + lsmFsPageRelease(pPg->pPage); + pPg->pPage = 0; + pCsr->iPg--; + while( pCsr->iPg>=0 ){ + pPg = &pCsr->aPg[pCsr->iPg]; + aData = fsPageData(pPg->pPage, &nData); + if( pPg->iCellpPage); + pCsr->iPg--; + } + + /* Read the key */ + rc = btreeCursorLoadKey(pCsr); + + /* Unless the cursor is at EOF, descend to cell -1 (yes, negative one) of + ** the left-most most descendent. */ + if( pCsr->iPg>=0 ){ + pCsr->aPg[pCsr->iPg].iCell++; + + iLoad = btreeCursorPtr(aData, nData, pPg->iCell); + do { + Page *pLoad; + pCsr->iPg++; + rc = lsmFsDbPageGet(pCsr->pFS, pCsr->pSeg, iLoad, &pLoad); + pCsr->aPg[pCsr->iPg].pPage = pLoad; + pCsr->aPg[pCsr->iPg].iCell = 0; + if( rc==LSM_OK ){ + if( pCsr->iPg==(pCsr->nDepth-1) ) break; + aData = fsPageData(pLoad, &nData); + iLoad = btreeCursorPtr(aData, nData, 0); + } + }while( rc==LSM_OK && pCsr->iPg<(pCsr->nDepth-1) ); + pCsr->aPg[pCsr->iPg].iCell = -1; + } + + }else{ + rc = btreeCursorLoadKey(pCsr); + } + + if( rc==LSM_OK && pCsr->iPg>=0 ){ + aData = fsPageData(pCsr->aPg[pCsr->iPg].pPage, &nData); + pCsr->iPtr = btreeCursorPtr(aData, nData, pCsr->aPg[pCsr->iPg].iCell+1); + } + + return rc; +} + +static void btreeCursorFree(BtreeCursor *pCsr){ + if( pCsr ){ + int i; + lsm_env *pEnv = lsmFsEnv(pCsr->pFS); + for(i=0; i<=pCsr->iPg; i++){ + lsmFsPageRelease(pCsr->aPg[i].pPage); + } + sortedBlobFree(&pCsr->blob); + lsmFree(pEnv, pCsr->aPg); + lsmFree(pEnv, pCsr); + } +} + +static int btreeCursorFirst(BtreeCursor *pCsr){ + int rc; + + Page *pPg = 0; + FileSystem *pFS = pCsr->pFS; + LsmPgno iPg = pCsr->pSeg->iRoot; + + do { + rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg); + assert( (rc==LSM_OK)==(pPg!=0) ); + if( rc==LSM_OK ){ + u8 *aData; + int nData; + int flags; + + aData = fsPageData(pPg, &nData); + flags = pageGetFlags(aData, nData); + if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; + + if( (pCsr->nDepth % 8)==0 ){ + int nNew = pCsr->nDepth + 8; + pCsr->aPg = (BtreePg *)lsmReallocOrFreeRc( + lsmFsEnv(pFS), pCsr->aPg, sizeof(BtreePg) * nNew, &rc + ); + if( rc==LSM_OK ){ + memset(&pCsr->aPg[pCsr->nDepth], 0, sizeof(BtreePg) * 8); + } + } + + if( rc==LSM_OK ){ + assert( pCsr->aPg[pCsr->nDepth].iCell==0 ); + pCsr->aPg[pCsr->nDepth].pPage = pPg; + pCsr->nDepth++; + iPg = pageGetRecordPtr(aData, nData, 0); + } + } + }while( rc==LSM_OK ); + lsmFsPageRelease(pPg); + pCsr->iPg = pCsr->nDepth-1; + + if( rc==LSM_OK && pCsr->nDepth ){ + pCsr->aPg[pCsr->iPg].iCell = -1; + rc = btreeCursorNext(pCsr); + } + + return rc; +} + +static void btreeCursorPosition(BtreeCursor *pCsr, MergeInput *p){ + if( pCsr->iPg>=0 ){ + p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); + p->iCell = ((pCsr->aPg[pCsr->iPg].iCell + 1) << 8) + pCsr->nDepth; + }else{ + p->iPg = 0; + p->iCell = 0; + } +} + +static void btreeCursorSplitkey(BtreeCursor *pCsr, MergeInput *p){ + int iCell = pCsr->aPg[pCsr->iPg].iCell; + if( iCell>=0 ){ + p->iCell = iCell; + p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); + }else{ + int i; + for(i=pCsr->iPg-1; i>=0; i--){ + if( pCsr->aPg[i].iCell>0 ) break; + } + assert( i>=0 ); + p->iCell = pCsr->aPg[i].iCell-1; + p->iPg = lsmFsPageNumber(pCsr->aPg[i].pPage); + } +} + +static int sortedKeyCompare( + int (*xCmp)(void *, int, void *, int), + int iLhsTopic, void *pLhsKey, int nLhsKey, + int iRhsTopic, void *pRhsKey, int nRhsKey +){ + int res = iLhsTopic - iRhsTopic; + if( res==0 ){ + res = xCmp(pLhsKey, nLhsKey, pRhsKey, nRhsKey); + } + return res; +} + +static int btreeCursorRestore( + BtreeCursor *pCsr, + int (*xCmp)(void *, int, void *, int), + MergeInput *p +){ + int rc = LSM_OK; + + if( p->iPg ){ + lsm_env *pEnv = lsmFsEnv(pCsr->pFS); + int iCell; /* Current cell number on leaf page */ + LsmPgno iLeaf; /* Page number of current leaf page */ + int nDepth; /* Depth of b-tree structure */ + Segment *pSeg = pCsr->pSeg; + + /* Decode the MergeInput structure */ + iLeaf = p->iPg; + nDepth = (p->iCell & 0x00FF); + iCell = (p->iCell >> 8) - 1; + + /* Allocate the BtreeCursor.aPg[] array */ + assert( pCsr->aPg==0 ); + pCsr->aPg = (BtreePg *)lsmMallocZeroRc(pEnv, sizeof(BtreePg) * nDepth, &rc); + + /* Populate the last entry of the aPg[] array */ + if( rc==LSM_OK ){ + Page **pp = &pCsr->aPg[nDepth-1].pPage; + pCsr->iPg = nDepth-1; + pCsr->nDepth = nDepth; + pCsr->aPg[pCsr->iPg].iCell = iCell; + rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); + } + + /* Populate any other aPg[] array entries */ + if( rc==LSM_OK && nDepth>1 ){ + LsmBlob blob = {0,0,0}; + void *pSeek; + int nSeek; + int iTopicSeek; + int iPg = 0; + LsmPgno iLoad = pSeg->iRoot; + Page *pPg = pCsr->aPg[nDepth-1].pPage; + + if( pageObjGetNRec(pPg)==0 ){ + /* This can happen when pPg is the right-most leaf in the b-tree. + ** In this case, set the iTopicSeek/pSeek/nSeek key to a value + ** greater than any real key. */ + assert( iCell==-1 ); + iTopicSeek = 1000; + pSeek = 0; + nSeek = 0; + }else{ + LsmPgno dummy; + rc = pageGetBtreeKey(pSeg, pPg, + 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob + ); + } + + do { + Page *pPg2; + rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLoad, &pPg2); + assert( rc==LSM_OK || pPg2==0 ); + if( rc==LSM_OK ){ + u8 *aData; /* Buffer containing page data */ + int nData; /* Size of aData[] in bytes */ + int iMin; + int iMax; + int iCell2; + + aData = fsPageData(pPg2, &nData); + assert( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); + + iLoad = pageGetPtr(aData, nData); + iCell2 = pageGetNRec(aData, nData); + iMax = iCell2-1; + iMin = 0; + + while( iMax>=iMin ){ + int iTry = (iMin+iMax)/2; + void *pKey; int nKey; /* Key for cell iTry */ + int iTopic; /* Topic for key pKeyT/nKeyT */ + LsmPgno iPtr; /* Pointer for cell iTry */ + int res; /* (pSeek - pKeyT) */ + + rc = pageGetBtreeKey( + pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob + ); + if( rc!=LSM_OK ) break; + + res = sortedKeyCompare( + xCmp, iTopicSeek, pSeek, nSeek, iTopic, pKey, nKey + ); + assert( res!=0 ); + + if( res<0 ){ + iLoad = iPtr; + iCell2 = iTry; + iMax = iTry-1; + }else{ + iMin = iTry+1; + } + } + + pCsr->aPg[iPg].pPage = pPg2; + pCsr->aPg[iPg].iCell = iCell2; + iPg++; + assert( iPg!=nDepth-1 + || lsmFsRedirectPage(pCsr->pFS, pSeg->pRedirect, iLoad)==iLeaf + ); + } + }while( rc==LSM_OK && iPg<(nDepth-1) ); + sortedBlobFree(&blob); + } + + /* Load the current key and pointer */ + if( rc==LSM_OK ){ + BtreePg *pBtreePg; + u8 *aData; + int nData; + + pBtreePg = &pCsr->aPg[pCsr->iPg]; + aData = fsPageData(pBtreePg->pPage, &nData); + pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1); + if( pBtreePg->iCell<0 ){ + LsmPgno dummy; + int i; + for(i=pCsr->iPg-1; i>=0; i--){ + if( pCsr->aPg[i].iCell>0 ) break; + } + assert( i>=0 ); + rc = pageGetBtreeKey(pSeg, + pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1, + &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob + ); + pCsr->eType |= LSM_SEPARATOR; + + }else{ + rc = btreeCursorLoadKey(pCsr); + } + } + } + return rc; +} + +static int btreeCursorNew( + lsm_db *pDb, + Segment *pSeg, + BtreeCursor **ppCsr +){ + int rc = LSM_OK; + BtreeCursor *pCsr; + + assert( pSeg->iRoot ); + pCsr = lsmMallocZeroRc(pDb->pEnv, sizeof(BtreeCursor), &rc); + if( pCsr ){ + pCsr->pFS = pDb->pFS; + pCsr->pSeg = pSeg; + pCsr->iPg = -1; + } + + *ppCsr = pCsr; + return rc; +} + +static void segmentPtrSetPage(SegmentPtr *pPtr, Page *pNext){ + lsmFsPageRelease(pPtr->pPg); + if( pNext ){ + int nData; + u8 *aData = fsPageData(pNext, &nData); + pPtr->nCell = pageGetNRec(aData, nData); + pPtr->flags = (u16)pageGetFlags(aData, nData); + pPtr->iPtr = pageGetPtr(aData, nData); + } + pPtr->pPg = pNext; +} + +/* +** Load a new page into the SegmentPtr object pPtr. +*/ +static int segmentPtrLoadPage( + FileSystem *pFS, + SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ + LsmPgno iNew /* Page number of new page */ +){ + Page *pPg = 0; /* The new page */ + int rc; /* Return Code */ + + rc = lsmFsDbPageGet(pFS, pPtr->pSeg, iNew, &pPg); + assert( rc==LSM_OK || pPg==0 ); + segmentPtrSetPage(pPtr, pPg); + + return rc; +} + +static int segmentPtrReadData( + SegmentPtr *pPtr, + int iOff, + int nByte, + void **ppData, + LsmBlob *pBlob +){ + return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); +} + +static int segmentPtrNextPage( + SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ + int eDir /* +1 for next(), -1 for prev() */ +){ + Page *pNext; /* New page to load */ + int rc; /* Return code */ + + assert( eDir==1 || eDir==-1 ); + assert( pPtr->pPg ); + assert( pPtr->pSeg || eDir>0 ); + + rc = lsmFsDbPageNext(pPtr->pSeg, pPtr->pPg, eDir, &pNext); + assert( rc==LSM_OK || pNext==0 ); + segmentPtrSetPage(pPtr, pNext); + return rc; +} + +static int segmentPtrLoadCell( + SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ + int iNew /* Cell number of new cell */ +){ + int rc = LSM_OK; + if( pPtr->pPg ){ + u8 *aData; /* Pointer to page data buffer */ + int iOff; /* Offset in aData[] to read from */ + int nPgsz; /* Size of page (aData[]) in bytes */ + + assert( iNewnCell ); + pPtr->iCell = iNew; + aData = fsPageData(pPtr->pPg, &nPgsz); + iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nPgsz, pPtr->iCell)]); + pPtr->eType = aData[iOff]; + iOff++; + iOff += GETVARINT64(&aData[iOff], pPtr->iPgPtr); + iOff += GETVARINT32(&aData[iOff], pPtr->nKey); + if( rtIsWrite(pPtr->eType) ){ + iOff += GETVARINT32(&aData[iOff], pPtr->nVal); + } + assert( pPtr->nKey>=0 ); + + rc = segmentPtrReadData( + pPtr, iOff, pPtr->nKey, &pPtr->pKey, &pPtr->blob1 + ); + if( rc==LSM_OK && rtIsWrite(pPtr->eType) ){ + rc = segmentPtrReadData( + pPtr, iOff+pPtr->nKey, pPtr->nVal, &pPtr->pVal, &pPtr->blob2 + ); + }else{ + pPtr->nVal = 0; + pPtr->pVal = 0; + } + } + + return rc; +} + + +static Segment *sortedSplitkeySegment(Level *pLevel){ + Merge *pMerge = pLevel->pMerge; + MergeInput *p = &pMerge->splitkey; + Segment *pSeg; + int i; + + for(i=0; inInput; i++){ + if( p->iPg==pMerge->aInput[i].iPg ) break; + } + if( pMerge->nInput==(pLevel->nRight+1) && i>=(pMerge->nInput-1) ){ + pSeg = &pLevel->pNext->lhs; + }else{ + pSeg = &pLevel->aRhs[i]; + } + + return pSeg; +} + +static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ + Segment *pSeg; + Page *pPg = 0; + lsm_env *pEnv = pDb->pEnv; /* Environment handle */ + int rc = *pRc; + Merge *pMerge = pLevel->pMerge; + + pSeg = sortedSplitkeySegment(pLevel); + if( rc==LSM_OK ){ + rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); + } + if( rc==LSM_OK ){ + int iTopic; + LsmBlob blob = {0, 0, 0, 0}; + u8 *aData; + int nData; + + aData = lsmFsPageData(pPg, &nData); + if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ + void *pKey; + int nKey; + LsmPgno dummy; + rc = pageGetBtreeKey(pSeg, + pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob + ); + if( rc==LSM_OK && blob.pData!=pKey ){ + rc = sortedBlobSet(pEnv, &blob, pKey, nKey); + } + }else{ + rc = pageGetKeyCopy( + pEnv, pSeg, pPg, pMerge->splitkey.iCell, &iTopic, &blob + ); + } + + pLevel->iSplitTopic = iTopic; + pLevel->pSplitKey = blob.pData; + pLevel->nSplitKey = blob.nData; + lsmFsPageRelease(pPg); + } + + *pRc = rc; +} + +/* +** Reset a segment cursor. Also free its buffers if they are nThreshold +** bytes or larger in size. +*/ +static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){ + lsmFsPageRelease(pPtr->pPg); + pPtr->pPg = 0; + pPtr->nCell = 0; + pPtr->pKey = 0; + pPtr->nKey = 0; + pPtr->pVal = 0; + pPtr->nVal = 0; + pPtr->eType = 0; + pPtr->iCell = 0; + if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1); + if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2); +} + +static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){ + return (pCsr->flags & CURSOR_READ_SEPARATORS)==0 + || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]); +} + +static int segmentPtrAdvance( + MultiCursor *pCsr, + SegmentPtr *pPtr, + int bReverse +){ + int eDir = (bReverse ? -1 : 1); + Level *pLvl = pPtr->pLevel; + do { + int rc; + int iCell; /* Number of new cell in page */ + int svFlags = 0; /* SegmentPtr.eType before advance */ + + iCell = pPtr->iCell + eDir; + assert( pPtr->pPg ); + assert( iCell<=pPtr->nCell && iCell>=-1 ); + + if( bReverse && pPtr->pSeg!=&pPtr->pLevel->lhs ){ + svFlags = pPtr->eType; + assert( svFlags ); + } + + if( iCell>=pPtr->nCell || iCell<0 ){ + do { + rc = segmentPtrNextPage(pPtr, eDir); + }while( rc==LSM_OK + && pPtr->pPg + && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG) ) + ); + if( rc!=LSM_OK ) return rc; + iCell = bReverse ? (pPtr->nCell-1) : 0; + } + rc = segmentPtrLoadCell(pPtr, iCell); + if( rc!=LSM_OK ) return rc; + + if( svFlags && pPtr->pPg ){ + int res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); + } + + if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ + Segment *pSeg = pPtr->pSeg; + rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); + if( rc!=LSM_OK ) return rc; + pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; + pPtr->eType |= (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); + pPtr->pKey = pLvl->pSplitKey; + pPtr->nKey = pLvl->nSplitKey; + } + + }while( pCsr + && pPtr->pPg + && segmentPtrIgnoreSeparators(pCsr, pPtr) + && rtIsSeparator(pPtr->eType) + ); + + return LSM_OK; +} + +static void segmentPtrEndPage( + FileSystem *pFS, + SegmentPtr *pPtr, + int bLast, + int *pRc +){ + if( *pRc==LSM_OK ){ + Segment *pSeg = pPtr->pSeg; + Page *pNew = 0; + if( bLast ){ + *pRc = lsmFsDbPageLast(pFS, pSeg, &pNew); + }else{ + *pRc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pNew); + } + segmentPtrSetPage(pPtr, pNew); + } +} + + +/* +** Try to move the segment pointer passed as the second argument so that it +** points at either the first (bLast==0) or last (bLast==1) cell in the valid +** region of the segment defined by pPtr->iFirst and pPtr->iLast. +** +** Return LSM_OK if successful or an lsm error code if something goes +** wrong (IO error, OOM etc.). +*/ +static int segmentPtrEnd(MultiCursor *pCsr, SegmentPtr *pPtr, int bLast){ + Level *pLvl = pPtr->pLevel; + int rc = LSM_OK; + FileSystem *pFS = pCsr->pDb->pFS; + int bIgnore; + + segmentPtrEndPage(pFS, pPtr, bLast, &rc); + while( rc==LSM_OK && pPtr->pPg + && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG)) + ){ + rc = segmentPtrNextPage(pPtr, (bLast ? -1 : 1)); + } + + if( rc==LSM_OK && pPtr->pPg ){ + rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); + if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ + int res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); + } + } + + bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); + if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ + rc = segmentPtrAdvance(pCsr, pPtr, bLast); + } + +#if 0 + if( bLast && rc==LSM_OK && pPtr->pPg + && pPtr->pSeg==&pLvl->lhs + && pLvl->nRight && (pPtr->eType & LSM_START_DELETE) + ){ + pPtr->iCell++; + pPtr->eType = LSM_END_DELETE | (pLvl->iSplitTopic); + pPtr->pKey = pLvl->pSplitKey; + pPtr->nKey = pLvl->nSplitKey; + pPtr->pVal = 0; + pPtr->nVal = 0; + } +#endif + + return rc; +} + +static void segmentPtrKey(SegmentPtr *pPtr, void **ppKey, int *pnKey){ + assert( pPtr->pPg ); + *ppKey = pPtr->pKey; + *pnKey = pPtr->nKey; +} + +#if 0 /* NOT USED */ +static char *keyToString(lsm_env *pEnv, void *pKey, int nKey){ + int i; + u8 *aKey = (u8 *)pKey; + char *zRet = (char *)lsmMalloc(pEnv, nKey+1); + + for(i=0; ipDb->pFS); + LsmBlob blob = {0, 0, 0}; + int eDir; + int iTopic = 0; /* TODO: Fix me */ + + for(eDir=-1; eDir<=1; eDir+=2){ + Page *pTest = pPtr->pPg; + + lsmFsPageRef(pTest); + while( pTest ){ + Segment *pSeg = pPtr->pSeg; + Page *pNext; + + int rc = lsmFsDbPageNext(pSeg, pTest, eDir, &pNext); + lsmFsPageRelease(pTest); + if( rc ) return 1; + pTest = pNext; + + if( pTest ){ + int nData; + u8 *aData = fsPageData(pTest, &nData); + int nCell = pageGetNRec(aData, nData); + int flags = pageGetFlags(aData, nData); + if( nCell && 0==(flags&SEGMENT_BTREE_FLAG) ){ + int nPgKey; + int iPgTopic; + u8 *pPgKey; + int res; + int iCell; + + iCell = ((eDir < 0) ? (nCell-1) : 0); + pPgKey = pageGetKey(pSeg, pTest, iCell, &iPgTopic, &nPgKey, &blob); + res = iTopic - iPgTopic; + if( res==0 ) res = pCsr->pDb->xCmp(pKey, nKey, pPgKey, nPgKey); + if( (eDir==1 && res>0) || (eDir==-1 && res<0) ){ + /* Taking this branch means something has gone wrong. */ + char *zMsg = lsmMallocPrintf(pEnv, "Key \"%s\" is not on page %d", + keyToString(pEnv, pKey, nKey), lsmFsPageNumber(pPtr->pPg) + ); + fprintf(stderr, "%s\n", zMsg); + assert( !"assertKeyLocation() failed" ); + } + lsmFsPageRelease(pTest); + pTest = 0; + } + } + } + } + + sortedBlobFree(&blob); + return 1; +} +#endif + +#ifndef NDEBUG +static int assertSeekResult( + MultiCursor *pCsr, + SegmentPtr *pPtr, + int iTopic, + void *pKey, + int nKey, + int eSeek +){ + if( pPtr->pPg ){ + int res; + res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey + ); + + if( eSeek==LSM_SEEK_EQ ) return (res==0); + if( eSeek==LSM_SEEK_LE ) return (res>=0); + if( eSeek==LSM_SEEK_GE ) return (res<=0); + } + + return 1; +} +#endif + +static int segmentPtrSearchOversized( + MultiCursor *pCsr, /* Cursor context */ + SegmentPtr *pPtr, /* Pointer to seek */ + int iTopic, /* Topic of key to search for */ + void *pKey, int nKey /* Key to seek to */ +){ + int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; + int rc = LSM_OK; + + /* If the OVERSIZED flag is set, then there is no pointer in the + ** upper level to the next page in the segment that contains at least + ** one key. So compare the largest key on the current page with the + ** key being sought (pKey/nKey). If (pKey/nKey) is larger, advance + ** to the next page in the segment that contains at least one key. + */ + while( rc==LSM_OK && (pPtr->flags & PGFTR_SKIP_NEXT_FLAG) ){ + u8 *pLastKey; + int nLastKey; + int iLastTopic; + int res; /* Result of comparison */ + Page *pNext; + + /* Load the last key on the current page. */ + pLastKey = pageGetKey(pPtr->pSeg, + pPtr->pPg, pPtr->nCell-1, &iLastTopic, &nLastKey, &pPtr->blob1 + ); + + /* If the loaded key is >= than (pKey/nKey), break out of the loop. + ** If (pKey/nKey) is present in this array, it must be on the current + ** page. */ + res = sortedKeyCompare( + xCmp, iLastTopic, pLastKey, nLastKey, iTopic, pKey, nKey + ); + if( res>=0 ) break; + + /* Advance to the next page that contains at least one key. */ + pNext = pPtr->pPg; + lsmFsPageRef(pNext); + while( 1 ){ + Page *pLoad; + u8 *aData; int nData; + + rc = lsmFsDbPageNext(pPtr->pSeg, pNext, 1, &pLoad); + lsmFsPageRelease(pNext); + pNext = pLoad; + if( pNext==0 ) break; + + assert( rc==LSM_OK ); + aData = lsmFsPageData(pNext, &nData); + if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 + && pageGetNRec(aData, nData)>0 + ){ + break; + } + } + if( pNext==0 ) break; + segmentPtrSetPage(pPtr, pNext); + + /* This should probably be an LSM_CORRUPT error. */ + assert( rc!=LSM_OK || (pPtr->flags & PGFTR_SKIP_THIS_FLAG) ); + } + + return rc; +} + +static int ptrFwdPointer( + Page *pPage, + int iCell, + Segment *pSeg, + LsmPgno *piPtr, + int *pbFound +){ + Page *pPg = pPage; + int iFirst = iCell; + int rc = LSM_OK; + + do { + Page *pNext = 0; + u8 *aData; + int nData; + + aData = lsmFsPageData(pPg, &nData); + if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 ){ + int i; + int nCell = pageGetNRec(aData, nData); + for(i=iFirst; ipPg && rc==LSM_OK ){ + int res = sortedKeyCompare(pCsr->pDb->xCmp, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey + ); + if( res<=0 ) break; + rc = segmentPtrAdvance(pCsr, pPtr, 0); + } + return rc; +} + + +/* +** This function is called as part of a SEEK_GE op on a multi-cursor if the +** FC pointer read from segment *pPtr comes from an entry with the +** LSM_START_DELETE flag set. In this case the pointer value cannot be +** trusted. Instead, the pointer that should be followed is that associated +** with the next entry in *pPtr that does not have LSM_START_DELETE set. +** +** Why the pointers can't be trusted: +** +** +** +** TODO: This is a stop-gap solution: +** +** At the moment, this function is called from within segmentPtrSeek(), +** as part of the initial lsmMCursorSeek() call. However, consider a +** database where the following has occurred: +** +** 1. A range delete removes keys 1..9999 using a range delete. +** 2. Keys 1 through 9999 are reinserted. +** 3. The levels containing the ops in 1. and 2. above are merged. Call +** this level N. Level N contains FC pointers to level N+1. +** +** Then, if the user attempts to query for (key>=2 LIMIT 10), the +** lsmMCursorSeek() call will iterate through 9998 entries searching for a +** pointer down to the level N+1 that is never actually used. It would be +** much better if the multi-cursor could do this lazily - only seek to the +** level (N+1) page after the user has moved the cursor on level N passed +** the big range-delete. +*/ +static int segmentPtrFwdPointer( + MultiCursor *pCsr, /* Multi-cursor pPtr belongs to */ + SegmentPtr *pPtr, /* Segment-pointer to extract FC ptr from */ + LsmPgno *piPtr /* OUT: FC pointer value */ +){ + Level *pLvl = pPtr->pLevel; + Level *pNext = pLvl->pNext; + Page *pPg = pPtr->pPg; + int rc; + int bFound; + LsmPgno iOut = 0; + + if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){ + if( pNext==0 + || (pNext->nRight==0 && pNext->lhs.iRoot) + || (pNext->nRight!=0 && pNext->aRhs[0].iRoot) + ){ + /* Do nothing. The pointer will not be used anyway. */ + return LSM_OK; + } + }else{ + if( pPtr[1].pSeg->iRoot ){ + return LSM_OK; + } + } + + /* Search for a pointer within the current segment. */ + lsmFsPageRef(pPg); + rc = ptrFwdPointer(pPg, pPtr->iCell, pPtr->pSeg, &iOut, &bFound); + + if( rc==LSM_OK && bFound==0 ){ + /* This case happens when pPtr points to the left-hand-side of a segment + ** currently undergoing an incremental merge. In this case, jump to the + ** oldest segment in the right-hand-side of the same level and continue + ** searching. But - do not consider any keys smaller than the levels + ** split-key. */ + SegmentPtr ptr; + + if( pPtr->pLevel->nRight==0 || pPtr->pSeg!=&pPtr->pLevel->lhs ){ + return LSM_CORRUPT_BKPT; + } + + memset(&ptr, 0, sizeof(SegmentPtr)); + ptr.pLevel = pPtr->pLevel; + ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; + rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); + if( rc==LSM_OK ){ + rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); + ptr.pPg = 0; + } + segmentPtrReset(&ptr, 0); + } + + *piPtr = iOut; + return rc; +} + +static int segmentPtrSeek( + MultiCursor *pCsr, /* Cursor context */ + SegmentPtr *pPtr, /* Pointer to seek */ + int iTopic, /* Key topic to seek to */ + void *pKey, int nKey, /* Key to seek to */ + int eSeek, /* Search bias - see above */ + LsmPgno *piPtr, /* OUT: FC pointer */ + int *pbStop +){ + int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; + int res = 0; /* Result of comparison operation */ + int rc = LSM_OK; + int iMin; + int iMax; + LsmPgno iPtrOut = 0; + + /* If the current page contains an oversized entry, then there are no + ** pointers to one or more of the subsequent pages in the sorted run. + ** The following call ensures that the segment-ptr points to the correct + ** page in this case. */ + rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey); + iPtrOut = pPtr->iPtr; + + /* Assert that this page is the right page of this segment for the key + ** that we are searching for. Do this by loading page (iPg-1) and testing + ** that pKey/nKey is greater than all keys on that page, and then by + ** loading (iPg+1) and testing that pKey/nKey is smaller than all + ** the keys it houses. + ** + ** TODO: With range-deletes in the tree, the test described above may fail. + */ +#if 0 + assert( assertKeyLocation(pCsr, pPtr, pKey, nKey) ); +#endif + + assert( pPtr->nCell>0 + || pPtr->pSeg->nSize==1 + || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) + ); + if( pPtr->nCell==0 ){ + segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); + }else{ + iMin = 0; + iMax = pPtr->nCell-1; + + while( 1 ){ + int iTry = (iMin+iMax)/2; + void *pKeyT; int nKeyT; /* Key for cell iTry */ + int iTopicT; + + assert( iTryeType); + + res = sortedKeyCompare(xCmp, iTopicT, pKeyT, nKeyT, iTopic, pKey, nKey); + if( res<=0 ){ + iPtrOut = pPtr->iPtr + pPtr->iPgPtr; + } + + if( res==0 || iMin==iMax ){ + break; + }else if( res>0 ){ + iMax = LSM_MAX(iTry-1, iMin); + }else{ + iMin = iTry+1; + } + } + + if( rc==LSM_OK ){ + assert( res==0 || (iMin==iMax && iMin>=0 && iMinnCell) ); + if( res ){ + rc = segmentPtrLoadCell(pPtr, iMin); + } + assert( rc!=LSM_OK || res>0 || iPtrOut==(pPtr->iPtr + pPtr->iPgPtr) ); + + if( rc==LSM_OK ){ + switch( eSeek ){ + case LSM_SEEK_EQ: { + int eType = pPtr->eType; + if( (res<0 && (eType & LSM_START_DELETE)) + || (res>0 && (eType & LSM_END_DELETE)) + || (res==0 && (eType & LSM_POINT_DELETE)) + ){ + *pbStop = 1; + }else if( res==0 && (eType & LSM_INSERT) ){ + lsm_env *pEnv = pCsr->pDb->pEnv; + *pbStop = 1; + pCsr->eType = pPtr->eType; + rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); + if( rc==LSM_OK ){ + rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); + } + pCsr->flags |= CURSOR_SEEK_EQ; + } + segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); + break; + } + case LSM_SEEK_LE: + if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1); + break; + case LSM_SEEK_GE: { + /* Figure out if we need to 'skip' the pointer forward or not */ + if( (res<=0 && (pPtr->eType & LSM_START_DELETE)) + || (res>0 && (pPtr->eType & LSM_END_DELETE)) + ){ + rc = segmentPtrFwdPointer(pCsr, pPtr, &iPtrOut); + } + if( res<0 && rc==LSM_OK ){ + rc = segmentPtrAdvance(pCsr, pPtr, 0); + } + break; + } + } + } + } + + /* If the cursor seek has found a separator key, and this cursor is + ** supposed to ignore separators keys, advance to the next entry. */ + if( rc==LSM_OK && pPtr->pPg + && segmentPtrIgnoreSeparators(pCsr, pPtr) + && rtIsSeparator(pPtr->eType) + ){ + assert( eSeek!=LSM_SEEK_EQ ); + rc = segmentPtrAdvance(pCsr, pPtr, eSeek==LSM_SEEK_LE); + } + } + + assert( rc!=LSM_OK || assertSeekResult(pCsr,pPtr,iTopic,pKey,nKey,eSeek) ); + *piPtr = iPtrOut; + return rc; +} + +static int seekInBtree( + MultiCursor *pCsr, /* Multi-cursor object */ + Segment *pSeg, /* Seek within this segment */ + int iTopic, + void *pKey, int nKey, /* Key to seek to */ + LsmPgno *aPg, /* OUT: Page numbers */ + Page **ppPg /* OUT: Leaf (sorted-run) page reference */ +){ + int i = 0; + int rc; + LsmPgno iPg; + Page *pPg = 0; + LsmBlob blob = {0, 0, 0}; + + iPg = pSeg->iRoot; + do { + LsmPgno *piFirst = 0; + if( aPg ){ + aPg[i++] = iPg; + piFirst = &aPg[i]; + } + + rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg); + assert( rc==LSM_OK || pPg==0 ); + if( rc==LSM_OK ){ + u8 *aData; /* Buffer containing page data */ + int nData; /* Size of aData[] in bytes */ + int iMin; + int iMax; + int nRec; + int flags; + + aData = fsPageData(pPg, &nData); + flags = pageGetFlags(aData, nData); + if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; + + iPg = pageGetPtr(aData, nData); + nRec = pageGetNRec(aData, nData); + + iMin = 0; + iMax = nRec-1; + while( iMax>=iMin ){ + int iTry = (iMin+iMax)/2; + void *pKeyT; int nKeyT; /* Key for cell iTry */ + int iTopicT; /* Topic for key pKeyT/nKeyT */ + LsmPgno iPtr; /* Pointer associated with cell iTry */ + int res; /* (pKey - pKeyT) */ + + rc = pageGetBtreeKey( + pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob + ); + if( rc!=LSM_OK ) break; + if( piFirst && pKeyT==blob.pData ){ + *piFirst = pageGetBtreeRef(pPg, iTry); + piFirst = 0; + i++; + } + + res = sortedKeyCompare( + pCsr->pDb->xCmp, iTopic, pKey, nKey, iTopicT, pKeyT, nKeyT + ); + if( res<0 ){ + iPg = iPtr; + iMax = iTry-1; + }else{ + iMin = iTry+1; + } + } + lsmFsPageRelease(pPg); + pPg = 0; + } + }while( rc==LSM_OK ); + + sortedBlobFree(&blob); + assert( (rc==LSM_OK)==(pPg!=0) ); + if( ppPg ){ + *ppPg = pPg; + }else{ + lsmFsPageRelease(pPg); + } + return rc; +} + +static int seekInSegment( + MultiCursor *pCsr, + SegmentPtr *pPtr, + int iTopic, + void *pKey, int nKey, + LsmPgno iPg, /* Page to search */ + int eSeek, /* Search bias - see above */ + LsmPgno *piPtr, /* OUT: FC pointer */ + int *pbStop /* OUT: Stop search flag */ +){ + LsmPgno iPtr = iPg; + int rc = LSM_OK; + + if( pPtr->pSeg->iRoot ){ + Page *pPg; + assert( pPtr->pSeg->iRoot!=0 ); + rc = seekInBtree(pCsr, pPtr->pSeg, iTopic, pKey, nKey, 0, &pPg); + if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg); + }else{ + if( iPtr==0 ){ + iPtr = pPtr->pSeg->iFirst; + } + if( rc==LSM_OK ){ + rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr); + } + } + + if( rc==LSM_OK ){ + rc = segmentPtrSeek(pCsr, pPtr, iTopic, pKey, nKey, eSeek, piPtr, pbStop); + } + return rc; +} + +/* +** Seek each segment pointer in the array of (pLvl->nRight+1) at aPtr[]. +** +** pbStop: +** This parameter is only significant if parameter eSeek is set to +** LSM_SEEK_EQ. In this case, it is set to true before returning if +** the seek operation is finished. This can happen in two ways: +** +** a) A key matching (pKey/nKey) is found, or +** b) A point-delete or range-delete deleting the key is found. +** +** In case (a), the multi-cursor CURSOR_SEEK_EQ flag is set and the pCsr->key +** and pCsr->val blobs populated before returning. +*/ +static int seekInLevel( + MultiCursor *pCsr, /* Sorted cursor object to seek */ + SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ + int eSeek, /* Search bias - see above */ + int iTopic, /* Key topic to search for */ + void *pKey, int nKey, /* Key to search for */ + LsmPgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ + int *pbStop /* OUT: See above */ +){ + Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ + int rc = LSM_OK; /* Return code */ + LsmPgno iOut = 0; /* Pointer to return to caller */ + int res = -1; /* Result of xCmp(pKey, split) */ + int nRhs = pLvl->nRight; /* Number of right-hand-side segments */ + int bStop = 0; + + /* If this is a composite level (one currently undergoing an incremental + ** merge), figure out if the search key is larger or smaller than the + ** levels split-key. */ + if( nRhs ){ + res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + } + + /* If (res<0), then key pKey/nKey is smaller than the split-key (or this + ** is not a composite level and there is no split-key). Search the + ** left-hand-side of the level in this case. */ + if( res<0 ){ + int i; + LsmPgno iPtr = 0; + if( nRhs==0 ) iPtr = *piPgno; + + rc = seekInSegment( + pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop + ); + if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){ + res = 0; + } + for(i=1; i<=nRhs; i++){ + segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); + } + } + + if( res>=0 ){ + int bHit = 0; /* True if at least one rhs is not EOF */ + LsmPgno iPtr = *piPgno; + int i; + segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD); + for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){ + SegmentPtr *pPtr = &aPtr[i]; + iOut = 0; + rc = seekInSegment( + pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop + ); + iPtr = iOut; + + /* If the segment-pointer has settled on a key that is smaller than + ** the splitkey, invalidate the segment-pointer. */ + if( pPtr->pPg ){ + res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + if( res<0 ){ + if( pPtr->eType & LSM_START_DELETE ){ + pPtr->eType &= ~LSM_INSERT; + pPtr->pKey = pLvl->pSplitKey; + pPtr->nKey = pLvl->nSplitKey; + pPtr->pVal = 0; + pPtr->nVal = 0; + }else{ + segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); + } + } + } + + if( aPtr[i].pKey ) bHit = 1; + } + + if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ + rc = segmentPtrEnd(pCsr, &aPtr[0], 1); + } + } + + assert( eSeek==LSM_SEEK_EQ || bStop==0 ); + *piPgno = iOut; + *pbStop = bStop; + return rc; +} + +static void multiCursorGetKey( + MultiCursor *pCsr, + int iKey, + int *peType, /* OUT: Key type (SORTED_WRITE etc.) */ + void **ppKey, /* OUT: Pointer to buffer containing key */ + int *pnKey /* OUT: Size of *ppKey in bytes */ +){ + int nKey = 0; + void *pKey = 0; + int eType = 0; + + switch( iKey ){ + case CURSOR_DATA_TREE0: + case CURSOR_DATA_TREE1: { + TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; + if( lsmTreeCursorValid(pTreeCsr) ){ + lsmTreeCursorKey(pTreeCsr, &eType, &pKey, &nKey); + } + break; + } + + case CURSOR_DATA_SYSTEM: { + Snapshot *pWorker = pCsr->pDb->pWorker; + if( pWorker && (pCsr->flags & CURSOR_FLUSH_FREELIST) ){ + int nEntry = pWorker->freelist.nEntry; + if( pCsr->iFree < (nEntry*2) ){ + FreelistEntry *aEntry = pWorker->freelist.aEntry; + int i = nEntry - 1 - (pCsr->iFree / 2); + u32 iKey2 = 0; + + if( (pCsr->iFree % 2) ){ + eType = LSM_END_DELETE|LSM_SYSTEMKEY; + iKey2 = aEntry[i].iBlk-1; + }else if( aEntry[i].iId>=0 ){ + eType = LSM_INSERT|LSM_SYSTEMKEY; + iKey2 = aEntry[i].iBlk; + + /* If the in-memory entry immediately before this one was a + ** DELETE, and the block number is one greater than the current + ** block number, mark this entry as an "end-delete-range". */ + if( i<(nEntry-1) && aEntry[i+1].iBlk==iKey2+1 && aEntry[i+1].iId<0 ){ + eType |= LSM_END_DELETE; + } + + }else{ + eType = LSM_START_DELETE|LSM_SYSTEMKEY; + iKey2 = aEntry[i].iBlk + 1; + } + + /* If the in-memory entry immediately after this one is a + ** DELETE, and the block number is one less than the current + ** key, mark this entry as an "start-delete-range". */ + if( i>0 && aEntry[i-1].iBlk==iKey2-1 && aEntry[i-1].iId<0 ){ + eType |= LSM_START_DELETE; + } + + pKey = pCsr->pSystemVal; + nKey = 4; + lsmPutU32(pKey, ~iKey2); + } + } + break; + } + + default: { + int iPtr = iKey - CURSOR_DATA_SEGMENT; + assert( iPtr>=0 ); + if( iPtr==pCsr->nPtr ){ + if( pCsr->pBtCsr ){ + pKey = pCsr->pBtCsr->pKey; + nKey = pCsr->pBtCsr->nKey; + eType = pCsr->pBtCsr->eType; + } + }else if( iPtrnPtr ){ + SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; + if( pPtr->pPg ){ + pKey = pPtr->pKey; + nKey = pPtr->nKey; + eType = pPtr->eType; + } + } + break; + } + } + + if( peType ) *peType = eType; + if( pnKey ) *pnKey = nKey; + if( ppKey ) *ppKey = pKey; +} + +static int sortedDbKeyCompare( + MultiCursor *pCsr, + int iLhsFlags, void *pLhsKey, int nLhsKey, + int iRhsFlags, void *pRhsKey, int nRhsKey +){ + int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; + int res; + + /* Compare the keys, including the system flag. */ + res = sortedKeyCompare(xCmp, + rtTopic(iLhsFlags), pLhsKey, nLhsKey, + rtTopic(iRhsFlags), pRhsKey, nRhsKey + ); + + /* If a key has the LSM_START_DELETE flag set, but not the LSM_INSERT or + ** LSM_POINT_DELETE flags, it is considered a delta larger. This prevents + ** the beginning of an open-ended set from masking a database entry or + ** delete at a lower level. */ + if( res==0 && (pCsr->flags & CURSOR_IGNORE_DELETE) ){ + const int m = LSM_POINT_DELETE|LSM_INSERT|LSM_END_DELETE |LSM_START_DELETE; + int iDel1 = 0; + int iDel2 = 0; + + if( LSM_START_DELETE==(iLhsFlags & m) ) iDel1 = +1; + if( LSM_END_DELETE ==(iLhsFlags & m) ) iDel1 = -1; + if( LSM_START_DELETE==(iRhsFlags & m) ) iDel2 = +1; + if( LSM_END_DELETE ==(iRhsFlags & m) ) iDel2 = -1; + + res = (iDel1 - iDel2); + } + + return res; +} + +static void multiCursorDoCompare(MultiCursor *pCsr, int iOut, int bReverse){ + int i1; + int i2; + int iRes; + void *pKey1; int nKey1; int eType1; + void *pKey2; int nKey2; int eType2; + const int mul = (bReverse ? -1 : 1); + + assert( pCsr->aTree && iOutnTree ); + if( iOut>=(pCsr->nTree/2) ){ + i1 = (iOut - pCsr->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pCsr->aTree[iOut*2]; + i2 = pCsr->aTree[iOut*2+1]; + } + + multiCursorGetKey(pCsr, i1, &eType1, &pKey1, &nKey1); + multiCursorGetKey(pCsr, i2, &eType2, &pKey2, &nKey2); + + if( pKey1==0 ){ + iRes = i2; + }else if( pKey2==0 ){ + iRes = i1; + }else{ + int res; + + /* Compare the keys */ + res = sortedDbKeyCompare(pCsr, + eType1, pKey1, nKey1, eType2, pKey2, nKey2 + ); + + res = res * mul; + if( res==0 ){ + /* The two keys are identical. Normally, this means that the key from + ** the newer run clobbers the old. However, if the newer key is a + ** separator key, or a range-delete-boundary only, do not allow it + ** to clobber an older entry. */ + int nc1 = (eType1 & (LSM_INSERT|LSM_POINT_DELETE))==0; + int nc2 = (eType2 & (LSM_INSERT|LSM_POINT_DELETE))==0; + iRes = (nc1 > nc2) ? i2 : i1; + }else if( res<0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pCsr->aTree[iOut] = iRes; +} + +/* +** This function advances segment pointer iPtr belonging to multi-cursor +** pCsr forward (bReverse==0) or backward (bReverse!=0). +** +** If the segment pointer points to a segment that is part of a composite +** level, then the following special case is handled. +** +** * If iPtr is the lhs of a composite level, and the cursor is being +** advanced forwards, and segment iPtr is at EOF, move all pointers +** that correspond to rhs segments of the same level to the first +** key in their respective data. +*/ +static int segmentCursorAdvance( + MultiCursor *pCsr, + int iPtr, + int bReverse +){ + int rc; + SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; + Level *pLvl = pPtr->pLevel; + int bComposite; /* True if pPtr is part of composite level */ + + /* Advance the segment-pointer object. */ + rc = segmentPtrAdvance(pCsr, pPtr, bReverse); + if( rc!=LSM_OK ) return rc; + + bComposite = (pLvl->nRight>0 && pCsr->nPtr>pLvl->nRight); + if( bComposite && pPtr->pPg==0 ){ + int bFix = 0; + if( (bReverse==0)==(pPtr->pSeg==&pLvl->lhs) ){ + int i; + if( bReverse ){ + SegmentPtr *pLhs = &pCsr->aPtr[iPtr - 1 - (pPtr->pSeg - pLvl->aRhs)]; + for(i=0; inRight; i++){ + if( pLhs[i+1].pPg ) break; + } + if( i==pLvl->nRight ){ + bFix = 1; + rc = segmentPtrEnd(pCsr, pLhs, 1); + } + }else{ + bFix = 1; + for(i=0; rc==LSM_OK && inRight; i++){ + rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); + } + } + } + + if( bFix ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bReverse); + } + } + } + +#if 0 + if( bComposite && pPtr->pSeg==&pLvl->lhs /* lhs of composite level */ + && bReverse==0 /* csr advanced forwards */ + && pPtr->pPg==0 /* segment at EOF */ + ){ + int i; + for(i=0; rc==LSM_OK && inRight; i++){ + rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); + } + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, 0); + } + } +#endif + + return rc; +} + +static void mcursorFreeComponents(MultiCursor *pCsr){ + int i; + lsm_env *pEnv = pCsr->pDb->pEnv; + + /* Close the tree cursor, if any. */ + lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); + lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); + + /* Reset the segment pointers */ + for(i=0; inPtr; i++){ + segmentPtrReset(&pCsr->aPtr[i], 0); + } + + /* And the b-tree cursor, if any */ + btreeCursorFree(pCsr->pBtCsr); + + /* Free allocations */ + lsmFree(pEnv, pCsr->aPtr); + lsmFree(pEnv, pCsr->aTree); + lsmFree(pEnv, pCsr->pSystemVal); + + /* Zero fields */ + pCsr->nPtr = 0; + pCsr->aPtr = 0; + pCsr->nTree = 0; + pCsr->aTree = 0; + pCsr->pSystemVal = 0; + pCsr->apTreeCsr[0] = 0; + pCsr->apTreeCsr[1] = 0; + pCsr->pBtCsr = 0; +} + +void lsmMCursorFreeCache(lsm_db *pDb){ + MultiCursor *p; + MultiCursor *pNext; + for(p=pDb->pCsrCache; p; p=pNext){ + pNext = p->pNext; + lsmMCursorClose(p, 0); + } + pDb->pCsrCache = 0; +} + +/* +** Close the cursor passed as the first argument. +** +** If the bCache parameter is true, then shift the cursor to the pCsrCache +** list for possible reuse instead of actually deleting it. +*/ +void lsmMCursorClose(MultiCursor *pCsr, int bCache){ + if( pCsr ){ + lsm_db *pDb = pCsr->pDb; + MultiCursor **pp; /* Iterator variable */ + + /* The cursor may or may not be currently part of the linked list + ** starting at lsm_db.pCsr. If it is, extract it. */ + for(pp=&pDb->pCsr; *pp; pp=&((*pp)->pNext)){ + if( *pp==pCsr ){ + *pp = pCsr->pNext; + break; + } + } + + if( bCache ){ + int i; /* Used to iterate through segment-pointers */ + + /* Release any page references held by this cursor. */ + assert( !pCsr->pBtCsr ); + for(i=0; inPtr; i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + lsmFsPageRelease(pPtr->pPg); + pPtr->pPg = 0; + } + + /* Reset the tree cursors */ + lsmTreeCursorReset(pCsr->apTreeCsr[0]); + lsmTreeCursorReset(pCsr->apTreeCsr[1]); + + /* Add the cursor to the pCsrCache list */ + pCsr->pNext = pDb->pCsrCache; + pDb->pCsrCache = pCsr; + }else{ + /* Free the allocation used to cache the current key, if any. */ + sortedBlobFree(&pCsr->key); + sortedBlobFree(&pCsr->val); + + /* Free the component cursors */ + mcursorFreeComponents(pCsr); + + /* Free the cursor structure itself */ + lsmFree(pDb->pEnv, pCsr); + } + } +} + +#define TREE_NONE 0 +#define TREE_OLD 1 +#define TREE_BOTH 2 + +/* +** Parameter eTree is one of TREE_OLD or TREE_BOTH. +*/ +static int multiCursorAddTree(MultiCursor *pCsr, Snapshot *pSnap, int eTree){ + int rc = LSM_OK; + lsm_db *db = pCsr->pDb; + + /* Add a tree cursor on the 'old' tree, if it exists. */ + if( eTree!=TREE_NONE + && lsmTreeHasOld(db) + && db->treehdr.iOldLog!=pSnap->iLogOff + ){ + rc = lsmTreeCursorNew(db, 1, &pCsr->apTreeCsr[1]); + } + + /* Add a tree cursor on the 'current' tree, if required. */ + if( rc==LSM_OK && eTree==TREE_BOTH ){ + rc = lsmTreeCursorNew(db, 0, &pCsr->apTreeCsr[0]); + } + + return rc; +} + +static int multiCursorAddRhs(MultiCursor *pCsr, Level *pLvl){ + int i; + int nRhs = pLvl->nRight; + + assert( pLvl->nRight>0 ); + assert( pCsr->aPtr==0 ); + pCsr->aPtr = lsmMallocZero(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nRhs); + if( !pCsr->aPtr ) return LSM_NOMEM_BKPT; + pCsr->nPtr = nRhs; + + for(i=0; iaPtr[i].pSeg = &pLvl->aRhs[i]; + pCsr->aPtr[i].pLevel = pLvl; + } + + return LSM_OK; +} + +static void multiCursorAddOne(MultiCursor *pCsr, Level *pLvl, int *pRc){ + if( *pRc==LSM_OK ){ + int iPtr = pCsr->nPtr; + int i; + pCsr->aPtr[iPtr].pLevel = pLvl; + pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; + iPtr++; + for(i=0; inRight; i++){ + pCsr->aPtr[iPtr].pLevel = pLvl; + pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; + iPtr++; + } + + if( pLvl->nRight && pLvl->pSplitKey==0 ){ + sortedSplitkey(pCsr->pDb, pLvl, pRc); + } + pCsr->nPtr = iPtr; + } +} + +static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ + Level *pLvl; + int nPtr = 0; + int rc = LSM_OK; + + for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ + /* If the LEVEL_INCOMPLETE flag is set, then this function is being + ** called (indirectly) from within a sortedNewToplevel() call to + ** construct pLvl. In this case ignore pLvl - this cursor is going to + ** be used to retrieve a freelist entry from the LSM, and the partially + ** complete level may confuse it. */ + if( pLvl->flags & LEVEL_INCOMPLETE ) continue; + nPtr += (1 + pLvl->nRight); + } + + assert( pCsr->aPtr==0 ); + pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); + + for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ + if( (pLvl->flags & LEVEL_INCOMPLETE)==0 ){ + multiCursorAddOne(pCsr, pLvl, &rc); + } + } + + return rc; +} + +static int multiCursorInit(MultiCursor *pCsr, Snapshot *pSnap){ + int rc; + rc = multiCursorAddAll(pCsr, pSnap); + if( rc==LSM_OK ){ + rc = multiCursorAddTree(pCsr, pSnap, TREE_BOTH); + } + pCsr->flags |= (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); + return rc; +} + +static MultiCursor *multiCursorNew(lsm_db *db, int *pRc){ + MultiCursor *pCsr; + pCsr = (MultiCursor *)lsmMallocZeroRc(db->pEnv, sizeof(MultiCursor), pRc); + if( pCsr ){ + pCsr->pNext = db->pCsr; + db->pCsr = pCsr; + pCsr->pDb = db; + } + return pCsr; +} + + +void lsmSortedRemap(lsm_db *pDb){ + MultiCursor *pCsr; + for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ + int iPtr; + if( pCsr->pBtCsr ){ + btreeCursorLoadKey(pCsr->pBtCsr); + } + for(iPtr=0; iPtrnPtr; iPtr++){ + segmentPtrLoadCell(&pCsr->aPtr[iPtr], pCsr->aPtr[iPtr].iCell); + } + } +} + +static void multiCursorReadSeparators(MultiCursor *pCsr){ + if( pCsr->nPtr>0 ){ + pCsr->flags |= CURSOR_READ_SEPARATORS; + } +} + +/* +** Have this cursor skip over SORTED_DELETE entries. +*/ +static void multiCursorIgnoreDelete(MultiCursor *pCsr){ + if( pCsr ) pCsr->flags |= CURSOR_IGNORE_DELETE; +} + +/* +** If the free-block list is not empty, then have this cursor visit a key +** with (a) the system bit set, and (b) the key "FREELIST" and (c) a value +** blob containing the serialized free-block list. +*/ +static int multiCursorVisitFreelist(MultiCursor *pCsr){ + int rc = LSM_OK; + pCsr->flags |= CURSOR_FLUSH_FREELIST; + pCsr->pSystemVal = lsmMallocRc(pCsr->pDb->pEnv, 4 + 8, &rc); + return rc; +} + +/* +** Allocate and return a new database cursor. +** +** This method should only be called to allocate user cursors. As it may +** recycle a cursor from lsm_db.pCsrCache. +*/ +int lsmMCursorNew( + lsm_db *pDb, /* Database handle */ + MultiCursor **ppCsr /* OUT: Allocated cursor */ +){ + MultiCursor *pCsr = 0; + int rc = LSM_OK; + + if( pDb->pCsrCache ){ + int bOld; /* True if there is an old in-memory tree */ + + /* Remove a cursor from the pCsrCache list and add it to the open list. */ + pCsr = pDb->pCsrCache; + pDb->pCsrCache = pCsr->pNext; + pCsr->pNext = pDb->pCsr; + pDb->pCsr = pCsr; + + /* The cursor can almost be used as is, except that the old in-memory + ** tree cursor may be present and not required, or required and not + ** present. Fix this if required. */ + bOld = (lsmTreeHasOld(pDb) && pDb->treehdr.iOldLog!=pDb->pClient->iLogOff); + if( !bOld && pCsr->apTreeCsr[1] ){ + lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); + pCsr->apTreeCsr[1] = 0; + }else if( bOld && !pCsr->apTreeCsr[1] ){ + rc = lsmTreeCursorNew(pDb, 1, &pCsr->apTreeCsr[1]); + } + + pCsr->flags = (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); + + }else{ + pCsr = multiCursorNew(pDb, &rc); + if( rc==LSM_OK ) rc = multiCursorInit(pCsr, pDb->pClient); + } + + if( rc!=LSM_OK ){ + lsmMCursorClose(pCsr, 0); + pCsr = 0; + } + assert( (rc==LSM_OK)==(pCsr!=0) ); + *ppCsr = pCsr; + return rc; +} + +static int multiCursorGetVal( + MultiCursor *pCsr, + int iVal, + void **ppVal, + int *pnVal +){ + int rc = LSM_OK; + + *ppVal = 0; + *pnVal = 0; + + switch( iVal ){ + case CURSOR_DATA_TREE0: + case CURSOR_DATA_TREE1: { + TreeCursor *pTreeCsr = pCsr->apTreeCsr[iVal-CURSOR_DATA_TREE0]; + if( lsmTreeCursorValid(pTreeCsr) ){ + lsmTreeCursorValue(pTreeCsr, ppVal, pnVal); + }else{ + *ppVal = 0; + *pnVal = 0; + } + break; + } + + case CURSOR_DATA_SYSTEM: { + Snapshot *pWorker = pCsr->pDb->pWorker; + if( pWorker + && (pCsr->iFree % 2)==0 + && pCsr->iFree < (pWorker->freelist.nEntry*2) + ){ + int iEntry = pWorker->freelist.nEntry - 1 - (pCsr->iFree / 2); + u8 *aVal = &((u8 *)(pCsr->pSystemVal))[4]; + lsmPutU64(aVal, pWorker->freelist.aEntry[iEntry].iId); + *ppVal = aVal; + *pnVal = 8; + } + break; + } + + default: { + int iPtr = iVal-CURSOR_DATA_SEGMENT; + if( iPtrnPtr ){ + SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; + if( pPtr->pPg ){ + *ppVal = pPtr->pVal; + *pnVal = pPtr->nVal; + } + } + } + } + + assert( rc==LSM_OK || (*ppVal==0 && *pnVal==0) ); + return rc; +} + +static int multiCursorAdvance(MultiCursor *pCsr, int bReverse); + +/* +** This function is called by worker connections to walk the part of the +** free-list stored within the LSM data structure. +*/ +int lsmSortedWalkFreelist( + lsm_db *pDb, /* Database handle */ + int bReverse, /* True to iterate from largest to smallest */ + int (*x)(void *, int, i64), /* Callback function */ + void *pCtx /* First argument to pass to callback */ +){ + MultiCursor *pCsr; /* Cursor used to read db */ + int rc = LSM_OK; /* Return Code */ + Snapshot *pSnap = 0; + + assert( pDb->pWorker ); + if( pDb->bIncrMerge ){ + rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); + if( rc!=LSM_OK ) return rc; + }else{ + pSnap = pDb->pWorker; + } + + pCsr = multiCursorNew(pDb, &rc); + if( pCsr ){ + rc = multiCursorAddAll(pCsr, pSnap); + pCsr->flags |= CURSOR_IGNORE_DELETE; + } + + if( rc==LSM_OK ){ + if( bReverse==0 ){ + rc = lsmMCursorLast(pCsr); + }else{ + rc = lsmMCursorSeek(pCsr, 1, "", 0, LSM_SEEK_GE); + } + + while( rc==LSM_OK && lsmMCursorValid(pCsr) && rtIsSystem(pCsr->eType) ){ + void *pKey; int nKey; + void *pVal = 0; int nVal = 0; + + rc = lsmMCursorKey(pCsr, &pKey, &nKey); + if( rc==LSM_OK ) rc = lsmMCursorValue(pCsr, &pVal, &nVal); + if( rc==LSM_OK && (nKey!=4 || nVal!=8) ) rc = LSM_CORRUPT_BKPT; + + if( rc==LSM_OK ){ + int iBlk; + i64 iSnap; + iBlk = (int)(~(lsmGetU32((u8 *)pKey))); + iSnap = (i64)lsmGetU64((u8 *)pVal); + if( x(pCtx, iBlk, iSnap) ) break; + rc = multiCursorAdvance(pCsr, !bReverse); + } + } + } + + lsmMCursorClose(pCsr, 0); + if( pSnap!=pDb->pWorker ){ + lsmFreeSnapshot(pDb->pEnv, pSnap); + } + + return rc; +} + +int lsmSortedLoadFreelist( + lsm_db *pDb, /* Database handle (must be worker) */ + void **ppVal, /* OUT: Blob containing LSM free-list */ + int *pnVal /* OUT: Size of *ppVal blob in bytes */ +){ + MultiCursor *pCsr; /* Cursor used to retreive free-list */ + int rc = LSM_OK; /* Return Code */ + + assert( pDb->pWorker ); + assert( *ppVal==0 && *pnVal==0 ); + + pCsr = multiCursorNew(pDb, &rc); + if( pCsr ){ + rc = multiCursorAddAll(pCsr, pDb->pWorker); + pCsr->flags |= CURSOR_IGNORE_DELETE; + } + + if( rc==LSM_OK ){ + rc = lsmMCursorLast(pCsr); + if( rc==LSM_OK + && rtIsWrite(pCsr->eType) && rtIsSystem(pCsr->eType) + && pCsr->key.nData==8 + && 0==memcmp(pCsr->key.pData, "FREELIST", 8) + ){ + void *pVal; int nVal; /* Value read from database */ + rc = lsmMCursorValue(pCsr, &pVal, &nVal); + if( rc==LSM_OK ){ + *ppVal = lsmMallocRc(pDb->pEnv, nVal, &rc); + if( *ppVal ){ + memcpy(*ppVal, pVal, nVal); + *pnVal = nVal; + } + } + } + + lsmMCursorClose(pCsr, 0); + } + + return rc; +} + +static int multiCursorAllocTree(MultiCursor *pCsr){ + int rc = LSM_OK; + if( pCsr->aTree==0 ){ + int nByte; /* Bytes of space to allocate */ + int nMin; /* Total number of cursors being merged */ + + nMin = CURSOR_DATA_SEGMENT + pCsr->nPtr + (pCsr->pBtCsr!=0); + pCsr->nTree = 2; + while( pCsr->nTreenTree = pCsr->nTree*2; + } + + nByte = sizeof(int)*pCsr->nTree*2; + pCsr->aTree = (int *)lsmMallocZeroRc(pCsr->pDb->pEnv, nByte, &rc); + } + return rc; +} + +static void multiCursorCacheKey(MultiCursor *pCsr, int *pRc){ + if( *pRc==LSM_OK ){ + void *pKey; + int nKey; + multiCursorGetKey(pCsr, pCsr->aTree[1], &pCsr->eType, &pKey, &nKey); + *pRc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->key, pKey, nKey); + } +} + +#ifdef LSM_DEBUG_EXPENSIVE +static void assertCursorTree(MultiCursor *pCsr){ + int bRev = !!(pCsr->flags & CURSOR_PREV_OK); + int *aSave = pCsr->aTree; + int nSave = pCsr->nTree; + int rc; + + pCsr->aTree = 0; + pCsr->nTree = 0; + rc = multiCursorAllocTree(pCsr); + if( rc==LSM_OK ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bRev); + } + + assert( nSave==pCsr->nTree + && 0==memcmp(aSave, pCsr->aTree, sizeof(int)*nSave) + ); + + lsmFree(pCsr->pDb->pEnv, pCsr->aTree); + } + + pCsr->aTree = aSave; + pCsr->nTree = nSave; +} +#else +# define assertCursorTree(x) +#endif + +static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){ + int eType = pCsr->eType; + int iKey; + int i; + int rdmask; + + assert( pCsr->flags & (CURSOR_NEXT_OK|CURSOR_PREV_OK) ); + assertCursorTree(pCsr); + + rdmask = (pCsr->flags & CURSOR_NEXT_OK) ? LSM_END_DELETE : LSM_START_DELETE; + + /* If the cursor does not currently point to an actual database key (i.e. + ** it points to a delete key, or the start or end of a range-delete), and + ** the CURSOR_IGNORE_DELETE flag is set, skip past this entry. */ + if( (pCsr->flags & CURSOR_IGNORE_DELETE) && bDeleteOk==0 ){ + if( (eType & LSM_INSERT)==0 ) return 0; + } + + /* If the cursor points to a system key (free-list entry), and the + ** CURSOR_IGNORE_SYSTEM flag is set, skip thie entry. */ + if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){ + return 0; + } + +#ifndef NDEBUG + /* This block fires assert() statements to check one of the assumptions + ** in the comment below - that if the lhs sub-cursor of a level undergoing + ** a merge is valid, then all the rhs sub-cursors must be at EOF. + ** + ** Also assert that all rhs sub-cursors are either at EOF or point to + ** a key that is not less than the level split-key. */ + for(i=0; inPtr; i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + Level *pLvl = pPtr->pLevel; + if( pLvl->nRight && pPtr->pPg ){ + if( pPtr->pSeg==&pLvl->lhs ){ + int j; + for(j=0; jnRight; j++) assert( pPtr[j+1].pPg==0 ); + }else{ + int res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + assert( res>=0 ); + } + } + } +#endif + + /* Now check if this key has already been deleted by a range-delete. If + ** so, skip past it. + ** + ** Assume, for the moment, that the tree contains no levels currently + ** undergoing incremental merge, and that this cursor is iterating forwards + ** through the database keys. The cursor currently points to a key in + ** level L. This key has already been deleted if any of the sub-cursors + ** that point to levels newer than L (or to the in-memory tree) point to + ** a key greater than the current key with the LSM_END_DELETE flag set. + ** + ** Or, if the cursor is iterating backwards through data keys, if any + ** such sub-cursor points to a key smaller than the current key with the + ** LSM_START_DELETE flag set. + ** + ** Why it works with levels undergoing a merge too: + ** + ** When a cursor iterates forwards, the sub-cursors for the rhs of a + ** level are only activated once the lhs reaches EOF. So when iterating + ** forwards, the keys visited are the same as if the level was completely + ** merged. + ** + ** If the cursor is iterating backwards, then the lhs sub-cursor is not + ** initialized until the last of the rhs sub-cursors has reached EOF. + ** Additionally, if the START_DELETE flag is set on the last entry (in + ** reverse order - so the entry with the smallest key) of a rhs sub-cursor, + ** then a pseudo-key equal to the levels split-key with the END_DELETE + ** flag set is visited by the sub-cursor. + */ + iKey = pCsr->aTree[1]; + for(i=0; iflags & CURSOR_IGNORE_DELETE)==0 + ){ + void *pKey; int nKey; + multiCursorGetKey(pCsr, i, 0, &pKey, &nKey); + if( 0==sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(eType), pCsr->key.pData, pCsr->key.nData, + rtTopic(csrflags), pKey, nKey + )){ + continue; + } + } + return 0; + } + } + + /* The current cursor position is one this cursor should visit. Return 1. */ + return 1; +} + +static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ + int rc; + + rc = multiCursorAllocTree(pCsr); + if( rc==LSM_OK ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bRev); + } + } + + assertCursorTree(pCsr); + multiCursorCacheKey(pCsr, &rc); + + if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ + rc = multiCursorAdvance(pCsr, bRev); + } + return rc; +} + + +static int multiCursorEnd(MultiCursor *pCsr, int bLast){ + int rc = LSM_OK; + int i; + + pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); + pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK); + pCsr->iFree = 0; + + /* Position the two in-memory tree cursors */ + for(i=0; rc==LSM_OK && i<2; i++){ + if( pCsr->apTreeCsr[i] ){ + rc = lsmTreeCursorEnd(pCsr->apTreeCsr[i], bLast); + } + } + + for(i=0; rc==LSM_OK && inPtr; i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + Level *pLvl = pPtr->pLevel; + int iRhs; + int bHit = 0; + + if( bLast ){ + for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ + rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); + if( pPtr[iRhs+1].pPg ) bHit = 1; + } + if( bHit==0 && rc==LSM_OK ){ + rc = segmentPtrEnd(pCsr, pPtr, 1); + }else{ + segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); + } + }else{ + int bLhs = (pPtr->pSeg==&pLvl->lhs); + assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); + + if( bLhs ){ + rc = segmentPtrEnd(pCsr, pPtr, 0); + if( pPtr->pKey ) bHit = 1; + } + for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ + if( bHit ){ + segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD); + }else{ + rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); + } + } + } + i += pLvl->nRight; + } + + /* And the b-tree cursor, if applicable */ + if( rc==LSM_OK && pCsr->pBtCsr ){ + assert( bLast==0 ); + rc = btreeCursorFirst(pCsr->pBtCsr); + } + + if( rc==LSM_OK ){ + rc = multiCursorSetupTree(pCsr, bLast); + } + + return rc; +} + + +int mcursorSave(MultiCursor *pCsr){ + int rc = LSM_OK; + if( pCsr->aTree ){ + int iTree = pCsr->aTree[1]; + if( iTree==CURSOR_DATA_TREE0 || iTree==CURSOR_DATA_TREE1 ){ + multiCursorCacheKey(pCsr, &rc); + } + } + mcursorFreeComponents(pCsr); + return rc; +} + +int mcursorRestore(lsm_db *pDb, MultiCursor *pCsr){ + int rc; + rc = multiCursorInit(pCsr, pDb->pClient); + if( rc==LSM_OK && pCsr->key.pData ){ + rc = lsmMCursorSeek(pCsr, + rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, +1 + ); + } + return rc; +} + +int lsmSaveCursors(lsm_db *pDb){ + int rc = LSM_OK; + MultiCursor *pCsr; + + for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ + rc = mcursorSave(pCsr); + } + return rc; +} + +int lsmRestoreCursors(lsm_db *pDb){ + int rc = LSM_OK; + MultiCursor *pCsr; + + for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ + rc = mcursorRestore(pDb, pCsr); + } + return rc; +} + +int lsmMCursorFirst(MultiCursor *pCsr){ + return multiCursorEnd(pCsr, 0); +} + +int lsmMCursorLast(MultiCursor *pCsr){ + return multiCursorEnd(pCsr, 1); +} + +lsm_db *lsmMCursorDb(MultiCursor *pCsr){ + return pCsr->pDb; +} + +void lsmMCursorReset(MultiCursor *pCsr){ + int i; + lsmTreeCursorReset(pCsr->apTreeCsr[0]); + lsmTreeCursorReset(pCsr->apTreeCsr[1]); + for(i=0; inPtr; i++){ + segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); + } + pCsr->key.nData = 0; +} + +static int treeCursorSeek( + MultiCursor *pCsr, + TreeCursor *pTreeCsr, + void *pKey, int nKey, + int eSeek, + int *pbStop +){ + int rc = LSM_OK; + if( pTreeCsr ){ + int res = 0; + lsmTreeCursorSeek(pTreeCsr, pKey, nKey, &res); + switch( eSeek ){ + case LSM_SEEK_EQ: { + int eType = lsmTreeCursorFlags(pTreeCsr); + if( (res<0 && (eType & LSM_START_DELETE)) + || (res>0 && (eType & LSM_END_DELETE)) + || (res==0 && (eType & LSM_POINT_DELETE)) + ){ + *pbStop = 1; + }else if( res==0 && (eType & LSM_INSERT) ){ + lsm_env *pEnv = pCsr->pDb->pEnv; + void *p; int n; /* Key/value from tree-cursor */ + *pbStop = 1; + pCsr->flags |= CURSOR_SEEK_EQ; + rc = lsmTreeCursorKey(pTreeCsr, &pCsr->eType, &p, &n); + if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->key, p, n); + if( rc==LSM_OK ) rc = lsmTreeCursorValue(pTreeCsr, &p, &n); + if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->val, p, n); + } + lsmTreeCursorReset(pTreeCsr); + break; + } + case LSM_SEEK_GE: + if( res<0 && lsmTreeCursorValid(pTreeCsr) ){ + lsmTreeCursorNext(pTreeCsr); + } + break; + default: + if( res>0 ){ + assert( lsmTreeCursorValid(pTreeCsr) ); + lsmTreeCursorPrev(pTreeCsr); + } + break; + } + } + return rc; +} + + +/* +** Seek the cursor. +*/ +int lsmMCursorSeek( + MultiCursor *pCsr, + int iTopic, + void *pKey, int nKey, + int eSeek +){ + int eESeek = eSeek; /* Effective eSeek parameter */ + int bStop = 0; /* Set to true to halt search operation */ + int rc = LSM_OK; /* Return code */ + int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ + LsmPgno iPgno = 0; /* FC pointer value */ + + assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); + assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); + + if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; + + assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE ); + assert( (pCsr->flags & CURSOR_FLUSH_FREELIST)==0 ); + assert( pCsr->nPtr==0 || pCsr->aPtr[0].pLevel ); + + pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); + rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[0], pKey, nKey, eESeek, &bStop); + if( rc==LSM_OK && bStop==0 ){ + rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[1], pKey, nKey, eESeek, &bStop); + } + + /* Seek all segment pointers. */ + for(iPtr=0; iPtrnPtr && rc==LSM_OK && bStop==0; iPtr++){ + SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; + assert( pPtr->pSeg==&pPtr->pLevel->lhs ); + rc = seekInLevel(pCsr, pPtr, eESeek, iTopic, pKey, nKey, &iPgno, &bStop); + iPtr += pPtr->pLevel->nRight; + } + + if( eSeek!=LSM_SEEK_EQ ){ + if( rc==LSM_OK ){ + rc = multiCursorAllocTree(pCsr); + } + if( rc==LSM_OK ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, eESeek==LSM_SEEK_LE); + } + if( eSeek==LSM_SEEK_GE ) pCsr->flags |= CURSOR_NEXT_OK; + if( eSeek==LSM_SEEK_LE ) pCsr->flags |= CURSOR_PREV_OK; + } + + multiCursorCacheKey(pCsr, &rc); + if( rc==LSM_OK && eSeek!=LSM_SEEK_LEFAST && 0==mcursorLocationOk(pCsr, 0) ){ + switch( eESeek ){ + case LSM_SEEK_EQ: + lsmMCursorReset(pCsr); + break; + case LSM_SEEK_GE: + rc = lsmMCursorNext(pCsr); + break; + default: + rc = lsmMCursorPrev(pCsr); + break; + } + } + } + + return rc; +} + +int lsmMCursorValid(MultiCursor *pCsr){ + int res = 0; + if( pCsr->flags & CURSOR_SEEK_EQ ){ + res = 1; + }else if( pCsr->aTree ){ + int iKey = pCsr->aTree[1]; + if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ + res = lsmTreeCursorValid(pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]); + }else{ + void *pKey; + multiCursorGetKey(pCsr, iKey, 0, &pKey, 0); + res = pKey!=0; + } + } + return res; +} + +static int mcursorAdvanceOk( + MultiCursor *pCsr, + int bReverse, + int *pRc +){ + void *pNew; /* Pointer to buffer containing new key */ + int nNew; /* Size of buffer pNew in bytes */ + int eNewType; /* Type of new record */ + + if( *pRc ) return 1; + + /* Check the current key value. If it is not greater than (if bReverse==0) + ** or less than (if bReverse!=0) the key currently cached in pCsr->key, + ** then the cursor has not yet been successfully advanced. + */ + multiCursorGetKey(pCsr, pCsr->aTree[1], &eNewType, &pNew, &nNew); + if( pNew ){ + int typemask = (pCsr->flags & CURSOR_IGNORE_DELETE) ? ~(0) : LSM_SYSTEMKEY; + int res = sortedDbKeyCompare(pCsr, + eNewType & typemask, pNew, nNew, + pCsr->eType & typemask, pCsr->key.pData, pCsr->key.nData + ); + + if( (bReverse==0 && res<=0) || (bReverse!=0 && res>=0) ){ + return 0; + } + + multiCursorCacheKey(pCsr, pRc); + assert( pCsr->eType==eNewType ); + + /* If this cursor is configured to skip deleted keys, and the current + ** cursor points to a SORTED_DELETE entry, then the cursor has not been + ** successfully advanced. + ** + ** Similarly, if the cursor is configured to skip system keys and the + ** current cursor points to a system key, it has not yet been advanced. + */ + if( *pRc==LSM_OK && 0==mcursorLocationOk(pCsr, 0) ) return 0; + } + return 1; +} + +static void flCsrAdvance(MultiCursor *pCsr){ + assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); + if( pCsr->iFree % 2 ){ + pCsr->iFree++; + }else{ + int nEntry = pCsr->pDb->pWorker->freelist.nEntry; + FreelistEntry *aEntry = pCsr->pDb->pWorker->freelist.aEntry; + + int i = nEntry - 1 - (pCsr->iFree / 2); + + /* If the current entry is a delete and the "end-delete" key will not + ** be attached to the next entry, increment iFree by 1 only. */ + if( aEntry[i].iId<0 ){ + while( 1 ){ + if( i==0 || aEntry[i-1].iBlk!=aEntry[i].iBlk-1 ){ + pCsr->iFree--; + break; + } + if( aEntry[i-1].iId>=0 ) break; + pCsr->iFree += 2; + i--; + } + } + pCsr->iFree += 2; + } +} + +static int multiCursorAdvance(MultiCursor *pCsr, int bReverse){ + int rc = LSM_OK; /* Return Code */ + if( lsmMCursorValid(pCsr) ){ + do { + int iKey = pCsr->aTree[1]; + + assertCursorTree(pCsr); + + /* If this multi-cursor is advancing forwards, and the sub-cursor + ** being advanced is the one that separator keys may be being read + ** from, record the current absolute pointer value. */ + if( pCsr->pPrevMergePtr ){ + if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ + assert( pCsr->pBtCsr ); + *pCsr->pPrevMergePtr = pCsr->pBtCsr->iPtr; + }else if( pCsr->pBtCsr==0 && pCsr->nPtr>0 + && iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr-1) + ){ + SegmentPtr *pPtr = &pCsr->aPtr[iKey-CURSOR_DATA_SEGMENT]; + *pCsr->pPrevMergePtr = pPtr->iPtr+pPtr->iPgPtr; + } + } + + if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ + TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; + if( bReverse ){ + rc = lsmTreeCursorPrev(pTreeCsr); + }else{ + rc = lsmTreeCursorNext(pTreeCsr); + } + }else if( iKey==CURSOR_DATA_SYSTEM ){ + assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); + assert( bReverse==0 ); + flCsrAdvance(pCsr); + }else if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ + assert( bReverse==0 && pCsr->pBtCsr ); + rc = btreeCursorNext(pCsr->pBtCsr); + }else{ + rc = segmentCursorAdvance(pCsr, iKey-CURSOR_DATA_SEGMENT, bReverse); + } + if( rc==LSM_OK ){ + int i; + for(i=(iKey+pCsr->nTree)/2; i>0; i=i/2){ + multiCursorDoCompare(pCsr, i, bReverse); + } + assertCursorTree(pCsr); + } + }while( mcursorAdvanceOk(pCsr, bReverse, &rc)==0 ); + } + return rc; +} + +int lsmMCursorNext(MultiCursor *pCsr){ + if( (pCsr->flags & CURSOR_NEXT_OK)==0 ) return LSM_MISUSE_BKPT; + return multiCursorAdvance(pCsr, 0); +} + +int lsmMCursorPrev(MultiCursor *pCsr){ + if( (pCsr->flags & CURSOR_PREV_OK)==0 ) return LSM_MISUSE_BKPT; + return multiCursorAdvance(pCsr, 1); +} + +int lsmMCursorKey(MultiCursor *pCsr, void **ppKey, int *pnKey){ + if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ + *pnKey = pCsr->key.nData; + *ppKey = pCsr->key.pData; + }else{ + int iKey = pCsr->aTree[1]; + + if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ + TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; + lsmTreeCursorKey(pTreeCsr, 0, ppKey, pnKey); + }else{ + int nKey; + +#ifndef NDEBUG + void *pKey; + int eType; + multiCursorGetKey(pCsr, iKey, &eType, &pKey, &nKey); + assert( eType==pCsr->eType ); + assert( nKey==pCsr->key.nData ); + assert( memcmp(pKey, pCsr->key.pData, nKey)==0 ); +#endif + + nKey = pCsr->key.nData; + if( nKey==0 ){ + *ppKey = 0; + }else{ + *ppKey = pCsr->key.pData; + } + *pnKey = nKey; + } + } + return LSM_OK; +} + +/* +** Compare the current key that cursor csr points to with pKey/nKey. Set +** *piRes to the result and return LSM_OK. +*/ +int lsm_csr_cmp(lsm_cursor *csr, const void *pKey, int nKey, int *piRes){ + MultiCursor *pCsr = (MultiCursor *)csr; + void *pCsrkey; int nCsrkey; + int rc; + rc = lsmMCursorKey(pCsr, &pCsrkey, &nCsrkey); + if( rc==LSM_OK ){ + int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; + *piRes = sortedKeyCompare(xCmp, 0, pCsrkey, nCsrkey, 0, (void *)pKey, nKey); + } + return rc; +} + +int lsmMCursorValue(MultiCursor *pCsr, void **ppVal, int *pnVal){ + void *pVal; + int nVal; + int rc; + if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ + rc = LSM_OK; + nVal = pCsr->val.nData; + pVal = pCsr->val.pData; + }else{ + + assert( pCsr->aTree ); + assert( mcursorLocationOk(pCsr, (pCsr->flags & CURSOR_IGNORE_DELETE)) ); + + rc = multiCursorGetVal(pCsr, pCsr->aTree[1], &pVal, &nVal); + if( pVal && rc==LSM_OK ){ + rc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->val, pVal, nVal); + pVal = pCsr->val.pData; + } + + if( rc!=LSM_OK ){ + pVal = 0; + nVal = 0; + } + } + *ppVal = pVal; + *pnVal = nVal; + return rc; +} + +int lsmMCursorType(MultiCursor *pCsr, int *peType){ + assert( pCsr->aTree ); + multiCursorGetKey(pCsr, pCsr->aTree[1], peType, 0, 0); + return LSM_OK; +} + +/* +** Buffer aData[], size nData, is assumed to contain a valid b-tree +** hierarchy page image. Return the offset in aData[] of the next free +** byte in the data area (where a new cell may be written if there is +** space). +*/ +static int mergeWorkerPageOffset(u8 *aData, int nData){ + int nRec; + int iOff; + int nKey; + int eType; + i64 nDummy; + + + nRec = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); + iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec-1)]); + eType = aData[iOff++]; + assert( eType==0 + || eType==(LSM_SYSTEMKEY|LSM_SEPARATOR) + || eType==(LSM_SEPARATOR) + ); + + iOff += lsmVarintGet64(&aData[iOff], &nDummy); + iOff += lsmVarintGet32(&aData[iOff], &nKey); + + return iOff + (eType ? nKey : 0); +} + +/* +** Following a checkpoint operation, database pages that are part of the +** checkpointed state of the LSM are deemed read-only. This includes the +** right-most page of the b-tree hierarchy of any separators array under +** construction, and all pages between it and the b-tree root, inclusive. +** This is a problem, as when further pages are appended to the separators +** array, entries must be added to the indicated b-tree hierarchy pages. +** +** This function copies all such b-tree pages to new locations, so that +** they can be modified as required. +** +** The complication is that not all database pages are the same size - due +** to the way the file.c module works some (the first and last in each block) +** are 4 bytes smaller than the others. +*/ +static int mergeWorkerMoveHierarchy( + MergeWorker *pMW, /* Merge worker */ + int bSep /* True for separators run */ +){ + lsm_db *pDb = pMW->pDb; /* Database handle */ + int rc = LSM_OK; /* Return code */ + int i; + Page **apHier = pMW->hier.apHier; + int nHier = pMW->hier.nHier; + + for(i=0; rc==LSM_OK && ipFS, pDb->pWorker, pMW->pLevel, 1, &pNew); + assert( rc==LSM_OK ); + + if( rc==LSM_OK ){ + u8 *a1; int n1; + u8 *a2; int n2; + + a1 = fsPageData(pNew, &n1); + a2 = fsPageData(apHier[i], &n2); + + assert( n1==n2 || n1+4==n2 ); + + if( n1==n2 ){ + memcpy(a1, a2, n2); + }else{ + int nEntry = pageGetNRec(a2, n2); + int iEof1 = SEGMENT_EOF(n1, nEntry); + int iEof2 = SEGMENT_EOF(n2, nEntry); + + memcpy(a1, a2, iEof2 - 4); + memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); + } + + lsmFsPageRelease(apHier[i]); + apHier[i] = pNew; + +#if 0 + assert( n1==n2 || n1+4==n2 || n2+4==n1 ); + if( n1>=n2 ){ + /* If n1 (size of the new page) is equal to or greater than n2 (the + ** size of the old page), then copy the data into the new page. If + ** n1==n2, this could be done with a single memcpy(). However, + ** since sometimes n1>n2, the page content and footer must be copied + ** separately. */ + int nEntry = pageGetNRec(a2, n2); + int iEof1 = SEGMENT_EOF(n1, nEntry); + int iEof2 = SEGMENT_EOF(n2, nEntry); + memcpy(a1, a2, iEof2); + memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); + lsmFsPageRelease(apHier[i]); + apHier[i] = pNew; + }else{ + lsmPutU16(&a1[SEGMENT_FLAGS_OFFSET(n1)], SEGMENT_BTREE_FLAG); + lsmPutU16(&a1[SEGMENT_NRECORD_OFFSET(n1)], 0); + lsmPutU64(&a1[SEGMENT_POINTER_OFFSET(n1)], 0); + i = i - 1; + lsmFsPageRelease(pNew); + } +#endif + } + } + +#ifdef LSM_DEBUG + if( rc==LSM_OK ){ + for(i=0; ipLevel->lhs; + p = &pMW->hier; + + if( p->apHier==0 && pSeg->iRoot!=0 ){ + FileSystem *pFS = pMW->pDb->pFS; + lsm_env *pEnv = pMW->pDb->pEnv; + Page **apHier = 0; + int nHier = 0; + LsmPgno iPg = pSeg->iRoot; + + do { + Page *pPg = 0; + u8 *aData; + int nData; + int flags; + + rc = lsmFsDbPageGet(pFS, pSeg, iPg, &pPg); + if( rc!=LSM_OK ) break; + + aData = fsPageData(pPg, &nData); + flags = pageGetFlags(aData, nData); + if( flags&SEGMENT_BTREE_FLAG ){ + Page **apNew = (Page **)lsmRealloc( + pEnv, apHier, sizeof(Page *)*(nHier+1) + ); + if( apNew==0 ){ + rc = LSM_NOMEM_BKPT; + break; + } + apHier = apNew; + memmove(&apHier[1], &apHier[0], sizeof(Page *) * nHier); + nHier++; + + apHier[0] = pPg; + iPg = pageGetPtr(aData, nData); + }else{ + lsmFsPageRelease(pPg); + break; + } + }while( 1 ); + + if( rc==LSM_OK ){ + u8 *aData; + int nData; + aData = fsPageData(apHier[0], &nData); + pMW->aSave[0].iPgno = pageGetPtr(aData, nData); + p->nHier = nHier; + p->apHier = apHier; + rc = mergeWorkerMoveHierarchy(pMW, 0); + }else{ + int i; + for(i=0; ihier; + lsm_db *pDb = pMW->pDb; /* Database handle */ + int rc = LSM_OK; /* Return Code */ + int iLevel; /* Level of b-tree hierachy to write to */ + int nData; /* Size of aData[] in bytes */ + u8 *aData; /* Page data for level iLevel */ + int iOff; /* Offset on b-tree page to write record to */ + int nRec; /* Initial number of records on b-tree page */ + + /* iKeyPg should be zero for an ordinary b-tree key, or non-zero for an + ** indirect key. The flags byte for an indirect key is 0x00. */ + assert( (eType==0)==(iKeyPg!=0) ); + + /* The MergeWorker.apHier[] array contains the right-most leaf of the b-tree + ** hierarchy, the root node, and all nodes that lie on the path between. + ** apHier[0] is the right-most leaf and apHier[pMW->nHier-1] is the current + ** root page. + ** + ** This loop searches for a node with enough space to store the key on, + ** starting with the leaf and iterating up towards the root. When the loop + ** exits, the key may be written to apHier[iLevel]. */ + for(iLevel=0; iLevel<=p->nHier; iLevel++){ + int nByte; /* Number of free bytes required */ + + if( iLevel==p->nHier ){ + /* Extend the array and allocate a new root page. */ + Page **aNew; + aNew = (Page **)lsmRealloc( + pMW->pDb->pEnv, p->apHier, sizeof(Page *)*(p->nHier+1) + ); + if( !aNew ){ + return LSM_NOMEM_BKPT; + } + p->apHier = aNew; + }else{ + Page *pOld; + int nFree; + + /* If the key will fit on this page, break out of the loop here. + ** The new entry will be written to page apHier[iLevel]. */ + pOld = p->apHier[iLevel]; + assert( lsmFsPageWritable(pOld) ); + aData = fsPageData(pOld, &nData); + if( eType==0 ){ + nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen64(iKeyPg); + }else{ + nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen32(nKey) + nKey; + } + + nRec = pageGetNRec(aData, nData); + nFree = SEGMENT_EOF(nData, nRec) - mergeWorkerPageOffset(aData, nData); + if( nByte<=nFree ) break; + + /* Otherwise, this page is full. Set the right-hand-child pointer + ** to iPtr and release it. */ + lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); + assert( lsmFsPageNumber(pOld)==0 ); + rc = lsmFsPagePersist(pOld); + if( rc==LSM_OK ){ + iPtr = lsmFsPageNumber(pOld); + lsmFsPageRelease(pOld); + } + } + + /* Allocate a new page for apHier[iLevel]. */ + p->apHier[iLevel] = 0; + if( rc==LSM_OK ){ + rc = lsmFsSortedAppend( + pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] + ); + } + if( rc!=LSM_OK ) return rc; + + aData = fsPageData(p->apHier[iLevel], &nData); + memset(aData, 0, nData); + lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], SEGMENT_BTREE_FLAG); + lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); + + if( iLevel==p->nHier ){ + p->nHier++; + break; + } + } + + /* Write the key into page apHier[iLevel]. */ + aData = fsPageData(p->apHier[iLevel], &nData); + iOff = mergeWorkerPageOffset(aData, nData); + nRec = pageGetNRec(aData, nData); + lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); + lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); + if( eType==0 ){ + aData[iOff++] = 0x00; + iOff += lsmVarintPut64(&aData[iOff], iPtr); + iOff += lsmVarintPut64(&aData[iOff], iKeyPg); + }else{ + aData[iOff++] = eType; + iOff += lsmVarintPut64(&aData[iOff], iPtr); + iOff += lsmVarintPut32(&aData[iOff], nKey); + memcpy(&aData[iOff], pKey, nKey); + } + + return rc; +} + +static int mergeWorkerBtreeIndirect(MergeWorker *pMW){ + int rc = LSM_OK; + if( pMW->iIndirect ){ + LsmPgno iKeyPg = pMW->aSave[1].iPgno; + rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0); + pMW->iIndirect = 0; + } + return rc; +} + +/* +** Append the database key (iTopic/pKey/nKey) to the b-tree under +** construction. This key has not yet been written to a segment page. +** The pointer that will accompany the new key in the b-tree - that +** points to the completed segment page that contains keys smaller than +** (pKey/nKey) is currently stored in pMW->aSave[0].iPgno. +*/ +static int mergeWorkerPushHierarchy( + MergeWorker *pMW, /* Merge worker object */ + int iTopic, /* Topic value for this key */ + void *pKey, /* Pointer to key buffer */ + int nKey /* Size of pKey buffer in bytes */ +){ + int rc = LSM_OK; /* Return Code */ + LsmPgno iPtr; /* Pointer value to accompany pKey/nKey */ + + assert( pMW->aSave[0].bStore==0 ); + assert( pMW->aSave[1].bStore==0 ); + rc = mergeWorkerBtreeIndirect(pMW); + + /* Obtain the absolute pointer value to store along with the key in the + ** page body. This pointer points to a page that contains keys that are + ** smaller than pKey/nKey. */ + iPtr = pMW->aSave[0].iPgno; + assert( iPtr!=0 ); + + /* Determine if the indirect format should be used. */ + if( (nKey*4 > lsmFsPageSize(pMW->pDb->pFS)) ){ + pMW->iIndirect = iPtr; + pMW->aSave[1].bStore = 1; + }else{ + rc = mergeWorkerBtreeWrite( + pMW, (u8)(iTopic | LSM_SEPARATOR), iPtr, 0, pKey, nKey + ); + } + + /* Ensure that the SortedRun.iRoot field is correct. */ + return rc; +} + +static int mergeWorkerFinishHierarchy( + MergeWorker *pMW /* Merge worker object */ +){ + int i; /* Used to loop through apHier[] */ + int rc = LSM_OK; /* Return code */ + LsmPgno iPtr; /* New right-hand-child pointer value */ + + iPtr = pMW->aSave[0].iPgno; + for(i=0; ihier.nHier && rc==LSM_OK; i++){ + Page *pPg = pMW->hier.apHier[i]; + int nData; /* Size of aData[] in bytes */ + u8 *aData; /* Page data for pPg */ + + aData = fsPageData(pPg, &nData); + lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); + + rc = lsmFsPagePersist(pPg); + iPtr = lsmFsPageNumber(pPg); + lsmFsPageRelease(pPg); + } + + if( pMW->hier.nHier ){ + pMW->pLevel->lhs.iRoot = iPtr; + lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); + pMW->hier.apHier = 0; + pMW->hier.nHier = 0; + } + + return rc; +} + +static int mergeWorkerAddPadding( + MergeWorker *pMW /* Merge worker object */ +){ + FileSystem *pFS = pMW->pDb->pFS; + return lsmFsSortedPadding(pFS, pMW->pDb->pWorker, &pMW->pLevel->lhs); +} + +/* +** Release all page references currently held by the merge-worker passed +** as the only argument. Unless an error has occurred, all pages have +** already been released. +*/ +static void mergeWorkerReleaseAll(MergeWorker *pMW){ + int i; + lsmFsPageRelease(pMW->pPage); + pMW->pPage = 0; + + for(i=0; ihier.nHier; i++){ + lsmFsPageRelease(pMW->hier.apHier[i]); + pMW->hier.apHier[i] = 0; + } + lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); + pMW->hier.apHier = 0; + pMW->hier.nHier = 0; +} + +static int keyszToSkip(FileSystem *pFS, int nKey){ + int nPgsz; /* Nominal database page size */ + nPgsz = lsmFsPageSize(pFS); + return LSM_MIN(((nKey * 4) / nPgsz), 3); +} + +/* +** Release the reference to the current output page of merge-worker *pMW +** (reference pMW->pPage). Set the page number values in aSave[] as +** required (see comments above struct MergeWorker for details). +*/ +static int mergeWorkerPersistAndRelease(MergeWorker *pMW){ + int rc; + int i; + + assert( pMW->pPage || (pMW->aSave[0].bStore==0 && pMW->aSave[1].bStore==0) ); + + /* Persist the page */ + rc = lsmFsPagePersist(pMW->pPage); + + /* If required, save the page number. */ + for(i=0; i<2; i++){ + if( pMW->aSave[i].bStore ){ + pMW->aSave[i].iPgno = lsmFsPageNumber(pMW->pPage); + pMW->aSave[i].bStore = 0; + } + } + + /* Release the completed output page. */ + lsmFsPageRelease(pMW->pPage); + pMW->pPage = 0; + return rc; +} + +/* +** Advance to the next page of an output run being populated by merge-worker +** pMW. The footer of the new page is initialized to indicate that it contains +** zero records. The flags field is cleared. The page footer pointer field +** is set to iFPtr. +** +** If successful, LSM_OK is returned. Otherwise, an error code. +*/ +static int mergeWorkerNextPage( + MergeWorker *pMW, /* Merge worker object to append page to */ + LsmPgno iFPtr /* Pointer value for footer of new page */ +){ + int rc = LSM_OK; /* Return code */ + Page *pNext = 0; /* New page appended to run */ + lsm_db *pDb = pMW->pDb; /* Database handle */ + + rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); + assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); + + if( rc==LSM_OK ){ + u8 *aData; /* Data buffer belonging to page pNext */ + int nData; /* Size of aData[] in bytes */ + + rc = mergeWorkerPersistAndRelease(pMW); + + pMW->pPage = pNext; + pMW->pLevel->pMerge->iOutputOff = 0; + aData = fsPageData(pNext, &nData); + lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); + lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], 0); + lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iFPtr); + pMW->nWork++; + } + + return rc; +} + +/* +** Write a blob of data into an output segment being populated by a +** merge-worker object. If argument bSep is true, write into the separators +** array. Otherwise, the main array. +** +** This function is used to write the blobs of data for keys and values. +*/ +static int mergeWorkerData( + MergeWorker *pMW, /* Merge worker object */ + int bSep, /* True to write to separators run */ + LsmPgno iFPtr, /* Footer ptr for new pages */ + u8 *aWrite, /* Write data from this buffer */ + int nWrite /* Size of aWrite[] in bytes */ +){ + int rc = LSM_OK; /* Return code */ + int nRem = nWrite; /* Number of bytes still to write */ + + while( rc==LSM_OK && nRem>0 ){ + Merge *pMerge = pMW->pLevel->pMerge; + int nCopy; /* Number of bytes to copy */ + u8 *aData; /* Pointer to buffer of current output page */ + int nData; /* Size of aData[] in bytes */ + int nRec; /* Number of records on current output page */ + int iOff; /* Offset in aData[] to write to */ + + assert( lsmFsPageWritable(pMW->pPage) ); + + aData = fsPageData(pMW->pPage, &nData); + nRec = pageGetNRec(aData, nData); + iOff = pMerge->iOutputOff; + nCopy = LSM_MIN(nRem, SEGMENT_EOF(nData, nRec) - iOff); + + memcpy(&aData[iOff], &aWrite[nWrite-nRem], nCopy); + nRem -= nCopy; + + if( nRem>0 ){ + rc = mergeWorkerNextPage(pMW, iFPtr); + }else{ + pMerge->iOutputOff = iOff + nCopy; + } + } + + return rc; +} + + +/* +** The MergeWorker passed as the only argument is working to merge two or +** more existing segments together (not to flush an in-memory tree). It +** has not yet written the first key to the first page of the output. +*/ +static int mergeWorkerFirstPage(MergeWorker *pMW){ + int rc = LSM_OK; /* Return code */ + Page *pPg = 0; /* First page of run pSeg */ + LsmPgno iFPtr = 0; /* Pointer value read from footer of pPg */ + MultiCursor *pCsr = pMW->pCsr; + + assert( pMW->pPage==0 ); + + if( pCsr->pBtCsr ){ + rc = LSM_OK; + iFPtr = pMW->pLevel->pNext->lhs.iFirst; + }else if( pCsr->nPtr>0 ){ + Segment *pSeg; + pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg; + rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg, pSeg->iFirst, &pPg); + if( rc==LSM_OK ){ + u8 *aData; /* Buffer for page pPg */ + int nData; /* Size of aData[] in bytes */ + aData = fsPageData(pPg, &nData); + iFPtr = pageGetPtr(aData, nData); + lsmFsPageRelease(pPg); + } + } + + if( rc==LSM_OK ){ + rc = mergeWorkerNextPage(pMW, iFPtr); + if( pCsr->pPrevMergePtr ) *pCsr->pPrevMergePtr = iFPtr; + pMW->aSave[0].bStore = 1; + } + + return rc; +} + +static int mergeWorkerWrite( + MergeWorker *pMW, /* Merge worker object to write into */ + int eType, /* One of SORTED_SEPARATOR, WRITE or DELETE */ + void *pKey, int nKey, /* Key value */ + void *pVal, int nVal, /* Value value */ + LsmPgno iPtr /* Absolute value of page pointer, or 0 */ +){ + int rc = LSM_OK; /* Return code */ + Merge *pMerge; /* Persistent part of level merge state */ + int nHdr; /* Space required for this record header */ + Page *pPg; /* Page to write to */ + u8 *aData; /* Data buffer for page pWriter->pPage */ + int nData = 0; /* Size of buffer aData[] in bytes */ + int nRec = 0; /* Number of records on page pPg */ + LsmPgno iFPtr = 0; /* Value of pointer in footer of pPg */ + LsmPgno iRPtr = 0; /* Value of pointer written into record */ + int iOff = 0; /* Current write offset within page pPg */ + Segment *pSeg; /* Segment being written */ + int flags = 0; /* If != 0, flags value for page footer */ + int bFirst = 0; /* True for first key of output run */ + + pMerge = pMW->pLevel->pMerge; + pSeg = &pMW->pLevel->lhs; + + if( pSeg->iFirst==0 && pMW->pPage==0 ){ + rc = mergeWorkerFirstPage(pMW); + bFirst = 1; + } + pPg = pMW->pPage; + if( pPg ){ + aData = fsPageData(pPg, &nData); + nRec = pageGetNRec(aData, nData); + iFPtr = pageGetPtr(aData, nData); + iRPtr = iPtr ? (iPtr - iFPtr) : 0; + } + + /* Figure out how much space is required by the new record. The space + ** required is divided into two sections: the header and the body. The + ** header consists of the intial varint fields. The body are the blobs + ** of data that correspond to the key and value data. The entire header + ** must be stored on the page. The body may overflow onto the next and + ** subsequent pages. + ** + ** The header space is: + ** + ** 1) record type - 1 byte. + ** 2) Page-pointer-offset - 1 varint + ** 3) Key size - 1 varint + ** 4) Value size - 1 varint (only if LSM_INSERT flag is set) + */ + if( rc==LSM_OK ){ + nHdr = 1 + lsmVarintLen64(iRPtr) + lsmVarintLen32(nKey); + if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal); + + /* If the entire header will not fit on page pPg, or if page pPg is + ** marked read-only, advance to the next page of the output run. */ + iOff = pMerge->iOutputOff; + if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ + if( iOff>=0 && pPg ){ + /* Zero any free space on the page */ + assert( aData ); + memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff); + } + iFPtr = *pMW->pCsr->pPrevMergePtr; + iRPtr = iPtr ? (iPtr - iFPtr) : 0; + iOff = 0; + nRec = 0; + rc = mergeWorkerNextPage(pMW, iFPtr); + pPg = pMW->pPage; + } + } + + /* If this record header will be the first on the page, and the page is + ** not the very first in the entire run, add a copy of the key to the + ** b-tree hierarchy. + */ + if( rc==LSM_OK && nRec==0 && bFirst==0 ){ + assert( pMerge->nSkip>=0 ); + + if( pMerge->nSkip==0 ){ + rc = mergeWorkerPushHierarchy(pMW, rtTopic(eType), pKey, nKey); + assert( pMW->aSave[0].bStore==0 ); + pMW->aSave[0].bStore = 1; + pMerge->nSkip = keyszToSkip(pMW->pDb->pFS, nKey); + }else{ + pMerge->nSkip--; + flags = PGFTR_SKIP_THIS_FLAG; + } + + if( pMerge->nSkip ) flags |= PGFTR_SKIP_NEXT_FLAG; + } + + /* Update the output segment */ + if( rc==LSM_OK ){ + aData = fsPageData(pPg, &nData); + + /* Update the page footer. */ + lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); + lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); + if( flags ) lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], (u16)flags); + + /* Write the entry header into the current page. */ + aData[iOff++] = (u8)eType; /* 1 */ + iOff += lsmVarintPut64(&aData[iOff], iRPtr); /* 2 */ + iOff += lsmVarintPut32(&aData[iOff], nKey); /* 3 */ + if( rtIsWrite(eType) ) iOff += lsmVarintPut32(&aData[iOff], nVal); /* 4 */ + pMerge->iOutputOff = iOff; + + /* Write the key and data into the segment. */ + assert( iFPtr==pageGetPtr(aData, nData) ); + rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pKey, nKey); + if( rc==LSM_OK && rtIsWrite(eType) ){ + if( rc==LSM_OK ){ + rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pVal, nVal); + } + } + } + + return rc; +} + + +/* +** Free all resources allocated by mergeWorkerInit(). +*/ +static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ + int i; /* Iterator variable */ + int rc = *pRc; + MultiCursor *pCsr = pMW->pCsr; + + /* Unless the merge has finished, save the cursor position in the + ** Merge.aInput[] array. See function mergeWorkerInit() for the + ** code to restore a cursor position based on aInput[]. */ + if( rc==LSM_OK && pCsr ){ + Merge *pMerge = pMW->pLevel->pMerge; + if( lsmMCursorValid(pCsr) ){ + int bBtree = (pCsr->pBtCsr!=0); + int iPtr; + + /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ + assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); + assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); + + for(i=0; i<(pMerge->nInput-bBtree); i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + if( pPtr->pPg ){ + pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); + pMerge->aInput[i].iCell = pPtr->iCell; + }else{ + pMerge->aInput[i].iPg = 0; + pMerge->aInput[i].iCell = 0; + } + } + if( bBtree && pMerge->nInput ){ + assert( i==pCsr->nPtr ); + btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); + } + + /* Store the location of the split-key */ + iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; + if( iPtrnPtr ){ + pMerge->splitkey = pMerge->aInput[iPtr]; + }else{ + btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); + } + } + + /* Zero any free space left on the final page. This helps with + ** compression if using a compression hook. And prevents valgrind + ** from complaining about uninitialized byte passed to write(). */ + if( pMW->pPage ){ + int nData; + u8 *aData = fsPageData(pMW->pPage, &nData); + int iOff = pMerge->iOutputOff; + int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData)); + memset(&aData[iOff], 0, iEof - iOff); + } + + pMerge->iOutputOff = -1; + } + + lsmMCursorClose(pCsr, 0); + + /* Persist and release the output page. */ + if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); + if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); + if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); + if( rc==LSM_OK ) rc = mergeWorkerAddPadding(pMW); + lsmFsFlushWaiting(pMW->pDb->pFS, &rc); + mergeWorkerReleaseAll(pMW); + + lsmFree(pMW->pDb->pEnv, pMW->aGobble); + pMW->aGobble = 0; + pMW->pCsr = 0; + + *pRc = rc; +} + +/* +** The cursor passed as the first argument is being used as the input for +** a merge operation. When this function is called, *piFlags contains the +** database entry flags for the current entry. The entry about to be written +** to the output. +** +** Note that this function only has to work for cursors configured to +** iterate forwards (not backwards). +*/ +static void mergeRangeDeletes(MultiCursor *pCsr, int *piVal, int *piFlags){ + int f = *piFlags; + int iKey = pCsr->aTree[1]; + int i; + + assert( pCsr->flags & CURSOR_NEXT_OK ); + if( pCsr->flags & CURSOR_IGNORE_DELETE ){ + /* The ignore-delete flag is set when the output of the merge will form + ** the oldest level in the database. In this case there is no point in + ** retaining any range-delete flags. */ + assert( (f & LSM_POINT_DELETE)==0 ); + f &= ~(LSM_START_DELETE|LSM_END_DELETE); + }else{ + for(i=0; i<(CURSOR_DATA_SEGMENT + pCsr->nPtr); i++){ + if( i!=iKey ){ + int eType; + void *pKey; + int nKey; + int res; + multiCursorGetKey(pCsr, i, &eType, &pKey, &nKey); + + if( pKey ){ + res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, + rtTopic(eType), pKey, nKey + ); + assert( res<=0 ); + if( res==0 ){ + if( (f & (LSM_INSERT|LSM_POINT_DELETE))==0 ){ + if( eType & LSM_INSERT ){ + f |= LSM_INSERT; + *piVal = i; + } + else if( eType & LSM_POINT_DELETE ){ + f |= LSM_POINT_DELETE; + } + } + f |= (eType & (LSM_END_DELETE|LSM_START_DELETE)); + } + + if( i>iKey && (eType & LSM_END_DELETE) && res<0 ){ + if( f & (LSM_INSERT|LSM_POINT_DELETE) ){ + f |= (LSM_END_DELETE|LSM_START_DELETE); + }else{ + f = 0; + } + break; + } + } + } + } + + assert( (f & LSM_INSERT)==0 || (f & LSM_POINT_DELETE)==0 ); + if( (f & LSM_START_DELETE) + && (f & LSM_END_DELETE) + && (f & LSM_POINT_DELETE ) + ){ + f = 0; + } + } + + *piFlags = f; +} + +static int mergeWorkerStep(MergeWorker *pMW){ + lsm_db *pDb = pMW->pDb; /* Database handle */ + MultiCursor *pCsr; /* Cursor to read input data from */ + int rc = LSM_OK; /* Return code */ + int eType; /* SORTED_SEPARATOR, WRITE or DELETE */ + void *pKey; int nKey; /* Key */ + LsmPgno iPtr; + int iVal; + + pCsr = pMW->pCsr; + + /* Pull the next record out of the source cursor. */ + lsmMCursorKey(pCsr, &pKey, &nKey); + eType = pCsr->eType; + + /* Figure out if the output record may have a different pointer value + ** than the previous. This is the case if the current key is identical to + ** a key that appears in the lowest level run being merged. If so, set + ** iPtr to the absolute pointer value. If not, leave iPtr set to zero, + ** indicating that the output pointer value should be a copy of the pointer + ** value written with the previous key. */ + iPtr = (pCsr->pPrevMergePtr ? *pCsr->pPrevMergePtr : 0); + if( pCsr->pBtCsr ){ + BtreeCursor *pBtCsr = pCsr->pBtCsr; + if( pBtCsr->pKey ){ + int res = rtTopic(pBtCsr->eType) - rtTopic(eType); + if( res==0 ) res = pDb->xCmp(pBtCsr->pKey, pBtCsr->nKey, pKey, nKey); + if( 0==res ) iPtr = pBtCsr->iPtr; + assert( res>=0 ); + } + }else if( pCsr->nPtr ){ + SegmentPtr *pPtr = &pCsr->aPtr[pCsr->nPtr-1]; + if( pPtr->pPg + && 0==pDb->xCmp(pPtr->pKey, pPtr->nKey, pKey, nKey) + ){ + iPtr = pPtr->iPtr+pPtr->iPgPtr; + } + } + + iVal = pCsr->aTree[1]; + mergeRangeDeletes(pCsr, &iVal, &eType); + + if( eType!=0 ){ + if( pMW->aGobble ){ + int iGobble = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; + if( iGobblenPtr && iGobble>=0 ){ + SegmentPtr *pGobble = &pCsr->aPtr[iGobble]; + if( (pGobble->flags & PGFTR_SKIP_THIS_FLAG)==0 ){ + pMW->aGobble[iGobble] = lsmFsPageNumber(pGobble->pPg); + } + } + } + + /* If this is a separator key and we know that the output pointer has not + ** changed, there is no point in writing an output record. Otherwise, + ** proceed. */ + if( rc==LSM_OK && (rtIsSeparator(eType)==0 || iPtr!=0) ){ + /* Write the record into the main run. */ + void *pVal; int nVal; + rc = multiCursorGetVal(pCsr, iVal, &pVal, &nVal); + if( pVal && rc==LSM_OK ){ + assert( nVal>=0 ); + rc = sortedBlobSet(pDb->pEnv, &pCsr->val, pVal, nVal); + pVal = pCsr->val.pData; + } + if( rc==LSM_OK ){ + rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, iPtr); + } + } + } + + /* Advance the cursor to the next input record (assuming one exists). */ + assert( lsmMCursorValid(pMW->pCsr) ); + if( rc==LSM_OK ) rc = lsmMCursorNext(pMW->pCsr); + + return rc; +} + +static int mergeWorkerDone(MergeWorker *pMW){ + return pMW->pCsr==0 || !lsmMCursorValid(pMW->pCsr); +} + +static void sortedFreeLevel(lsm_env *pEnv, Level *p){ + if( p ){ + lsmFree(pEnv, p->pSplitKey); + lsmFree(pEnv, p->pMerge); + lsmFree(pEnv, p->aRhs); + lsmFree(pEnv, p); + } +} + +static void sortedInvokeWorkHook(lsm_db *pDb){ + if( pDb->xWork ){ + pDb->xWork(pDb, pDb->pWorkCtx); + } +} + +static int sortedNewToplevel( + lsm_db *pDb, /* Connection handle */ + int eTree, /* One of the TREE_XXX constants */ + int *pnWrite /* OUT: Number of database pages written */ +){ + int rc = LSM_OK; /* Return Code */ + MultiCursor *pCsr = 0; + Level *pNext = 0; /* The current top level */ + Level *pNew; /* The new level itself */ + Segment *pLinked = 0; /* Delete separators from this segment */ + Level *pDel = 0; /* Delete this entire level */ + int nWrite = 0; /* Number of database pages written */ + Freelist freelist; + + if( eTree!=TREE_NONE ){ + rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); + } + + assert( pDb->bUseFreelist==0 ); + pDb->pFreelist = &freelist; + pDb->bUseFreelist = 1; + memset(&freelist, 0, sizeof(freelist)); + + /* Allocate the new level structure to write to. */ + pNext = lsmDbSnapshotLevel(pDb->pWorker); + pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); + if( pNew ){ + pNew->pNext = pNext; + lsmDbSnapshotSetLevel(pDb->pWorker, pNew); + } + + /* Create a cursor to gather the data required by the new segment. The new + ** segment contains everything in the tree and pointers to the next segment + ** in the database (if any). */ + pCsr = multiCursorNew(pDb, &rc); + if( pCsr ){ + pCsr->pDb = pDb; + rc = multiCursorVisitFreelist(pCsr); + if( rc==LSM_OK ){ + rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); + } + if( rc==LSM_OK && pNext && pNext->pMerge==0 ){ + if( (pNext->flags & LEVEL_FREELIST_ONLY) ){ + pDel = pNext; + pCsr->aPtr = lsmMallocZeroRc(pDb->pEnv, sizeof(SegmentPtr), &rc); + multiCursorAddOne(pCsr, pNext, &rc); + }else if( eTree!=TREE_NONE && pNext->lhs.iRoot ){ + pLinked = &pNext->lhs; + rc = btreeCursorNew(pDb, pLinked, &pCsr->pBtCsr); + } + } + + /* If this will be the only segment in the database, discard any delete + ** markers present in the in-memory tree. */ + if( pNext==0 ){ + multiCursorIgnoreDelete(pCsr); + } + } + + if( rc!=LSM_OK ){ + lsmMCursorClose(pCsr, 0); + }else{ + LsmPgno iLeftPtr = 0; + Merge merge; /* Merge object used to create new level */ + MergeWorker mergeworker; /* MergeWorker object for the same purpose */ + + memset(&merge, 0, sizeof(Merge)); + memset(&mergeworker, 0, sizeof(MergeWorker)); + + pNew->pMerge = &merge; + pNew->flags |= LEVEL_INCOMPLETE; + mergeworker.pDb = pDb; + mergeworker.pLevel = pNew; + mergeworker.pCsr = pCsr; + pCsr->pPrevMergePtr = &iLeftPtr; + + /* Mark the separators array for the new level as a "phantom". */ + mergeworker.bFlush = 1; + + /* Do the work to create the new merged segment on disk */ + if( rc==LSM_OK ) rc = lsmMCursorFirst(pCsr); + while( rc==LSM_OK && mergeWorkerDone(&mergeworker)==0 ){ + rc = mergeWorkerStep(&mergeworker); + } + mergeWorkerShutdown(&mergeworker, &rc); + assert( rc!=LSM_OK || mergeworker.nWork==0 || pNew->lhs.iFirst ); + if( rc==LSM_OK && pNew->lhs.iFirst ){ + rc = lsmFsSortedFinish(pDb->pFS, &pNew->lhs); + } + nWrite = mergeworker.nWork; + pNew->flags &= ~LEVEL_INCOMPLETE; + if( eTree==TREE_NONE ){ + pNew->flags |= LEVEL_FREELIST_ONLY; + } + pNew->pMerge = 0; + } + + if( rc!=LSM_OK || pNew->lhs.iFirst==0 ){ + assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); + lsmDbSnapshotSetLevel(pDb->pWorker, pNext); + sortedFreeLevel(pDb->pEnv, pNew); + }else{ + if( pLinked ){ + pLinked->iRoot = 0; + }else if( pDel ){ + assert( pNew->pNext==pDel ); + pNew->pNext = pDel->pNext; + lsmFsSortedDelete(pDb->pFS, pDb->pWorker, 1, &pDel->lhs); + sortedFreeLevel(pDb->pEnv, pDel); + } + +#if LSM_LOG_STRUCTURE + lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "new-toplevel"); +#endif + + if( freelist.nEntry ){ + Freelist *p = &pDb->pWorker->freelist; + lsmFree(pDb->pEnv, p->aEntry); + memcpy(p, &freelist, sizeof(freelist)); + freelist.aEntry = 0; + }else{ + pDb->pWorker->freelist.nEntry = 0; + } + + assertBtreeOk(pDb, &pNew->lhs); + sortedInvokeWorkHook(pDb); + } + + if( pnWrite ) *pnWrite = nWrite; + pDb->pWorker->nWrite += nWrite; + pDb->pFreelist = 0; + pDb->bUseFreelist = 0; + lsmFree(pDb->pEnv, freelist.aEntry); + return rc; +} + +/* +** The nMerge levels in the LSM beginning with pLevel consist of a +** left-hand-side segment only. Replace these levels with a single new +** level consisting of a new empty segment on the left-hand-side and the +** nMerge segments from the replaced levels on the right-hand-side. +** +** Also, allocate and populate a Merge object and set Level.pMerge to +** point to it. +*/ +static int sortedMergeSetup( + lsm_db *pDb, /* Database handle */ + Level *pLevel, /* First level to merge */ + int nMerge, /* Merge this many levels together */ + Level **ppNew /* New, merged, level */ +){ + int rc = LSM_OK; /* Return Code */ + Level *pNew; /* New Level object */ + int bUseNext = 0; /* True to link in next separators */ + Merge *pMerge; /* New Merge object */ + int nByte; /* Bytes of space allocated at pMerge */ + +#ifdef LSM_DEBUG + int iLevel; + Level *pX = pLevel; + for(iLevel=0; iLevelnRight==0 ); + pX = pX->pNext; + } +#endif + + /* Allocate the new Level object */ + pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); + if( pNew ){ + pNew->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, + nMerge * sizeof(Segment), &rc); + } + + /* Populate the new Level object */ + if( rc==LSM_OK ){ + Level *pNext = 0; /* Level following pNew */ + int i; + int bFreeOnly = 1; + Level *pTopLevel; + Level *p = pLevel; + Level **pp; + pNew->nRight = nMerge; + pNew->iAge = pLevel->iAge+1; + for(i=0; inRight==0 ); + pNext = p->pNext; + pNew->aRhs[i] = p->lhs; + if( (p->flags & LEVEL_FREELIST_ONLY)==0 ) bFreeOnly = 0; + sortedFreeLevel(pDb->pEnv, p); + p = pNext; + } + + if( bFreeOnly ) pNew->flags |= LEVEL_FREELIST_ONLY; + + /* Replace the old levels with the new. */ + pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); + pNew->pNext = p; + for(pp=&pTopLevel; *pp!=pLevel; pp=&((*pp)->pNext)); + *pp = pNew; + lsmDbSnapshotSetLevel(pDb->pWorker, pTopLevel); + + /* Determine whether or not the next separators will be linked in */ + if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot && pNext + && (bFreeOnly==0 || (pNext->flags & LEVEL_FREELIST_ONLY)) + ){ + bUseNext = 1; + } + } + + /* Allocate the merge object */ + nByte = sizeof(Merge) + sizeof(MergeInput) * (nMerge + bUseNext); + pMerge = (Merge *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); + if( pMerge ){ + pMerge->aInput = (MergeInput *)&pMerge[1]; + pMerge->nInput = nMerge + bUseNext; + pNew->pMerge = pMerge; + } + + *ppNew = pNew; + return rc; +} + +static int mergeWorkerInit( + lsm_db *pDb, /* Db connection to do merge work */ + Level *pLevel, /* Level to work on merging */ + MergeWorker *pMW /* Object to initialize */ +){ + int rc = LSM_OK; /* Return code */ + Merge *pMerge = pLevel->pMerge; /* Persistent part of merge state */ + MultiCursor *pCsr = 0; /* Cursor opened for pMW */ + Level *pNext = pLevel->pNext; /* Next level in LSM */ + + assert( pDb->pWorker ); + assert( pLevel->pMerge ); + assert( pLevel->nRight>0 ); + + memset(pMW, 0, sizeof(MergeWorker)); + pMW->pDb = pDb; + pMW->pLevel = pLevel; + pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc); + + /* Create a multi-cursor to read the data to write to the new + ** segment. The new segment contains: + ** + ** 1. Records from LHS of each of the nMerge levels being merged. + ** 2. Separators from either the last level being merged, or the + ** separators attached to the LHS of the following level, or neither. + ** + ** If the new level is the lowest (oldest) in the db, discard any + ** delete keys. Key annihilation. + */ + pCsr = multiCursorNew(pDb, &rc); + if( pCsr ){ + pCsr->flags |= CURSOR_NEXT_OK; + rc = multiCursorAddRhs(pCsr, pLevel); + } + if( rc==LSM_OK && pMerge->nInput > pLevel->nRight ){ + rc = btreeCursorNew(pDb, &pNext->lhs, &pCsr->pBtCsr); + }else if( pNext ){ + multiCursorReadSeparators(pCsr); + }else{ + multiCursorIgnoreDelete(pCsr); + } + + assert( rc!=LSM_OK || pMerge->nInput==(pCsr->nPtr+(pCsr->pBtCsr!=0)) ); + pMW->pCsr = pCsr; + + /* Load the b-tree hierarchy into memory. */ + if( rc==LSM_OK ) rc = mergeWorkerLoadHierarchy(pMW); + if( rc==LSM_OK && pMW->hier.nHier==0 ){ + pMW->aSave[0].iPgno = pLevel->lhs.iFirst; + } + + /* Position the cursor. */ + if( rc==LSM_OK ){ + pCsr->pPrevMergePtr = &pMerge->iCurrentPtr; + if( pLevel->lhs.iFirst==0 ){ + /* The output array is still empty. So position the cursor at the very + ** start of the input. */ + rc = multiCursorEnd(pCsr, 0); + }else{ + /* The output array is non-empty. Position the cursor based on the + ** page/cell data saved in the Merge.aInput[] array. */ + int i; + for(i=0; rc==LSM_OK && inPtr; i++){ + MergeInput *pInput = &pMerge->aInput[i]; + if( pInput->iPg ){ + SegmentPtr *pPtr; + assert( pCsr->aPtr[i].pPg==0 ); + pPtr = &pCsr->aPtr[i]; + rc = segmentPtrLoadPage(pDb->pFS, pPtr, pInput->iPg); + if( rc==LSM_OK && pPtr->nCell>0 ){ + rc = segmentPtrLoadCell(pPtr, pInput->iCell); + } + } + } + + if( rc==LSM_OK && pCsr->pBtCsr ){ + int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; + assert( i==pCsr->nPtr ); + rc = btreeCursorRestore(pCsr->pBtCsr, xCmp, &pMerge->aInput[i]); + } + + if( rc==LSM_OK ){ + rc = multiCursorSetupTree(pCsr, 0); + } + } + pCsr->flags |= CURSOR_NEXT_OK; + } + + return rc; +} + +static int sortedBtreeGobble( + lsm_db *pDb, /* Worker connection */ + MultiCursor *pCsr, /* Multi-cursor being used for a merge */ + int iGobble /* pCsr->aPtr[] entry to operate on */ +){ + int rc = LSM_OK; + if( rtTopic(pCsr->eType)==0 ){ + Segment *pSeg = pCsr->aPtr[iGobble].pSeg; + LsmPgno *aPg; + int nPg; + + /* Seek from the root of the b-tree to the segment leaf that may contain + ** a key equal to the one multi-cursor currently points to. Record the + ** page number of each b-tree page and the leaf. The segment may be + ** gobbled up to (but not including) the first of these page numbers. + */ + assert( pSeg->iRoot>0 ); + aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc); + if( rc==LSM_OK ){ + rc = seekInBtree(pCsr, pSeg, + rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 + ); + } + + if( rc==LSM_OK ){ + for(nPg=0; aPg[nPg]; nPg++); + lsmFsGobble(pDb, pSeg, aPg, nPg); + } + + lsmFree(pDb->pEnv, aPg); + } + return rc; +} + +/* +** Argument p points to a level of age N. Return the number of levels in +** the linked list starting at p that have age=N (always at least 1). +*/ +static int sortedCountLevels(Level *p){ + int iAge = p->iAge; + int nRet = 0; + do { + nRet++; + p = p->pNext; + }while( p && p->iAge==iAge ); + return nRet; +} + +static int sortedSelectLevel(lsm_db *pDb, int nMerge, Level **ppOut){ + Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); + int rc = LSM_OK; + Level *pLevel = 0; /* Output value */ + Level *pBest = 0; /* Best level to work on found so far */ + int nBest; /* Number of segments merged at pBest */ + Level *pThis = 0; /* First in run of levels with age=iAge */ + int nThis = 0; /* Number of levels starting at pThis */ + + assert( nMerge>=1 ); + nBest = LSM_MAX(1, nMerge-1); + + /* Find the longest contiguous run of levels not currently undergoing a + ** merge with the same age in the structure. Or the level being merged + ** with the largest number of right-hand segments. Work on it. */ + for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ + if( pLevel->nRight==0 && pThis && pLevel->iAge==pThis->iAge ){ + nThis++; + }else{ + if( nThis>nBest ){ + if( (pLevel->iAge!=pThis->iAge+1) + || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) + ){ + pBest = pThis; + nBest = nThis; + } + } + if( pLevel->nRight ){ + if( pLevel->nRight>nBest ){ + nBest = pLevel->nRight; + pBest = pLevel; + } + nThis = 0; + pThis = 0; + }else{ + pThis = pLevel; + nThis = 1; + } + } + } + if( nThis>nBest ){ + assert( pThis ); + pBest = pThis; + nBest = nThis; + } + + if( pBest==0 && nMerge==1 ){ + int nFree = 0; + int nUsr = 0; + for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ + assert( !pLevel->nRight ); + if( pLevel->flags & LEVEL_FREELIST_ONLY ){ + nFree++; + }else{ + nUsr++; + } + } + if( nUsr>1 ){ + pBest = pTopLevel; + nBest = nFree + nUsr; + } + } + + if( pBest ){ + if( pBest->nRight==0 ){ + rc = sortedMergeSetup(pDb, pBest, nBest, ppOut); + }else{ + *ppOut = pBest; + } + } + + return rc; +} + +static int sortedDbIsFull(lsm_db *pDb){ + Level *pTop = lsmDbSnapshotLevel(pDb->pWorker); + + if( lsmDatabaseFull(pDb) ) return 1; + if( pTop && pTop->iAge==0 + && (pTop->nRight || sortedCountLevels(pTop)>=pDb->nMerge) + ){ + return 1; + } + return 0; +} + +typedef struct MoveBlockCtx MoveBlockCtx; +struct MoveBlockCtx { + int iSeen; /* Previous free block on list */ + int iFrom; /* Total number of blocks in file */ +}; + +static int moveBlockCb(void *pCtx, int iBlk, i64 iSnapshot){ + MoveBlockCtx *p = (MoveBlockCtx *)pCtx; + assert( p->iFrom==0 ); + if( iBlk==(p->iSeen-1) ){ + p->iSeen = iBlk; + return 0; + } + p->iFrom = p->iSeen-1; + return 1; +} + +/* +** This function is called to further compact a database for which all +** of the content has already been merged into a single segment. If +** possible, it moves the contents of a single block from the end of the +** file to a free-block that lies closer to the start of the file (allowing +** the file to be eventually truncated). +*/ +static int sortedMoveBlock(lsm_db *pDb, int *pnWrite){ + Snapshot *p = pDb->pWorker; + Level *pLvl = lsmDbSnapshotLevel(p); + int iFrom; /* Block to move */ + int iTo; /* Destination to move block to */ + int rc; /* Return code */ + + MoveBlockCtx sCtx; + + assert( pLvl->pNext==0 && pLvl->nRight==0 ); + assert( p->redirect.n<=LSM_MAX_BLOCK_REDIRECTS ); + + *pnWrite = 0; + + /* Check that the redirect array is not already full. If it is, return + ** without moving any database content. */ + if( p->redirect.n>=LSM_MAX_BLOCK_REDIRECTS ) return LSM_OK; + + /* Find the last block of content in the database file. Do this by + ** traversing the free-list in reverse (descending block number) order. + ** The first block not on the free list is the one that will be moved. + ** Since the db consists of a single segment, there is no ambiguity as + ** to which segment the block belongs to. */ + sCtx.iSeen = p->nBlock+1; + sCtx.iFrom = 0; + rc = lsmWalkFreelist(pDb, 1, moveBlockCb, &sCtx); + if( rc!=LSM_OK || sCtx.iFrom==0 ) return rc; + iFrom = sCtx.iFrom; + + /* Find the first free block in the database, ignoring block 1. Block + ** 1 is tricky as it is smaller than the other blocks. */ + rc = lsmBlockAllocate(pDb, iFrom, &iTo); + if( rc!=LSM_OK || iTo==0 ) return rc; + assert( iTo!=1 && iTopFS, &pLvl->lhs, iTo, iFrom); + if( rc==LSM_OK ){ + if( p->redirect.a==0 ){ + int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; + p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); + } + if( rc==LSM_OK ){ + + /* Check if the block just moved was already redirected. */ + int i; + for(i=0; iredirect.n; i++){ + if( p->redirect.a[i].iTo==iFrom ) break; + } + + if( i==p->redirect.n ){ + /* Block iFrom was not already redirected. Add a new array entry. */ + memmove(&p->redirect.a[1], &p->redirect.a[0], + sizeof(struct RedirectEntry) * p->redirect.n + ); + p->redirect.a[0].iFrom = iFrom; + p->redirect.a[0].iTo = iTo; + p->redirect.n++; + }else{ + /* Block iFrom was already redirected. Overwrite existing entry. */ + p->redirect.a[i].iTo = iTo; + } + + rc = lsmBlockFree(pDb, iFrom); + + *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); + pLvl->lhs.pRedirect = &p->redirect; + } + } + +#if LSM_LOG_STRUCTURE + if( rc==LSM_OK ){ + char aBuf[64]; + sprintf(aBuf, "move-block %d/%d", p->redirect.n-1, LSM_MAX_BLOCK_REDIRECTS); + lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, aBuf); + } +#endif + return rc; +} + +/* +*/ +static int mergeInsertFreelistSegments( + lsm_db *pDb, + int nFree, + MergeWorker *pMW +){ + int rc = LSM_OK; + if( nFree>0 ){ + MultiCursor *pCsr = pMW->pCsr; + Level *pLvl = pMW->pLevel; + SegmentPtr *aNew1; + Segment *aNew2; + + Level *pIter; + Level *pNext; + int i = 0; + + aNew1 = (SegmentPtr *)lsmMallocZeroRc( + pDb->pEnv, sizeof(SegmentPtr) * (pCsr->nPtr+nFree), &rc + ); + if( rc ) return rc; + memcpy(&aNew1[nFree], pCsr->aPtr, sizeof(SegmentPtr)*pCsr->nPtr); + pCsr->nPtr += nFree; + lsmFree(pDb->pEnv, pCsr->aTree); + lsmFree(pDb->pEnv, pCsr->aPtr); + pCsr->aTree = 0; + pCsr->aPtr = aNew1; + + aNew2 = (Segment *)lsmMallocZeroRc( + pDb->pEnv, sizeof(Segment) * (pLvl->nRight+nFree), &rc + ); + if( rc ) return rc; + memcpy(&aNew2[nFree], pLvl->aRhs, sizeof(Segment)*pLvl->nRight); + pLvl->nRight += nFree; + lsmFree(pDb->pEnv, pLvl->aRhs); + pLvl->aRhs = aNew2; + + for(pIter=pDb->pWorker->pLevel; rc==LSM_OK && pIter!=pLvl; pIter=pNext){ + Segment *pSeg = &pLvl->aRhs[i]; + memcpy(pSeg, &pIter->lhs, sizeof(Segment)); + + pCsr->aPtr[i].pSeg = pSeg; + pCsr->aPtr[i].pLevel = pLvl; + rc = segmentPtrEnd(pCsr, &pCsr->aPtr[i], 0); + + pDb->pWorker->pLevel = pNext = pIter->pNext; + sortedFreeLevel(pDb->pEnv, pIter); + i++; + } + assert( i==nFree ); + assert( rc!=LSM_OK || pDb->pWorker->pLevel==pLvl ); + + for(i=nFree; inPtr; i++){ + pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; + } + + lsmFree(pDb->pEnv, pMW->aGobble); + pMW->aGobble = 0; + } + return rc; +} + +static int sortedWork( + lsm_db *pDb, /* Database handle. Must be worker. */ + int nWork, /* Number of pages of work to do */ + int nMerge, /* Try to merge this many levels at once */ + int bFlush, /* Set if call is to make room for a flush */ + int *pnWrite /* OUT: Actual number of pages written */ +){ + int rc = LSM_OK; /* Return Code */ + int nRemaining = nWork; /* Units of work to do before returning */ + Snapshot *pWorker = pDb->pWorker; + + assert( pWorker ); + if( lsmDbSnapshotLevel(pWorker)==0 ) return LSM_OK; + + while( nRemaining>0 ){ + Level *pLevel = 0; + + /* Find a level to work on. */ + rc = sortedSelectLevel(pDb, nMerge, &pLevel); + assert( rc==LSM_OK || pLevel==0 ); + + if( pLevel==0 ){ + int nDone = 0; + Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); + if( bFlush==0 && nMerge==1 && pTopLevel && pTopLevel->pNext==0 ){ + rc = sortedMoveBlock(pDb, &nDone); + } + nRemaining -= nDone; + + /* Could not find any work to do. Finished. */ + if( nDone==0 ) break; + }else{ + int bSave = 0; + Freelist freelist = {0, 0, 0}; + MergeWorker mergeworker; /* State used to work on the level merge */ + + assert( pDb->bIncrMerge==0 ); + assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); + + pDb->bIncrMerge = 1; + rc = mergeWorkerInit(pDb, pLevel, &mergeworker); + assert( mergeworker.nWork==0 ); + + while( rc==LSM_OK + && 0==mergeWorkerDone(&mergeworker) + && (mergeworker.nWorkbUseFreelist) + ){ + int eType = rtTopic(mergeworker.pCsr->eType); + rc = mergeWorkerStep(&mergeworker); + + /* If the cursor now points at the first entry past the end of the + ** user data (i.e. either to EOF or to the first free-list entry + ** that will be added to the run), then check if it is possible to + ** merge in any free-list entries that are either in-memory or in + ** free-list-only blocks. */ + if( rc==LSM_OK && nMerge==1 && eType==0 + && (rtTopic(mergeworker.pCsr->eType) || mergeWorkerDone(&mergeworker)) + ){ + int nFree = 0; /* Number of free-list-only levels to merge */ + Level *pLvl; + assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); + + /* Now check if all levels containing data newer than this one + ** are single-segment free-list only levels. If so, they will be + ** merged in now. */ + for(pLvl=pDb->pWorker->pLevel; + pLvl!=mergeworker.pLevel && (pLvl->flags & LEVEL_FREELIST_ONLY); + pLvl=pLvl->pNext + ){ + assert( pLvl->nRight==0 ); + nFree++; + } + if( pLvl==mergeworker.pLevel ){ + + rc = mergeInsertFreelistSegments(pDb, nFree, &mergeworker); + if( rc==LSM_OK ){ + rc = multiCursorVisitFreelist(mergeworker.pCsr); + } + if( rc==LSM_OK ){ + rc = multiCursorSetupTree(mergeworker.pCsr, 0); + pDb->pFreelist = &freelist; + pDb->bUseFreelist = 1; + } + } + } + } + nRemaining -= LSM_MAX(mergeworker.nWork, 1); + + if( rc==LSM_OK ){ + /* Check if the merge operation is completely finished. If not, + ** gobble up (declare eligible for recycling) any pages from rhs + ** segments for which the content has been completely merged into + ** the lhs of the level. */ + if( mergeWorkerDone(&mergeworker)==0 ){ + int i; + for(i=0; inRight; i++){ + SegmentPtr *pGobble = &mergeworker.pCsr->aPtr[i]; + if( pGobble->pSeg->iRoot ){ + rc = sortedBtreeGobble(pDb, mergeworker.pCsr, i); + }else if( mergeworker.aGobble[i] ){ + lsmFsGobble(pDb, pGobble->pSeg, &mergeworker.aGobble[i], 1); + } + } + }else{ + int i; + int bEmpty; + mergeWorkerShutdown(&mergeworker, &rc); + bEmpty = (pLevel->lhs.iFirst==0); + + if( bEmpty==0 && rc==LSM_OK ){ + rc = lsmFsSortedFinish(pDb->pFS, &pLevel->lhs); + } + + if( pDb->bUseFreelist ){ + Freelist *p = &pDb->pWorker->freelist; + lsmFree(pDb->pEnv, p->aEntry); + memcpy(p, &freelist, sizeof(freelist)); + pDb->bUseFreelist = 0; + pDb->pFreelist = 0; + bSave = 1; + } + + for(i=0; inRight; i++){ + lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); + } + + if( bEmpty ){ + /* If the new level is completely empty, remove it from the + ** database snapshot. This can only happen if all input keys were + ** annihilated. Since keys are only annihilated if the new level + ** is the last in the linked list (contains the most ancient of + ** database content), this guarantees that pLevel->pNext==0. */ + Level *pTop; /* Top level of worker snapshot */ + Level **pp; /* Read/write iterator for Level.pNext list */ + + assert( pLevel->pNext==0 ); + + /* Remove the level from the worker snapshot. */ + pTop = lsmDbSnapshotLevel(pWorker); + for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); + *pp = pLevel->pNext; + lsmDbSnapshotSetLevel(pWorker, pTop); + + /* Free the Level structure. */ + sortedFreeLevel(pDb->pEnv, pLevel); + }else{ + + /* Free the separators of the next level, if required. */ + if( pLevel->pMerge->nInput > pLevel->nRight ){ + assert( pLevel->pNext->lhs.iRoot ); + pLevel->pNext->lhs.iRoot = 0; + } + + /* Zero the right-hand-side of pLevel */ + lsmFree(pDb->pEnv, pLevel->aRhs); + pLevel->nRight = 0; + pLevel->aRhs = 0; + + /* Free the Merge object */ + lsmFree(pDb->pEnv, pLevel->pMerge); + pLevel->pMerge = 0; + } + + if( bSave && rc==LSM_OK ){ + pDb->bIncrMerge = 0; + rc = lsmSaveWorker(pDb, 0); + } + } + } + + /* Clean up the MergeWorker object initialized above. If no error + ** has occurred, invoke the work-hook to inform the application that + ** the database structure has changed. */ + mergeWorkerShutdown(&mergeworker, &rc); + pDb->bIncrMerge = 0; + if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); + +#if LSM_LOG_STRUCTURE + lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "work"); +#endif + assertBtreeOk(pDb, &pLevel->lhs); + assertRunInOrder(pDb, &pLevel->lhs); + + /* If bFlush is true and the database is no longer considered "full", + ** break out of the loop even if nRemaining is still greater than + ** zero. The caller has an in-memory tree to flush to disk. */ + if( bFlush && sortedDbIsFull(pDb)==0 ) break; + } + } + + if( pnWrite ) *pnWrite = (nWork - nRemaining); + pWorker->nWrite += (nWork - nRemaining); + +#ifdef LSM_LOG_WORK + lsmLogMessage(pDb, rc, "sortedWork(): %d pages", (nWork-nRemaining)); +#endif + return rc; +} + +/* +** The database connection passed as the first argument must be a worker +** connection. This function checks if there exists an "old" in-memory tree +** ready to be flushed to disk. If so, true is returned. Otherwise false. +** +** If an error occurs, *pRc is set to an LSM error code before returning. +** It is assumed that *pRc is set to LSM_OK when this function is called. +*/ +static int sortedTreeHasOld(lsm_db *pDb, int *pRc){ + int rc = LSM_OK; + int bRet = 0; + + assert( pDb->pWorker ); + if( *pRc==LSM_OK ){ + if( rc==LSM_OK + && pDb->treehdr.iOldShmid + && pDb->treehdr.iOldLog!=pDb->pWorker->iLogOff + ){ + bRet = 1; + }else{ + bRet = 0; + } + *pRc = rc; + } + assert( *pRc==LSM_OK || bRet==0 ); + return bRet; +} + +/* +** Create a new free-list only top-level segment. Return LSM_OK if successful +** or an LSM error code if some error occurs. +*/ +static int sortedNewFreelistOnly(lsm_db *pDb){ + return sortedNewToplevel(pDb, TREE_NONE, 0); +} + +int lsmSaveWorker(lsm_db *pDb, int bFlush){ + Snapshot *p = pDb->pWorker; + if( p->freelist.nEntry>pDb->nMaxFreelist ){ + int rc = sortedNewFreelistOnly(pDb); + if( rc!=LSM_OK ) return rc; + } + return lsmCheckpointSaveWorker(pDb, bFlush); +} + +static int doLsmSingleWork( + lsm_db *pDb, + int bShutdown, + int nMerge, /* Minimum segments to merge together */ + int nPage, /* Number of pages to write to disk */ + int *pnWrite, /* OUT: Pages actually written to disk */ + int *pbCkpt /* OUT: True if an auto-checkpoint is req. */ +){ + Snapshot *pWorker; /* Worker snapshot */ + int rc = LSM_OK; /* Return code */ + int bDirty = 0; + int nMax = nPage; /* Maximum pages to write to disk */ + int nRem = nPage; + int bCkpt = 0; + + assert( nPage>0 ); + + /* Open the worker 'transaction'. It will be closed before this function + ** returns. */ + assert( pDb->pWorker==0 ); + rc = lsmBeginWork(pDb); + if( rc!=LSM_OK ) return rc; + pWorker = pDb->pWorker; + + /* If this connection is doing auto-checkpoints, set nMax (and nRem) so + ** that this call stops writing when the auto-checkpoint is due. The + ** caller will do the checkpoint, then possibly call this function again. */ + if( bShutdown==0 && pDb->nAutockpt ){ + u32 nSync; + u32 nUnsync; + int nPgsz; + + lsmCheckpointSynced(pDb, 0, 0, &nSync); + nUnsync = lsmCheckpointNWrite(pDb->pShmhdr->aSnap1, 0); + nPgsz = lsmCheckpointPgsz(pDb->pShmhdr->aSnap1); + + nMax = (int)LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (int)(nUnsync-nSync)); + if( nMaxnTransOpen==0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + } + if( sortedTreeHasOld(pDb, &rc) ){ + /* sortedDbIsFull() returns non-zero if either (a) there are too many + ** levels in total in the db, or (b) there are too many levels with the + ** the same age in the db. Either way, call sortedWork() to merge + ** existing segments together until this condition is cleared. */ + if( sortedDbIsFull(pDb) ){ + int nPg = 0; + rc = sortedWork(pDb, nRem, nMerge, 1, &nPg); + nRem -= nPg; + assert( rc!=LSM_OK || nRem<=0 || !sortedDbIsFull(pDb) ); + bDirty = 1; + } + + if( rc==LSM_OK && nRem>0 ){ + int nPg = 0; + rc = sortedNewToplevel(pDb, TREE_OLD, &nPg); + nRem -= nPg; + if( rc==LSM_OK ){ + if( pDb->nTransOpen>0 ){ + lsmTreeDiscardOld(pDb); + } + rc = lsmSaveWorker(pDb, 1); + bDirty = 0; + } + } + } + + /* If nPage is still greater than zero, do some merging. */ + if( rc==LSM_OK && nRem>0 && bShutdown==0 ){ + int nPg = 0; + rc = sortedWork(pDb, nRem, nMerge, 0, &nPg); + nRem -= nPg; + if( nPg ) bDirty = 1; + } + + /* If the in-memory part of the free-list is too large, write a new + ** top-level containing just the in-memory free-list entries to disk. */ + if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){ + while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ + int nPg = 0; + rc = sortedWork(pDb, 16, nMerge, 1, &nPg); + nRem -= nPg; + } + if( rc==LSM_OK ){ + rc = sortedNewFreelistOnly(pDb); + } + bDirty = 1; + } + + if( rc==LSM_OK ){ + *pnWrite = (nMax - nRem); + *pbCkpt = (bCkpt && nRem<=0); + if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0 + && pWorker->pLevel + && pWorker->pLevel->nRight==0 + && pWorker->pLevel->pNext==0 + ){ + *pbCkpt = 1; + } + } + + if( rc==LSM_OK && bDirty ){ + lsmFinishWork(pDb, 0, &rc); + }else{ + int rcdummy = LSM_BUSY; + lsmFinishWork(pDb, 0, &rcdummy); + *pnWrite = 0; + } + assert( pDb->pWorker==0 ); + return rc; +} + +static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){ + int rc = LSM_OK; /* Return code */ + int nWrite = 0; /* Number of pages written */ + + assert( nMerge>=1 ); + + if( nPage!=0 ){ + int bCkpt = 0; + do { + int nThis = 0; + int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); + + bCkpt = 0; + rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); + nWrite += nThis; + if( rc==LSM_OK && bCkpt ){ + rc = lsm_checkpoint(pDb, 0); + } + }while( rc==LSM_OK && bCkpt && (nWritenTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; + if( nMerge<=0 ) nMerge = pDb->nMerge; + + lsmFsPurgeCache(pDb->pFS); + + /* Convert from KB to pages */ + nPgsz = lsmFsPageSize(pDb->pFS); + if( nKB>=0 ){ + nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; + }else{ + nPage = -1; + } + + rc = doLsmWork(pDb, nMerge, nPage, &nWrite); + + if( pnWrite ){ + /* Convert back from pages to KB */ + *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); + } + return rc; +} + +int lsm_flush(lsm_db *db){ + int rc; + + if( db->nTransOpen>0 || db->pCsr ){ + rc = LSM_MISUSE_BKPT; + }else{ + rc = lsmBeginWriteTrans(db); + if( rc==LSM_OK ){ + lsmFlushTreeToDisk(db); + lsmTreeDiscardOld(db); + lsmTreeMakeOld(db); + lsmTreeDiscardOld(db); + } + + if( rc==LSM_OK ){ + rc = lsmFinishWriteTrans(db, 1); + }else{ + lsmFinishWriteTrans(db, 0); + } + lsmFinishReadTrans(db); + } + + return rc; +} + +/* +** This function is called in auto-work mode to perform merging work on +** the data structure. It performs enough merging work to prevent the +** height of the tree from growing indefinitely assuming that roughly +** nUnit database pages worth of data have been written to the database +** (i.e. the in-memory tree) since the last call. +*/ +int lsmSortedAutoWork( + lsm_db *pDb, /* Database handle */ + int nUnit /* Pages of data written to in-memory tree */ +){ + int rc = LSM_OK; /* Return code */ + int nDepth = 0; /* Current height of tree (longest path) */ + Level *pLevel; /* Used to iterate through levels */ + int bRestore = 0; + + assert( pDb->pWorker==0 ); + assert( pDb->nTransOpen>0 ); + + /* Determine how many units of work to do before returning. One unit of + ** work is achieved by writing one page (~4KB) of merged data. */ + for(pLevel=lsmDbSnapshotLevel(pDb->pClient); pLevel; pLevel=pLevel->pNext){ + /* nDepth += LSM_MAX(1, pLevel->nRight); */ + nDepth += 1; + } + if( lsmTreeHasOld(pDb) ){ + nDepth += 1; + bRestore = 1; + rc = lsmSaveCursors(pDb); + if( rc!=LSM_OK ) return rc; + } + + if( nDepth>0 ){ + int nRemaining; /* Units of work to do before returning */ + + nRemaining = nUnit * nDepth; +#ifdef LSM_LOG_WORK + lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", + nUnit, nDepth, nRemaining); +#endif + assert( nRemaining>=0 ); + rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); + if( rc==LSM_BUSY ) rc = LSM_OK; + + if( bRestore && pDb->pCsr ){ + lsmMCursorFreeCache(pDb); + lsmFreeSnapshot(pDb->pEnv, pDb->pClient); + pDb->pClient = 0; + if( rc==LSM_OK ){ + rc = lsmCheckpointLoad(pDb, 0); + } + if( rc==LSM_OK ){ + rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot, &pDb->pClient); + } + if( rc==LSM_OK ){ + rc = lsmRestoreCursors(pDb); + } + } + } + + return rc; +} + +/* +** This function is only called during system shutdown. The contents of +** any in-memory trees present (old or current) are written out to disk. +*/ +int lsmFlushTreeToDisk(lsm_db *pDb){ + int rc; + + rc = lsmBeginWork(pDb); + while( rc==LSM_OK && sortedDbIsFull(pDb) ){ + rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); + } + + if( rc==LSM_OK ){ + rc = sortedNewToplevel(pDb, TREE_BOTH, 0); + } + + lsmFinishWork(pDb, 1, &rc); + return rc; +} + +/* +** Return a string representation of the segment passed as the only argument. +** Space for the returned string is allocated using lsmMalloc(), and should +** be freed by the caller using lsmFree(). +*/ +static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){ + LsmPgno nSize = pSeg->nSize; + LsmPgno iRoot = pSeg->iRoot; + LsmPgno iFirst = pSeg->iFirst; + LsmPgno iLast = pSeg->iLastPg; + char *z; + + char *z1; + char *z2; + int nPad; + + z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast); + if( iRoot ){ + z2 = lsmMallocPrintf(pEnv, "root=%lld", iRoot); + }else{ + z2 = lsmMallocPrintf(pEnv, "size=%lld", nSize); + } + + nPad = nMin - 2 - strlen(z1) - 1 - strlen(z2); + nPad = LSM_MAX(0, nPad); + + if( iRoot ){ + z = lsmMallocPrintf(pEnv, "/%s %*s%s\\", z1, nPad, "", z2); + }else{ + z = lsmMallocPrintf(pEnv, "|%s %*s%s|", z1, nPad, "", z2); + } + lsmFree(pEnv, z1); + lsmFree(pEnv, z2); + + return z; +} + +static int fileToString( + lsm_db *pDb, /* For xMalloc() */ + char *aBuf, + int nBuf, + int nMin, + Segment *pSeg +){ + int i = 0; + if( pSeg ){ + char *zSeg; + + zSeg = segToString(pDb->pEnv, pSeg, nMin); + snprintf(&aBuf[i], nBuf-i, "%s", zSeg); + i += strlen(&aBuf[i]); + lsmFree(pDb->pEnv, zSeg); + +#ifdef LSM_LOG_FREELIST + lsmInfoArrayStructure(pDb, 1, pSeg->iFirst, &zSeg); + snprintf(&aBuf[i], nBuf-1, " (%s)", zSeg); + i += strlen(&aBuf[i]); + lsmFree(pDb->pEnv, zSeg); +#endif + aBuf[nBuf] = 0; + }else{ + aBuf[0] = '\0'; + } + + return i; +} + +void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ + LsmBlob blob = {0, 0, 0}; /* LsmBlob used for keys */ + LsmString s; + int i; + + int nRec; + LsmPgno iPtr; + int flags; + u8 *aData; + int nData; + + aData = fsPageData(pPg, &nData); + + nRec = pageGetNRec(aData, nData); + iPtr = pageGetPtr(aData, nData); + flags = pageGetFlags(aData, nData); + + lsmStringInit(&s, pDb->pEnv); + lsmStringAppendf(&s,"nCell=%d iPtr=%lld flags=%d {", nRec, iPtr, flags); + if( flags&SEGMENT_BTREE_FLAG ) iPtr = 0; + + for(i=0; ipFS, pRun, iRef, &pRef); + aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); + }else{ + aCell += lsmVarintGet32(aCell, &nKey); + if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); + sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); + aVal = &aKey[nKey]; + iTopic = eType; + } + + lsmStringAppendf(&s, "%s%2X:", (i==0?"":" "), iTopic); + for(iChar=0; iChar0 && bVals ){ + lsmStringAppendf(&s, "##"); + for(iChar=0; iCharpFS, pSeg, iRef, &pRef); + pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); + aKey = (u8 *)pBlob->pData; + nKey = pBlob->nData; + lsmFsPageRelease(pRef); + }else{ + aKey = (u8 *)""; + nKey = 11; + } + }else{ + aCell += lsmVarintGet32(aCell, &nKey); + if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); + sortedReadData(pSeg, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); + aVal = &aKey[nKey]; + } + + if( peType ) *peType = eType; + if( piPgPtr ) *piPgPtr = iPgPtr; + if( paKey ) *paKey = aKey; + if( paVal ) *paVal = aVal; + if( pnKey ) *pnKey = nKey; + if( pnVal ) *pnVal = nVal; +} + +static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){ + int iChar; + for(iChar=0; iCharpClient || pDb->pWorker ); + pSnap = pDb->pClient; + if( pSnap==0 ) pSnap = pDb->pWorker; + if( pSnap->redirect.n>0 ){ + Level *pLvl; + int bUse = 0; + for(pLvl=pSnap->pLevel; pLvl->pNext; pLvl=pLvl->pNext); + pSeg = (pLvl->nRight==0 ? &pLvl->lhs : &pLvl->aRhs[pLvl->nRight-1]); + rc = lsmFsSegmentContainsPg(pDb->pFS, pSeg, iPg, &bUse); + if( bUse==0 ){ + pSeg = 0; + } + } + + /* iPg is a real page number (not subject to redirection). So it is safe + ** to pass a NULL in place of the segment pointer as the second argument + ** to lsmFsDbPageGet() here. */ + if( rc==LSM_OK ){ + rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); + } + + if( rc==LSM_OK ){ + LsmBlob blob = {0, 0, 0, 0}; + int nKeyWidth = 0; + LsmString str; + int nRec; + LsmPgno iPtr; + int flags2; + int iCell; + u8 *aData; int nData; /* Page data and size thereof */ + + aData = fsPageData(pPg, &nData); + nRec = pageGetNRec(aData, nData); + iPtr = pageGetPtr(aData, nData); + flags2 = pageGetFlags(aData, nData); + + lsmStringInit(&str, pDb->pEnv); + lsmStringAppendf(&str, "Page : %lld (%d bytes)\n", iPg, nData); + lsmStringAppendf(&str, "nRec : %d\n", nRec); + lsmStringAppendf(&str, "iPtr : %lld\n", iPtr); + lsmStringAppendf(&str, "flags: %04x\n", flags2); + lsmStringAppendf(&str, "\n"); + + for(iCell=0; iCellnKeyWidth ) nKeyWidth = nKey; + } + if( bHex ) nKeyWidth = nKeyWidth * 2; + + for(iCell=0; iCell0 && bValues ){ + lsmStringAppendf(&str, "%*s", nKeyWidth - (nKey*(1+bHex)), ""); + lsmStringAppendf(&str, " "); + infoAppendBlob(&str, bHex, aVal, nVal); + } + if( rtTopic(eType) ){ + int iBlk = (int)~lsmGetU32(aKey); + lsmStringAppendf(&str, " (block=%d", iBlk); + if( nVal>0 ){ + i64 iSnap = lsmGetU64(aVal); + lsmStringAppendf(&str, " snapshot=%lld", iSnap); + } + lsmStringAppendf(&str, ")"); + } + lsmStringAppendf(&str, "\n"); + } + + if( bData ){ + lsmStringAppendf(&str, "\n-------------------" + "-------------------------------------------------------------\n"); + lsmStringAppendf(&str, "Page %d\n", + iPg, (iPg-1)*nData, iPg*nData - 1); + for(i=0; inData ){ + lsmStringAppendf(&str, " "); + }else{ + lsmStringAppendf(&str, "%02x ", aData[i+j]); + } + } + lsmStringAppendf(&str, " "); + for(j=0; jnData ){ + lsmStringAppendf(&str, " "); + }else{ + lsmStringAppendf(&str,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + } + } + lsmStringAppendf(&str,"\n"); + } + } + + *pzOut = str.z; + sortedBlobFree(&blob); + lsmFsPageRelease(pPg); + } + + return rc; +} + +int lsmInfoPageDump( + lsm_db *pDb, /* Database handle */ + LsmPgno iPg, /* Page number of page to dump */ + int bHex, /* True to output key/value in hex form */ + char **pzOut /* OUT: lsmMalloc'd string */ +){ + int flags = INFO_PAGE_DUMP_DATA | INFO_PAGE_DUMP_VALUES; + if( bHex ) flags |= INFO_PAGE_DUMP_HEX; + return infoPageDump(pDb, iPg, flags, pzOut); +} + +void sortedDumpSegment(lsm_db *pDb, Segment *pRun, int bVals){ + assert( pDb->xLog ); + if( pRun && pRun->iFirst ){ + int flags = (bVals ? INFO_PAGE_DUMP_VALUES : 0); + char *zSeg; + Page *pPg; + + zSeg = segToString(pDb->pEnv, pRun, 0); + lsmLogMessage(pDb, LSM_OK, "Segment: %s", zSeg); + lsmFree(pDb->pEnv, zSeg); + + lsmFsDbPageGet(pDb->pFS, pRun, pRun->iFirst, &pPg); + while( pPg ){ + Page *pNext; + char *z = 0; + infoPageDump(pDb, lsmFsPageNumber(pPg), flags, &z); + lsmLogMessage(pDb, LSM_OK, "%s", z); + lsmFree(pDb->pEnv, z); +#if 0 + sortedDumpPage(pDb, pRun, pPg, bVals); +#endif + lsmFsDbPageNext(pRun, pPg, 1, &pNext); + lsmFsPageRelease(pPg); + pPg = pNext; + } + } +} + +/* +** Invoke the log callback zero or more times with messages that describe +** the current database structure. +*/ +void lsmSortedDumpStructure( + lsm_db *pDb, /* Database handle (used for xLog callback) */ + Snapshot *pSnap, /* Snapshot to dump */ + int bKeys, /* Output the keys from each segment */ + int bVals, /* Output the values from each segment */ + const char *zWhy /* Caption to print near top of dump */ +){ + Snapshot *pDump = pSnap; + Level *pTopLevel; + char *zFree = 0; + + assert( pSnap ); + pTopLevel = lsmDbSnapshotLevel(pDump); + if( pDb->xLog && pTopLevel ){ + static int nCall = 0; + Level *pLevel; + int iLevel = 0; + + nCall++; + lsmLogMessage(pDb, LSM_OK, "Database structure %d (%s)", nCall, zWhy); + +#if 0 + if( nCall==1031 || nCall==1032 ) bKeys=1; +#endif + + for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ + char zLeft[1024]; + char zRight[1024]; + int i = 0; + + Segment *aLeft[24]; + Segment *aRight[24]; + + int nLeft = 0; + int nRight = 0; + + Segment *pSeg = &pLevel->lhs; + aLeft[nLeft++] = pSeg; + + for(i=0; inRight; i++){ + aRight[nRight++] = &pLevel->aRhs[i]; + } + +#ifdef LSM_LOG_FREELIST + if( nRight ){ + memmove(&aRight[1], aRight, sizeof(aRight[0])*nRight); + aRight[0] = 0; + nRight++; + } +#endif + + for(i=0; iiAge, (int)pLevel->flags + ); + }else{ + zLevel[0] = '\0'; + } + + if( nRight==0 ){ + iPad = 10; + } + + lsmLogMessage(pDb, LSM_OK, "% 25s % *s% -35s %s", + zLevel, iPad, "", zLeft, zRight + ); + } + + iLevel++; + } + + if( bKeys ){ + for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ + int i; + sortedDumpSegment(pDb, &pLevel->lhs, bVals); + for(i=0; inRight; i++){ + sortedDumpSegment(pDb, &pLevel->aRhs[i], bVals); + } + } + } + } + + lsmInfoFreelist(pDb, &zFree); + lsmLogMessage(pDb, LSM_OK, "Freelist: %s", zFree); + lsmFree(pDb->pEnv, zFree); + + assert( lsmFsIntegrityCheck(pDb) ); +} + +void lsmSortedFreeLevel(lsm_env *pEnv, Level *pLevel){ + Level *pNext; + Level *p; + + for(p=pLevel; p; p=pNext){ + pNext = p->pNext; + sortedFreeLevel(pEnv, p); + } +} + +void lsmSortedSaveTreeCursors(lsm_db *pDb){ + MultiCursor *pCsr; + for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ + lsmTreeCursorSave(pCsr->apTreeCsr[0]); + lsmTreeCursorSave(pCsr->apTreeCsr[1]); + } +} + +void lsmSortedExpandBtreePage(Page *pPg, int nOrig){ + u8 *aData; + int nData; + int nEntry; + int iHdr; + + aData = lsmFsPageData(pPg, &nData); + nEntry = pageGetNRec(aData, nOrig); + iHdr = SEGMENT_EOF(nOrig, nEntry); + memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr); +} + +#ifdef LSM_DEBUG_EXPENSIVE +static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){ + Page *pPg = 0; + LsmBlob blob1 = {0, 0, 0, 0}; + LsmBlob blob2 = {0, 0, 0, 0}; + + lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg); + while( pPg ){ + u8 *aData; int nData; + Page *pNext; + + aData = lsmFsPageData(pPg, &nData); + if( 0==(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ){ + int i; + int nRec = pageGetNRec(aData, nData); + for(i=0; ipEnv, pSeg, pPg, i, &iTopic1, &blob1); + + if( i==0 && blob2.nData ){ + assert( sortedKeyCompare( + pDb->xCmp, iTopic2, blob2.pData, blob2.nData, + iTopic1, blob1.pData, blob1.nData + )<0 ); + } + + if( i<(nRec-1) ){ + pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i+1, &iTopic2, &blob2); + assert( sortedKeyCompare( + pDb->xCmp, iTopic1, blob1.pData, blob1.nData, + iTopic2, blob2.pData, blob2.nData + )<0 ); + } + } + } + + lsmFsDbPageNext(pSeg, pPg, 1, &pNext); + lsmFsPageRelease(pPg); + pPg = pNext; + } + + sortedBlobFree(&blob1); + sortedBlobFree(&blob2); +} +#endif + +#ifdef LSM_DEBUG_EXPENSIVE +/* +** This function is only included in the build if LSM_DEBUG_EXPENSIVE is +** defined. Its only purpose is to evaluate various assert() statements to +** verify that the database is well formed in certain respects. +** +** More specifically, it checks that the array pOne contains the required +** pointers to pTwo. Array pTwo must be a main array. pOne may be either a +** separators array or another main array. If pOne does not contain the +** correct set of pointers, an assert() statement fails. +*/ +static int assertPointersOk( + lsm_db *pDb, /* Database handle */ + Segment *pOne, /* Segment containing pointers */ + Segment *pTwo, /* Segment containing pointer targets */ + int bRhs /* True if pTwo may have been Gobble()d */ +){ + int rc = LSM_OK; /* Error code */ + SegmentPtr ptr1; /* Iterates through pOne */ + SegmentPtr ptr2; /* Iterates through pTwo */ + LsmPgno iPrev; + + assert( pOne && pTwo ); + + memset(&ptr1, 0, sizeof(ptr1)); + memset(&ptr2, 0, sizeof(ptr1)); + ptr1.pSeg = pOne; + ptr2.pSeg = pTwo; + segmentPtrEndPage(pDb->pFS, &ptr1, 0, &rc); + segmentPtrEndPage(pDb->pFS, &ptr2, 0, &rc); + + /* Check that the footer pointer of the first page of pOne points to + ** the first page of pTwo. */ + iPrev = pTwo->iFirst; + if( ptr1.iPtr!=iPrev && !bRhs ){ + assert( 0 ); + } + + if( rc==LSM_OK && ptr1.nCell>0 ){ + rc = segmentPtrLoadCell(&ptr1, 0); + } + + while( rc==LSM_OK && ptr2.pPg ){ + LsmPgno iThis; + + /* Advance to the next page of segment pTwo that contains at least + ** one cell. Break out of the loop if the iterator reaches EOF. */ + do{ + rc = segmentPtrNextPage(&ptr2, 1); + assert( rc==LSM_OK ); + }while( rc==LSM_OK && ptr2.pPg && ptr2.nCell==0 ); + if( rc!=LSM_OK || ptr2.pPg==0 ) break; + iThis = lsmFsPageNumber(ptr2.pPg); + + if( (ptr2.flags & (PGFTR_SKIP_THIS_FLAG|SEGMENT_BTREE_FLAG))==0 ){ + + /* Load the first cell in the array pTwo page. */ + rc = segmentPtrLoadCell(&ptr2, 0); + + /* Iterate forwards through pOne, searching for a key that matches the + ** key ptr2.pKey/nKey. This key should have a pointer to the page that + ** ptr2 currently points to. */ + while( rc==LSM_OK ){ + int res = rtTopic(ptr1.eType) - rtTopic(ptr2.eType); + if( res==0 ){ + res = pDb->xCmp(ptr1.pKey, ptr1.nKey, ptr2.pKey, ptr2.nKey); + } + + if( res<0 ){ + assert( bRhs || ptr1.iPtr+ptr1.iPgPtr==iPrev ); + }else if( res>0 ){ + assert( 0 ); + }else{ + assert( ptr1.iPtr+ptr1.iPgPtr==iThis ); + iPrev = iThis; + break; + } + + rc = segmentPtrAdvance(0, &ptr1, 0); + if( ptr1.pPg==0 ){ + assert( 0 ); + } + } + } + } + + segmentPtrReset(&ptr1, 0); + segmentPtrReset(&ptr2, 0); + return LSM_OK; +} + +/* +** This function is only included in the build if LSM_DEBUG_EXPENSIVE is +** defined. Its only purpose is to evaluate various assert() statements to +** verify that the database is well formed in certain respects. +** +** More specifically, it checks that the b-tree embedded in array pRun +** contains the correct keys. If not, an assert() fails. +*/ +static int assertBtreeOk( + lsm_db *pDb, + Segment *pSeg +){ + int rc = LSM_OK; /* Return code */ + if( pSeg->iRoot ){ + LsmBlob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */ + FileSystem *pFS = pDb->pFS; /* File system to read from */ + Page *pPg = 0; /* Main run page */ + BtreeCursor *pCsr = 0; /* Btree cursor */ + + rc = btreeCursorNew(pDb, pSeg, &pCsr); + if( rc==LSM_OK ){ + rc = btreeCursorFirst(pCsr); + } + if( rc==LSM_OK ){ + rc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pPg); + } + + while( rc==LSM_OK ){ + Page *pNext; + u8 *aData; + int nData; + int flags; + + rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); + lsmFsPageRelease(pPg); + pPg = pNext; + if( pPg==0 ) break; + aData = fsPageData(pPg, &nData); + flags = pageGetFlags(aData, nData); + if( rc==LSM_OK + && 0==((SEGMENT_BTREE_FLAG|PGFTR_SKIP_THIS_FLAG) & flags) + && 0!=pageGetNRec(aData, nData) + ){ + u8 *pKey; + int nKey; + int iTopic; + pKey = pageGetKey(pSeg, pPg, 0, &iTopic, &nKey, &blob); + assert( nKey==pCsr->nKey && 0==memcmp(pKey, pCsr->pKey, nKey) ); + assert( lsmFsPageNumber(pPg)==pCsr->iPtr ); + rc = btreeCursorNext(pCsr); + } + } + assert( rc!=LSM_OK || pCsr->pKey==0 ); + + if( pPg ) lsmFsPageRelease(pPg); + + btreeCursorFree(pCsr); + sortedBlobFree(&blob); + } + + return rc; +} +#endif /* ifdef LSM_DEBUG_EXPENSIVE */ diff --git a/ext/lsm1/lsm_str.c b/ext/lsm1/lsm_str.c new file mode 100644 index 0000000000000000000000000000000000000000..9b1b63cee248afaecb3e48d5bee20b83c539a883 --- /dev/null +++ b/ext/lsm1/lsm_str.c @@ -0,0 +1,148 @@ +/* +** 2012-04-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Dynamic string functions. +*/ +#include "lsmInt.h" + +/* +** Turn bulk and uninitialized memory into an LsmString object +*/ +void lsmStringInit(LsmString *pStr, lsm_env *pEnv){ + memset(pStr, 0, sizeof(pStr[0])); + pStr->pEnv = pEnv; +} + +/* +** Increase the memory allocated for holding the string. Realloc as needed. +** +** If a memory allocation error occurs, set pStr->n to -1 and free the existing +** allocation. If a prior memory allocation has occurred, this routine is a +** no-op. +*/ +int lsmStringExtend(LsmString *pStr, int nNew){ + assert( nNew>0 ); + if( pStr->n<0 ) return LSM_NOMEM; + if( pStr->n + nNew >= pStr->nAlloc ){ + int nAlloc = pStr->n + nNew + 100; + char *zNew = lsmRealloc(pStr->pEnv, pStr->z, nAlloc); + if( zNew==0 ){ + lsmFree(pStr->pEnv, pStr->z); + nAlloc = 0; + pStr->n = -1; + } + pStr->nAlloc = nAlloc; + pStr->z = zNew; + } + return (pStr->z ? LSM_OK : LSM_NOMEM_BKPT); +} + +/* +** Clear an LsmString object, releasing any allocated memory that it holds. +** This also clears the error indication (if any). +*/ +void lsmStringClear(LsmString *pStr){ + lsmFree(pStr->pEnv, pStr->z); + lsmStringInit(pStr, pStr->pEnv); +} + +/* +** Append N bytes of text to the end of an LsmString object. If +** N is negative, append the entire string. +** +** If the string is in an error state, this routine is a no-op. +*/ +int lsmStringAppend(LsmString *pStr, const char *z, int N){ + int rc; + if( N<0 ) N = (int)strlen(z); + rc = lsmStringExtend(pStr, N+1); + if( pStr->nAlloc ){ + memcpy(pStr->z+pStr->n, z, N+1); + pStr->n += N; + } + return rc; +} + +int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n){ + int rc; + rc = lsmStringExtend(pStr, n); + if( pStr->nAlloc ){ + memcpy(pStr->z+pStr->n, a, n); + pStr->n += n; + } + return rc; +} + +/* +** Append printf-formatted content to an LsmString. +*/ +void lsmStringVAppendf( + LsmString *pStr, + const char *zFormat, + va_list ap1, + va_list ap2 +){ +#if (!defined(__STDC_VERSION__) || (__STDC_VERSION__<199901L)) && \ + !defined(__APPLE__) + extern int vsnprintf(char *str, size_t size, const char *format, va_list ap) + /* Compatibility crutch for C89 compilation mode. sqlite3_vsnprintf() + does not work identically and causes test failures if used here. + For the time being we are assuming that the target has vsnprintf(), + but that is not guaranteed to be the case for pure C89 platforms. + */; +#endif + int nWrite; + int nAvail; + + nAvail = pStr->nAlloc - pStr->n; + nWrite = vsnprintf(pStr->z + pStr->n, nAvail, zFormat, ap1); + + if( nWrite>=nAvail ){ + lsmStringExtend(pStr, nWrite+1); + if( pStr->nAlloc==0 ) return; + nWrite = vsnprintf(pStr->z + pStr->n, nWrite+1, zFormat, ap2); + } + + pStr->n += nWrite; + pStr->z[pStr->n] = 0; +} + +void lsmStringAppendf(LsmString *pStr, const char *zFormat, ...){ + va_list ap, ap2; + va_start(ap, zFormat); + va_start(ap2, zFormat); + lsmStringVAppendf(pStr, zFormat, ap, ap2); + va_end(ap); + va_end(ap2); +} + +int lsmStrlen(const char *zName){ + int nRet = 0; + while( zName[nRet] ) nRet++; + return nRet; +} + +/* +** Write into memory obtained from lsm_malloc(). +*/ +char *lsmMallocPrintf(lsm_env *pEnv, const char *zFormat, ...){ + LsmString s; + va_list ap, ap2; + lsmStringInit(&s, pEnv); + va_start(ap, zFormat); + va_start(ap2, zFormat); + lsmStringVAppendf(&s, zFormat, ap, ap2); + va_end(ap); + va_end(ap2); + if( s.n<0 ) return 0; + return (char *)lsmReallocOrFree(pEnv, s.z, s.n+1); +} diff --git a/ext/lsm1/lsm_tree.c b/ext/lsm1/lsm_tree.c new file mode 100644 index 0000000000000000000000000000000000000000..1a199fc1ce52782c64d77217c5d36bee64622a64 --- /dev/null +++ b/ext/lsm1/lsm_tree.c @@ -0,0 +1,2465 @@ +/* +** 2011-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of an in-memory tree structure. +** +** Technically the tree is a B-tree of order 4 (in the Knuth sense - each +** node may have up to 4 children). Keys are stored within B-tree nodes by +** reference. This may be slightly slower than a conventional red-black +** tree, but it is simpler. It is also an easier structure to modify to +** create a version that supports nested transaction rollback. +** +** This tree does not currently support a delete operation. One is not +** required. When LSM deletes a key from a database, it inserts a DELETE +** marker into the data structure. As a result, although the value associated +** with a key stored in the in-memory tree structure may be modified, no +** keys are ever removed. +*/ + +/* +** MVCC NOTES +** +** The in-memory tree structure supports SQLite-style MVCC. This means +** that while one client is writing to the tree structure, other clients +** may still be querying an older snapshot of the tree. +** +** One way to implement this is to use an append-only b-tree. In this +** case instead of modifying nodes in-place, a copy of the node is made +** and the required modifications made to the copy. The parent of the +** node is then modified (to update the pointer so that it points to +** the new copy), which causes a copy of the parent to be made, and so on. +** This means that each time the tree is written to a new root node is +** created. A snapshot is identified by the root node that it uses. +** +** The problem with the above is that each time the tree is written to, +** a copy of the node structure modified and all of its ancestor nodes +** is made. This may prove excessive with large tree structures. +** +** To reduce this overhead, the data structure used for a tree node is +** designed so that it may be edited in place exactly once without +** affecting existing users. In other words, the node structure is capable +** of storing two separate versions of the node at the same time. +** When a node is to be edited, if the node structure already contains +** two versions, a copy is made as in the append-only approach. Or, if +** it only contains a single version, it is edited in place. +** +** This reduces the overhead so that, roughly, one new node structure +** must be allocated for each write (on top of those allocations that +** would have been required by a non-MVCC tree). Logic: Assume that at +** any time, 50% of nodes in the tree already contain 2 versions. When +** a new entry is written to a node, there is a 50% chance that a copy +** of the node will be required. And a 25% chance that a copy of its +** parent is required. And so on. +** +** ROLLBACK +** +** The in-memory tree also supports transaction and sub-transaction +** rollback. In order to rollback to point in time X, the following is +** necessary: +** +** 1. All memory allocated since X must be freed, and +** 2. All "v2" data adding to nodes that existed at X should be zeroed. +** 3. The root node must be restored to its X value. +** +** The Mempool object used to allocate memory for the tree supports +** operation (1) - see the lsmPoolMark() and lsmPoolRevert() functions. +** +** To support (2), all nodes that have v2 data are part of a singly linked +** list, sorted by the age of the v2 data (nodes that have had data added +** most recently are at the end of the list). So to zero all v2 data added +** since X, the linked list is traversed from the first node added following +** X onwards. +** +*/ + +#ifndef _LSM_INT_H +# include "lsmInt.h" +#endif + +#include + +#define MAX_DEPTH 32 + +typedef struct TreeKey TreeKey; +typedef struct TreeNode TreeNode; +typedef struct TreeLeaf TreeLeaf; +typedef struct NodeVersion NodeVersion; + +struct TreeOld { + u32 iShmid; /* Last shared-memory chunk in use by old */ + u32 iRoot; /* Offset of root node in shm file */ + u32 nHeight; /* Height of tree structure */ +}; + +#if 0 +/* +** assert() that a TreeKey.flags value is sane. Usage: +** +** assert( lsmAssertFlagsOk(pTreeKey->flags) ); +*/ +static int lsmAssertFlagsOk(u8 keyflags){ + /* At least one flag must be set. Otherwise, what is this key doing? */ + assert( keyflags!=0 ); + + /* The POINT_DELETE and INSERT flags cannot both be set. */ + assert( (keyflags & LSM_POINT_DELETE)==0 || (keyflags & LSM_INSERT)==0 ); + + /* If both the START_DELETE and END_DELETE flags are set, then the INSERT + ** flag must also be set. In other words - the three DELETE flags cannot + ** all be set */ + assert( (keyflags & LSM_END_DELETE)==0 + || (keyflags & LSM_START_DELETE)==0 + || (keyflags & LSM_POINT_DELETE)==0 + ); + + return 1; +} +#endif +static int assert_delete_ranges_match(lsm_db *); +static int treeCountEntries(lsm_db *db); + +/* +** Container for a key-value pair. Within the *-shm file, each key/value +** pair is stored in a single allocation (which may not actually be +** contiguous in memory). Layout is the TreeKey structure, followed by +** the nKey bytes of key blob, followed by the nValue bytes of value blob +** (if nValue is non-negative). +*/ +struct TreeKey { + int nKey; /* Size of pKey in bytes */ + int nValue; /* Size of pValue. Or negative. */ + u8 flags; /* Various LSM_XXX flags */ +}; + +#define TKV_KEY(p) ((void *)&(p)[1]) +#define TKV_VAL(p) ((void *)(((u8 *)&(p)[1]) + (p)->nKey)) + +/* +** A single tree node. A node structure may contain up to 3 key/value +** pairs. Internal (non-leaf) nodes have up to 4 children. +** +** TODO: Update the format of this to be more compact. Get it working +** first though... +*/ +struct TreeNode { + u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ + + /* The following fields are present for interior nodes only, not leaves. */ + u32 aiChildPtr[4]; /* Array of pointers to child nodes */ + + /* The extra child pointer slot. */ + u32 iV2; /* Transaction number of v2 */ + u8 iV2Child; /* apChild[] entry replaced by pV2Ptr */ + u32 iV2Ptr; /* Substitute pointer */ +}; + +struct TreeLeaf { + u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ +}; + +typedef struct TreeBlob TreeBlob; +struct TreeBlob { + int n; + u8 *a; +}; + +/* +** Cursor for searching a tree structure. +** +** If a cursor does not point to any element (a.k.a. EOF), then the +** TreeCursor.iNode variable is set to a negative value. Otherwise, the +** cursor currently points to key aiCell[iNode] on node apTreeNode[iNode]. +** +** Entries in the apTreeNode[] and aiCell[] arrays contain the node and +** index of the TreeNode.apChild[] pointer followed to descend to the +** current element. Hence apTreeNode[0] always contains the root node of +** the tree. +*/ +struct TreeCursor { + lsm_db *pDb; /* Database handle for this cursor */ + TreeRoot *pRoot; /* Root node and height of tree to access */ + int iNode; /* Cursor points at apTreeNode[iNode] */ + TreeNode *apTreeNode[MAX_DEPTH];/* Current position in tree */ + u8 aiCell[MAX_DEPTH]; /* Current position in tree */ + TreeKey *pSave; /* Saved key */ + TreeBlob blob; /* Dynamic storage for a key */ +}; + +/* +** A value guaranteed to be larger than the largest possible transaction +** id (TreeHeader.iTransId). +*/ +#define WORKING_VERSION (1<<30) + +static int tblobGrow(lsm_db *pDb, TreeBlob *p, int n, int *pRc){ + if( n>p->n ){ + lsmFree(pDb->pEnv, p->a); + p->a = lsmMallocRc(pDb->pEnv, n, pRc); + p->n = n; + } + return (p->a==0); +} +static void tblobFree(lsm_db *pDb, TreeBlob *p){ + lsmFree(pDb->pEnv, p->a); +} + + +/*********************************************************************** +** Start of IntArray methods. */ +/* +** Append value iVal to the contents of IntArray *p. Return LSM_OK if +** successful, or LSM_NOMEM if an OOM condition is encountered. +*/ +static int intArrayAppend(lsm_env *pEnv, IntArray *p, u32 iVal){ + assert( p->nArray<=p->nAlloc ); + if( p->nArray>=p->nAlloc ){ + u32 *aNew; + int nNew = p->nArray ? p->nArray*2 : 128; + aNew = lsmRealloc(pEnv, p->aArray, nNew*sizeof(u32)); + if( !aNew ) return LSM_NOMEM_BKPT; + p->aArray = aNew; + p->nAlloc = nNew; + } + + p->aArray[p->nArray++] = iVal; + return LSM_OK; +} + +/* +** Zero the IntArray object. +*/ +static void intArrayFree(lsm_env *pEnv, IntArray *p){ + p->nArray = 0; +} + +/* +** Return the number of entries currently in the int-array object. +*/ +static int intArraySize(IntArray *p){ + return p->nArray; +} + +/* +** Return a copy of the iIdx'th entry in the int-array. +*/ +static u32 intArrayEntry(IntArray *p, int iIdx){ + return p->aArray[iIdx]; +} + +/* +** Truncate the int-array so that all but the first nVal values are +** discarded. +*/ +static void intArrayTruncate(IntArray *p, int nVal){ + p->nArray = nVal; +} +/* End of IntArray methods. +***********************************************************************/ + +static int treeKeycmp(void *p1, int n1, void *p2, int n2){ + int res; + res = memcmp(p1, p2, LSM_MIN(n1, n2)); + if( res==0 ) res = (n1-n2); + return res; +} + +/* +** The pointer passed as the first argument points to an interior node, +** not a leaf. This function returns the offset of the iCell'th child +** sub-tree of the node. +*/ +static u32 getChildPtr(TreeNode *p, int iVersion, int iCell){ + assert( iVersion>=0 ); + assert( iCell>=0 && iCell<=array_size(p->aiChildPtr) ); + if( p->iV2 && p->iV2<=(u32)iVersion && iCell==p->iV2Child ) return p->iV2Ptr; + return p->aiChildPtr[iCell]; +} + +/* +** Given an offset within the *-shm file, return the associated chunk number. +*/ +static int treeOffsetToChunk(u32 iOff){ + assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); + return (int)(iOff>>15); +} + +#define treeShmptrUnsafe(pDb, iPtr) \ +(&((u8*)((pDb)->apShm[(iPtr)>>15]))[(iPtr) & (LSM_SHM_CHUNK_SIZE-1)]) + +/* +** Return a pointer to the mapped memory location associated with *-shm +** file offset iPtr. +*/ +static void *treeShmptr(lsm_db *pDb, u32 iPtr){ + + assert( (iPtr>>15)<(u32)pDb->nShm ); + assert( pDb->apShm[iPtr>>15] ); + + return iPtr ? treeShmptrUnsafe(pDb, iPtr) : 0; +} + +static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ + return (ShmChunk *)(pDb->apShm[iChunk]); +} + +static ShmChunk * treeShmChunkRc(lsm_db *pDb, int iChunk, int *pRc){ + assert( *pRc==LSM_OK ); + if( iChunknShm || LSM_OK==(*pRc = lsmShmCacheChunks(pDb, iChunk+1)) ){ + return (ShmChunk *)(pDb->apShm[iChunk]); + } + return 0; +} + + +#ifndef NDEBUG +static void assertIsWorkingChild( + lsm_db *db, + TreeNode *pNode, + TreeNode *pParent, + int iCell +){ + TreeNode *p; + u32 iPtr = getChildPtr(pParent, WORKING_VERSION, iCell); + p = treeShmptr(db, iPtr); + assert( p==pNode ); +} +#else +# define assertIsWorkingChild(w,x,y,z) +#endif + +/* Values for the third argument to treeShmkey(). */ +#define TKV_LOADKEY 1 +#define TKV_LOADVAL 2 + +static TreeKey *treeShmkey( + lsm_db *pDb, /* Database handle */ + u32 iPtr, /* Shmptr to TreeKey struct */ + int eLoad, /* Either zero or a TREEKEY_LOADXXX value */ + TreeBlob *pBlob, /* Used if dynamic memory is required */ + int *pRc /* IN/OUT: Error code */ +){ + TreeKey *pRet; + + assert( eLoad==TKV_LOADKEY || eLoad==TKV_LOADVAL ); + pRet = (TreeKey *)treeShmptr(pDb, iPtr); + if( pRet ){ + int nReq; /* Bytes of space required at pRet */ + int nAvail; /* Bytes of space available at pRet */ + + nReq = sizeof(TreeKey) + pRet->nKey; + if( eLoad==TKV_LOADVAL && pRet->nValue>0 ){ + nReq += pRet->nValue; + } + assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); + nAvail = LSM_SHM_CHUNK_SIZE - (iPtr & (LSM_SHM_CHUNK_SIZE-1)); + + if( nAvaila[nLoad], p, n); + nLoad += n; + if( nLoad==nReq ) break; + + pChunk = treeShmChunk(pDb, treeOffsetToChunk(iPtr)); + assert( pChunk ); + iPtr = (pChunk->iNext * LSM_SHM_CHUNK_SIZE) + LSM_SHM_CHUNK_HDR; + nAvail = LSM_SHM_CHUNK_SIZE - LSM_SHM_CHUNK_HDR; + } + } + pRet = (TreeKey *)(pBlob->a); + } + } + + return pRet; +} + +#if defined(LSM_DEBUG) && defined(LSM_EXPENSIVE_ASSERT) +void assert_leaf_looks_ok(TreeNode *pNode){ + assert( pNode->apKey[1] ); +} + +void assert_node_looks_ok(TreeNode *pNode, int nHeight){ + if( pNode ){ + assert( pNode->apKey[1] ); + if( nHeight>1 ){ + int i; + assert( getChildPtr(pNode, WORKING_VERSION, 1) ); + assert( getChildPtr(pNode, WORKING_VERSION, 2) ); + for(i=0; i<4; i++){ + assert_node_looks_ok(getChildPtr(pNode, WORKING_VERSION, i), nHeight-1); + } + } + } +} + +/* +** Run various assert() statements to check that the working-version of the +** tree is correct in the following respects: +** +** * todo... +*/ +void assert_tree_looks_ok(int rc, Tree *pTree){ +} +#else +# define assert_tree_looks_ok(x,y) +#endif + +void lsmFlagsToString(int flags, char *zFlags){ + + zFlags[0] = (flags & LSM_END_DELETE) ? ']' : '.'; + + /* Only one of LSM_POINT_DELETE, LSM_INSERT and LSM_SEPARATOR should ever + ** be set. If this is not true, write a '?' to the output. */ + switch( flags & (LSM_POINT_DELETE|LSM_INSERT|LSM_SEPARATOR) ){ + case 0: zFlags[1] = '.'; break; + case LSM_POINT_DELETE: zFlags[1] = '-'; break; + case LSM_INSERT: zFlags[1] = '+'; break; + case LSM_SEPARATOR: zFlags[1] = '^'; break; + default: zFlags[1] = '?'; break; + } + + zFlags[2] = (flags & LSM_SYSTEMKEY) ? '*' : '.'; + zFlags[3] = (flags & LSM_START_DELETE) ? '[' : '.'; + zFlags[4] = '\0'; +} + +#ifdef LSM_DEBUG + +/* +** Pointer pBlob points to a buffer containing a blob of binary data +** nBlob bytes long. Append the contents of this blob to *pStr, with +** each octet represented by a 2-digit hexadecimal number. For example, +** if the input blob is three bytes in size and contains {0x01, 0x44, 0xFF}, +** then "0144ff" is appended to *pStr. +*/ +static void lsmAppendStrBlob(LsmString *pStr, void *pBlob, int nBlob){ + int i; + lsmStringExtend(pStr, nBlob*2); + if( pStr->nAlloc==0 ) return; + for(i=0; i='a' && c<='z' ){ + pStr->z[pStr->n++] = c; + }else if( c!=0 || nBlob==1 || i!=(nBlob-1) ){ + pStr->z[pStr->n++] = "0123456789abcdef"[(c>>4)&0xf]; + pStr->z[pStr->n++] = "0123456789abcdef"[c&0xf]; + } + } + pStr->z[pStr->n] = 0; +} + +#if 0 /* NOT USED */ +/* +** Append nIndent space (0x20) characters to string *pStr. +*/ +static void lsmAppendIndent(LsmString *pStr, int nIndent){ + int i; + lsmStringExtend(pStr, nIndent); + for(i=0; ipEnv); + + /* Append each key to string s. */ + for(i=0; i<3; i++){ + u32 iPtr = pNode->aiKeyPtr[i]; + if( iPtr ){ + TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i],TKV_LOADKEY, &b,&rc); + strAppendFlags(&s, pKey->flags); + lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); + lsmStringAppend(&s, " ", -1); + } + } + + printf("% 6d %.*sleaf%.*s: %s\n", + iNode, nPath, zPath, 20-nPath-4, zSpace, s.z + ); + lsmStringClear(&s); + }else{ + for(i=0; i<4 && nHeight>0; i++){ + u32 iPtr = getChildPtr(pNode, pDb->treehdr.root.iTransId, i); + zPath[nPath] = (char)(i+'0'); + zPath[nPath+1] = '/'; + + if( iPtr ){ + dump_node_contents(pDb, iPtr, zPath, nPath+2, nHeight-1); + } + if( i!=3 && pNode->aiKeyPtr[i] ){ + TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TKV_LOADKEY,&b,&rc); + lsmStringInit(&s, pDb->pEnv); + strAppendFlags(&s, pKey->flags); + lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); + printf("% 6d %.*s%.*s: %s\n", + iNode, nPath+1, zPath, 20-nPath-1, zSpace, s.z); + lsmStringClear(&s); + } + } + } + + tblobFree(pDb, &b); +} + +void dump_tree_contents(lsm_db *pDb, const char *zCaption){ + char zPath[64]; + TreeRoot *p = &pDb->treehdr.root; + printf("\n%s\n", zCaption); + zPath[0] = '/'; + if( p->iRoot ){ + dump_node_contents(pDb, p->iRoot, zPath, 1, p->nHeight-1); + } + fflush(stdout); +} + +#endif + +/* +** Initialize a cursor object, the space for which has already been +** allocated. +*/ +static void treeCursorInit(lsm_db *pDb, int bOld, TreeCursor *pCsr){ + memset(pCsr, 0, sizeof(TreeCursor)); + pCsr->pDb = pDb; + if( bOld ){ + pCsr->pRoot = &pDb->treehdr.oldroot; + }else{ + pCsr->pRoot = &pDb->treehdr.root; + } + pCsr->iNode = -1; +} + +/* +** Return a pointer to the mapping of the TreeKey object that the cursor +** is pointing to. +*/ +static TreeKey *csrGetKey(TreeCursor *pCsr, TreeBlob *pBlob, int *pRc){ + TreeKey *pRet; + lsm_db *pDb = pCsr->pDb; + u32 iPtr = pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]]; + + assert( iPtr ); + pRet = (TreeKey*)treeShmptrUnsafe(pDb, iPtr); + if( !(pRet->flags & LSM_CONTIGUOUS) ){ + pRet = treeShmkey(pDb, iPtr, TKV_LOADVAL, pBlob, pRc); + } + + return pRet; +} + +/* +** Save the current position of tree cursor pCsr. +*/ +int lsmTreeCursorSave(TreeCursor *pCsr){ + int rc = LSM_OK; + if( pCsr && pCsr->pSave==0 ){ + int iNode = pCsr->iNode; + if( iNode>=0 ){ + pCsr->pSave = csrGetKey(pCsr, &pCsr->blob, &rc); + } + pCsr->iNode = -1; + } + return rc; +} + +/* +** Restore the position of a saved tree cursor. +*/ +static int treeCursorRestore(TreeCursor *pCsr, int *pRes){ + int rc = LSM_OK; + if( pCsr->pSave ){ + TreeKey *pKey = pCsr->pSave; + pCsr->pSave = 0; + if( pRes ){ + rc = lsmTreeCursorSeek(pCsr, TKV_KEY(pKey), pKey->nKey, pRes); + } + } + return rc; +} + +/* +** Allocate nByte bytes of space within the *-shm file. If successful, +** return LSM_OK and set *piPtr to the offset within the file at which +** the allocated space is located. +*/ +static u32 treeShmalloc(lsm_db *pDb, int bAlign, int nByte, int *pRc){ + u32 iRet = 0; + if( *pRc==LSM_OK ){ + const static int CHUNK_SIZE = LSM_SHM_CHUNK_SIZE; + const static int CHUNK_HDR = LSM_SHM_CHUNK_HDR; + u32 iWrite; /* Current write offset */ + u32 iEof; /* End of current chunk */ + int iChunk; /* Current chunk */ + + assert( nByte <= (CHUNK_SIZE-CHUNK_HDR) ); + + /* Check if there is enough space on the current chunk to fit the + ** new allocation. If not, link in a new chunk and put the new + ** allocation at the start of it. */ + iWrite = pDb->treehdr.iWrite; + if( bAlign ){ + iWrite = (iWrite + 3) & ~0x0003; + assert( (iWrite % 4)==0 ); + } + + assert( iWrite ); + iChunk = treeOffsetToChunk(iWrite-1); + iEof = (iChunk+1) * CHUNK_SIZE; + assert( iEof>=iWrite && (iEof-iWrite)<(u32)CHUNK_SIZE ); + if( (iWrite+nByte)>iEof ){ + ShmChunk *pHdr; /* Header of chunk just finished (iChunk) */ + ShmChunk *pFirst; /* Header of chunk treehdr.iFirst */ + ShmChunk *pNext; /* Header of new chunk */ + int iNext = 0; /* Next chunk */ + int rc = LSM_OK; + + pFirst = treeShmChunk(pDb, pDb->treehdr.iFirst); + + assert( shm_sequence_ge(pDb->treehdr.iUsedShmid, pFirst->iShmid) ); + assert( (pDb->treehdr.iNextShmid+1-pDb->treehdr.nChunk)==pFirst->iShmid ); + + /* Check if the chunk at the start of the linked list is still in + ** use. If not, reuse it. If so, allocate a new chunk by appending + ** to the *-shm file. */ + if( pDb->treehdr.iUsedShmid!=pFirst->iShmid ){ + int bInUse; + rc = lsmTreeInUse(pDb, pFirst->iShmid, &bInUse); + if( rc!=LSM_OK ){ + *pRc = rc; + return 0; + } + if( bInUse==0 ){ + iNext = pDb->treehdr.iFirst; + pDb->treehdr.iFirst = pFirst->iNext; + assert( pDb->treehdr.iFirst ); + } + } + if( iNext==0 ) iNext = pDb->treehdr.nChunk++; + + /* Set the header values for the new chunk */ + pNext = treeShmChunkRc(pDb, iNext, &rc); + if( pNext ){ + pNext->iNext = 0; + pNext->iShmid = (pDb->treehdr.iNextShmid++); + }else{ + *pRc = rc; + return 0; + } + + /* Set the header values for the chunk just finished */ + pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE); + pHdr->iNext = iNext; + + /* Advance to the next chunk */ + iWrite = iNext * CHUNK_SIZE + CHUNK_HDR; + } + + /* Allocate space at iWrite. */ + iRet = iWrite; + pDb->treehdr.iWrite = iWrite + nByte; + pDb->treehdr.root.nByte += nByte; + } + return iRet; +} + +/* +** Allocate and zero nByte bytes of space within the *-shm file. +*/ +static void *treeShmallocZero(lsm_db *pDb, int nByte, u32 *piPtr, int *pRc){ + u32 iPtr; + void *p; + iPtr = treeShmalloc(pDb, 1, nByte, pRc); + p = treeShmptr(pDb, iPtr); + if( p ){ + assert( *pRc==LSM_OK ); + memset(p, 0, nByte); + *piPtr = iPtr; + } + return p; +} + +static TreeNode *newTreeNode(lsm_db *pDb, u32 *piPtr, int *pRc){ + return treeShmallocZero(pDb, sizeof(TreeNode), piPtr, pRc); +} + +static TreeLeaf *newTreeLeaf(lsm_db *pDb, u32 *piPtr, int *pRc){ + return treeShmallocZero(pDb, sizeof(TreeLeaf), piPtr, pRc); +} + +static TreeKey *newTreeKey( + lsm_db *pDb, + u32 *piPtr, + void *pKey, int nKey, /* Key data */ + void *pVal, int nVal, /* Value data (or nVal<0 for delete) */ + int *pRc +){ + TreeKey *p; + u32 iPtr; + u32 iEnd; + int nRem; + u8 *a; + int n; + + /* Allocate space for the TreeKey structure itself */ + *piPtr = iPtr = treeShmalloc(pDb, 1, sizeof(TreeKey), pRc); + p = treeShmptr(pDb, iPtr); + if( *pRc ) return 0; + p->nKey = nKey; + p->nValue = nVal; + + /* Allocate and populate the space required for the key and value. */ + n = nRem = nKey; + a = (u8 *)pKey; + while( a ){ + while( nRem>0 ){ + u8 *aAlloc; + int nAlloc; + u32 iWrite; + + iWrite = (pDb->treehdr.iWrite & (LSM_SHM_CHUNK_SIZE-1)); + iWrite = LSM_MAX(iWrite, LSM_SHM_CHUNK_HDR); + nAlloc = LSM_MIN((LSM_SHM_CHUNK_SIZE-iWrite), (u32)nRem); + + aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc)); + if( aAlloc==0 ) break; + memcpy(aAlloc, &a[n-nRem], nAlloc); + nRem -= nAlloc; + } + a = pVal; + n = nRem = nVal; + pVal = 0; + } + + iEnd = iPtr + sizeof(TreeKey) + nKey + LSM_MAX(0, nVal); + if( (iPtr & ~(LSM_SHM_CHUNK_SIZE-1))!=(iEnd & ~(LSM_SHM_CHUNK_SIZE-1)) ){ + p->flags = 0; + }else{ + p->flags = LSM_CONTIGUOUS; + } + + if( *pRc ) return 0; +#if 0 + printf("store: %d %s\n", (int)iPtr, (char *)pKey); +#endif + return p; +} + +static TreeNode *copyTreeNode( + lsm_db *pDb, + TreeNode *pOld, + u32 *piNew, + int *pRc +){ + TreeNode *pNew; + + pNew = newTreeNode(pDb, piNew, pRc); + if( pNew ){ + memcpy(pNew->aiKeyPtr, pOld->aiKeyPtr, sizeof(pNew->aiKeyPtr)); + memcpy(pNew->aiChildPtr, pOld->aiChildPtr, sizeof(pNew->aiChildPtr)); + if( pOld->iV2 ) pNew->aiChildPtr[pOld->iV2Child] = pOld->iV2Ptr; + } + return pNew; +} + +static TreeNode *copyTreeLeaf( + lsm_db *pDb, + TreeLeaf *pOld, + u32 *piNew, + int *pRc +){ + TreeLeaf *pNew; + pNew = newTreeLeaf(pDb, piNew, pRc); + if( pNew ){ + memcpy(pNew, pOld, sizeof(TreeLeaf)); + } + return (TreeNode *)pNew; +} + +/* +** The tree cursor passed as the second argument currently points to an +** internal node (not a leaf). Specifically, to a sub-tree pointer. This +** function replaces the sub-tree that the cursor currently points to +** with sub-tree pNew. +** +** The sub-tree may be replaced either by writing the "v2 data" on the +** internal node, or by allocating a new TreeNode structure and then +** calling this function on the parent of the internal node. +*/ +static int treeUpdatePtr(lsm_db *pDb, TreeCursor *pCsr, u32 iNew){ + int rc = LSM_OK; + if( pCsr->iNode<0 ){ + /* iNew is the new root node */ + pDb->treehdr.root.iRoot = iNew; + }else{ + /* If this node already has version 2 content, allocate a copy and + ** update the copy with the new pointer value. Otherwise, store the + ** new pointer as v2 data within the current node structure. */ + + TreeNode *p; /* The node to be modified */ + int iChildPtr; /* apChild[] entry to modify */ + + p = pCsr->apTreeNode[pCsr->iNode]; + iChildPtr = pCsr->aiCell[pCsr->iNode]; + + if( p->iV2 ){ + /* The "allocate new TreeNode" option */ + u32 iCopy; + TreeNode *pCopy; + pCopy = copyTreeNode(pDb, p, &iCopy, &rc); + if( pCopy ){ + assert( rc==LSM_OK ); + pCopy->aiChildPtr[iChildPtr] = iNew; + pCsr->iNode--; + rc = treeUpdatePtr(pDb, pCsr, iCopy); + } + }else{ + /* The "v2 data" option */ + u32 iPtr; + assert( pDb->treehdr.root.iTransId>0 ); + + if( pCsr->iNode ){ + iPtr = getChildPtr( + pCsr->apTreeNode[pCsr->iNode-1], + pDb->treehdr.root.iTransId, pCsr->aiCell[pCsr->iNode-1] + ); + }else{ + iPtr = pDb->treehdr.root.iRoot; + } + rc = intArrayAppend(pDb->pEnv, &pDb->rollback, iPtr); + + if( rc==LSM_OK ){ + p->iV2 = pDb->treehdr.root.iTransId; + p->iV2Child = (u8)iChildPtr; + p->iV2Ptr = iNew; + } + } + } + + return rc; +} + +/* +** Cursor pCsr points at a node that is part of pTree. This function +** inserts a new key and optionally child node pointer into that node. +** +** The position into which the new key and pointer are inserted is +** determined by the iSlot parameter. The new key will be inserted to +** the left of the key currently stored in apKey[iSlot]. Or, if iSlot is +** greater than the index of the rightmost key in the node. +** +** Pointer pLeftPtr points to a child tree that contains keys that are +** smaller than pTreeKey. +*/ +static int treeInsert( + lsm_db *pDb, /* Database handle */ + TreeCursor *pCsr, /* Cursor indicating path to insert at */ + u32 iLeftPtr, /* Left child pointer */ + u32 iTreeKey, /* Location of key to insert */ + u32 iRightPtr, /* Right child pointer */ + int iSlot /* Position to insert key into */ +){ + int rc = LSM_OK; + TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; + + /* Check if the node is currently full. If so, split pNode in two and + ** call this function recursively to add a key to the parent. Otherwise, + ** insert the new key directly into pNode. */ + assert( pNode->aiKeyPtr[1] ); + if( pNode->aiKeyPtr[0] && pNode->aiKeyPtr[2] ){ + u32 iLeft; TreeNode *pLeft; /* New left-hand sibling node */ + u32 iRight; TreeNode *pRight; /* New right-hand sibling node */ + + pLeft = newTreeNode(pDb, &iLeft, &rc); + pRight = newTreeNode(pDb, &iRight, &rc); + if( rc ) return rc; + + pLeft->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 0); + pLeft->aiKeyPtr[1] = pNode->aiKeyPtr[0]; + pLeft->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 1); + + pRight->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 2); + pRight->aiKeyPtr[1] = pNode->aiKeyPtr[2]; + pRight->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 3); + + if( pCsr->iNode==0 ){ + /* pNode is the root of the tree. Grow the tree by one level. */ + u32 iRoot; TreeNode *pRoot; /* New root node */ + + pRoot = newTreeNode(pDb, &iRoot, &rc); + pRoot->aiKeyPtr[1] = pNode->aiKeyPtr[1]; + pRoot->aiChildPtr[1] = iLeft; + pRoot->aiChildPtr[2] = iRight; + + pDb->treehdr.root.iRoot = iRoot; + pDb->treehdr.root.nHeight++; + }else{ + + pCsr->iNode--; + rc = treeInsert(pDb, pCsr, + iLeft, pNode->aiKeyPtr[1], iRight, pCsr->aiCell[pCsr->iNode] + ); + } + + assert( pLeft->iV2==0 ); + assert( pRight->iV2==0 ); + switch( iSlot ){ + case 0: + pLeft->aiKeyPtr[0] = iTreeKey; + pLeft->aiChildPtr[0] = iLeftPtr; + if( iRightPtr ) pLeft->aiChildPtr[1] = iRightPtr; + break; + case 1: + pLeft->aiChildPtr[3] = (iRightPtr ? iRightPtr : pLeft->aiChildPtr[2]); + pLeft->aiKeyPtr[2] = iTreeKey; + pLeft->aiChildPtr[2] = iLeftPtr; + break; + case 2: + pRight->aiKeyPtr[0] = iTreeKey; + pRight->aiChildPtr[0] = iLeftPtr; + if( iRightPtr ) pRight->aiChildPtr[1] = iRightPtr; + break; + case 3: + pRight->aiChildPtr[3] = (iRightPtr ? iRightPtr : pRight->aiChildPtr[2]); + pRight->aiKeyPtr[2] = iTreeKey; + pRight->aiChildPtr[2] = iLeftPtr; + break; + } + + }else{ + TreeNode *pNew; + u32 *piKey; + u32 *piChild; + u32 iStore = 0; + u32 iNew = 0; + int i; + + /* Allocate a new version of node pNode. */ + pNew = newTreeNode(pDb, &iNew, &rc); + if( rc ) return rc; + + piKey = pNew->aiKeyPtr; + piChild = pNew->aiChildPtr; + + for(i=0; iaiKeyPtr[i] ){ + *(piKey++) = pNode->aiKeyPtr[i]; + *(piChild++) = getChildPtr(pNode, WORKING_VERSION, i); + } + } + + *piKey++ = iTreeKey; + *piChild++ = iLeftPtr; + + iStore = iRightPtr; + for(i=iSlot; i<3; i++){ + if( pNode->aiKeyPtr[i] ){ + *(piKey++) = pNode->aiKeyPtr[i]; + *(piChild++) = iStore ? iStore : getChildPtr(pNode, WORKING_VERSION, i); + iStore = 0; + } + } + + if( iStore ){ + *piChild = iStore; + }else{ + *piChild = getChildPtr(pNode, WORKING_VERSION, + (pNode->aiKeyPtr[2] ? 3 : 2) + ); + } + pCsr->iNode--; + rc = treeUpdatePtr(pDb, pCsr, iNew); + } + + return rc; +} + +static int treeInsertLeaf( + lsm_db *pDb, /* Database handle */ + TreeCursor *pCsr, /* Cursor structure */ + u32 iTreeKey, /* Key pointer to insert */ + int iSlot /* Insert key to the left of this */ +){ + int rc = LSM_OK; /* Return code */ + TreeNode *pLeaf = pCsr->apTreeNode[pCsr->iNode]; + TreeLeaf *pNew; + u32 iNew; + + assert( iSlot>=0 && iSlot<=4 ); + assert( pCsr->iNode>0 ); + assert( pLeaf->aiKeyPtr[1] ); + + pCsr->iNode--; + + pNew = newTreeLeaf(pDb, &iNew, &rc); + if( pNew ){ + if( pLeaf->aiKeyPtr[0] && pLeaf->aiKeyPtr[2] ){ + /* The leaf is full. Split it in two. */ + TreeLeaf *pRight; + u32 iRight; + pRight = newTreeLeaf(pDb, &iRight, &rc); + if( pRight ){ + assert( rc==LSM_OK ); + pNew->aiKeyPtr[1] = pLeaf->aiKeyPtr[0]; + pRight->aiKeyPtr[1] = pLeaf->aiKeyPtr[2]; + switch( iSlot ){ + case 0: pNew->aiKeyPtr[0] = iTreeKey; break; + case 1: pNew->aiKeyPtr[2] = iTreeKey; break; + case 2: pRight->aiKeyPtr[0] = iTreeKey; break; + case 3: pRight->aiKeyPtr[2] = iTreeKey; break; + } + + rc = treeInsert(pDb, pCsr, iNew, pLeaf->aiKeyPtr[1], iRight, + pCsr->aiCell[pCsr->iNode] + ); + } + }else{ + int iOut = 0; + int i; + for(i=0; i<4; i++){ + if( i==iSlot ) pNew->aiKeyPtr[iOut++] = iTreeKey; + if( i<3 && pLeaf->aiKeyPtr[i] ){ + pNew->aiKeyPtr[iOut++] = pLeaf->aiKeyPtr[i]; + } + } + rc = treeUpdatePtr(pDb, pCsr, iNew); + } + } + + return rc; +} + +void lsmTreeMakeOld(lsm_db *pDb){ + + /* A write transaction must be open. Otherwise the code below that + ** assumes (pDb->pClient->iLogOff) is current may malfunction. + ** + ** Update: currently this assert fails due to lsm_flush(), which does + ** not set nTransOpen. + */ + assert( /* pDb->nTransOpen>0 && */ pDb->iReader>=0 ); + + if( pDb->treehdr.iOldShmid==0 ){ + pDb->treehdr.iOldLog = (pDb->treehdr.log.aRegion[2].iEnd << 1); + pDb->treehdr.iOldLog |= (~(pDb->pClient->iLogOff) & (i64)0x0001); + + pDb->treehdr.oldcksum0 = pDb->treehdr.log.cksum0; + pDb->treehdr.oldcksum1 = pDb->treehdr.log.cksum1; + pDb->treehdr.iOldShmid = pDb->treehdr.iNextShmid-1; + memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); + + pDb->treehdr.root.iTransId = 1; + pDb->treehdr.root.iRoot = 0; + pDb->treehdr.root.nHeight = 0; + pDb->treehdr.root.nByte = 0; + } +} + +void lsmTreeDiscardOld(lsm_db *pDb){ + assert( lsmShmAssertLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL) + || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) + ); + pDb->treehdr.iUsedShmid = pDb->treehdr.iOldShmid; + pDb->treehdr.iOldShmid = 0; +} + +int lsmTreeHasOld(lsm_db *pDb){ + return pDb->treehdr.iOldShmid!=0; +} + +/* +** This function is called during recovery to initialize the +** tree header. Only the database connections private copy of the tree-header +** is initialized here - it will be copied into shared memory if log file +** recovery is successful. +*/ +int lsmTreeInit(lsm_db *pDb){ + ShmChunk *pOne; + int rc = LSM_OK; + + memset(&pDb->treehdr, 0, sizeof(TreeHeader)); + pDb->treehdr.root.iTransId = 1; + pDb->treehdr.iFirst = 1; + pDb->treehdr.nChunk = 2; + pDb->treehdr.iWrite = LSM_SHM_CHUNK_SIZE + LSM_SHM_CHUNK_HDR; + pDb->treehdr.iNextShmid = 2; + pDb->treehdr.iUsedShmid = 1; + + pOne = treeShmChunkRc(pDb, 1, &rc); + if( pOne ){ + pOne->iNext = 0; + pOne->iShmid = 1; + } + return rc; +} + +static void treeHeaderChecksum( + TreeHeader *pHdr, + u32 *aCksum +){ + u32 cksum1 = 0x12345678; + u32 cksum2 = 0x9ABCDEF0; + u32 *a = (u32 *)pHdr; + int i; + + assert( (offsetof(TreeHeader, aCksum) + sizeof(u32)*2)==sizeof(TreeHeader) ); + assert( (sizeof(TreeHeader) % (sizeof(u32)*2))==0 ); + + for(i=0; i<(offsetof(TreeHeader, aCksum) / sizeof(u32)); i+=2){ + cksum1 += a[i]; + cksum2 += (cksum1 + a[i+1]); + } + aCksum[0] = cksum1; + aCksum[1] = cksum2; +} + +/* +** Return true if the checksum stored in TreeHeader object *pHdr is +** consistent with the contents of its other fields. +*/ +static int treeHeaderChecksumOk(TreeHeader *pHdr){ + u32 aCksum[2]; + treeHeaderChecksum(pHdr, aCksum); + return (0==memcmp(aCksum, pHdr->aCksum, sizeof(aCksum))); +} + +/* +** This type is used by functions lsmTreeRepair() and treeSortByShmid() to +** make relinking the linked list of shared-memory chunks easier. +*/ +typedef struct ShmChunkLoc ShmChunkLoc; +struct ShmChunkLoc { + ShmChunk *pShm; + u32 iLoc; +}; + +/* +** This function checks that the linked list of shared memory chunks +** that starts at chunk db->treehdr.iFirst: +** +** 1) Includes all chunks in the shared-memory region, and +** 2) Links them together in order of ascending shm-id. +** +** If no error occurs and the conditions above are met, LSM_OK is returned. +** +** If either of the conditions are untrue, LSM_CORRUPT is returned. Or, if +** an error is encountered before the checks are completed, another LSM error +** code (i.e. LSM_IOERR or LSM_NOMEM) may be returned. +*/ +static int treeCheckLinkedList(lsm_db *db){ + int rc = LSM_OK; + int nVisit = 0; + ShmChunk *p; + + p = treeShmChunkRc(db, db->treehdr.iFirst, &rc); + while( rc==LSM_OK && p ){ + if( p->iNext ){ + if( p->iNext>=db->treehdr.nChunk ){ + rc = LSM_CORRUPT_BKPT; + }else{ + ShmChunk *pNext = treeShmChunkRc(db, p->iNext, &rc); + if( rc==LSM_OK ){ + if( pNext->iShmid!=p->iShmid+1 ){ + rc = LSM_CORRUPT_BKPT; + } + p = pNext; + } + } + }else{ + p = 0; + } + nVisit++; + } + + if( rc==LSM_OK && (u32)nVisit!=db->treehdr.nChunk-1 ){ + rc = LSM_CORRUPT_BKPT; + } + return rc; +} + +/* +** Iterate through the current in-memory tree. If there are any v2-pointers +** with transaction ids larger than db->treehdr.iTransId, zero them. +*/ +static int treeRepairPtrs(lsm_db *db){ + int rc = LSM_OK; + + if( db->treehdr.root.nHeight>1 ){ + TreeCursor csr; /* Cursor used to iterate through tree */ + u32 iTransId = db->treehdr.root.iTransId; + + /* Initialize the cursor structure. Also decrement the nHeight variable + ** in the tree-header. This will prevent the cursor from visiting any + ** leaf nodes. */ + db->treehdr.root.nHeight--; + treeCursorInit(db, 0, &csr); + + rc = lsmTreeCursorEnd(&csr, 0); + while( rc==LSM_OK && lsmTreeCursorValid(&csr) ){ + TreeNode *pNode = csr.apTreeNode[csr.iNode]; + if( pNode->iV2>iTransId ){ + pNode->iV2Child = 0; + pNode->iV2Ptr = 0; + pNode->iV2 = 0; + } + rc = lsmTreeCursorNext(&csr); + } + tblobFree(csr.pDb, &csr.blob); + + db->treehdr.root.nHeight++; + } + + return rc; +} + +static int treeRepairList(lsm_db *db){ + int rc = LSM_OK; + int i; + ShmChunk *p; + ShmChunk *pMin = 0; + u32 iMin = 0; + + /* Iterate through all shm chunks. Find the smallest shm-id present in + ** the shared-memory region. */ + for(i=1; rc==LSM_OK && (u32)itreehdr.nChunk; i++){ + p = treeShmChunkRc(db, i, &rc); + if( p && (pMin==0 || shm_sequence_ge(pMin->iShmid, p->iShmid)) ){ + pMin = p; + iMin = i; + } + } + + /* Fix the shm-id values on any chunks with a shm-id greater than or + ** equal to treehdr.iNextShmid. Then do a merge-sort of all chunks to + ** fix the ShmChunk.iNext pointers. + */ + if( rc==LSM_OK ){ + int nSort; + int nByte; + u32 iPrevShmid; + ShmChunkLoc *aSort; + + /* Allocate space for a merge sort. */ + nSort = 1; + while( (u32)nSort < (db->treehdr.nChunk-1) ) nSort = nSort * 2; + nByte = sizeof(ShmChunkLoc) * nSort * 2; + aSort = lsmMallocZeroRc(db->pEnv, nByte, &rc); + iPrevShmid = pMin->iShmid; + + /* Fix all shm-ids, if required. */ + if( rc==LSM_OK ){ + iPrevShmid = pMin->iShmid-1; + for(i=1; (u32)itreehdr.nChunk; i++){ + p = treeShmChunk(db, i); + aSort[i-1].pShm = p; + aSort[i-1].iLoc = i; + if( (u32)i!=db->treehdr.iFirst ){ + if( shm_sequence_ge(p->iShmid, db->treehdr.iNextShmid) ){ + p->iShmid = iPrevShmid--; + } + } + } + if( iMin!=db->treehdr.iFirst ){ + p = treeShmChunk(db, db->treehdr.iFirst); + p->iShmid = iPrevShmid; + } + } + + if( rc==LSM_OK ){ + ShmChunkLoc *aSpace = &aSort[nSort]; + for(i=0; iiShmid, iPrevShmid) ); + assert( aSpace[aSort[i].pShm->iShmid - iPrevShmid].pShm==0 ); + aSpace[aSort[i].pShm->iShmid - iPrevShmid] = aSort[i]; + } + } + + if( aSpace[nSort-1].pShm ) aSpace[nSort-1].pShm->iNext = 0; + for(i=0; iiNext = aSpace[i+1].iLoc; + } + } + + rc = treeCheckLinkedList(db); + lsmFree(db->pEnv, aSort); + } + } + + return rc; +} + +/* +** This function is called as part of opening a write-transaction if the +** writer-flag is already set - indicating that the previous writer +** failed before ending its transaction. +*/ +int lsmTreeRepair(lsm_db *db){ + int rc = LSM_OK; + TreeHeader hdr; + ShmHeader *pHdr = db->pShmhdr; + + /* Ensure that the two tree-headers are consistent. Copy one over the other + ** if necessary. Prefer the data from a tree-header for which the checksum + ** computes. Or, if they both compute, prefer tree-header-1. */ + if( memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)) ){ + if( treeHeaderChecksumOk(&pHdr->hdr1) ){ + memcpy(&pHdr->hdr2, &pHdr->hdr1, sizeof(TreeHeader)); + }else{ + memcpy(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)); + } + } + + /* Save the connections current copy of the tree-header. It will be + ** restored before returning. */ + memcpy(&hdr, &db->treehdr, sizeof(TreeHeader)); + + /* Walk the tree. Zero any v2 pointers with a transaction-id greater than + ** the transaction-id currently in the tree-headers. */ + rc = treeRepairPtrs(db); + + /* Repair the linked list of shared-memory chunks. */ + if( rc==LSM_OK ){ + rc = treeRepairList(db); + } + + memcpy(&db->treehdr, &hdr, sizeof(TreeHeader)); + return rc; +} + +static void treeOverwriteKey(lsm_db *db, TreeCursor *pCsr, u32 iKey, int *pRc){ + if( *pRc==LSM_OK ){ + TreeRoot *p = &db->treehdr.root; + TreeNode *pNew; + u32 iNew; + TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; + int iCell = pCsr->aiCell[pCsr->iNode]; + + /* Create a copy of this node */ + if( (pCsr->iNode>0 && (u32)pCsr->iNode==(p->nHeight-1)) ){ + pNew = copyTreeLeaf(db, (TreeLeaf *)pNode, &iNew, pRc); + }else{ + pNew = copyTreeNode(db, pNode, &iNew, pRc); + } + + if( pNew ){ + /* Modify the value in the new version */ + pNew->aiKeyPtr[iCell] = iKey; + + /* Change the pointer in the parent (if any) to point at the new + ** TreeNode */ + pCsr->iNode--; + treeUpdatePtr(db, pCsr, iNew); + } + } +} + +static int treeNextIsEndDelete(lsm_db *db, TreeCursor *pCsr){ + int iNode = pCsr->iNode; + int iCell = pCsr->aiCell[iNode]+1; + + /* Cursor currently points to a leaf node. */ + assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); + + while( iNode>=0 ){ + TreeNode *pNode = pCsr->apTreeNode[iNode]; + if( iCell<3 && pNode->aiKeyPtr[iCell] ){ + int rc = LSM_OK; + TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); + assert( rc==LSM_OK ); + return ((pKey->flags & LSM_END_DELETE) ? 1 : 0); + } + iNode--; + iCell = pCsr->aiCell[iNode]; + } + + return 0; +} + +static int treePrevIsStartDelete(lsm_db *db, TreeCursor *pCsr){ + int iNode = pCsr->iNode; + + /* Cursor currently points to a leaf node. */ + assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); + + while( iNode>=0 ){ + TreeNode *pNode = pCsr->apTreeNode[iNode]; + int iCell = pCsr->aiCell[iNode]-1; + if( iCell>=0 && pNode->aiKeyPtr[iCell] ){ + int rc = LSM_OK; + TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); + assert( rc==LSM_OK ); + return ((pKey->flags & LSM_START_DELETE) ? 1 : 0); + } + iNode--; + } + + return 0; +} + + +static int treeInsertEntry( + lsm_db *pDb, /* Database handle */ + int flags, /* Flags associated with entry */ + void *pKey, /* Pointer to key data */ + int nKey, /* Size of key data in bytes */ + void *pVal, /* Pointer to value data (or NULL) */ + int nVal /* Bytes in value data (or -ve for delete) */ +){ + int rc = LSM_OK; /* Return Code */ + TreeKey *pTreeKey; /* New key-value being inserted */ + u32 iTreeKey; + TreeRoot *p = &pDb->treehdr.root; + TreeCursor csr; /* Cursor to seek to pKey/nKey */ + int res = 0; /* Result of seek operation on csr */ + + assert( nVal>=0 || pVal==0 ); + assert_tree_looks_ok(LSM_OK, pTree); + assert( flags==LSM_INSERT || flags==LSM_POINT_DELETE + || flags==LSM_START_DELETE || flags==LSM_END_DELETE + ); + assert( (flags & LSM_CONTIGUOUS)==0 ); +#if 0 + dump_tree_contents(pDb, "before"); +#endif + + if( p->iRoot ){ + TreeKey *pRes; /* Key at end of seek operation */ + treeCursorInit(pDb, 0, &csr); + + /* Seek to the leaf (or internal node) that the new key belongs on */ + rc = lsmTreeCursorSeek(&csr, pKey, nKey, &res); + pRes = csrGetKey(&csr, &csr.blob, &rc); + if( rc!=LSM_OK ) return rc; + assert( pRes ); + + if( flags==LSM_START_DELETE ){ + /* When inserting a start-delete-range entry, if the key that + ** occurs immediately before the new entry is already a START_DELETE, + ** then the new entry is not required. */ + if( (res<=0 && (pRes->flags & LSM_START_DELETE)) + || (res>0 && treePrevIsStartDelete(pDb, &csr)) + ){ + goto insert_entry_out; + } + }else if( flags==LSM_END_DELETE ){ + /* When inserting an start-delete-range entry, if the key that + ** occurs immediately after the new entry is already an END_DELETE, + ** then the new entry is not required. */ + if( (res<0 && treeNextIsEndDelete(pDb, &csr)) + || (res>=0 && (pRes->flags & LSM_END_DELETE)) + ){ + goto insert_entry_out; + } + } + + if( res==0 && (flags & (LSM_END_DELETE|LSM_START_DELETE)) ){ + if( pRes->flags & LSM_INSERT ){ + nVal = pRes->nValue; + pVal = TKV_VAL(pRes); + } + flags = flags | pRes->flags; + } + + if( flags & (LSM_INSERT|LSM_POINT_DELETE) ){ + if( (res<0 && (pRes->flags & LSM_START_DELETE)) + || (res>0 && (pRes->flags & LSM_END_DELETE)) + ){ + flags = flags | (LSM_END_DELETE|LSM_START_DELETE); + }else if( res==0 ){ + flags = flags | (pRes->flags & (LSM_END_DELETE|LSM_START_DELETE)); + } + } + }else{ + memset(&csr, 0, sizeof(TreeCursor)); + } + + /* Allocate and populate a new key-value pair structure */ + pTreeKey = newTreeKey(pDb, &iTreeKey, pKey, nKey, pVal, nVal, &rc); + if( rc!=LSM_OK ) return rc; + assert( pTreeKey->flags==0 || pTreeKey->flags==LSM_CONTIGUOUS ); + pTreeKey->flags |= flags; + + if( p->iRoot==0 ){ + /* The tree is completely empty. Add a new root node and install + ** (pKey/nKey) as the middle entry. Even though it is a leaf at the + ** moment, use newTreeNode() to allocate the node (i.e. allocate enough + ** space for the fields used by interior nodes). This is because the + ** treeInsert() routine may convert this node to an interior node. */ + TreeNode *pRoot = newTreeNode(pDb, &p->iRoot, &rc); + if( rc==LSM_OK ){ + assert( p->nHeight==0 ); + pRoot->aiKeyPtr[1] = iTreeKey; + p->nHeight = 1; + } + }else{ + if( res==0 ){ + /* The search found a match within the tree. */ + treeOverwriteKey(pDb, &csr, iTreeKey, &rc); + }else{ + /* The cursor now points to the leaf node into which the new entry should + ** be inserted. There may or may not be a free slot within the leaf for + ** the new key-value pair. + ** + ** iSlot is set to the index of the key within pLeaf that the new key + ** should be inserted to the left of (or to a value 1 greater than the + ** index of the rightmost key if the new key is larger than all keys + ** currently stored in the node). + */ + int iSlot = csr.aiCell[csr.iNode] + (res<0); + if( csr.iNode==0 ){ + rc = treeInsert(pDb, &csr, 0, iTreeKey, 0, iSlot); + }else{ + rc = treeInsertLeaf(pDb, &csr, iTreeKey, iSlot); + } + } + } + +#if 0 + dump_tree_contents(pDb, "after"); +#endif + insert_entry_out: + tblobFree(pDb, &csr.blob); + assert_tree_looks_ok(rc, pTree); + return rc; +} + +/* +** Insert a new entry into the in-memory tree. +** +** If the value of the 5th parameter, nVal, is negative, then a delete-marker +** is inserted into the tree. In this case the value pointer, pVal, must be +** NULL. +*/ +int lsmTreeInsert( + lsm_db *pDb, /* Database handle */ + void *pKey, /* Pointer to key data */ + int nKey, /* Size of key data in bytes */ + void *pVal, /* Pointer to value data (or NULL) */ + int nVal /* Bytes in value data (or -ve for delete) */ +){ + int flags; + if( nVal<0 ){ + flags = LSM_POINT_DELETE; + }else{ + flags = LSM_INSERT; + } + + return treeInsertEntry(pDb, flags, pKey, nKey, pVal, nVal); +} + +static int treeDeleteEntry(lsm_db *db, TreeCursor *pCsr, u32 iNewptr){ + TreeRoot *p = &db->treehdr.root; + TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; + int iSlot = pCsr->aiCell[pCsr->iNode]; + int bLeaf; + int rc = LSM_OK; + + assert( pNode->aiKeyPtr[1] ); + assert( pNode->aiKeyPtr[iSlot] ); + assert( iSlot==0 || iSlot==1 || iSlot==2 ); + assert( ((u32)pCsr->iNode==(db->treehdr.root.nHeight-1))==(iNewptr==0) ); + + bLeaf = ((u32)pCsr->iNode==(p->nHeight-1) && p->nHeight>1); + + if( pNode->aiKeyPtr[0] || pNode->aiKeyPtr[2] ){ + /* There are currently at least 2 keys on this node. So just create + ** a new copy of the node with one of the keys removed. If the node + ** happens to be the root node of the tree, allocate an entire + ** TreeNode structure instead of just a TreeLeaf. */ + TreeNode *pNew; + u32 iNew; + + if( bLeaf ){ + pNew = (TreeNode *)newTreeLeaf(db, &iNew, &rc); + }else{ + pNew = newTreeNode(db, &iNew, &rc); + } + if( pNew ){ + int i; + int iOut = 1; + for(i=0; i<4; i++){ + if( i==iSlot ){ + i++; + if( bLeaf==0 ) pNew->aiChildPtr[iOut] = iNewptr; + if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; + iOut++; + }else if( bLeaf || p->nHeight==1 ){ + if( i<3 && pNode->aiKeyPtr[i] ){ + pNew->aiKeyPtr[iOut++] = pNode->aiKeyPtr[i]; + } + }else{ + if( getChildPtr(pNode, WORKING_VERSION, i) ){ + pNew->aiChildPtr[iOut] = getChildPtr(pNode, WORKING_VERSION, i); + if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; + iOut++; + } + } + } + assert( iOut<=4 ); + assert( bLeaf || pNew->aiChildPtr[0]==0 ); + pCsr->iNode--; + rc = treeUpdatePtr(db, pCsr, iNew); + } + + }else if( pCsr->iNode==0 ){ + /* Removing the only key in the root node. iNewptr is the new root. */ + assert( iSlot==1 ); + db->treehdr.root.iRoot = iNewptr; + db->treehdr.root.nHeight--; + + }else{ + /* There is only one key on this node and the node is not the root + ** node. Find a peer for this node. Then redistribute the contents of + ** the peer and the parent cell between the parent and either one or + ** two new nodes. */ + TreeNode *pParent; /* Parent tree node */ + int iPSlot; + u32 iPeer; /* Pointer to peer leaf node */ + int iDir; + TreeNode *pPeer; /* The peer leaf node */ + TreeNode *pNew1; u32 iNew1; /* First new leaf node */ + + assert( iSlot==1 ); + + pParent = pCsr->apTreeNode[pCsr->iNode-1]; + iPSlot = pCsr->aiCell[pCsr->iNode-1]; + + if( iPSlot>0 && getChildPtr(pParent, WORKING_VERSION, iPSlot-1) ){ + iDir = -1; + }else{ + iDir = +1; + } + iPeer = getChildPtr(pParent, WORKING_VERSION, iPSlot+iDir); + pPeer = (TreeNode *)treeShmptr(db, iPeer); + assertIsWorkingChild(db, pNode, pParent, iPSlot); + + /* Allocate the first new leaf node. This is always required. */ + if( bLeaf ){ + pNew1 = (TreeNode *)newTreeLeaf(db, &iNew1, &rc); + }else{ + pNew1 = (TreeNode *)newTreeNode(db, &iNew1, &rc); + } + + if( pPeer->aiKeyPtr[0] && pPeer->aiKeyPtr[2] ){ + /* Peer node is completely full. This means that two new leaf nodes + ** and a new parent node are required. */ + + TreeNode *pNew2; u32 iNew2; /* Second new leaf node */ + TreeNode *pNewP; u32 iNewP; /* New parent node */ + + if( bLeaf ){ + pNew2 = (TreeNode *)newTreeLeaf(db, &iNew2, &rc); + }else{ + pNew2 = (TreeNode *)newTreeNode(db, &iNew2, &rc); + } + pNewP = copyTreeNode(db, pParent, &iNewP, &rc); + + if( iDir==-1 ){ + pNew1->aiKeyPtr[1] = pPeer->aiKeyPtr[0]; + if( bLeaf==0 ){ + pNew1->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 0); + pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 1); + } + + pNewP->aiChildPtr[iPSlot-1] = iNew1; + pNewP->aiKeyPtr[iPSlot-1] = pPeer->aiKeyPtr[1]; + pNewP->aiChildPtr[iPSlot] = iNew2; + + pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[2]; + pNew2->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot-1]; + if( bLeaf==0 ){ + pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 2); + pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 3); + pNew2->aiChildPtr[2] = iNewptr; + } + }else{ + pNew1->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot]; + if( bLeaf==0 ){ + pNew1->aiChildPtr[1] = iNewptr; + pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 0); + } + + pNewP->aiChildPtr[iPSlot] = iNew1; + pNewP->aiKeyPtr[iPSlot] = pPeer->aiKeyPtr[0]; + pNewP->aiChildPtr[iPSlot+1] = iNew2; + + pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[1]; + pNew2->aiKeyPtr[1] = pPeer->aiKeyPtr[2]; + if( bLeaf==0 ){ + pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 1); + pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 2); + pNew2->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 3); + } + } + assert( pCsr->iNode>=1 ); + pCsr->iNode -= 2; + if( rc==LSM_OK ){ + assert( pNew1->aiKeyPtr[1] && pNew2->aiKeyPtr[1] ); + rc = treeUpdatePtr(db, pCsr, iNewP); + } + }else{ + int iKOut = 0; + int iPOut = 0; + int i; + + pCsr->iNode--; + + if( iDir==1 ){ + pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; + if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; + } + for(i=0; i<3; i++){ + if( pPeer->aiKeyPtr[i] ){ + pNew1->aiKeyPtr[iKOut++] = pPeer->aiKeyPtr[i]; + } + } + if( bLeaf==0 ){ + for(i=0; i<4; i++){ + if( getChildPtr(pPeer, WORKING_VERSION, i) ){ + pNew1->aiChildPtr[iPOut++] = getChildPtr(pPeer, WORKING_VERSION, i); + } + } + } + if( iDir==-1 ){ + iPSlot--; + pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; + if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; + pCsr->aiCell[pCsr->iNode] = (u8)iPSlot; + } + + rc = treeDeleteEntry(db, pCsr, iNew1); + } + } + + return rc; +} + +/* +** Delete a range of keys from the tree structure (i.e. the lsm_delete_range() +** function, not lsm_delete()). +** +** This is a two step process: +** +** 1) Remove all entries currently stored in the tree that have keys +** that fall into the deleted range. +** +** TODO: There are surely good ways to optimize this step - removing +** a range of keys from a b-tree. But for now, this function removes +** them one at a time using the usual approach. +** +** 2) Unless the largest key smaller than or equal to (pKey1/nKey1) is +** already marked as START_DELETE, insert a START_DELETE key. +** Similarly, unless the smallest key greater than or equal to +** (pKey2/nKey2) is already START_END, insert a START_END key. +*/ +int lsmTreeDelete( + lsm_db *db, + void *pKey1, int nKey1, /* Start of range */ + void *pKey2, int nKey2 /* End of range */ +){ + int rc = LSM_OK; + int bDone = 0; + TreeRoot *p = &db->treehdr.root; + TreeBlob blob = {0, 0}; + + /* The range must be sensible - that (key1 < key2). */ + assert( treeKeycmp(pKey1, nKey1, pKey2, nKey2)<0 ); + assert( assert_delete_ranges_match(db) ); + +#if 0 + static int nCall = 0; + printf("\n"); + nCall++; + printf("%d delete %s .. %s\n", nCall, (char *)pKey1, (char *)pKey2); + dump_tree_contents(db, "before delete"); +#endif + + /* Step 1. This loop runs until the tree contains no keys within the + ** range being deleted. Or until an error occurs. */ + while( bDone==0 && rc==LSM_OK ){ + int res; + TreeCursor csr; /* Cursor to seek to first key in range */ + void *pDel; int nDel; /* Key to (possibly) delete this iteration */ +#ifndef NDEBUG + int nEntry = treeCountEntries(db); +#endif + + /* Seek the cursor to the first entry in the tree greater than pKey1. */ + treeCursorInit(db, 0, &csr); + lsmTreeCursorSeek(&csr, pKey1, nKey1, &res); + if( res<=0 && lsmTreeCursorValid(&csr) ) lsmTreeCursorNext(&csr); + + /* If there is no such entry, or if it is greater than pKey2, then the + ** tree now contains no keys in the range being deleted. In this case + ** break out of the loop. */ + bDone = 1; + if( lsmTreeCursorValid(&csr) ){ + lsmTreeCursorKey(&csr, 0, &pDel, &nDel); + if( treeKeycmp(pDel, nDel, pKey2, nKey2)<0 ) bDone = 0; + } + + if( bDone==0 ){ + if( (u32)csr.iNode==(p->nHeight-1) ){ + /* The element to delete already lies on a leaf node */ + rc = treeDeleteEntry(db, &csr, 0); + }else{ + /* 1. Overwrite the current key with a copy of the next key in the + ** tree (key N). + ** + ** 2. Seek to key N (cursor will stop at the internal node copy of + ** N). Move to the next key (original copy of N). Delete + ** this entry. + */ + u32 iKey; + TreeKey *pKey; + int iNode = csr.iNode; + lsmTreeCursorNext(&csr); + assert( (u32)csr.iNode==(p->nHeight-1) ); + + iKey = csr.apTreeNode[csr.iNode]->aiKeyPtr[csr.aiCell[csr.iNode]]; + lsmTreeCursorPrev(&csr); + + treeOverwriteKey(db, &csr, iKey, &rc); + pKey = treeShmkey(db, iKey, TKV_LOADKEY, &blob, &rc); + if( pKey ){ + rc = lsmTreeCursorSeek(&csr, TKV_KEY(pKey), pKey->nKey, &res); + } + if( rc==LSM_OK ){ + assert( res==0 && csr.iNode==iNode ); + rc = lsmTreeCursorNext(&csr); + if( rc==LSM_OK ){ + rc = treeDeleteEntry(db, &csr, 0); + } + } + } + } + + /* Clean up any memory allocated by the cursor. */ + tblobFree(db, &csr.blob); +#if 0 + dump_tree_contents(db, "ddd delete"); +#endif + assert( bDone || treeCountEntries(db)==(nEntry-1) ); + } + +#if 0 + dump_tree_contents(db, "during delete"); +#endif + + /* Now insert the START_DELETE and END_DELETE keys. */ + if( rc==LSM_OK ){ + rc = treeInsertEntry(db, LSM_START_DELETE, pKey1, nKey1, 0, -1); + } +#if 0 + dump_tree_contents(db, "during delete 2"); +#endif + if( rc==LSM_OK ){ + rc = treeInsertEntry(db, LSM_END_DELETE, pKey2, nKey2, 0, -1); + } + +#if 0 + dump_tree_contents(db, "after delete"); +#endif + + tblobFree(db, &blob); + assert( assert_delete_ranges_match(db) ); + return rc; +} + +/* +** Return, in bytes, the amount of memory currently used by the tree +** structure. +*/ +int lsmTreeSize(lsm_db *pDb){ + return pDb->treehdr.root.nByte; +} + +/* +** Open a cursor on the in-memory tree pTree. +*/ +int lsmTreeCursorNew(lsm_db *pDb, int bOld, TreeCursor **ppCsr){ + TreeCursor *pCsr; + *ppCsr = pCsr = lsmMalloc(pDb->pEnv, sizeof(TreeCursor)); + if( pCsr ){ + treeCursorInit(pDb, bOld, pCsr); + return LSM_OK; + } + return LSM_NOMEM_BKPT; +} + +/* +** Close an in-memory tree cursor. +*/ +void lsmTreeCursorDestroy(TreeCursor *pCsr){ + if( pCsr ){ + tblobFree(pCsr->pDb, &pCsr->blob); + lsmFree(pCsr->pDb->pEnv, pCsr); + } +} + +void lsmTreeCursorReset(TreeCursor *pCsr){ + if( pCsr ){ + pCsr->iNode = -1; + pCsr->pSave = 0; + } +} + +#ifndef NDEBUG +static int treeCsrCompare(TreeCursor *pCsr, void *pKey, int nKey, int *pRc){ + TreeKey *p; + int cmp = 0; + assert( pCsr->iNode>=0 ); + p = csrGetKey(pCsr, &pCsr->blob, pRc); + if( p ){ + cmp = treeKeycmp(TKV_KEY(p), p->nKey, pKey, nKey); + } + return cmp; +} +#endif + + +/* +** Attempt to seek the cursor passed as the first argument to key (pKey/nKey) +** in the tree structure. If an exact match for the key is found, leave the +** cursor pointing to it and set *pRes to zero before returning. If an +** exact match cannot be found, do one of the following: +** +** * Leave the cursor pointing to the smallest element in the tree that +** is larger than the key and set *pRes to +1, or +** +** * Leave the cursor pointing to the largest element in the tree that +** is smaller than the key and set *pRes to -1, or +** +** * If the tree is empty, leave the cursor at EOF and set *pRes to -1. +*/ +int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes){ + int rc = LSM_OK; /* Return code */ + lsm_db *pDb = pCsr->pDb; + TreeRoot *pRoot = pCsr->pRoot; + u32 iNodePtr; /* Location of current node in search */ + + /* Discard any saved position data */ + treeCursorRestore(pCsr, 0); + + iNodePtr = pRoot->iRoot; + if( iNodePtr==0 ){ + /* Either an error occurred or the tree is completely empty. */ + assert( rc!=LSM_OK || pRoot->iRoot==0 ); + *pRes = -1; + pCsr->iNode = -1; + }else{ + TreeBlob b = {0, 0}; + int res = 0; /* Result of comparison function */ + int iNode = -1; + while( iNodePtr ){ + TreeNode *pNode; /* Node at location iNodePtr */ + int iTest; /* Index of second key to test (0 or 2) */ + u32 iTreeKey; + TreeKey *pTreeKey; /* Key to compare against */ + + pNode = (TreeNode *)treeShmptrUnsafe(pDb, iNodePtr); + iNode++; + pCsr->apTreeNode[iNode] = pNode; + + /* Compare (pKey/nKey) with the key in the middle slot of B-tree node + ** pNode. The middle slot is never empty. If the comparison is a match, + ** then the search is finished. Break out of the loop. */ + pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, pNode->aiKeyPtr[1]); + if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ + pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); + if( rc!=LSM_OK ) break; + } + res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); + if( res==0 ){ + pCsr->aiCell[iNode] = 1; + break; + } + + /* Based on the results of the previous comparison, compare (pKey/nKey) + ** to either the left or right key of the B-tree node, if such a key + ** exists. */ + iTest = (res>0 ? 0 : 2); + iTreeKey = pNode->aiKeyPtr[iTest]; + if( iTreeKey ){ + pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, iTreeKey); + if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ + pTreeKey = treeShmkey(pDb, iTreeKey, TKV_LOADKEY, &b, &rc); + if( rc ) break; + } + res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); + if( res==0 ){ + pCsr->aiCell[iNode] = (u8)iTest; + break; + } + }else{ + iTest = 1; + } + + if( (u32)iNode<(pRoot->nHeight-1) ){ + iNodePtr = getChildPtr(pNode, pRoot->iTransId, iTest + (res<0)); + }else{ + iNodePtr = 0; + } + pCsr->aiCell[iNode] = (u8)(iTest + (iNodePtr && (res<0))); + } + + *pRes = res; + pCsr->iNode = iNode; + tblobFree(pDb, &b); + } + + /* assert() that *pRes has been set properly */ +#ifndef NDEBUG + if( rc==LSM_OK && lsmTreeCursorValid(pCsr) ){ + int cmp = treeCsrCompare(pCsr, pKey, nKey, &rc); + assert( rc!=LSM_OK || *pRes==cmp || (*pRes ^ cmp)>0 ); + } +#endif + + return rc; +} + +int lsmTreeCursorNext(TreeCursor *pCsr){ +#ifndef NDEBUG + TreeKey *pK1; + TreeBlob key1 = {0, 0}; +#endif + lsm_db *pDb = pCsr->pDb; + TreeRoot *pRoot = pCsr->pRoot; + const int iLeaf = pRoot->nHeight-1; + int iCell; + int rc = LSM_OK; + TreeNode *pNode; + + /* Restore the cursor position, if required */ + int iRestore = 0; + treeCursorRestore(pCsr, &iRestore); + if( iRestore>0 ) return LSM_OK; + + /* Save a pointer to the current key. This is used in an assert() at the + ** end of this function - to check that the 'next' key really is larger + ** than the current key. */ +#ifndef NDEBUG + pK1 = csrGetKey(pCsr, &key1, &rc); + if( rc!=LSM_OK ) return rc; +#endif + + assert( lsmTreeCursorValid(pCsr) ); + assert( pCsr->aiCell[pCsr->iNode]<3 ); + + pNode = pCsr->apTreeNode[pCsr->iNode]; + iCell = ++pCsr->aiCell[pCsr->iNode]; + + /* If the current node is not a leaf, and the current cell has sub-tree + ** associated with it, descend to the left-most key on the left-most + ** leaf of the sub-tree. */ + if( pCsr->iNodeiTransId, iCell) ){ + do { + u32 iNodePtr; + pCsr->iNode++; + iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + pCsr->apTreeNode[pCsr->iNode] = pNode; + iCell = pCsr->aiCell[pCsr->iNode] = (pNode->aiKeyPtr[0]==0); + }while( pCsr->iNode < iLeaf ); + } + + /* Otherwise, the next key is found by following pointer up the tree + ** until there is a key immediately to the right of the pointer followed + ** to reach the sub-tree containing the current key. */ + else if( iCell>=3 || pNode->aiKeyPtr[iCell]==0 ){ + while( (--pCsr->iNode)>=0 ){ + iCell = pCsr->aiCell[pCsr->iNode]; + if( iCell<3 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; + } + } + +#ifndef NDEBUG + if( pCsr->iNode>=0 ){ + TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); + assert( rc||treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); + } + tblobFree(pDb, &key1); +#endif + + return rc; +} + +int lsmTreeCursorPrev(TreeCursor *pCsr){ +#ifndef NDEBUG + TreeKey *pK1; + TreeBlob key1 = {0, 0}; +#endif + lsm_db *pDb = pCsr->pDb; + TreeRoot *pRoot = pCsr->pRoot; + const int iLeaf = pRoot->nHeight-1; + int iCell; + int rc = LSM_OK; + TreeNode *pNode; + + /* Restore the cursor position, if required */ + int iRestore = 0; + treeCursorRestore(pCsr, &iRestore); + if( iRestore<0 ) return LSM_OK; + + /* Save a pointer to the current key. This is used in an assert() at the + ** end of this function - to check that the 'next' key really is smaller + ** than the current key. */ +#ifndef NDEBUG + pK1 = csrGetKey(pCsr, &key1, &rc); + if( rc!=LSM_OK ) return rc; +#endif + + assert( lsmTreeCursorValid(pCsr) ); + pNode = pCsr->apTreeNode[pCsr->iNode]; + iCell = pCsr->aiCell[pCsr->iNode]; + assert( iCell>=0 && iCell<3 ); + + /* If the current node is not a leaf, and the current cell has sub-tree + ** associated with it, descend to the right-most key on the right-most + ** leaf of the sub-tree. */ + if( pCsr->iNodeiTransId, iCell) ){ + do { + u32 iNodePtr; + pCsr->iNode++; + iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + if( rc!=LSM_OK ) break; + pCsr->apTreeNode[pCsr->iNode] = pNode; + iCell = 1 + (pNode->aiKeyPtr[2]!=0) + (pCsr->iNode < iLeaf); + pCsr->aiCell[pCsr->iNode] = (u8)iCell; + }while( pCsr->iNode < iLeaf ); + } + + /* Otherwise, the next key is found by following pointer up the tree until + ** there is a key immediately to the left of the pointer followed to reach + ** the sub-tree containing the current key. */ + else{ + do { + iCell = pCsr->aiCell[pCsr->iNode]-1; + if( iCell>=0 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; + }while( (--pCsr->iNode)>=0 ); + pCsr->aiCell[pCsr->iNode] = (u8)iCell; + } + +#ifndef NDEBUG + if( pCsr->iNode>=0 ){ + TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); + assert( rc || treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)<0 ); + } + tblobFree(pDb, &key1); +#endif + + return rc; +} + +/* +** Move the cursor to the first (bLast==0) or last (bLast!=0) entry in the +** in-memory tree. +*/ +int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast){ + lsm_db *pDb = pCsr->pDb; + TreeRoot *pRoot = pCsr->pRoot; + int rc = LSM_OK; + + u32 iNodePtr; + pCsr->iNode = -1; + + /* Discard any saved position data */ + treeCursorRestore(pCsr, 0); + + iNodePtr = pRoot->iRoot; + while( iNodePtr ){ + int iCell; + TreeNode *pNode; + + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + if( rc ) break; + + if( bLast ){ + iCell = ((pNode->aiKeyPtr[2]==0) ? 2 : 3); + }else{ + iCell = ((pNode->aiKeyPtr[0]==0) ? 1 : 0); + } + pCsr->iNode++; + pCsr->apTreeNode[pCsr->iNode] = pNode; + + if( (u32)pCsr->iNodenHeight-1 ){ + iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); + }else{ + iNodePtr = 0; + } + pCsr->aiCell[pCsr->iNode] = (u8)(iCell - (iNodePtr==0 && bLast)); + } + + return rc; +} + +int lsmTreeCursorFlags(TreeCursor *pCsr){ + int flags = 0; + if( pCsr && pCsr->iNode>=0 ){ + int rc = LSM_OK; + TreeKey *pKey = (TreeKey *)treeShmptrUnsafe(pCsr->pDb, + pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] + ); + assert( rc==LSM_OK ); + flags = (pKey->flags & ~LSM_CONTIGUOUS); + } + return flags; +} + +int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey){ + TreeKey *pTreeKey; + int rc = LSM_OK; + + assert( lsmTreeCursorValid(pCsr) ); + + pTreeKey = pCsr->pSave; + if( !pTreeKey ){ + pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); + } + if( rc==LSM_OK ){ + *pnKey = pTreeKey->nKey; + if( pFlags ) *pFlags = pTreeKey->flags; + *ppKey = (void *)&pTreeKey[1]; + } + + return rc; +} + +int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal){ + int res = 0; + int rc; + + rc = treeCursorRestore(pCsr, &res); + if( res==0 ){ + TreeKey *pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); + if( rc==LSM_OK ){ + if( pTreeKey->flags & LSM_INSERT ){ + *pnVal = pTreeKey->nValue; + *ppVal = TKV_VAL(pTreeKey); + }else{ + *ppVal = 0; + *pnVal = -1; + } + } + }else{ + *ppVal = 0; + *pnVal = 0; + } + + return rc; +} + +/* +** Return true if the cursor currently points to a valid entry. +*/ +int lsmTreeCursorValid(TreeCursor *pCsr){ + return (pCsr && (pCsr->pSave || pCsr->iNode>=0)); +} + +/* +** Store a mark in *pMark. Later on, a call to lsmTreeRollback() with a +** pointer to the same TreeMark structure may be used to roll the tree +** contents back to their current state. +*/ +void lsmTreeMark(lsm_db *pDb, TreeMark *pMark){ + pMark->iRoot = pDb->treehdr.root.iRoot; + pMark->nHeight = pDb->treehdr.root.nHeight; + pMark->iWrite = pDb->treehdr.iWrite; + pMark->nChunk = pDb->treehdr.nChunk; + pMark->iNextShmid = pDb->treehdr.iNextShmid; + pMark->iRollback = intArraySize(&pDb->rollback); +} + +/* +** Roll back to mark pMark. Structure *pMark should have been previously +** populated by a call to lsmTreeMark(). +*/ +void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark){ + int iIdx; + int nIdx; + u32 iNext; + ShmChunk *pChunk; + u32 iChunk; + u32 iShmid; + + /* Revert all required v2 pointers. */ + nIdx = intArraySize(&pDb->rollback); + for(iIdx = pMark->iRollback; iIdxrollback, iIdx)); + assert( pNode ); + pNode->iV2 = 0; + pNode->iV2Child = 0; + pNode->iV2Ptr = 0; + } + intArrayTruncate(&pDb->rollback, pMark->iRollback); + + /* Restore the free-chunk list. */ + assert( pMark->iWrite!=0 ); + iChunk = treeOffsetToChunk(pMark->iWrite-1); + pChunk = treeShmChunk(pDb, iChunk); + iNext = pChunk->iNext; + pChunk->iNext = 0; + + pChunk = treeShmChunk(pDb, pDb->treehdr.iFirst); + iShmid = pChunk->iShmid-1; + + while( iNext ){ + u32 iFree = iNext; /* Current chunk being rollback-freed */ + ShmChunk *pFree; /* Pointer to chunk iFree */ + + pFree = treeShmChunk(pDb, iFree); + iNext = pFree->iNext; + + if( iFreenChunk ){ + pFree->iNext = pDb->treehdr.iFirst; + pFree->iShmid = iShmid--; + pDb->treehdr.iFirst = iFree; + } + } + + /* Restore the tree-header fields */ + pDb->treehdr.root.iRoot = pMark->iRoot; + pDb->treehdr.root.nHeight = pMark->nHeight; + pDb->treehdr.iWrite = pMark->iWrite; + pDb->treehdr.nChunk = pMark->nChunk; + pDb->treehdr.iNextShmid = pMark->iNextShmid; +} + +/* +** Load the in-memory tree header from shared-memory into pDb->treehdr. +** If the header cannot be loaded, return LSM_PROTOCOL. +** +** If the header is successfully loaded and parameter piRead is not NULL, +** is is set to 1 if the header was loaded from ShmHeader.hdr1, or 2 if +** the header was loaded from ShmHeader.hdr2. +*/ +int lsmTreeLoadHeader(lsm_db *pDb, int *piRead){ + int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; + while( (nRem--)>0 ){ + ShmHeader *pShm = pDb->pShmhdr; + + memcpy(&pDb->treehdr, &pShm->hdr1, sizeof(TreeHeader)); + if( treeHeaderChecksumOk(&pDb->treehdr) ){ + if( piRead ) *piRead = 1; + return LSM_OK; + } + memcpy(&pDb->treehdr, &pShm->hdr2, sizeof(TreeHeader)); + if( treeHeaderChecksumOk(&pDb->treehdr) ){ + if( piRead ) *piRead = 2; + return LSM_OK; + } + + lsmShmBarrier(pDb); + } + return LSM_PROTOCOL_BKPT; +} + +int lsmTreeLoadHeaderOk(lsm_db *pDb, int iRead){ + TreeHeader *p = (iRead==1) ? &pDb->pShmhdr->hdr1 : &pDb->pShmhdr->hdr2; + assert( iRead==1 || iRead==2 ); + return (0==memcmp(pDb->treehdr.aCksum, p->aCksum, sizeof(u32)*2)); +} + +/* +** This function is called to conclude a transaction. If argument bCommit +** is true, the transaction is committed. Otherwise it is rolled back. +*/ +int lsmTreeEndTransaction(lsm_db *pDb, int bCommit){ + ShmHeader *pShm = pDb->pShmhdr; + + treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); + memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); + lsmShmBarrier(pDb); + memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); + pShm->bWriter = 0; + intArrayFree(pDb->pEnv, &pDb->rollback); + + return LSM_OK; +} + +#ifndef NDEBUG +static int assert_delete_ranges_match(lsm_db *db){ + int prev = 0; + TreeBlob blob = {0, 0}; + TreeCursor csr; /* Cursor used to iterate through tree */ + int rc; + + treeCursorInit(db, 0, &csr); + for( rc = lsmTreeCursorEnd(&csr, 0); + rc==LSM_OK && lsmTreeCursorValid(&csr); + rc = lsmTreeCursorNext(&csr) + ){ + TreeKey *pKey = csrGetKey(&csr, &blob, &rc); + if( rc!=LSM_OK ) break; + assert( ((prev&LSM_START_DELETE)==0)==((pKey->flags&LSM_END_DELETE)==0) ); + prev = pKey->flags; + } + + tblobFree(csr.pDb, &csr.blob); + tblobFree(csr.pDb, &blob); + + return 1; +} + +static int treeCountEntries(lsm_db *db){ + TreeCursor csr; /* Cursor used to iterate through tree */ + int rc; + int nEntry = 0; + + treeCursorInit(db, 0, &csr); + for( rc = lsmTreeCursorEnd(&csr, 0); + rc==LSM_OK && lsmTreeCursorValid(&csr); + rc = lsmTreeCursorNext(&csr) + ){ + nEntry++; + } + + tblobFree(csr.pDb, &csr.blob); + + return nEntry; +} +#endif diff --git a/ext/lsm1/lsm_unix.c b/ext/lsm1/lsm_unix.c new file mode 100644 index 0000000000000000000000000000000000000000..88952d15fc721ac19747a0cf284b7f7b28520c45 --- /dev/null +++ b/ext/lsm1/lsm_unix.c @@ -0,0 +1,753 @@ +/* +** 2011-12-03 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Unix-specific run-time environment implementation for LSM. +*/ + +#ifndef _WIN32 + +#if defined(__GNUC__) || defined(__TINYC__) +/* workaround for ftruncate() visibility on gcc. */ +# ifndef _XOPEN_SOURCE +# define _XOPEN_SOURCE 500 +# endif +#endif + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include "lsmInt.h" + +/* There is no fdatasync() call on Android */ +#ifdef __ANDROID__ +# define fdatasync(x) fsync(x) +#endif + +/* +** An open file is an instance of the following object +*/ +typedef struct PosixFile PosixFile; +struct PosixFile { + lsm_env *pEnv; /* The run-time environment */ + const char *zName; /* Full path to file */ + int fd; /* The open file descriptor */ + int shmfd; /* Shared memory file-descriptor */ + void *pMap; /* Pointer to mapping of file fd */ + off_t nMap; /* Size of mapping at pMap in bytes */ + int nShm; /* Number of entries in array apShm[] */ + void **apShm; /* Array of 32K shared memory segments */ +}; + +static char *posixShmFile(PosixFile *p){ + char *zShm; + int nName = strlen(p->zName); + zShm = (char *)lsmMalloc(p->pEnv, nName+4+1); + if( zShm ){ + memcpy(zShm, p->zName, nName); + memcpy(&zShm[nName], "-shm", 5); + } + return zShm; +} + +static int lsmPosixOsOpen( + lsm_env *pEnv, + const char *zFile, + int flags, + lsm_file **ppFile +){ + int rc = LSM_OK; + PosixFile *p; + + p = lsm_malloc(pEnv, sizeof(PosixFile)); + if( p==0 ){ + rc = LSM_NOMEM; + }else{ + int bReadonly = (flags & LSM_OPEN_READONLY); + int oflags = (bReadonly ? O_RDONLY : (O_RDWR|O_CREAT)); + memset(p, 0, sizeof(PosixFile)); + p->zName = zFile; + p->pEnv = pEnv; + p->fd = open(zFile, oflags, 0644); + if( p->fd<0 ){ + lsm_free(pEnv, p); + p = 0; + if( errno==ENOENT ){ + rc = lsmErrorBkpt(LSM_IOERR_NOENT); + }else{ + rc = LSM_IOERR_BKPT; + } + } + } + + *ppFile = (lsm_file *)p; + return rc; +} + +static int lsmPosixOsWrite( + lsm_file *pFile, /* File to write to */ + lsm_i64 iOff, /* Offset to write to */ + void *pData, /* Write data from this buffer */ + int nData /* Bytes of data to write */ +){ + int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + off_t offset; + + offset = lseek(p->fd, (off_t)iOff, SEEK_SET); + if( offset!=iOff ){ + rc = LSM_IOERR_BKPT; + }else{ + ssize_t prc = write(p->fd, pData, (size_t)nData); + if( prc<0 ) rc = LSM_IOERR_BKPT; + } + + return rc; +} + +static int lsmPosixOsTruncate( + lsm_file *pFile, /* File to write to */ + lsm_i64 nSize /* Size to truncate file to */ +){ + PosixFile *p = (PosixFile *)pFile; + int rc = LSM_OK; /* Return code */ + int prc; /* Posix Return Code */ + struct stat sStat; /* Result of fstat() invocation */ + + prc = fstat(p->fd, &sStat); + if( prc==0 && sStat.st_size>nSize ){ + prc = ftruncate(p->fd, (off_t)nSize); + } + if( prc<0 ) rc = LSM_IOERR_BKPT; + + return rc; +} + +static int lsmPosixOsRead( + lsm_file *pFile, /* File to read from */ + lsm_i64 iOff, /* Offset to read from */ + void *pData, /* Read data into this buffer */ + int nData /* Bytes of data to read */ +){ + int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + off_t offset; + + offset = lseek(p->fd, (off_t)iOff, SEEK_SET); + if( offset!=iOff ){ + rc = LSM_IOERR_BKPT; + }else{ + ssize_t prc = read(p->fd, pData, (size_t)nData); + if( prc<0 ){ + rc = LSM_IOERR_BKPT; + }else if( prcpMap ){ + prc = msync(p->pMap, p->nMap, MS_SYNC); + } + if( prc==0 ) prc = fdatasync(p->fd); + if( prc<0 ) rc = LSM_IOERR_BKPT; +#else + (void)pFile; +#endif + + return rc; +} + +static int lsmPosixOsSectorSize(lsm_file *pFile){ + return 512; +} + +static int lsmPosixOsRemap( + lsm_file *pFile, + lsm_i64 iMin, + void **ppOut, + lsm_i64 *pnOut +){ + off_t iSz; + int prc; + PosixFile *p = (PosixFile *)pFile; + struct stat buf; + + /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. + ** Thereafter, in chunks of 1MB at a time. */ + const int aIncrSz[] = {256*1024, 1024*1024}; + int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; + + if( p->pMap ){ + munmap(p->pMap, p->nMap); + *ppOut = p->pMap = 0; + *pnOut = p->nMap = 0; + } + + if( iMin>=0 ){ + memset(&buf, 0, sizeof(buf)); + prc = fstat(p->fd, &buf); + if( prc!=0 ) return LSM_IOERR_BKPT; + iSz = buf.st_size; + if( iSzfd, iSz); + if( prc!=0 ) return LSM_IOERR_BKPT; + } + + p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); + if( p->pMap==MAP_FAILED ){ + p->pMap = 0; + return LSM_IOERR_BKPT; + } + p->nMap = iSz; + } + + *ppOut = p->pMap; + *pnOut = p->nMap; + return LSM_OK; +} + +static int lsmPosixOsFullpath( + lsm_env *pEnv, + const char *zName, + char *zOut, + int *pnOut +){ + int nBuf = *pnOut; + int nReq; + + if( zName[0]!='/' ){ + char *z; + char *zTmp; + int nTmp = 512; + zTmp = lsmMalloc(pEnv, nTmp); + while( zTmp ){ + z = getcwd(zTmp, nTmp); + if( z || errno!=ERANGE ) break; + nTmp = nTmp*2; + zTmp = lsmReallocOrFree(pEnv, zTmp, nTmp); + } + if( zTmp==0 ) return LSM_NOMEM_BKPT; + if( z==0 ) return LSM_IOERR_BKPT; + assert( z==zTmp ); + + nTmp = strlen(zTmp); + nReq = nTmp + 1 + strlen(zName) + 1; + if( nReq<=nBuf ){ + memcpy(zOut, zTmp, nTmp); + zOut[nTmp] = '/'; + memcpy(&zOut[nTmp+1], zName, strlen(zName)+1); + } + lsmFree(pEnv, zTmp); + }else{ + nReq = strlen(zName)+1; + if( nReq<=nBuf ){ + memcpy(zOut, zName, strlen(zName)+1); + } + } + + *pnOut = nReq; + return LSM_OK; +} + +static int lsmPosixOsFileid( + lsm_file *pFile, + void *pBuf, + int *pnBuf +){ + int prc; + int nBuf; + int nReq; + PosixFile *p = (PosixFile *)pFile; + struct stat buf; + + nBuf = *pnBuf; + nReq = (sizeof(buf.st_dev) + sizeof(buf.st_ino)); + *pnBuf = nReq; + if( nReq>nBuf ) return LSM_OK; + + memset(&buf, 0, sizeof(buf)); + prc = fstat(p->fd, &buf); + if( prc!=0 ) return LSM_IOERR_BKPT; + + memcpy(pBuf, &buf.st_dev, sizeof(buf.st_dev)); + memcpy(&(((u8 *)pBuf)[sizeof(buf.st_dev)]), &buf.st_ino, sizeof(buf.st_ino)); + return LSM_OK; +} + +static int lsmPosixOsUnlink(lsm_env *pEnv, const char *zFile){ + int prc = unlink(zFile); + return prc ? LSM_IOERR_BKPT : LSM_OK; +} + +static int lsmPosixOsLock(lsm_file *pFile, int iLock, int eType){ + int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + static const short aType[3] = { F_UNLCK, F_RDLCK, F_WRLCK }; + struct flock lock; + + assert( aType[LSM_LOCK_UNLOCK]==F_UNLCK ); + assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); + assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); + assert( eType>=0 && eType0 && iLock<=32 ); + + memset(&lock, 0, sizeof(lock)); + lock.l_whence = SEEK_SET; + lock.l_len = 1; + lock.l_type = aType[eType]; + lock.l_start = (4096-iLock); + + if( fcntl(p->fd, F_SETLK, &lock) ){ + int e = errno; + if( e==EACCES || e==EAGAIN ){ + rc = LSM_BUSY; + }else{ + rc = LSM_IOERR_BKPT; + } + } + + return rc; +} + +static int lsmPosixOsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ + int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + static const short aType[3] = { 0, F_RDLCK, F_WRLCK }; + struct flock lock; + + assert( eType==LSM_LOCK_SHARED || eType==LSM_LOCK_EXCL ); + assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); + assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); + assert( eType>=0 && eType0 && iLock<=32 ); + + memset(&lock, 0, sizeof(lock)); + lock.l_whence = SEEK_SET; + lock.l_len = nLock; + lock.l_type = aType[eType]; + lock.l_start = (4096-iLock-nLock+1); + + if( fcntl(p->fd, F_GETLK, &lock) ){ + rc = LSM_IOERR_BKPT; + }else if( lock.l_type!=F_UNLCK ){ + rc = LSM_BUSY; + } + + return rc; +} + +static int lsmPosixOsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ + PosixFile *p = (PosixFile *)pFile; + + *ppShm = 0; + assert( sz==LSM_SHM_CHUNK_SIZE ); + if( iChunk>=p->nShm ){ + int i; + void **apNew; + int nNew = iChunk+1; + off_t nReq = nNew * LSM_SHM_CHUNK_SIZE; + struct stat sStat; + + /* If the shared-memory file has not been opened, open it now. */ + if( p->shmfd<=0 ){ + char *zShm = posixShmFile(p); + if( !zShm ) return LSM_NOMEM_BKPT; + p->shmfd = open(zShm, O_RDWR|O_CREAT, 0644); + lsmFree(p->pEnv, zShm); + if( p->shmfd<0 ){ + return LSM_IOERR_BKPT; + } + } + + /* If the shared-memory file is not large enough to contain the + ** requested chunk, cause it to grow. */ + if( fstat(p->shmfd, &sStat) ){ + return LSM_IOERR_BKPT; + } + if( sStat.st_sizeshmfd, nReq) ){ + return LSM_IOERR_BKPT; + } + } + + apNew = (void **)lsmRealloc(p->pEnv, p->apShm, sizeof(void *) * nNew); + if( !apNew ) return LSM_NOMEM_BKPT; + for(i=p->nShm; iapShm = apNew; + p->nShm = nNew; + } + + if( p->apShm[iChunk]==0 ){ + p->apShm[iChunk] = mmap(0, LSM_SHM_CHUNK_SIZE, + PROT_READ|PROT_WRITE, MAP_SHARED, p->shmfd, iChunk*LSM_SHM_CHUNK_SIZE + ); + if( p->apShm[iChunk]==MAP_FAILED ){ + p->apShm[iChunk] = 0; + return LSM_IOERR_BKPT; + } + } + + *ppShm = p->apShm[iChunk]; + return LSM_OK; +} + +static void lsmPosixOsShmBarrier(void){ +} + +static int lsmPosixOsShmUnmap(lsm_file *pFile, int bDelete){ + PosixFile *p = (PosixFile *)pFile; + if( p->shmfd>0 ){ + int i; + for(i=0; inShm; i++){ + if( p->apShm[i] ){ + munmap(p->apShm[i], LSM_SHM_CHUNK_SIZE); + p->apShm[i] = 0; + } + } + close(p->shmfd); + p->shmfd = 0; + if( bDelete ){ + char *zShm = posixShmFile(p); + if( zShm ) unlink(zShm); + lsmFree(p->pEnv, zShm); + } + } + return LSM_OK; +} + + +static int lsmPosixOsClose(lsm_file *pFile){ + PosixFile *p = (PosixFile *)pFile; + lsmPosixOsShmUnmap(pFile, 0); + if( p->pMap ) munmap(p->pMap, p->nMap); + close(p->fd); + lsm_free(p->pEnv, p->apShm); + lsm_free(p->pEnv, p); + return LSM_OK; +} + +static int lsmPosixOsSleep(lsm_env *pEnv, int us){ +#if 0 + /* Apparently on Android usleep() returns void */ + if( usleep(us) ) return LSM_IOERR; +#endif + usleep(us); + return LSM_OK; +} + +/**************************************************************************** +** Memory allocation routines. +*/ +#define BLOCK_HDR_SIZE ROUND8( sizeof(size_t) ) + +static void *lsmPosixOsMalloc(lsm_env *pEnv, size_t N){ + unsigned char * m; + N += BLOCK_HDR_SIZE; + m = (unsigned char *)malloc(N); + *((size_t*)m) = N; + return m + BLOCK_HDR_SIZE; +} + +static void lsmPosixOsFree(lsm_env *pEnv, void *p){ + if(p){ + free( ((unsigned char *)p) - BLOCK_HDR_SIZE ); + } +} + +static void *lsmPosixOsRealloc(lsm_env *pEnv, void *p, size_t N){ + unsigned char * m = (unsigned char *)p; + if(1>N){ + lsmPosixOsFree( pEnv, p ); + return NULL; + }else if(NULL==p){ + return lsmPosixOsMalloc(pEnv, N); + }else{ + void * re = NULL; + m -= BLOCK_HDR_SIZE; +#if 0 /* arguable: don't shrink */ + size_t * sz = (size_t*)m; + if(*sz >= (size_t)N){ + return p; + } +#endif + re = realloc( m, N + BLOCK_HDR_SIZE ); + if(re){ + m = (unsigned char *)re; + *((size_t*)m) = N; + return m + BLOCK_HDR_SIZE; + }else{ + return NULL; + } + } +} + +static size_t lsmPosixOsMSize(lsm_env *pEnv, void *p){ + unsigned char * m = (unsigned char *)p; + return *((size_t*)(m-BLOCK_HDR_SIZE)); +} +#undef BLOCK_HDR_SIZE + + +#ifdef LSM_MUTEX_PTHREADS +/************************************************************************* +** Mutex methods for pthreads based systems. If LSM_MUTEX_PTHREADS is +** missing then a no-op implementation of mutexes found in lsm_mutex.c +** will be used instead. +*/ +#include + +typedef struct PthreadMutex PthreadMutex; +struct PthreadMutex { + lsm_env *pEnv; + pthread_mutex_t mutex; +#ifdef LSM_DEBUG + pthread_t owner; +#endif +}; + +#ifdef LSM_DEBUG +# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER, 0 } +#else +# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER } +#endif + +static int lsmPosixOsMutexStatic( + lsm_env *pEnv, + int iMutex, + lsm_mutex **ppStatic +){ + static PthreadMutex sMutex[2] = { + LSM_PTHREAD_STATIC_MUTEX, + LSM_PTHREAD_STATIC_MUTEX + }; + + assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); + assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); + + *ppStatic = (lsm_mutex *)&sMutex[iMutex-1]; + return LSM_OK; +} + +static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ + PthreadMutex *pMutex; /* Pointer to new mutex */ + pthread_mutexattr_t attr; /* Attributes object */ + + pMutex = (PthreadMutex *)lsmMallocZero(pEnv, sizeof(PthreadMutex)); + if( !pMutex ) return LSM_NOMEM_BKPT; + + pMutex->pEnv = pEnv; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&pMutex->mutex, &attr); + pthread_mutexattr_destroy(&attr); + + *ppNew = (lsm_mutex *)pMutex; + return LSM_OK; +} + +static void lsmPosixOsMutexDel(lsm_mutex *p){ + PthreadMutex *pMutex = (PthreadMutex *)p; + pthread_mutex_destroy(&pMutex->mutex); + lsmFree(pMutex->pEnv, pMutex); +} + +static void lsmPosixOsMutexEnter(lsm_mutex *p){ + PthreadMutex *pMutex = (PthreadMutex *)p; + pthread_mutex_lock(&pMutex->mutex); + +#ifdef LSM_DEBUG + assert( !pthread_equal(pMutex->owner, pthread_self()) ); + pMutex->owner = pthread_self(); + assert( pthread_equal(pMutex->owner, pthread_self()) ); +#endif +} + +static int lsmPosixOsMutexTry(lsm_mutex *p){ + int ret; + PthreadMutex *pMutex = (PthreadMutex *)p; + ret = pthread_mutex_trylock(&pMutex->mutex); +#ifdef LSM_DEBUG + if( ret==0 ){ + assert( !pthread_equal(pMutex->owner, pthread_self()) ); + pMutex->owner = pthread_self(); + assert( pthread_equal(pMutex->owner, pthread_self()) ); + } +#endif + return ret; +} + +static void lsmPosixOsMutexLeave(lsm_mutex *p){ + PthreadMutex *pMutex = (PthreadMutex *)p; +#ifdef LSM_DEBUG + assert( pthread_equal(pMutex->owner, pthread_self()) ); + pMutex->owner = 0; + assert( !pthread_equal(pMutex->owner, pthread_self()) ); +#endif + pthread_mutex_unlock(&pMutex->mutex); +} + +#ifdef LSM_DEBUG +static int lsmPosixOsMutexHeld(lsm_mutex *p){ + PthreadMutex *pMutex = (PthreadMutex *)p; + return pMutex ? pthread_equal(pMutex->owner, pthread_self()) : 1; +} +static int lsmPosixOsMutexNotHeld(lsm_mutex *p){ + PthreadMutex *pMutex = (PthreadMutex *)p; + return pMutex ? !pthread_equal(pMutex->owner, pthread_self()) : 1; +} +#endif +/* +** End of pthreads mutex implementation. +*************************************************************************/ +#else +/************************************************************************* +** Noop mutex implementation +*/ +typedef struct NoopMutex NoopMutex; +struct NoopMutex { + lsm_env *pEnv; /* Environment handle (for xFree()) */ + int bHeld; /* True if mutex is held */ + int bStatic; /* True for a static mutex */ +}; +static NoopMutex aStaticNoopMutex[2] = { + {0, 0, 1}, + {0, 0, 1}, +}; + +static int lsmPosixOsMutexStatic( + lsm_env *pEnv, + int iMutex, + lsm_mutex **ppStatic +){ + assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); + *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; + return LSM_OK; +} +static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ + NoopMutex *p; + p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); + if( p ) p->pEnv = pEnv; + *ppNew = (lsm_mutex *)p; + return (p ? LSM_OK : LSM_NOMEM_BKPT); +} +static void lsmPosixOsMutexDel(lsm_mutex *pMutex) { + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bStatic==0 && p->pEnv ); + lsmFree(p->pEnv, p); +} +static void lsmPosixOsMutexEnter(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bHeld==0 ); + p->bHeld = 1; +} +static int lsmPosixOsMutexTry(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bHeld==0 ); + p->bHeld = 1; + return 0; +} +static void lsmPosixOsMutexLeave(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bHeld==1 ); + p->bHeld = 0; +} +#ifdef LSM_DEBUG +static int lsmPosixOsMutexHeld(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + return p ? p->bHeld : 1; +} +static int lsmPosixOsMutexNotHeld(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + return p ? !p->bHeld : 1; +} +#endif +/***************************************************************************/ +#endif /* else LSM_MUTEX_NONE */ + +/* Without LSM_DEBUG, the MutexHeld tests are never called */ +#ifndef LSM_DEBUG +# define lsmPosixOsMutexHeld 0 +# define lsmPosixOsMutexNotHeld 0 +#endif + +lsm_env *lsm_default_env(void){ + static lsm_env posix_env = { + sizeof(lsm_env), /* nByte */ + 1, /* iVersion */ + /***** file i/o ******************/ + 0, /* pVfsCtx */ + lsmPosixOsFullpath, /* xFullpath */ + lsmPosixOsOpen, /* xOpen */ + lsmPosixOsRead, /* xRead */ + lsmPosixOsWrite, /* xWrite */ + lsmPosixOsTruncate, /* xTruncate */ + lsmPosixOsSync, /* xSync */ + lsmPosixOsSectorSize, /* xSectorSize */ + lsmPosixOsRemap, /* xRemap */ + lsmPosixOsFileid, /* xFileid */ + lsmPosixOsClose, /* xClose */ + lsmPosixOsUnlink, /* xUnlink */ + lsmPosixOsLock, /* xLock */ + lsmPosixOsTestLock, /* xTestLock */ + lsmPosixOsShmMap, /* xShmMap */ + lsmPosixOsShmBarrier, /* xShmBarrier */ + lsmPosixOsShmUnmap, /* xShmUnmap */ + /***** memory allocation *********/ + 0, /* pMemCtx */ + lsmPosixOsMalloc, /* xMalloc */ + lsmPosixOsRealloc, /* xRealloc */ + lsmPosixOsFree, /* xFree */ + lsmPosixOsMSize, /* xSize */ + /***** mutexes *********************/ + 0, /* pMutexCtx */ + lsmPosixOsMutexStatic, /* xMutexStatic */ + lsmPosixOsMutexNew, /* xMutexNew */ + lsmPosixOsMutexDel, /* xMutexDel */ + lsmPosixOsMutexEnter, /* xMutexEnter */ + lsmPosixOsMutexTry, /* xMutexTry */ + lsmPosixOsMutexLeave, /* xMutexLeave */ + lsmPosixOsMutexHeld, /* xMutexHeld */ + lsmPosixOsMutexNotHeld, /* xMutexNotHeld */ + /***** other *********************/ + lsmPosixOsSleep, /* xSleep */ + }; + return &posix_env; +} + +#endif diff --git a/ext/lsm1/lsm_varint.c b/ext/lsm1/lsm_varint.c new file mode 100644 index 0000000000000000000000000000000000000000..f690e3b0630fa05023c84d318af62286c8ee5c19 --- /dev/null +++ b/ext/lsm1/lsm_varint.c @@ -0,0 +1,201 @@ + +/* +** 2012-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** SQLite4-compatible varint implementation. +*/ +#include "lsmInt.h" + +/************************************************************************* +** The following is a copy of the varint.c module from SQLite 4. +*/ + +/* +** Decode the varint in z[]. Write the integer value into *pResult and +** return the number of bytes in the varint. +*/ +static int lsmSqlite4GetVarint64(const unsigned char *z, u64 *pResult){ + unsigned int x; + if( z[0]<=240 ){ + *pResult = z[0]; + return 1; + } + if( z[0]<=248 ){ + *pResult = (z[0]-241)*256 + z[1] + 240; + return 2; + } + if( z[0]==249 ){ + *pResult = 2288 + 256*z[1] + z[2]; + return 3; + } + if( z[0]==250 ){ + *pResult = (z[1]<<16) + (z[2]<<8) + z[3]; + return 4; + } + x = (z[1]<<24) + (z[2]<<16) + (z[3]<<8) + z[4]; + if( z[0]==251 ){ + *pResult = x; + return 5; + } + if( z[0]==252 ){ + *pResult = (((u64)x)<<8) + z[5]; + return 6; + } + if( z[0]==253 ){ + *pResult = (((u64)x)<<16) + (z[5]<<8) + z[6]; + return 7; + } + if( z[0]==254 ){ + *pResult = (((u64)x)<<24) + (z[5]<<16) + (z[6]<<8) + z[7]; + return 8; + } + *pResult = (((u64)x)<<32) + + (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); + return 9; +} + +/* +** Write a 32-bit unsigned integer as 4 big-endian bytes. +*/ +static void lsmVarintWrite32(unsigned char *z, unsigned int y){ + z[0] = (unsigned char)(y>>24); + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); +} + +/* +** Write a varint into z[]. The buffer z[] must be at least 9 characters +** long to accommodate the largest possible varint. Return the number of +** bytes of z[] used. +*/ +static int lsmSqlite4PutVarint64(unsigned char *z, u64 x){ + unsigned int w, y; + if( x<=240 ){ + z[0] = (unsigned char)x; + return 1; + } + if( x<=2287 ){ + y = (unsigned int)(x - 240); + z[0] = (unsigned char)(y/256 + 241); + z[1] = (unsigned char)(y%256); + return 2; + } + if( x<=67823 ){ + y = (unsigned int)(x - 2288); + z[0] = 249; + z[1] = (unsigned char)(y/256); + z[2] = (unsigned char)(y%256); + return 3; + } + y = (unsigned int)x; + w = (unsigned int)(x>>32); + if( w==0 ){ + if( y<=16777215 ){ + z[0] = 250; + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); + return 4; + } + z[0] = 251; + lsmVarintWrite32(z+1, y); + return 5; + } + if( w<=255 ){ + z[0] = 252; + z[1] = (unsigned char)w; + lsmVarintWrite32(z+2, y); + return 6; + } + if( w<=32767 ){ + z[0] = 253; + z[1] = (unsigned char)(w>>8); + z[2] = (unsigned char)w; + lsmVarintWrite32(z+3, y); + return 7; + } + if( w<=16777215 ){ + z[0] = 254; + z[1] = (unsigned char)(w>>16); + z[2] = (unsigned char)(w>>8); + z[3] = (unsigned char)w; + lsmVarintWrite32(z+4, y); + return 8; + } + z[0] = 255; + lsmVarintWrite32(z+1, w); + lsmVarintWrite32(z+5, y); + return 9; +} + +/* +** End of SQLite 4 code. +*************************************************************************/ + +int lsmVarintPut64(u8 *aData, i64 iVal){ + return lsmSqlite4PutVarint64(aData, (u64)iVal); +} + +int lsmVarintGet64(const u8 *aData, i64 *piVal){ + return lsmSqlite4GetVarint64(aData, (u64 *)piVal); +} + +int lsmVarintPut32(u8 *aData, int iVal){ + return lsmSqlite4PutVarint64(aData, (u64)iVal); +} + +int lsmVarintGet32(u8 *z, int *piVal){ + u64 i; + int ret; + + if( z[0]<=240 ){ + *piVal = z[0]; + return 1; + } + if( z[0]<=248 ){ + *piVal = (z[0]-241)*256 + z[1] + 240; + return 2; + } + if( z[0]==249 ){ + *piVal = 2288 + 256*z[1] + z[2]; + return 3; + } + if( z[0]==250 ){ + *piVal = (z[1]<<16) + (z[2]<<8) + z[3]; + return 4; + } + + ret = lsmSqlite4GetVarint64(z, &i); + *piVal = (int)i; + return ret; +} + +int lsmVarintLen32(int n){ + u8 aData[9]; + return lsmVarintPut32(aData, n); +} + +int lsmVarintLen64(i64 n){ + u8 aData[9]; + return lsmVarintPut64(aData, n); +} + +/* +** The argument is the first byte of a varint. This function returns the +** total number of bytes in the entire varint (including the first byte). +*/ +int lsmVarintSize(u8 c){ + if( c<241 ) return 1; + if( c<249 ) return 2; + return (int)(c - 246); +} diff --git a/ext/lsm1/lsm_vtab.c b/ext/lsm1/lsm_vtab.c new file mode 100644 index 0000000000000000000000000000000000000000..8c21923e1a7a77e8759abfb6b54e6bd569d95540 --- /dev/null +++ b/ext/lsm1/lsm_vtab.c @@ -0,0 +1,1084 @@ +/* +** 2015-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a virtual table for SQLite3 around the LSM +** storage engine from SQLite4. +** +** USAGE +** +** CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...); +** +** The filename parameter is the name of the LSM database file, which is +** separate and distinct from the SQLite3 database file. +** +** The keytype must be one of: UINT, TEXT, BLOB. All keys must be of that +** one type. "UINT" means unsigned integer. The values may be of any +** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL. +** +** The virtual table contains read-only hidden columns: +** +** lsm1_key A BLOB which is the raw LSM key. If the "keytype" +** is BLOB or TEXT then this column is exactly the +** same as the key. For the UINT keytype, this column +** will be a variable-length integer encoding of the key. +** +** lsm1_value A BLOB which is the raw LSM value. All of the value +** columns are packed into this BLOB using the encoding +** described below. +** +** Attempts to write values into the lsm1_key and lsm1_value columns are +** silently ignored. +** +** EXAMPLE +** +** The virtual table declared this way: +** +** CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d); +** +** Results in a new virtual table named "demo2" that acts as if it has +** the following schema: +** +** CREATE TABLE demo2( +** id UINT PRIMARY KEY ON CONFLICT REPLACE, +** a ANY, +** b ANY, +** c ANY, +** d ANY, +** lsm1_key BLOB HIDDEN, +** lsm1_value BLOB HIDDEN +** ) WITHOUT ROWID; +** +** +** +** INTERNALS +** +** The key encoding for BLOB and TEXT is just a copy of the blob or text. +** UTF-8 is used for text. The key encoding for UINT is the variable-length +** integer format at https://sqlite.org/src4/doc/trunk/www/varint.wiki. +** +** The values are encoded as a single blob (since that is what lsm stores as +** its content). There is a "type integer" followed by "content" for each +** value, alternating back and forth. The content might be empty. +** +** TYPE1 CONTENT1 TYPE2 CONTENT2 TYPE3 CONTENT3 .... +** +** Each "type integer" is encoded as a variable-length integer in the +** format of the link above. Let the type integer be T. The actual +** datatype is an integer 0-5 equal to T%6. Values 1 through 5 correspond +** to SQLITE_INTEGER through SQLITE_NULL. The size of the content in bytes +** is T/6. Type value 0 means that the value is an integer whose actual +** values is T/6 and there is no content. The type-value-0 integer format +** only works for integers in the range of 0 through 40. +** +** There is no content for NULL or type-0 integers. For BLOB and TEXT +** values, the content is the blob data or the UTF-8 text data. For +** non-negative integers X, the content is a variable-length integer X*2. +** For negative integers Y, the content is varaible-length integer (1-Y)*2+1. +** For FLOAT values, the content is the IEEE754 floating point value in +** native byte-order. This means that FLOAT values will be corrupted when +** database file is moved between big-endian and little-endian machines. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include "lsm.h" +#include +#include + +/* Forward declaration of subclasses of virtual table objects */ +typedef struct lsm1_vtab lsm1_vtab; +typedef struct lsm1_cursor lsm1_cursor; +typedef struct lsm1_vblob lsm1_vblob; + +/* Primitive types */ +typedef unsigned char u8; +typedef unsigned int u32; +typedef sqlite3_uint64 u64; + +/* An open connection to an LSM table */ +struct lsm1_vtab { + sqlite3_vtab base; /* Base class - must be first */ + lsm_db *pDb; /* Open connection to the LSM table */ + u8 keyType; /* SQLITE_BLOB, _TEXT, or _INTEGER */ + u32 nVal; /* Number of value columns */ +}; + + +/* lsm1_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +struct lsm1_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + lsm_cursor *pLsmCur; /* The LSM cursor */ + u8 isDesc; /* 0: scan forward. 1: scan reverse */ + u8 atEof; /* True if the scan is complete */ + u8 bUnique; /* True if no more than one row of output */ + u8 *zData; /* Content of the current row */ + u32 nData; /* Number of bytes in the current row */ + u8 *aeType; /* Types for all column values */ + u32 *aiOfst; /* Offsets to the various fields */ + u32 *aiLen; /* Length of each field */ + u8 *pKey2; /* Loop termination key, or NULL */ + u32 nKey2; /* Length of the loop termination key */ +}; + +/* An extensible buffer object. +** +** Content can be appended. Space to hold new content is automatically +** allocated. +*/ +struct lsm1_vblob { + u8 *a; /* Space to hold content, from sqlite3_malloc64() */ + u64 n; /* Bytes of space used */ + u64 nAlloc; /* Bytes of space allocated */ + u8 errNoMem; /* True if a memory allocation error has been seen */ +}; + +#if defined(__GNUC__) +# define LSM1_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define LSM1_NOINLINE __declspec(noinline) +#else +# define LSM1_NOINLINE +#endif + + +/* Increase the available space in the vblob object so that it can hold +** at least N more bytes. Return the number of errors. +*/ +static int lsm1VblobEnlarge(lsm1_vblob *p, u32 N){ + if( p->n+N>p->nAlloc ){ + if( p->errNoMem ) return 1; + p->nAlloc += N + (p->nAlloc ? p->nAlloc : N); + p->a = sqlite3_realloc64(p->a, p->nAlloc); + if( p->a==0 ){ + p->n = 0; + p->nAlloc = 0; + p->errNoMem = 1; + return 1; + } + p->nAlloc = sqlite3_msize(p->a); + } + return 0; +} + +/* Append N bytes to a vblob after first enlarging it */ +static LSM1_NOINLINE void lsm1VblobEnlargeAndAppend( + lsm1_vblob *p, + const u8 *pData, + u32 N +){ + if( p->n+N>p->nAlloc && lsm1VblobEnlarge(p, N) ) return; + memcpy(p->a+p->n, pData, N); + p->n += N; +} + +/* Append N bytes to a vblob */ +static void lsm1VblobAppend(lsm1_vblob *p, const u8 *pData, u32 N){ + sqlite3_int64 n = p->n; + if( n+N>p->nAlloc ){ + lsm1VblobEnlargeAndAppend(p, pData, N); + }else{ + p->n += N; + memcpy(p->a+n, pData, N); + } +} + +/* append text to a vblob */ +static void lsm1VblobAppendText(lsm1_vblob *p, const char *z){ + lsm1VblobAppend(p, (u8*)z, (u32)strlen(z)); +} + +/* Dequote the string */ +static void lsm1Dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; ikeyType = keyType; + rc = lsm_new(0, &pNew->pDb); + if( rc ){ + *pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc); + rc = SQLITE_ERROR; + goto connect_failed; + } + zFilename = sqlite3_mprintf("%s", argv[3]); + lsm1Dequote(zFilename); + rc = lsm_open(pNew->pDb, zFilename); + sqlite3_free(zFilename); + if( rc ){ + *pzErr = sqlite3_mprintf("lsm_open failed with %d", rc); + rc = SQLITE_ERROR; + goto connect_failed; + } + + memset(&sql, 0, sizeof(sql)); + lsm1VblobAppendText(&sql, "CREATE TABLE x("); + lsm1VblobAppendText(&sql, argv[4]); + lsm1VblobAppendText(&sql, " "); + lsm1VblobAppendText(&sql, argv[5]); + lsm1VblobAppendText(&sql, " PRIMARY KEY"); + for(i=6; inVal++; + } + lsm1VblobAppendText(&sql, + ", lsm1_command HIDDEN" + ", lsm1_key HIDDEN" + ", lsm1_value HIDDEN) WITHOUT ROWID"); + lsm1VblobAppend(&sql, (u8*)"", 1); + if( sql.errNoMem ){ + rc = SQLITE_NOMEM; + goto connect_failed; + } + rc = sqlite3_declare_vtab(db, (const char*)sql.a); + sqlite3_free(sql.a); + +connect_failed: + if( rc!=SQLITE_OK ){ + if( pNew ){ + if( pNew->pDb ) lsm_close(pNew->pDb); + sqlite3_free(pNew); + } + *ppVtab = 0; + } + return rc; +} + +/* +** This method is the destructor for lsm1_cursor objects. +*/ +static int lsm1Disconnect(sqlite3_vtab *pVtab){ + lsm1_vtab *p = (lsm1_vtab*)pVtab; + lsm_close(p->pDb); + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new lsm1_cursor object. +*/ +static int lsm1Open(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ + lsm1_vtab *p = (lsm1_vtab*)pVtab; + lsm1_cursor *pCur; + int rc; + pCur = sqlite3_malloc64( sizeof(*pCur) + + p->nVal*(sizeof(pCur->aiOfst)+sizeof(pCur->aiLen)+1) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->aiOfst = (u32*)&pCur[1]; + pCur->aiLen = &pCur->aiOfst[p->nVal]; + pCur->aeType = (u8*)&pCur->aiLen[p->nVal]; + *ppCursor = &pCur->base; + rc = lsm_csr_open(p->pDb, &pCur->pLsmCur); + if( rc==LSM_OK ){ + rc = SQLITE_OK; + }else{ + sqlite3_free(pCur); + *ppCursor = 0; + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Destructor for a lsm1_cursor. +*/ +static int lsm1Close(sqlite3_vtab_cursor *cur){ + lsm1_cursor *pCur = (lsm1_cursor*)cur; + sqlite3_free(pCur->pKey2); + lsm_csr_close(pCur->pLsmCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a lsm1_cursor to its next row of output. +*/ +static int lsm1Next(sqlite3_vtab_cursor *cur){ + lsm1_cursor *pCur = (lsm1_cursor*)cur; + int rc = LSM_OK; + if( pCur->bUnique ){ + pCur->atEof = 1; + }else{ + if( pCur->isDesc ){ + rc = lsm_csr_prev(pCur->pLsmCur); + }else{ + rc = lsm_csr_next(pCur->pLsmCur); + } + if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)==0 ){ + pCur->atEof = 1; + } + if( pCur->pKey2 && pCur->atEof==0 ){ + const u8 *pVal; + u32 nVal; + assert( pCur->isDesc==0 ); + rc = lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, (int*)&nVal); + if( rc==LSM_OK ){ + u32 len = pCur->nKey2; + int c; + if( len>nVal ) len = nVal; + c = memcmp(pVal, pCur->pKey2, len); + if( c==0 ) c = nVal - pCur->nKey2; + if( c>0 ) pCur->atEof = 1; + } + } + pCur->zData = 0; + } + return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int lsm1Eof(sqlite3_vtab_cursor *cur){ + lsm1_cursor *pCur = (lsm1_cursor*)cur; + return pCur->atEof; +} + +/* +** Rowids are not supported by the underlying virtual table. So always +** return 0 for the rowid. +*/ +static int lsm1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + *pRowid = 0; + return SQLITE_OK; +} + +/* +** Type prefixes on LSM keys +*/ +#define LSM1_TYPE_NEGATIVE 0 +#define LSM1_TYPE_POSITIVE 1 +#define LSM1_TYPE_TEXT 2 +#define LSM1_TYPE_BLOB 3 + +/* +** Write a 32-bit unsigned integer as 4 big-endian bytes. +*/ +static void varintWrite32(unsigned char *z, unsigned int y){ + z[0] = (unsigned char)(y>>24); + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); +} + +/* +** Write a varint into z[]. The buffer z[] must be at least 9 characters +** long to accommodate the largest possible varint. Return the number of +** bytes of z[] used. +*/ +static int lsm1PutVarint64(unsigned char *z, sqlite3_uint64 x){ + unsigned int w, y; + if( x<=240 ){ + z[0] = (unsigned char)x; + return 1; + } + if( x<=2287 ){ + y = (unsigned int)(x - 240); + z[0] = (unsigned char)(y/256 + 241); + z[1] = (unsigned char)(y%256); + return 2; + } + if( x<=67823 ){ + y = (unsigned int)(x - 2288); + z[0] = 249; + z[1] = (unsigned char)(y/256); + z[2] = (unsigned char)(y%256); + return 3; + } + y = (unsigned int)x; + w = (unsigned int)(x>>32); + if( w==0 ){ + if( y<=16777215 ){ + z[0] = 250; + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); + return 4; + } + z[0] = 251; + varintWrite32(z+1, y); + return 5; + } + if( w<=255 ){ + z[0] = 252; + z[1] = (unsigned char)w; + varintWrite32(z+2, y); + return 6; + } + if( w<=65535 ){ + z[0] = 253; + z[1] = (unsigned char)(w>>8); + z[2] = (unsigned char)w; + varintWrite32(z+3, y); + return 7; + } + if( w<=16777215 ){ + z[0] = 254; + z[1] = (unsigned char)(w>>16); + z[2] = (unsigned char)(w>>8); + z[3] = (unsigned char)w; + varintWrite32(z+4, y); + return 8; + } + z[0] = 255; + varintWrite32(z+1, w); + varintWrite32(z+5, y); + return 9; +} + +/* Append non-negative integer x as a variable-length integer. +*/ +static void lsm1VblobAppendVarint(lsm1_vblob *p, sqlite3_uint64 x){ + sqlite3_int64 n = p->n; + if( n+9>p->nAlloc && lsm1VblobEnlarge(p, 9) ) return; + p->n += lsm1PutVarint64(p->a+p->n, x); +} + +/* +** Decode the varint in the first n bytes z[]. Write the integer value +** into *pResult and return the number of bytes in the varint. +** +** If the decode fails because there are not enough bytes in z[] then +** return 0; +*/ +static int lsm1GetVarint64( + const unsigned char *z, + int n, + sqlite3_uint64 *pResult +){ + unsigned int x; + if( n<1 ) return 0; + if( z[0]<=240 ){ + *pResult = z[0]; + return 1; + } + if( z[0]<=248 ){ + if( n<2 ) return 0; + *pResult = (z[0]-241)*256 + z[1] + 240; + return 2; + } + if( n=0 ){ + u = (sqlite3_uint64)v; + return lsm1PutVarint64(z, u*2); + }else{ + u = (sqlite3_uint64)(-1-v); + return lsm1PutVarint64(z, u*2+1); + } +} + +/* Decoded a signed varint. */ +static int lsm1GetSignedVarint64( + const unsigned char *z, + int n, + sqlite3_int64 *pResult +){ + sqlite3_uint64 u = 0; + n = lsm1GetVarint64(z, n, &u); + if( u&1 ){ + *pResult = -1 - (sqlite3_int64)(u>>1); + }else{ + *pResult = (sqlite3_int64)(u>>1); + } + return n; +} + + +/* +** Read the value part of the key-value pair and decode it into columns. +*/ +static int lsm1DecodeValues(lsm1_cursor *pCur){ + lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); + int i, n; + int rc; + u8 eType; + sqlite3_uint64 v; + + if( pCur->zData ) return 1; + rc = lsm_csr_value(pCur->pLsmCur, (const void**)&pCur->zData, + (int*)&pCur->nData); + if( rc ) return 0; + for(i=n=0; inVal; i++){ + v = 0; + n += lsm1GetVarint64(pCur->zData+n, pCur->nData-n, &v); + pCur->aeType[i] = eType = (u8)(v%6); + if( eType==0 ){ + pCur->aiOfst[i] = (u32)(v/6); + pCur->aiLen[i] = 0; + }else{ + pCur->aiOfst[i] = n; + n += (pCur->aiLen[i] = (u32)(v/6)); + } + if( n>pCur->nData ) break; + } + if( inVal ){ + pCur->zData = 0; + return 0; + } + return 1; +} + +/* +** Return values of columns for the row at which the lsm1_cursor +** is currently pointing. +*/ +static int lsm1Column( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + lsm1_cursor *pCur = (lsm1_cursor*)cur; + lsm1_vtab *pTab = (lsm1_vtab*)(cur->pVtab); + if( i==0 ){ + /* The key column */ + const void *pVal; + int nVal; + if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ + if( pTab->keyType==SQLITE_BLOB ){ + sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); + }else if( pTab->keyType==SQLITE_TEXT ){ + sqlite3_result_text(ctx,(const char*)pVal, nVal, SQLITE_TRANSIENT); + }else{ + const unsigned char *z = (const unsigned char*)pVal; + sqlite3_uint64 v1; + lsm1GetVarint64(z, nVal, &v1); + sqlite3_result_int64(ctx, (sqlite3_int64)v1); + } + } + }else if( i>pTab->nVal ){ + if( i==pTab->nVal+2 ){ /* lsm1_key */ + const void *pVal; + int nVal; + if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ + sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); + } + }else if( i==pTab->nVal+3 ){ /* lsm1_value */ + const void *pVal; + int nVal; + if( lsm_csr_value(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ + sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); + } + } + }else if( lsm1DecodeValues(pCur) ){ + /* The i-th value column (where leftmost is 1) */ + const u8 *zData; + u32 nData; + i--; + zData = pCur->zData + pCur->aiOfst[i]; + nData = pCur->aiLen[i]; + switch( pCur->aeType[i] ){ + case 0: { /* in-line integer */ + sqlite3_result_int(ctx, pCur->aiOfst[i]); + break; + } + case SQLITE_INTEGER: { + sqlite3_int64 v; + lsm1GetSignedVarint64(zData, nData, &v); + sqlite3_result_int64(ctx, v); + break; + } + case SQLITE_FLOAT: { + double v; + if( nData==sizeof(v) ){ + memcpy(&v, zData, sizeof(v)); + sqlite3_result_double(ctx, v); + } + break; + } + case SQLITE_TEXT: { + sqlite3_result_text(ctx, (const char*)zData, nData, SQLITE_TRANSIENT); + break; + } + case SQLITE_BLOB: { + sqlite3_result_blob(ctx, zData, nData, SQLITE_TRANSIENT); + break; + } + default: { + /* A NULL. Do nothing */ + } + } + } + return SQLITE_OK; +} + +/* Parameter "pValue" contains an SQL value that is to be used as +** a key in an LSM table. The type of the key is determined by +** "keyType". Extract the raw bytes used for the key in LSM1. +*/ +static void lsm1KeyFromValue( + int keyType, /* The key type */ + sqlite3_value *pValue, /* The key value */ + u8 *pBuf, /* Storage space for a generated key */ + const u8 **ppKey, /* OUT: the bytes of the key */ + int *pnKey /* OUT: size of the key */ +){ + if( keyType==SQLITE_BLOB ){ + *ppKey = (const u8*)sqlite3_value_blob(pValue); + *pnKey = sqlite3_value_bytes(pValue); + }else if( keyType==SQLITE_TEXT ){ + *ppKey = (const u8*)sqlite3_value_text(pValue); + *pnKey = sqlite3_value_bytes(pValue); + }else{ + sqlite3_int64 v = sqlite3_value_int64(pValue); + if( v<0 ) v = 0; + *pnKey = lsm1PutVarint64(pBuf, v); + *ppKey = pBuf; + } +} + +/* Move to the first row to return. +*/ +static int lsm1Filter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + lsm1_cursor *pCur = (lsm1_cursor *)pVtabCursor; + lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); + int rc = LSM_OK; + int seekType = -1; + const u8 *pVal = 0; + int nVal; + u8 keyType = pTab->keyType; + u8 aKey1[16]; + + pCur->atEof = 1; + sqlite3_free(pCur->pKey2); + pCur->pKey2 = 0; + if( idxNum<99 ){ + lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal); + } + switch( idxNum ){ + case 0: { /* key==argv[0] */ + assert( argc==1 ); + seekType = LSM_SEEK_EQ; + pCur->isDesc = 0; + pCur->bUnique = 1; + break; + } + case 1: { /* key>=argv[0] AND key<=argv[1] */ + u8 aKey[12]; + seekType = LSM_SEEK_GE; + pCur->isDesc = 0; + pCur->bUnique = 0; + if( keyType==SQLITE_INTEGER ){ + sqlite3_int64 v = sqlite3_value_int64(argv[1]); + if( v<0 ) v = 0; + pCur->nKey2 = lsm1PutVarint64(aKey, (sqlite3_uint64)v); + pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); + if( pCur->pKey2==0 ) return SQLITE_NOMEM; + memcpy(pCur->pKey2, aKey, pCur->nKey2); + }else{ + pCur->nKey2 = sqlite3_value_bytes(argv[1]); + pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); + if( pCur->pKey2==0 ) return SQLITE_NOMEM; + if( keyType==SQLITE_BLOB ){ + memcpy(pCur->pKey2, sqlite3_value_blob(argv[1]), pCur->nKey2); + }else{ + memcpy(pCur->pKey2, sqlite3_value_text(argv[1]), pCur->nKey2); + } + } + break; + } + case 2: { /* key>=argv[0] */ + seekType = LSM_SEEK_GE; + pCur->isDesc = 0; + pCur->bUnique = 0; + break; + } + case 3: { /* key<=argv[0] */ + seekType = LSM_SEEK_LE; + pCur->isDesc = 1; + pCur->bUnique = 0; + break; + } + default: { /* full table scan */ + pCur->isDesc = 0; + pCur->bUnique = 0; + break; + } + } + if( pVal ){ + rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, seekType); + }else{ + rc = lsm_csr_first(pCur->pLsmCur); + } + if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)!=0 ){ + pCur->atEof = 0; + } + return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; +} + +/* +** Only comparisons against the key are allowed. The idxNum defines +** which comparisons are available: +** +** 0 key==?1 +** 1 key>=?1 AND key<=?2 +** 2 key>?1 or key>=?1 +** 3 keyaConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->iColumn!=0 ) continue; + switch( pConstraint->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: { + if( idxNum>0 ){ + argIdx = i; + iIdx2 = -1; + idxNum = 0; + omit1 = 1; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum==99 ){ + argIdx = i; + idxNum = 2; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; + }else if( idxNum==3 ){ + iIdx2 = idxNum; + omit2 = omit1; + argIdx = i; + idxNum = 1; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum==99 ){ + argIdx = i; + idxNum = 3; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; + }else if( idxNum==2 ){ + iIdx2 = i; + idxNum = 1; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; + } + break; + } + } + } + if( argIdx>=0 ){ + pIdxInfo->aConstraintUsage[argIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[argIdx].omit = omit1; + } + if( iIdx2>=0 ){ + pIdxInfo->aConstraintUsage[iIdx2].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[iIdx2].omit = omit2; + } + if( idxNum==0 ){ + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 1; + pIdxInfo->orderByConsumed = 1; + }else if( idxNum==1 ){ + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + }else if( idxNum<99 ){ + pIdxInfo->estimatedCost = (double)5000; + pIdxInfo->estimatedRows = 5000; + }else{ + /* Full table scan */ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + } + pIdxInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** The xUpdate method is normally used for INSERT, REPLACE, UPDATE, and +** DELETE. But this virtual table only supports INSERT and REPLACE. +** DELETE is accomplished by inserting a record with a value of NULL. +** UPDATE is achieved by using REPLACE. +*/ +int lsm1Update( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + lsm1_vtab *p = (lsm1_vtab*)pVTab; + int nKey, nKey2; + int i; + int rc = LSM_OK; + const u8 *pKey, *pKey2; + unsigned char aKey[16]; + unsigned char pSpace[16]; + lsm1_vblob val; + + if( argc==1 ){ + /* DELETE the record whose key is argv[0] */ + lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); + lsm_delete(p->pDb, pKey, nKey); + return SQLITE_OK; + } + + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + /* An UPDATE */ + lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); + lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2); + if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){ + /* The UPDATE changes the PRIMARY KEY value. DELETE the old key */ + lsm_delete(p->pDb, pKey, nKey); + } + /* Fall through into the INSERT case to complete the UPDATE */ + } + + /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement + ** special commands. + */ + if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){ + return SQLITE_OK; + } + lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey); + memset(&val, 0, sizeof(val)); + for(i=0; inVal; i++){ + sqlite3_value *pArg = argv[3+i]; + u8 eType = sqlite3_value_type(pArg); + switch( eType ){ + case SQLITE_NULL: { + lsm1VblobAppendVarint(&val, SQLITE_NULL); + break; + } + case SQLITE_INTEGER: { + sqlite3_int64 v = sqlite3_value_int64(pArg); + if( v>=0 && v<=240/6 ){ + lsm1VblobAppendVarint(&val, v*6); + }else{ + int n = lsm1PutSignedVarint64(pSpace, v); + lsm1VblobAppendVarint(&val, SQLITE_INTEGER + n*6); + lsm1VblobAppend(&val, pSpace, n); + } + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + lsm1VblobAppendVarint(&val, SQLITE_FLOAT + 8*6); + lsm1VblobAppend(&val, (u8*)&r, sizeof(r)); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_value_bytes(pArg); + lsm1VblobAppendVarint(&val, n*6 + SQLITE_BLOB); + lsm1VblobAppend(&val, sqlite3_value_blob(pArg), n); + break; + } + case SQLITE_TEXT: { + int n = sqlite3_value_bytes(pArg); + lsm1VblobAppendVarint(&val, n*6 + SQLITE_TEXT); + lsm1VblobAppend(&val, sqlite3_value_text(pArg), n); + break; + } + } + } + if( val.errNoMem ){ + return SQLITE_NOMEM; + } + rc = lsm_insert(p->pDb, pKey, nKey, val.a, val.n); + sqlite3_free(val.a); + return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; +} + +/* Begin a transaction +*/ +static int lsm1Begin(sqlite3_vtab *pVtab){ + lsm1_vtab *p = (lsm1_vtab*)pVtab; + int rc = lsm_begin(p->pDb, 1); + return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; +} + +/* Phase 1 of a transaction commit. +*/ +static int lsm1Sync(sqlite3_vtab *pVtab){ + return SQLITE_OK; +} + +/* Commit a transaction +*/ +static int lsm1Commit(sqlite3_vtab *pVtab){ + lsm1_vtab *p = (lsm1_vtab*)pVtab; + int rc = lsm_commit(p->pDb, 0); + return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; +} + +/* Rollback a transaction +*/ +static int lsm1Rollback(sqlite3_vtab *pVtab){ + lsm1_vtab *p = (lsm1_vtab*)pVtab; + int rc = lsm_rollback(p->pDb, 0); + return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; +} + +/* +** This following structure defines all the methods for the +** generate_lsm1 virtual table. +*/ +static sqlite3_module lsm1Module = { + 0, /* iVersion */ + lsm1Connect, /* xCreate */ + lsm1Connect, /* xConnect */ + lsm1BestIndex, /* xBestIndex */ + lsm1Disconnect, /* xDisconnect */ + lsm1Disconnect, /* xDestroy */ + lsm1Open, /* xOpen - open a cursor */ + lsm1Close, /* xClose - close a cursor */ + lsm1Filter, /* xFilter - configure scan constraints */ + lsm1Next, /* xNext - advance a cursor */ + lsm1Eof, /* xEof - check for end of scan */ + lsm1Column, /* xColumn - read data */ + lsm1Rowid, /* xRowid - read data */ + lsm1Update, /* xUpdate */ + lsm1Begin, /* xBegin */ + lsm1Sync, /* xSync */ + lsm1Commit, /* xCommit */ + lsm1Rollback, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_lsm_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "lsm1", &lsm1Module, 0); + return rc; +} diff --git a/ext/lsm1/lsm_win32.c b/ext/lsm1/lsm_win32.c new file mode 100644 index 0000000000000000000000000000000000000000..6c5d06b4c8e1db924b592fb480cb9da533bc5a6b --- /dev/null +++ b/ext/lsm1/lsm_win32.c @@ -0,0 +1,1063 @@ +/* +** 2011-12-03 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Win32-specific run-time environment implementation for LSM. +*/ + +#ifdef _WIN32 + +#include +#include + +#include +#include +#include +#include + +#include "windows.h" + +#include "lsmInt.h" + +/* +** An open file is an instance of the following object +*/ +typedef struct Win32File Win32File; +struct Win32File { + lsm_env *pEnv; /* The run-time environment */ + const char *zName; /* Full path to file */ + + HANDLE hFile; /* Open file handle */ + HANDLE hShmFile; /* File handle for *-shm file */ + + SYSTEM_INFO sysInfo; /* Operating system information */ + HANDLE hMap; /* File handle for mapping */ + LPVOID pMap; /* Pointer to mapping of file fd */ + size_t nMap; /* Size of mapping at pMap in bytes */ + int nShm; /* Number of entries in ahShm[]/apShm[] */ + LPHANDLE ahShm; /* Array of handles for shared mappings */ + LPVOID *apShm; /* Array of 32K shared memory segments */ +}; + +static char *win32ShmFile(Win32File *pWin32File){ + char *zShm; + int nName = strlen(pWin32File->zName); + zShm = (char *)lsmMallocZero(pWin32File->pEnv, nName+4+1); + if( zShm ){ + memcpy(zShm, pWin32File->zName, nName); + memcpy(&zShm[nName], "-shm", 5); + } + return zShm; +} + +static int win32Sleep(int us){ + Sleep((us + 999) / 1000); + return LSM_OK; +} + +/* +** The number of times that an I/O operation will be retried following a +** locking error - probably caused by antivirus software. Also the initial +** delay before the first retry. The delay increases linearly with each +** retry. +*/ +#ifndef LSM_WIN32_IOERR_RETRY +# define LSM_WIN32_IOERR_RETRY 10 +#endif +#ifndef LSM_WIN32_IOERR_RETRY_DELAY +# define LSM_WIN32_IOERR_RETRY_DELAY 25000 +#endif +static int win32IoerrRetry = LSM_WIN32_IOERR_RETRY; +static int win32IoerrRetryDelay = LSM_WIN32_IOERR_RETRY_DELAY; + +/* +** The "win32IoerrCanRetry1" macro is used to determine if a particular +** I/O error code obtained via GetLastError() is eligible to be retried. +** It must accept the error code DWORD as its only argument and should +** return non-zero if the error code is transient in nature and the +** operation responsible for generating the original error might succeed +** upon being retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "win32IoerrCanRetry2" may be defined. If +** it is defined, it will be consulted only when the macro +** "win32IoerrCanRetry1" returns zero. The "win32IoerrCanRetry2" macro +** is completely optional and may be used to include additional error +** codes in the set that should result in the failing I/O operation being +** retried by the caller. If defined, the "win32IoerrCanRetry2" macro +** must exhibit external semantics identical to those of the +** "win32IoerrCanRetry1" macro. +*/ +#if !defined(win32IoerrCanRetry1) +#define win32IoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + +/* +** If an I/O error occurs, invoke this routine to see if it should be +** retried. Return TRUE to retry. Return FALSE to give up with an +** error. +*/ +static int win32RetryIoerr( + lsm_env *pEnv, + int *pnRetry +){ + DWORD lastErrno; + if( *pnRetry>=win32IoerrRetry ){ + return 0; + } + lastErrno = GetLastError(); + if( win32IoerrCanRetry1(lastErrno) ){ + win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#if defined(win32IoerrCanRetry2) + else if( win32IoerrCanRetry2(lastErrno) ){ + win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#endif + return 0; +} + +/* +** Convert a UTF-8 string to Microsoft Unicode. +** +** Space to hold the returned string is obtained from lsmMalloc(). +*/ +static LPWSTR win32Utf8ToUnicode(lsm_env *pEnv, const char *zText){ + int nChar; + LPWSTR zWideText; + + nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideText = lsmMallocZero(pEnv, nChar * sizeof(WCHAR)); + if( zWideText==0 ){ + return 0; + } + nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); + if( nChar==0 ){ + lsmFree(pEnv, zWideText); + zWideText = 0; + } + return zWideText; +} + +/* +** Convert a Microsoft Unicode string to UTF-8. +** +** Space to hold the returned string is obtained from lsmMalloc(). +*/ +static char *win32UnicodeToUtf8(lsm_env *pEnv, LPCWSTR zWideText){ + int nByte; + char *zText; + + nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zText = lsmMallocZero(pEnv, nByte); + if( zText==0 ){ + return 0; + } + nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); + if( nByte == 0 ){ + lsmFree(pEnv, zText); + zText = 0; + } + return zText; +} + +#if !defined(win32IsNotFound) +#define win32IsNotFound(a) (((a)==ERROR_FILE_NOT_FOUND) || \ + ((a)==ERROR_PATH_NOT_FOUND)) +#endif + +static int win32Open( + lsm_env *pEnv, + const char *zFile, + int flags, + LPHANDLE phFile +){ + int rc; + LPWSTR zConverted; + + zConverted = win32Utf8ToUnicode(pEnv, zFile); + if( zConverted==0 ){ + rc = LSM_NOMEM_BKPT; + }else{ + int bReadonly = (flags & LSM_OPEN_READONLY); + DWORD dwDesiredAccess; + DWORD dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + DWORD dwCreationDisposition; + DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + HANDLE hFile; + int nRetry = 0; + if( bReadonly ){ + dwDesiredAccess = GENERIC_READ; + dwCreationDisposition = OPEN_EXISTING; + }else{ + dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + dwCreationDisposition = OPEN_ALWAYS; + } + while( (hFile = CreateFileW((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL))==INVALID_HANDLE_VALUE && + win32RetryIoerr(pEnv, &nRetry) ){ + /* Noop */ + } + lsmFree(pEnv, zConverted); + if( hFile!=INVALID_HANDLE_VALUE ){ + *phFile = hFile; + rc = LSM_OK; + }else{ + if( win32IsNotFound(GetLastError()) ){ + rc = lsmErrorBkpt(LSM_IOERR_NOENT); + }else{ + rc = LSM_IOERR_BKPT; + } + } + } + return rc; +} + +static int lsmWin32OsOpen( + lsm_env *pEnv, + const char *zFile, + int flags, + lsm_file **ppFile +){ + int rc = LSM_OK; + Win32File *pWin32File; + + pWin32File = lsmMallocZero(pEnv, sizeof(Win32File)); + if( pWin32File==0 ){ + rc = LSM_NOMEM_BKPT; + }else{ + HANDLE hFile = NULL; + + rc = win32Open(pEnv, zFile, flags, &hFile); + if( rc==LSM_OK ){ + memset(&pWin32File->sysInfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&pWin32File->sysInfo); + pWin32File->pEnv = pEnv; + pWin32File->zName = zFile; + pWin32File->hFile = hFile; + }else{ + lsmFree(pEnv, pWin32File); + pWin32File = 0; + } + } + *ppFile = (lsm_file *)pWin32File; + return rc; +} + +static int lsmWin32OsWrite( + lsm_file *pFile, /* File to write to */ + lsm_i64 iOff, /* Offset to write to */ + void *pData, /* Write data from this buffer */ + int nData /* Bytes of data to write */ +){ + Win32File *pWin32File = (Win32File *)pFile; + OVERLAPPED overlapped; /* The offset for WriteFile. */ + u8 *aRem = (u8 *)pData; /* Data yet to be written */ + int nRem = nData; /* Number of bytes yet to be written */ + int nRetry = 0; /* Number of retrys */ + + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); + overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); + while( nRem>0 ){ + DWORD nWrite = 0; /* Bytes written using WriteFile */ + if( !WriteFile(pWin32File->hFile, aRem, nRem, &nWrite, &overlapped) ){ + if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; + break; + } + assert( nWrite==0 || nWrite<=(DWORD)nRem ); + if( nWrite==0 || nWrite>(DWORD)nRem ){ + break; + } + iOff += nWrite; + overlapped.Offset = (LONG)(iOff & 0xFFFFFFFF); + overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); + aRem += nWrite; + nRem -= nWrite; + } + if( nRem!=0 ) return LSM_IOERR_BKPT; + return LSM_OK; +} + +static int win32Truncate( + HANDLE hFile, + lsm_i64 nSize +){ + LARGE_INTEGER offset; + offset.QuadPart = nSize; + if( !SetFilePointerEx(hFile, offset, 0, FILE_BEGIN) ){ + return LSM_IOERR_BKPT; + } + if (!SetEndOfFile(hFile) ){ + return LSM_IOERR_BKPT; + } + return LSM_OK; +} + +static int lsmWin32OsTruncate( + lsm_file *pFile, /* File to write to */ + lsm_i64 nSize /* Size to truncate file to */ +){ + Win32File *pWin32File = (Win32File *)pFile; + return win32Truncate(pWin32File->hFile, nSize); +} + +static int lsmWin32OsRead( + lsm_file *pFile, /* File to read from */ + lsm_i64 iOff, /* Offset to read from */ + void *pData, /* Read data into this buffer */ + int nData /* Bytes of data to read */ +){ + Win32File *pWin32File = (Win32File *)pFile; + OVERLAPPED overlapped; /* The offset for ReadFile */ + DWORD nRead = 0; /* Bytes read using ReadFile */ + int nRetry = 0; /* Number of retrys */ + + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); + overlapped.OffsetHigh = (LONG)((iOff>>32) & 0X7FFFFFFF); + while( !ReadFile(pWin32File->hFile, pData, nData, &nRead, &overlapped) && + GetLastError()!=ERROR_HANDLE_EOF ){ + if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; + return LSM_IOERR_BKPT; + } + if( nRead<(DWORD)nData ){ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pData)[nRead], 0, nData - nRead); + } + return LSM_OK; +} + +static int lsmWin32OsSync(lsm_file *pFile){ + int rc = LSM_OK; + +#ifndef LSM_NO_SYNC + Win32File *pWin32File = (Win32File *)pFile; + + if( pWin32File->pMap!=NULL ){ + if( !FlushViewOfFile(pWin32File->pMap, 0) ){ + rc = LSM_IOERR_BKPT; + } + } + if( rc==LSM_OK && !FlushFileBuffers(pWin32File->hFile) ){ + rc = LSM_IOERR_BKPT; + } +#else + unused_parameter(pFile); +#endif + + return rc; +} + +static int lsmWin32OsSectorSize(lsm_file *pFile){ + return 512; +} + +static void win32Unmap(Win32File *pWin32File){ + if( pWin32File->pMap!=NULL ){ + UnmapViewOfFile(pWin32File->pMap); + pWin32File->pMap = NULL; + pWin32File->nMap = 0; + } + if( pWin32File->hMap!=NULL ){ + CloseHandle(pWin32File->hMap); + pWin32File->hMap = NULL; + } +} + +static int lsmWin32OsRemap( + lsm_file *pFile, + lsm_i64 iMin, + void **ppOut, + lsm_i64 *pnOut +){ + Win32File *pWin32File = (Win32File *)pFile; + + /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. + ** Thereafter, in chunks of 1MB at a time. */ + const int aIncrSz[] = {256*1024, 1024*1024}; + int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; + + *ppOut = NULL; + *pnOut = 0; + + win32Unmap(pWin32File); + if( iMin>=0 ){ + LARGE_INTEGER fileSize; + DWORD dwSizeHigh; + DWORD dwSizeLow; + HANDLE hMap; + LPVOID pMap; + memset(&fileSize, 0, sizeof(LARGE_INTEGER)); + if( !GetFileSizeEx(pWin32File->hFile, &fileSize) ){ + return LSM_IOERR_BKPT; + } + assert( fileSize.QuadPart>=0 ); + if( fileSize.QuadPart> 32); + hMap = CreateFileMappingW(pWin32File->hFile, NULL, PAGE_READWRITE, + dwSizeHigh, dwSizeLow, NULL); + if( hMap==NULL ){ + return LSM_IOERR_BKPT; + } + pWin32File->hMap = hMap; + assert( fileSize.QuadPart<=0xFFFFFFFF ); + pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, 0, 0, + (SIZE_T)fileSize.QuadPart); + if( pMap==NULL ){ + return LSM_IOERR_BKPT; + } + pWin32File->pMap = pMap; + pWin32File->nMap = (SIZE_T)fileSize.QuadPart; + } + *ppOut = pWin32File->pMap; + *pnOut = pWin32File->nMap; + return LSM_OK; +} + +static BOOL win32IsDriveLetterAndColon( + const char *zPathname +){ + return ( isalpha(zPathname[0]) && zPathname[1]==':' ); +} + +static int lsmWin32OsFullpath( + lsm_env *pEnv, + const char *zName, + char *zOut, + int *pnOut +){ + DWORD nByte; + void *zConverted; + LPWSTR zTempWide; + char *zTempUtf8; + + if( zName[0]=='/' && win32IsDriveLetterAndColon(zName+1) ){ + zName++; + } + zConverted = win32Utf8ToUnicode(pEnv, zName); + if( zConverted==0 ){ + return LSM_NOMEM_BKPT; + } + nByte = GetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); + if( nByte==0 ){ + lsmFree(pEnv, zConverted); + return LSM_IOERR_BKPT; + } + nByte += 3; + zTempWide = lsmMallocZero(pEnv, nByte * sizeof(zTempWide[0])); + if( zTempWide==0 ){ + lsmFree(pEnv, zConverted); + return LSM_NOMEM_BKPT; + } + nByte = GetFullPathNameW((LPCWSTR)zConverted, nByte, zTempWide, 0); + if( nByte==0 ){ + lsmFree(pEnv, zConverted); + lsmFree(pEnv, zTempWide); + return LSM_IOERR_BKPT; + } + lsmFree(pEnv, zConverted); + zTempUtf8 = win32UnicodeToUtf8(pEnv, zTempWide); + lsmFree(pEnv, zTempWide); + if( zTempUtf8 ){ + int nOut = *pnOut; + int nLen = strlen(zTempUtf8) + 1; + if( nLen<=nOut ){ + snprintf(zOut, nOut, "%s", zTempUtf8); + } + lsmFree(pEnv, zTempUtf8); + *pnOut = nLen; + return LSM_OK; + }else{ + return LSM_NOMEM_BKPT; + } +} + +static int lsmWin32OsFileid( + lsm_file *pFile, + void *pBuf, + int *pnBuf +){ + int nBuf; + int nReq; + u8 *pBuf2 = (u8 *)pBuf; + Win32File *pWin32File = (Win32File *)pFile; + BY_HANDLE_FILE_INFORMATION fileInfo; + + nBuf = *pnBuf; + nReq = (sizeof(fileInfo.dwVolumeSerialNumber) + + sizeof(fileInfo.nFileIndexHigh) + + sizeof(fileInfo.nFileIndexLow)); + *pnBuf = nReq; + if( nReq>nBuf ) return LSM_OK; + memset(&fileInfo, 0, sizeof(BY_HANDLE_FILE_INFORMATION)); + if( !GetFileInformationByHandle(pWin32File->hFile, &fileInfo) ){ + return LSM_IOERR_BKPT; + } + nReq = sizeof(fileInfo.dwVolumeSerialNumber); + memcpy(pBuf2, &fileInfo.dwVolumeSerialNumber, nReq); + pBuf2 += nReq; + nReq = sizeof(fileInfo.nFileIndexHigh); + memcpy(pBuf, &fileInfo.nFileIndexHigh, nReq); + pBuf2 += nReq; + nReq = sizeof(fileInfo.nFileIndexLow); + memcpy(pBuf2, &fileInfo.nFileIndexLow, nReq); + return LSM_OK; +} + +static int win32Delete( + lsm_env *pEnv, + const char *zFile +){ + int rc; + LPWSTR zConverted; + + zConverted = win32Utf8ToUnicode(pEnv, zFile); + if( zConverted==0 ){ + rc = LSM_NOMEM_BKPT; + }else{ + int nRetry = 0; + DWORD attr; + + do { + attr = GetFileAttributesW(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + rc = LSM_IOERR_BKPT; + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = LSM_IOERR_BKPT; /* Files only. */ + break; + } + if ( DeleteFileW(zConverted) ){ + rc = LSM_OK; /* Deleted OK. */ + break; + } + if ( !win32RetryIoerr(pEnv, &nRetry) ){ + rc = LSM_IOERR_BKPT; /* No more retries. */ + break; + } + }while( 1 ); + } + lsmFree(pEnv, zConverted); + return rc; +} + +static int lsmWin32OsUnlink(lsm_env *pEnv, const char *zFile){ + return win32Delete(pEnv, zFile); +} + +#if !defined(win32IsLockBusy) +#define win32IsLockBusy(a) (((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_IO_PENDING)) +#endif + +static int win32LockFile( + Win32File *pWin32File, + int iLock, + int nLock, + int eType +){ + OVERLAPPED ovlp; + + assert( LSM_LOCK_UNLOCK==0 ); + assert( LSM_LOCK_SHARED==1 ); + assert( LSM_LOCK_EXCL==2 ); + assert( eType>=LSM_LOCK_UNLOCK && eType<=LSM_LOCK_EXCL ); + assert( nLock>=0 ); + assert( iLock>0 && iLock<=32 ); + + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = (4096-iLock-nLock+1); + if( eType>LSM_LOCK_UNLOCK ){ + DWORD flags = LOCKFILE_FAIL_IMMEDIATELY; + if( eType>=LSM_LOCK_EXCL ) flags |= LOCKFILE_EXCLUSIVE_LOCK; + if( !LockFileEx(pWin32File->hFile, flags, 0, (DWORD)nLock, 0, &ovlp) ){ + if( win32IsLockBusy(GetLastError()) ){ + return LSM_BUSY; + }else{ + return LSM_IOERR_BKPT; + } + } + }else{ + if( !UnlockFileEx(pWin32File->hFile, 0, (DWORD)nLock, 0, &ovlp) ){ + return LSM_IOERR_BKPT; + } + } + return LSM_OK; +} + +static int lsmWin32OsLock(lsm_file *pFile, int iLock, int eType){ + Win32File *pWin32File = (Win32File *)pFile; + return win32LockFile(pWin32File, iLock, 1, eType); +} + +static int lsmWin32OsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ + int rc; + Win32File *pWin32File = (Win32File *)pFile; + rc = win32LockFile(pWin32File, iLock, nLock, eType); + if( rc!=LSM_OK ) return rc; + win32LockFile(pWin32File, iLock, nLock, LSM_LOCK_UNLOCK); + return LSM_OK; +} + +static int lsmWin32OsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ + int rc; + Win32File *pWin32File = (Win32File *)pFile; + int iOffset = iChunk * sz; + int iOffsetShift = iOffset % pWin32File->sysInfo.dwAllocationGranularity; + int nNew = iChunk + 1; + lsm_i64 nReq = nNew * sz; + + *ppShm = NULL; + assert( sz>=0 ); + assert( sz==LSM_SHM_CHUNK_SIZE ); + if( iChunk>=pWin32File->nShm ){ + LPHANDLE ahNew; + LPVOID *apNew; + LARGE_INTEGER fileSize; + + /* If the shared-memory file has not been opened, open it now. */ + if( pWin32File->hShmFile==NULL ){ + char *zShm = win32ShmFile(pWin32File); + if( !zShm ) return LSM_NOMEM_BKPT; + rc = win32Open(pWin32File->pEnv, zShm, 0, &pWin32File->hShmFile); + lsmFree(pWin32File->pEnv, zShm); + if( rc!=LSM_OK ){ + return rc; + } + } + + /* If the shared-memory file is not large enough to contain the + ** requested chunk, cause it to grow. */ + memset(&fileSize, 0, sizeof(LARGE_INTEGER)); + if( !GetFileSizeEx(pWin32File->hShmFile, &fileSize) ){ + return LSM_IOERR_BKPT; + } + assert( fileSize.QuadPart>=0 ); + if( fileSize.QuadParthShmFile, nReq); + if( rc!=LSM_OK ){ + return rc; + } + } + + ahNew = (LPHANDLE)lsmMallocZero(pWin32File->pEnv, sizeof(HANDLE) * nNew); + if( !ahNew ) return LSM_NOMEM_BKPT; + apNew = (LPVOID *)lsmMallocZero(pWin32File->pEnv, sizeof(LPVOID) * nNew); + if( !apNew ){ + lsmFree(pWin32File->pEnv, ahNew); + return LSM_NOMEM_BKPT; + } + memcpy(ahNew, pWin32File->ahShm, sizeof(HANDLE) * pWin32File->nShm); + memcpy(apNew, pWin32File->apShm, sizeof(LPVOID) * pWin32File->nShm); + lsmFree(pWin32File->pEnv, pWin32File->ahShm); + pWin32File->ahShm = ahNew; + lsmFree(pWin32File->pEnv, pWin32File->apShm); + pWin32File->apShm = apNew; + pWin32File->nShm = nNew; + } + + if( pWin32File->ahShm[iChunk]==NULL ){ + HANDLE hMap; + assert( nReq<=0xFFFFFFFF ); + hMap = CreateFileMappingW(pWin32File->hShmFile, NULL, PAGE_READWRITE, 0, + (DWORD)nReq, NULL); + if( hMap==NULL ){ + return LSM_IOERR_BKPT; + } + pWin32File->ahShm[iChunk] = hMap; + } + if( pWin32File->apShm[iChunk]==NULL ){ + LPVOID pMap; + pMap = MapViewOfFile(pWin32File->ahShm[iChunk], + FILE_MAP_WRITE | FILE_MAP_READ, 0, + iOffset - iOffsetShift, sz + iOffsetShift); + if( pMap==NULL ){ + return LSM_IOERR_BKPT; + } + pWin32File->apShm[iChunk] = pMap; + } + if( iOffsetShift!=0 ){ + char *p = (char *)pWin32File->apShm[iChunk]; + *ppShm = (void *)&p[iOffsetShift]; + }else{ + *ppShm = pWin32File->apShm[iChunk]; + } + return LSM_OK; +} + +static void lsmWin32OsShmBarrier(void){ + MemoryBarrier(); +} + +static int lsmWin32OsShmUnmap(lsm_file *pFile, int bDelete){ + Win32File *pWin32File = (Win32File *)pFile; + + if( pWin32File->hShmFile!=NULL ){ + int i; + for(i=0; inShm; i++){ + if( pWin32File->apShm[i]!=NULL ){ + UnmapViewOfFile(pWin32File->apShm[i]); + pWin32File->apShm[i] = NULL; + } + if( pWin32File->ahShm[i]!=NULL ){ + CloseHandle(pWin32File->ahShm[i]); + pWin32File->ahShm[i] = NULL; + } + } + CloseHandle(pWin32File->hShmFile); + pWin32File->hShmFile = NULL; + if( bDelete ){ + char *zShm = win32ShmFile(pWin32File); + if( zShm ){ win32Delete(pWin32File->pEnv, zShm); } + lsmFree(pWin32File->pEnv, zShm); + } + } + return LSM_OK; +} + +#define MX_CLOSE_ATTEMPT 3 +static int lsmWin32OsClose(lsm_file *pFile){ + int rc; + int nRetry = 0; + Win32File *pWin32File = (Win32File *)pFile; + lsmWin32OsShmUnmap(pFile, 0); + win32Unmap(pWin32File); + do{ + if( pWin32File->hFile==NULL ){ + rc = LSM_IOERR_BKPT; + break; + } + rc = CloseHandle(pWin32File->hFile); + if( rc ){ + pWin32File->hFile = NULL; + rc = LSM_OK; + break; + } + if( ++nRetry>=MX_CLOSE_ATTEMPT ){ + rc = LSM_IOERR_BKPT; + break; + } + }while( 1 ); + lsmFree(pWin32File->pEnv, pWin32File->ahShm); + lsmFree(pWin32File->pEnv, pWin32File->apShm); + lsmFree(pWin32File->pEnv, pWin32File); + return rc; +} + +static int lsmWin32OsSleep(lsm_env *pEnv, int us){ + unused_parameter(pEnv); + return win32Sleep(us); +} + +/**************************************************************************** +** Memory allocation routines. +*/ + +static void *lsmWin32OsMalloc(lsm_env *pEnv, size_t N){ + assert( HeapValidate(GetProcessHeap(), 0, NULL) ); + return HeapAlloc(GetProcessHeap(), 0, (SIZE_T)N); +} + +static void lsmWin32OsFree(lsm_env *pEnv, void *p){ + assert( HeapValidate(GetProcessHeap(), 0, NULL) ); + if( p ){ + HeapFree(GetProcessHeap(), 0, p); + } +} + +static void *lsmWin32OsRealloc(lsm_env *pEnv, void *p, size_t N){ + unsigned char *m = (unsigned char *)p; + assert( HeapValidate(GetProcessHeap(), 0, NULL) ); + if( 1>N ){ + lsmWin32OsFree(pEnv, p); + return NULL; + }else if( NULL==p ){ + return lsmWin32OsMalloc(pEnv, N); + }else{ +#if 0 /* arguable: don't shrink */ + SIZE_T sz = HeapSize(GetProcessHeap(), 0, m); + if( sz>=(SIZE_T)N ){ + return p; + } +#endif + return HeapReAlloc(GetProcessHeap(), 0, m, N); + } +} + +static size_t lsmWin32OsMSize(lsm_env *pEnv, void *p){ + assert( HeapValidate(GetProcessHeap(), 0, NULL) ); + return (size_t)HeapSize(GetProcessHeap(), 0, p); +} + + +#ifdef LSM_MUTEX_WIN32 +/************************************************************************* +** Mutex methods for Win32 based systems. If LSM_MUTEX_WIN32 is +** missing then a no-op implementation of mutexes found below will be +** used instead. +*/ +#include "windows.h" + +typedef struct Win32Mutex Win32Mutex; +struct Win32Mutex { + lsm_env *pEnv; + CRITICAL_SECTION mutex; +#ifdef LSM_DEBUG + DWORD owner; +#endif +}; + +#ifndef WIN32_MUTEX_INITIALIZER +# define WIN32_MUTEX_INITIALIZER { 0 } +#endif + +#ifdef LSM_DEBUG +# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER, 0 } +#else +# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER } +#endif + +static int lsmWin32OsMutexStatic( + lsm_env *pEnv, + int iMutex, + lsm_mutex **ppStatic +){ + static volatile LONG initialized = 0; + static Win32Mutex sMutex[2] = { + LSM_WIN32_STATIC_MUTEX, + LSM_WIN32_STATIC_MUTEX + }; + + assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); + assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); + + if( InterlockedCompareExchange(&initialized, 1, 0)==0 ){ + int i; + for(i=0; ipEnv = pEnv; + InitializeCriticalSection(&pMutex->mutex); + + *ppNew = (lsm_mutex *)pMutex; + return LSM_OK; +} + +static void lsmWin32OsMutexDel(lsm_mutex *p){ + Win32Mutex *pMutex = (Win32Mutex *)p; + DeleteCriticalSection(&pMutex->mutex); + lsmFree(pMutex->pEnv, pMutex); +} + +static void lsmWin32OsMutexEnter(lsm_mutex *p){ + Win32Mutex *pMutex = (Win32Mutex *)p; + EnterCriticalSection(&pMutex->mutex); + +#ifdef LSM_DEBUG + assert( pMutex->owner!=GetCurrentThreadId() ); + pMutex->owner = GetCurrentThreadId(); + assert( pMutex->owner==GetCurrentThreadId() ); +#endif +} + +static int lsmWin32OsMutexTry(lsm_mutex *p){ + BOOL bRet; + Win32Mutex *pMutex = (Win32Mutex *)p; + bRet = TryEnterCriticalSection(&pMutex->mutex); +#ifdef LSM_DEBUG + if( bRet ){ + assert( pMutex->owner!=GetCurrentThreadId() ); + pMutex->owner = GetCurrentThreadId(); + assert( pMutex->owner==GetCurrentThreadId() ); + } +#endif + return !bRet; +} + +static void lsmWin32OsMutexLeave(lsm_mutex *p){ + Win32Mutex *pMutex = (Win32Mutex *)p; +#ifdef LSM_DEBUG + assert( pMutex->owner==GetCurrentThreadId() ); + pMutex->owner = 0; + assert( pMutex->owner!=GetCurrentThreadId() ); +#endif + LeaveCriticalSection(&pMutex->mutex); +} + +#ifdef LSM_DEBUG +static int lsmWin32OsMutexHeld(lsm_mutex *p){ + Win32Mutex *pMutex = (Win32Mutex *)p; + return pMutex ? pMutex->owner==GetCurrentThreadId() : 1; +} +static int lsmWin32OsMutexNotHeld(lsm_mutex *p){ + Win32Mutex *pMutex = (Win32Mutex *)p; + return pMutex ? pMutex->owner!=GetCurrentThreadId() : 1; +} +#endif +/* +** End of Win32 mutex implementation. +*************************************************************************/ +#else +/************************************************************************* +** Noop mutex implementation +*/ +typedef struct NoopMutex NoopMutex; +struct NoopMutex { + lsm_env *pEnv; /* Environment handle (for xFree()) */ + int bHeld; /* True if mutex is held */ + int bStatic; /* True for a static mutex */ +}; +static NoopMutex aStaticNoopMutex[2] = { + {0, 0, 1}, + {0, 0, 1}, +}; + +static int lsmWin32OsMutexStatic( + lsm_env *pEnv, + int iMutex, + lsm_mutex **ppStatic +){ + assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); + *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; + return LSM_OK; +} +static int lsmWin32OsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ + NoopMutex *p; + p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); + if( p ) p->pEnv = pEnv; + *ppNew = (lsm_mutex *)p; + return (p ? LSM_OK : LSM_NOMEM_BKPT); +} +static void lsmWin32OsMutexDel(lsm_mutex *pMutex) { + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bStatic==0 && p->pEnv ); + lsmFree(p->pEnv, p); +} +static void lsmWin32OsMutexEnter(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bHeld==0 ); + p->bHeld = 1; +} +static int lsmWin32OsMutexTry(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bHeld==0 ); + p->bHeld = 1; + return 0; +} +static void lsmWin32OsMutexLeave(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + assert( p->bHeld==1 ); + p->bHeld = 0; +} +#ifdef LSM_DEBUG +static int lsmWin32OsMutexHeld(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + return p ? p->bHeld : 1; +} +static int lsmWin32OsMutexNotHeld(lsm_mutex *pMutex){ + NoopMutex *p = (NoopMutex *)pMutex; + return p ? !p->bHeld : 1; +} +#endif +/***************************************************************************/ +#endif /* else LSM_MUTEX_NONE */ + +/* Without LSM_DEBUG, the MutexHeld tests are never called */ +#ifndef LSM_DEBUG +# define lsmWin32OsMutexHeld 0 +# define lsmWin32OsMutexNotHeld 0 +#endif + +lsm_env *lsm_default_env(void){ + static lsm_env win32_env = { + sizeof(lsm_env), /* nByte */ + 1, /* iVersion */ + /***** file i/o ******************/ + 0, /* pVfsCtx */ + lsmWin32OsFullpath, /* xFullpath */ + lsmWin32OsOpen, /* xOpen */ + lsmWin32OsRead, /* xRead */ + lsmWin32OsWrite, /* xWrite */ + lsmWin32OsTruncate, /* xTruncate */ + lsmWin32OsSync, /* xSync */ + lsmWin32OsSectorSize, /* xSectorSize */ + lsmWin32OsRemap, /* xRemap */ + lsmWin32OsFileid, /* xFileid */ + lsmWin32OsClose, /* xClose */ + lsmWin32OsUnlink, /* xUnlink */ + lsmWin32OsLock, /* xLock */ + lsmWin32OsTestLock, /* xTestLock */ + lsmWin32OsShmMap, /* xShmMap */ + lsmWin32OsShmBarrier, /* xShmBarrier */ + lsmWin32OsShmUnmap, /* xShmUnmap */ + /***** memory allocation *********/ + 0, /* pMemCtx */ + lsmWin32OsMalloc, /* xMalloc */ + lsmWin32OsRealloc, /* xRealloc */ + lsmWin32OsFree, /* xFree */ + lsmWin32OsMSize, /* xSize */ + /***** mutexes *********************/ + 0, /* pMutexCtx */ + lsmWin32OsMutexStatic, /* xMutexStatic */ + lsmWin32OsMutexNew, /* xMutexNew */ + lsmWin32OsMutexDel, /* xMutexDel */ + lsmWin32OsMutexEnter, /* xMutexEnter */ + lsmWin32OsMutexTry, /* xMutexTry */ + lsmWin32OsMutexLeave, /* xMutexLeave */ + lsmWin32OsMutexHeld, /* xMutexHeld */ + lsmWin32OsMutexNotHeld, /* xMutexNotHeld */ + /***** other *********************/ + lsmWin32OsSleep, /* xSleep */ + }; + return &win32_env; +} + +#endif diff --git a/ext/lsm1/test/lsm1_common.tcl b/ext/lsm1/test/lsm1_common.tcl new file mode 100644 index 0000000000000000000000000000000000000000..0e6cd84e319eabcbd35cf106df528cbd07066a9a --- /dev/null +++ b/ext/lsm1/test/lsm1_common.tcl @@ -0,0 +1,38 @@ +# 2014 Dec 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. .. test] +} +source $testdir/tester.tcl + +# Check if the lsm1 extension has been compiled. +if {$::tcl_platform(platform) == "windows"} { + set lsm1 lsm.dll +} else { + set lsm1 lsm.so +} + +if {[file exists [file join .. $lsm1]]} { + proc return_if_no_lsm1 {} {} +} else { + proc return_if_no_lsm1 {} { + finish_test + return -code return + } + return +} + +proc load_lsm1_vtab {db} { + db enable_load_extension 1 + db eval {SELECT load_extension('../lsm')} +} diff --git a/ext/lsm1/test/lsm1_simple.test b/ext/lsm1/test/lsm1_simple.test new file mode 100644 index 0000000000000000000000000000000000000000..2eab50a83a410ef57d70d88042f7395d15fe65cc --- /dev/null +++ b/ext/lsm1/test/lsm1_simple.test @@ -0,0 +1,152 @@ +# 2017 July 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the lsm1 virtual table module. +# + +source [file join [file dirname [info script]] lsm1_common.tcl] +set testprefix lsm1_simple +return_if_no_lsm1 +load_lsm1_vtab db + +forcedelete testlsm.db + +do_execsql_test 100 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d); + PRAGMA table_info(x1); +} { + 0 a UINT 1 {} 1 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 +} + +do_execsql_test 110 { + INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), + (12,NULL,3.25,-559281390); + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33} +do_execsql_test 111 { + SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1; +} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'} + +do_execsql_test 120 { + UPDATE x1 SET d = d+1.0 WHERE a=15; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0} + +do_execsql_test 130 { + UPDATE x1 SET a=123456789 WHERE a=12; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390} +do_execsql_test 131 { + SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000; +} {X'FB075BCD15' 0x75bcd15} + +do_execsql_test 140 { + DELETE FROM x1 WHERE a=15; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390} + +do_test 150 { + lsort [glob testlsm.db*] +} {testlsm.db testlsm.db-log testlsm.db-shm} + +db close +do_test 160 { + lsort [glob testlsm.db*] +} {testlsm.db} + +forcedelete testlsm.db +forcedelete test.db +sqlite3 db test.db +load_lsm1_vtab db + + +do_execsql_test 200 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); + PRAGMA table_info(x1); +} { + 0 a TEXT 1 {} 1 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 +} +do_execsql_test 210 { + INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), + (12,NULL,3.25,-559281390); + SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1; +} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |} +do_execsql_test 211 { + SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1; +} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |} +do_execsql_test 212 { + SELECT quote(a), quote(lsm1_key), quote(lsm1_value) FROM x1 WHERE a='12'; +} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB'} + +#------------------------------------------------------------------------- +reset_db +forcedelete testlsm.db +load_lsm1_vtab db +do_execsql_test 300 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); +} +do_eqp_test 310 { + SELECT * FROM x1 WHERE a=? +} {SCAN TABLE x1 VIRTUAL TABLE INDEX 0:} + +do_eqp_test 320 { + SELECT * FROM x1 WHERE a>? +} {SCAN TABLE x1 VIRTUAL TABLE INDEX 2:} + +do_eqp_test 330 { + SELECT * FROM x1 WHERE a 'five'; +} {4 1 3 2} +do_execsql_test 421 { + SELECT b FROM x1 WHERE a <= 'three'; +} {3 1 4 5} + +finish_test diff --git a/ext/lsm1/tool/mklsm1c.tcl b/ext/lsm1/tool/mklsm1c.tcl new file mode 100644 index 0000000000000000000000000000000000000000..d4a317b7003238e49ebc5d9ce521d86200cf877b --- /dev/null +++ b/ext/lsm1/tool/mklsm1c.tcl @@ -0,0 +1,88 @@ +#!/bin/sh +# restart with tclsh \ +exec tclsh "$0" "$@" + +set srcdir [file dirname [file dirname [info script]]] +set G(src) [string map [list %dir% $srcdir] { + %dir%/lsm.h + %dir%/lsmInt.h + %dir%/lsm_vtab.c + %dir%/lsm_ckpt.c + %dir%/lsm_file.c + %dir%/lsm_log.c + %dir%/lsm_main.c + %dir%/lsm_mem.c + %dir%/lsm_mutex.c + %dir%/lsm_shared.c + %dir%/lsm_sorted.c + %dir%/lsm_str.c + %dir%/lsm_tree.c + %dir%/lsm_unix.c + %dir%/lsm_varint.c + %dir%/lsm_win32.c +}] + +set G(hdr) { + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +} + +set G(footer) { + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */ +} + +#------------------------------------------------------------------------- +# Read and return the entire contents of text file $zFile from disk. +# +proc readfile {zFile} { + set fd [open $zFile] + set data [read $fd] + close $fd + return $data +} + +proc lsm1c_init {zOut} { + global G + set G(fd) stdout + set G(fd) [open $zOut w] + + puts -nonewline $G(fd) $G(hdr) +} + +proc lsm1c_printfile {zIn} { + global G + set data [readfile $zIn] + set zTail [file tail $zIn] + puts $G(fd) "#line 1 \"$zTail\"" + + foreach line [split $data "\n"] { + if {[regexp {^# *include.*lsm} $line]} { + set line "/* $line */" + } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } { + set line "static $line" + } + puts $G(fd) $line + } +} + +proc lsm1c_close {} { + global G + puts -nonewline $G(fd) $G(footer) + if {$G(fd)!="stdout"} { + close $G(fd) + } +} + + +lsm1c_init lsm1.c +foreach f $G(src) { lsm1c_printfile $f } +lsm1c_close diff --git a/ext/misc/README.md b/ext/misc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..69cb230255e9cf78ecd6c4d99c9706a2f534b9d7 --- /dev/null +++ b/ext/misc/README.md @@ -0,0 +1,60 @@ +## Miscellaneous Extensions + +This folder contains a collection of smaller loadable extensions. +See for instructions on how +to compile and use loadable extensions. +Each extension in this folder is implemented in a single file of C code. + +Each source file contains a description in its header comment. See the +header comments for details about each extension. Additional notes are +as follows: + + * **carray.c** — This module implements the + [carray](https://www.sqlite.org/carray.html) table-valued function. + It is a good example of how to go about implementing a custom + [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). + + * **csv.c** — A [virtual table](https://sqlite.org/vtab.html) + for reading + [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values). + + * **dbdump.c** — This is not actually a loadable extension, but + rather a library that implements an approximate equivalent to the + ".dump" command of the + [command-line shell](https://www.sqlite.org/cli.html). + + * **json1.c** — Various SQL functions and table-valued functions + for processing JSON. This extension is already built into the + [SQLite amalgamation](https://sqlite.org/amalgamation.html). See + for additional information. + + * **memvfs.c** — This file implements a custom + [VFS](https://www.sqlite.org/vfs.html) that stores an entire database + file in a single block of RAM. It serves as a good example of how + to implement a simple custom VFS. + + * **rot13.c** — This file implements the very simple rot13() + substitution function. This file makes a good template for implementing + new custom SQL functions for SQLite. + + * **series.c** — This is an implementation of the + "generate_series" [virtual table](https://www.sqlite.org/vtab.html). + It can make a good template for new custom virtual table implementations. + + * **shathree.c** — An implementation of the sha3() and + sha3_query() SQL functions. The file is named "shathree.c" instead + of "sha3.c" because the default entry point names in SQLite are based + on the source filename with digits removed, so if we used the name + "sha3.c" then the entry point would conflict with the prior "sha1.c" + extension. + + * **unionvtab.c** — Implementation of the unionvtab and + [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables. + These virtual tables allow a single + large table to be spread out across multiple database files. In the + case of swarmvtab, the individual database files can be attached on + demand. + + * **zipfile.c** — A [virtual table](https://sqlite.org/vtab.html) + that can read and write a + [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29). diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c new file mode 100644 index 0000000000000000000000000000000000000000..dd9bee53d7ed2a58787ab24faae2ab51cddc5b4a --- /dev/null +++ b/ext/misc/amatch.c @@ -0,0 +1,1502 @@ +/* +** 2013-03-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a demonstration virtual table that finds +** "approximate matches" - strings from a finite set that are nearly the +** same as a single input string. The virtual table is called "amatch". +** +** A amatch virtual table is created like this: +** +** CREATE VIRTUAL TABLE f USING approximate_match( +** vocabulary_table=, -- V +** vocabulary_word=, -- W +** vocabulary_language=, -- L +** edit_distances= +** ); +** +** When it is created, the new amatch table must be supplied with the +** the name of a table V and columns V.W and V.L such that +** +** SELECT W FROM V WHERE L=$language +** +** returns the allowed vocabulary for the match. If the "vocabulary_language" +** or L columnname is left unspecified or is an empty string, then no +** filtering of the vocabulary by language is performed. +** +** For efficiency, it is essential that the vocabulary table be indexed: +** +** CREATE vocab_index ON V(W) +** +** A separate edit-cost-table provides scoring information that defines +** what it means for one string to be "close" to another. +** +** The edit-cost-table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. +** +** Each row in the edit-cost-table represents a single character +** transformation going from user input to the vocabulary. The leftmost +** column of the row (column 0) contains an integer identifier of the +** language to which the transformation rule belongs (see "MULTIPLE LANGUAGES" +** below). The second column of the row (column 1) contains the input +** character or characters - the characters of user input. The third +** column contains characters as they appear in the vocabulary table. +** And the fourth column contains the integer cost of making the +** transformation. For example: +** +** CREATE TABLE f_data(iLang, cFrom, cTo, Cost); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); +** +** The first row inserted into the edit-cost-table by the SQL script +** above indicates that the cost of having an extra 'a' in the vocabulary +** table that is missing in the user input 100. (All costs are integers. +** Overall cost must not exceed 16777216.) The second INSERT statement +** creates a rule saying that the cost of having a single letter 'b' in +** user input which is missing in the vocabulary table is 87. The third +** INSERT statement mean that the cost of matching an 'o' in user input +** against an 'oe' in the vocabulary table is 38. And so forth. +** +** The following rules are special: +** +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '', 97); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', '?', 98); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '?', 99); +** +** The '?' to '' rule is the cost of having any single character in the input +** that is not found in the vocabular. The '' to '?' rule is the cost of +** having a character in the vocabulary table that is missing from input. +** And the '?' to '?' rule is the cost of doing an arbitrary character +** substitution. These three generic rules apply across all languages. +** In other words, the iLang field is ignored for the generic substitution +** rules. If more than one cost is given for a generic substitution rule, +** then the lowest cost is used. +** +** Once it has been created, the amatch virtual table can be queried +** as follows: +** +** SELECT word, distance FROM f +** WHERE word MATCH 'abcdefg' +** AND distance<200; +** +** This query outputs the strings contained in the T(F) field that +** are close to "abcdefg" and in order of increasing distance. No string +** is output more than once. If there are multiple ways to transform the +** target string ("abcdefg") into a string in the vocabulary table then +** the lowest cost transform is the one that is returned. In this example, +** the search is limited to strings with a total distance of less than 200. +** +** For efficiency, it is important to put tight bounds on the distance. +** The time and memory space needed to perform this query is exponential +** in the maximum distance. A good rule of thumb is to limit the distance +** to no more than 1.5 or 2 times the maximum cost of any rule in the +** edit-cost-table. +** +** The amatch is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a amatch table will throw an error. +** +** It is important to put some kind of a limit on the amatch output. This +** can be either in the form of a LIMIT clause at the end of the query, +** or better, a "distance +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct amatch_vtab amatch_vtab; +typedef struct amatch_cursor amatch_cursor; +typedef struct amatch_rule amatch_rule; +typedef struct amatch_word amatch_word; +typedef struct amatch_avl amatch_avl; + + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct amatch_avl { + amatch_word *pWord; /* Points to the object being stored in the tree */ + char *zKey; /* Key. zero-terminated string. Must be unique */ + amatch_avl *pBefore; /* Other elements less than zKey */ + amatch_avl *pAfter; /* Other elements greater than zKey */ + amatch_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the amatch_avl.height and amatch_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void amatchAvlRecomputeHeight(amatch_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static amatch_avl *amatchAvlRotateBefore(amatch_avl *pP){ + amatch_avl *pB = pP->pBefore; + amatch_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static amatch_avl *amatchAvlRotateAfter(amatch_avl *pP){ + amatch_avl *pA = pP->pAfter; + amatch_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static amatch_avl **amatchAvlFromPtr(amatch_avl *p, amatch_avl **pp){ + amatch_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static amatch_avl *amatchAvlBalance(amatch_avl *p){ + amatch_avl *pTop = p; + amatch_avl **pp; + while( p ){ + amatchAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + amatch_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = amatchAvlRotateAfter(pB); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + amatch_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = amatchAvlRotateBefore(pA); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with zKey. Return a pointer +** to the entry or return NULL. +*/ +static amatch_avl *amatchAvlSearch(amatch_avl *p, const char *zKey){ + int c; + while( p && (c = strcmp(zKey, p->zKey))!=0 ){ + p = (c<0) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static amatch_avl *amatchAvlFirst(amatch_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +#if 0 /* NOT USED */ +/* Return the node with the next larger key after p. +*/ +static amatch_avl *amatchAvlNext(amatch_avl *p){ + amatch_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = amatchAvlFirst(p->pAfter); + } + return p; +} +#endif + +#if 0 /* NOT USED */ +/* Verify AVL tree integrity +*/ +static int amatchAvlIntegrity(amatch_avl *pHead){ + amatch_avl *p; + if( pHead==0 ) return 1; + if( (p = pHead->pBefore)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)<0 ); + while( p->pAfter ) p = p->pAfter; + assert( strcmp(p->zKey, pHead->zKey)<0 ); + } + if( (p = pHead->pAfter)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + p = amatchAvlFirst(p); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + } + return 1; +} +static int amatchAvlIntegrity2(amatch_avl *pHead){ + amatch_avl *p, *pNext; + for(p=amatchAvlFirst(pHead); p; p=pNext){ + pNext = amatchAvlNext(p); + if( pNext==0 ) break; + assert( strcmp(p->zKey, pNext->zKey)<0 ); + } + return 1; +} +#endif + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static amatch_avl *amatchAvlInsert(amatch_avl **ppHead, amatch_avl *pNew){ + int c; + amatch_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + c = strcmp(pNew->zKey, p->zKey); + if( c<0 ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( c>0 ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = amatchAvlBalance(p); + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ + return 0; +} + +/* Remove node pOld from the tree. pOld must be an element of the tree or +** the AVL tree will become corrupt. +*/ +static void amatchAvlRemove(amatch_avl **ppHead, amatch_avl *pOld){ + amatch_avl **ppParent; + amatch_avl *pBalance = 0; + /* assert( amatchAvlSearch(*ppHead, pOld->zKey)==pOld ); */ + ppParent = amatchAvlFromPtr(pOld, ppHead); + if( pOld->pBefore==0 && pOld->pAfter==0 ){ + *ppParent = 0; + pBalance = pOld->pUp; + }else if( pOld->pBefore && pOld->pAfter ){ + amatch_avl *pX, *pY; + pX = amatchAvlFirst(pOld->pAfter); + *amatchAvlFromPtr(pX, 0) = pX->pAfter; + if( pX->pAfter ) pX->pAfter->pUp = pX->pUp; + pBalance = pX->pUp; + pX->pAfter = pOld->pAfter; + if( pX->pAfter ){ + pX->pAfter->pUp = pX; + }else{ + assert( pBalance==pOld ); + pBalance = pX; + } + pX->pBefore = pY = pOld->pBefore; + if( pY ) pY->pUp = pX; + pX->pUp = pOld->pUp; + *ppParent = pX; + }else if( pOld->pBefore==0 ){ + *ppParent = pBalance = pOld->pAfter; + pBalance->pUp = pOld->pUp; + }else if( pOld->pAfter==0 ){ + *ppParent = pBalance = pOld->pBefore; + pBalance->pUp = pOld->pUp; + } + *ppHead = amatchAvlBalance(pBalance); + pOld->pUp = 0; + pOld->pBefore = 0; + pOld->pAfter = 0; + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + + +/* +** Various types. +** +** amatch_cost is the "cost" of an edit operation. +** +** amatch_len is the length of a matching string. +** +** amatch_langid is an ruleset identifier. +*/ +typedef int amatch_cost; +typedef signed char amatch_len; +typedef int amatch_langid; + +/* +** Limits +*/ +#define AMATCH_MX_LENGTH 50 /* Maximum length of a rule string */ +#define AMATCH_MX_LANGID 2147483647 /* Maximum rule ID */ +#define AMATCH_MX_COST 1000 /* Maximum single-rule cost */ + +/* +** A match or partial match +*/ +struct amatch_word { + amatch_word *pNext; /* Next on a list of all amatch_words */ + amatch_avl sCost; /* Linkage of this node into the cost tree */ + amatch_avl sWord; /* Linkage of this node into the word tree */ + amatch_cost rCost; /* Cost of the match so far */ + int iSeq; /* Sequence number */ + char zCost[10]; /* Cost key (text rendering of rCost) */ + short int nMatch; /* Input characters matched */ + char zWord[4]; /* Text of the word. Extra space appended as needed */ +}; + +/* +** Each transformation rule is stored as an instance of this object. +** All rules are kept on a linked list sorted by rCost. +*/ +struct amatch_rule { + amatch_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from (a string from user input) */ + amatch_cost rCost; /* Cost of this transformation */ + amatch_langid iLang; /* The langauge to which this rule belongs */ + amatch_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + char zTo[4]; /* Tranform to V.W value (extra space appended) */ +}; + +/* +** A amatch virtual-table object +*/ +struct amatch_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zClassName; /* Name of this class. Default: "amatch" */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zCostTab; /* Name of edit-cost-table */ + char *zVocabTab; /* Name of vocabulary table */ + char *zVocabWord; /* Name of vocabulary table word column */ + char *zVocabLang; /* Name of vocabulary table language column */ + amatch_rule *pRule; /* All active rules in this amatch */ + amatch_cost rIns; /* Generic insertion cost '' -> ? */ + amatch_cost rDel; /* Generic deletion cost ? -> '' */ + amatch_cost rSub; /* Generic substitution cost ? -> ? */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pVCheck; /* Query to check zVocabTab */ + int nCursor; /* Number of active cursors */ +}; + +/* A amatch cursor object */ +struct amatch_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid of the current word */ + amatch_langid iLang; /* Use this language ID */ + amatch_cost rLimit; /* Maximum cost of any term */ + int nBuf; /* Space allocated for zBuf */ + int oomErr; /* True following an OOM error */ + int nWord; /* Number of amatch_word objects */ + char *zBuf; /* Temp-use buffer space */ + char *zInput; /* Input word to match against */ + amatch_vtab *pVtab; /* The virtual table this cursor belongs to */ + amatch_word *pAllWords; /* List of all amatch_word objects */ + amatch_word *pCurrent; /* Most recent solution */ + amatch_avl *pCost; /* amatch_word objects keyed by iCost */ + amatch_avl *pWord; /* amatch_word objects keyed by zWord */ +}; + +/* +** The two input rule lists are both sorted in order of increasing +** cost. Merge them together into a single list, sorted by cost, and +** return a pointer to the head of that list. +*/ +static amatch_rule *amatchMergeRules(amatch_rule *pA, amatch_rule *pB){ + amatch_rule head; + amatch_rule *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCost<=pB->rCost ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Statement pStmt currently points to a row in the amatch data table. This +** function allocates and populates a amatch_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int amatchLoadOneRule( + amatch_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + amatch_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iLang = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + amatch_cost rCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + amatch_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + if( zFrom[0]=='?' && zFrom[1]==0 ){ + if( p->rSub==0 || p->rSub>rCost ) p->rSub = rCost; + } + *ppRule = 0; + return SQLITE_OK; + } + + if( rCost<=0 || rCost>AMATCH_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, AMATCH_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>AMATCH_MX_LENGTH || nTo>AMATCH_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, AMATCH_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iLang<0 || iLang>AMATCH_MX_LANGID ){ + *pzErr = sqlite3_mprintf("%s: iLang must be between 0 and %d", + p->zClassName, AMATCH_MX_LANGID + ); + rc = SQLITE_ERROR; + }else + if( strcmp(zFrom,"")==0 && strcmp(zTo,"?")==0 ){ + if( p->rIns==0 || p->rIns>rCost ) p->rIns = rCost; + }else + if( strcmp(zFrom,"?")==0 && strcmp(zTo,"")==0 ){ + if( p->rDel==0 || p->rDel>rCost ) p->rDel = rCost; + }else + { + pRule = sqlite3_malloc64( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = (amatch_len)nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = (amatch_len)nTo; + pRule->rCost = rCost; + pRule->iLang = (int)iLang; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Free all the content in the edit-cost-table +*/ +static void amatchFreeRules(amatch_vtab *p){ + while( p->pRule ){ + amatch_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + p->pRule = 0; +} + +/* +** Load the content of the amatch data table into memory. +*/ +static int amatchLoadRules( + sqlite3 *db, /* Database handle */ + amatch_vtab *p, /* Virtual amatch table to configure */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + amatch_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zCostTab); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, p->zCostTab, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + amatch_rule *pRule = 0; + rc = amatchLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets amatch_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + amatch_rule *pX; + amatch_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = amatchMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *amatchDequote(const char *zIn){ + sqlite3_int64 nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc64(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, (size_t)(nIn+1)); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInpVCheck ){ + sqlite3_finalize(p->pVCheck); + p->pVCheck = 0; + } +} + +/* +** Deallocate an amatch_vtab object +*/ +static void amatchFree(amatch_vtab *p){ + if( p ){ + amatchFreeRules(p); + amatchVCheckClear(p); + sqlite3_free(p->zClassName); + sqlite3_free(p->zDb); + sqlite3_free(p->zCostTab); + sqlite3_free(p->zVocabTab); + sqlite3_free(p->zVocabWord); + sqlite3_free(p->zVocabLang); + sqlite3_free(p->zSelf); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the amatch module. +*/ +static int amatchDisconnect(sqlite3_vtab *pVtab){ + amatch_vtab *p = (amatch_vtab*)pVtab; + assert( p->nCursor==0 ); + amatchFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *amatchValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("approximate_match") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int amatchConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + amatch_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zClassName = sqlite3_mprintf("%s", zModule); + if( pNew->zClassName==0 ) goto amatchConnectError; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto amatchConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto amatchConnectError; + for(i=3; izVocabTab); + pNew->zVocabTab = amatchDequote(zVal); + if( pNew->zVocabTab==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_word", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabWord); + pNew->zVocabWord = amatchDequote(zVal); + if( pNew->zVocabWord==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_language", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabLang); + pNew->zVocabLang = amatchDequote(zVal); + if( pNew->zVocabLang==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("edit_distances", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zCostTab); + pNew->zCostTab = amatchDequote(zVal); + if( pNew->zCostTab==0 ) goto amatchConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + amatchFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = SQLITE_OK; + if( pNew->zCostTab==0 ){ + *pzErr = sqlite3_mprintf("no edit_distances table specified"); + rc = SQLITE_ERROR; + }else{ + rc = amatchLoadRules(db, pNew, pzErr); + } + if( rc==SQLITE_OK ){ + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,distance,language," + "command HIDDEN,nword HIDDEN)" + ); +#define AMATCH_COL_WORD 0 +#define AMATCH_COL_DISTANCE 1 +#define AMATCH_COL_LANGUAGE 2 +#define AMATCH_COL_COMMAND 3 +#define AMATCH_COL_NWORD 4 + } + if( rc!=SQLITE_OK ){ + amatchFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +amatchConnectError: + amatchFree(pNew); + return rc; +} + +/* +** Open a new amatch cursor. +*/ +static int amatchOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + amatch_vtab *p = (amatch_vtab*)pVTab; + amatch_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void amatchClearCursor(amatch_cursor *pCur){ + amatch_word *pWord, *pNextWord; + for(pWord=pCur->pAllWords; pWord; pWord=pNextWord){ + pNextWord = pWord->pNext; + sqlite3_free(pWord); + } + pCur->pAllWords = 0; + sqlite3_free(pCur->zInput); + pCur->zInput = 0; + sqlite3_free(pCur->zBuf); + pCur->zBuf = 0; + pCur->nBuf = 0; + pCur->pCost = 0; + pCur->pWord = 0; + pCur->pCurrent = 0; + pCur->rLimit = 1000000; + pCur->iLang = 0; + pCur->nWord = 0; +} + +/* +** Close a amatch cursor. +*/ +static int amatchClose(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor *)cur; + amatchClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Render a 24-bit unsigned integer as a 4-byte base-64 number. +*/ +static void amatchEncodeInt(int x, char *z){ + static const char a[] = + "0123456789" + "ABCDEFGHIJ" + "KLMNOPQRST" + "UVWXYZ^abc" + "defghijklm" + "nopqrstuvw" + "xyz~"; + z[0] = a[(x>>18)&0x3f]; + z[1] = a[(x>>12)&0x3f]; + z[2] = a[(x>>6)&0x3f]; + z[3] = a[x&0x3f]; +} + +/* +** Write the zCost[] field for a amatch_word object +*/ +static void amatchWriteCost(amatch_word *pWord){ + amatchEncodeInt(pWord->rCost, pWord->zCost); + amatchEncodeInt(pWord->iSeq, pWord->zCost+4); + pWord->zCost[8] = 0; +} + +/* Circumvent compiler warnings about the use of strcpy() by supplying +** our own implementation. +*/ +static void amatchStrcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} +static void amatchStrcat(char *dest, const char *src){ + while( *dest ) dest++; + amatchStrcpy(dest, src); +} + +/* +** Add a new amatch_word object to the queue. +** +** If a prior amatch_word object with the same zWord, and nMatch +** already exists, update its rCost (if the new rCost is less) but +** otherwise leave it unchanged. Do not add a duplicate. +** +** Do nothing if the cost exceeds threshold. +*/ +static void amatchAddWord( + amatch_cursor *pCur, + amatch_cost rCost, + int nMatch, + const char *zWordBase, + const char *zWordTail +){ + amatch_word *pWord; + amatch_avl *pNode; + amatch_avl *pOther; + int nBase, nTail; + char zBuf[4]; + + if( rCost>pCur->rLimit ){ + return; + } + nBase = (int)strlen(zWordBase); + nTail = (int)strlen(zWordTail); + if( nBase+nTail+3>pCur->nBuf ){ + pCur->nBuf = nBase+nTail+100; + pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf); + if( pCur->zBuf==0 ){ + pCur->nBuf = 0; + return; + } + } + amatchEncodeInt(nMatch, zBuf); + memcpy(pCur->zBuf, zBuf+2, 2); + memcpy(pCur->zBuf+2, zWordBase, nBase); + memcpy(pCur->zBuf+2+nBase, zWordTail, nTail+1); + pNode = amatchAvlSearch(pCur->pWord, pCur->zBuf); + if( pNode ){ + pWord = pNode->pWord; + if( pWord->rCost>rCost ){ +#ifdef AMATCH_TRACE_1 + printf("UPDATE [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + pWord->rCost = rCost; + amatchWriteCost(pWord); +#ifdef AMATCH_TRACE_1 + printf(" ---> %d (\"%s\" \"%s\")\n", + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + } + return; + } + pWord = sqlite3_malloc64( sizeof(*pWord) + nBase + nTail - 1 ); + if( pWord==0 ) return; + memset(pWord, 0, sizeof(*pWord)); + pWord->rCost = rCost; + pWord->iSeq = pCur->nWord++; + amatchWriteCost(pWord); + pWord->nMatch = (short)nMatch; + pWord->pNext = pCur->pAllWords; + pCur->pAllWords = pWord; + pWord->sCost.zKey = pWord->zCost; + pWord->sCost.pWord = pWord; + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + pWord->sWord.zKey = pWord->zWord; + pWord->sWord.pWord = pWord; + amatchStrcpy(pWord->zWord, pCur->zBuf); + pOther = amatchAvlInsert(&pCur->pWord, &pWord->sWord); + assert( pOther==0 ); (void)pOther; +#ifdef AMATCH_TRACE_1 + printf("INSERT [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", pWord->zWord+2, + pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, rCost, + pWord->zWord, pWord->zCost); +#endif +} + + +/* +** Advance a cursor to its next row of output +*/ +static int amatchNext(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + amatch_word *pWord = 0; + amatch_avl *pNode; + int isMatch = 0; + amatch_vtab *p = pCur->pVtab; + int nWord; + int rc; + int i; + const char *zW; + amatch_rule *pRule; + char *zBuf = 0; + char nBuf = 0; + char zNext[8]; + char zNextIn[8]; + int nNextIn; + + if( p->pVCheck==0 ){ + char *zSql; + if( p->zVocabLang && p->zVocabLang[0] ){ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"", + " WHERE \"%w\">=?1 AND \"%w\"=?2" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord, p->zVocabLang + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"" + " WHERE \"%w\">=?1" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord + ); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->pVCheck, 0); + sqlite3_free(zSql); + if( rc ) return rc; + } + sqlite3_bind_int(p->pVCheck, 2, pCur->iLang); + + do{ + pNode = amatchAvlFirst(pCur->pCost); + if( pNode==0 ){ + pWord = 0; + break; + } + pWord = pNode->pWord; + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + +#ifdef AMATCH_TRACE_1 + printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + nWord = (int)strlen(pWord->zWord+2); + if( nWord+20>nBuf ){ + nBuf = (char)(nWord+100); + zBuf = sqlite3_realloc(zBuf, nBuf); + if( zBuf==0 ) return SQLITE_NOMEM; + } + amatchStrcpy(zBuf, pWord->zWord+2); + zNext[0] = 0; + zNextIn[0] = pCur->zInput[pWord->nMatch]; + if( zNextIn[0] ){ + for(i=1; i<=4 && (pCur->zInput[pWord->nMatch+i]&0xc0)==0x80; i++){ + zNextIn[i] = pCur->zInput[pWord->nMatch+i]; + } + zNextIn[i] = 0; + nNextIn = i; + }else{ + nNextIn = 0; + } + + if( zNextIn[0] && zNextIn[0]!='*' ){ + sqlite3_reset(p->pVCheck); + amatchStrcat(zBuf, zNextIn); + sqlite3_bind_text(p->pVCheck, 1, zBuf, nWord+nNextIn, SQLITE_STATIC); + rc = sqlite3_step(p->pVCheck); + if( rc==SQLITE_ROW ){ + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + if( strncmp(zBuf, zW, nWord+nNextIn)==0 ){ + amatchAddWord(pCur, pWord->rCost, pWord->nMatch+nNextIn, zBuf, ""); + } + } + zBuf[nWord] = 0; + } + + while( 1 ){ + amatchStrcpy(zBuf+nWord, zNext); + sqlite3_reset(p->pVCheck); + sqlite3_bind_text(p->pVCheck, 1, zBuf, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(p->pVCheck); + if( rc!=SQLITE_ROW ) break; + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + amatchStrcpy(zBuf+nWord, zNext); + if( strncmp(zW, zBuf, nWord)!=0 ) break; + if( (zNextIn[0]=='*' && zNextIn[1]==0) + || (zNextIn[0]==0 && zW[nWord]==0) + ){ + isMatch = 1; + zNextIn[0] = 0; + nNextIn = 0; + break; + } + zNext[0] = zW[nWord]; + for(i=1; i<=4 && (zW[nWord+i]&0xc0)==0x80; i++){ + zNext[i] = zW[nWord+i]; + } + zNext[i] = 0; + zBuf[nWord] = 0; + if( p->rIns>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rIns, pWord->nMatch, + zBuf, zNext); + } + if( p->rSub>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rSub, pWord->nMatch+nNextIn, + zBuf, zNext); + } + if( p->rIns<0 && p->rSub<0 ) break; + zNext[i-1]++; /* FIX ME */ + } + sqlite3_reset(p->pVCheck); + + if( p->rDel>0 ){ + zBuf[nWord] = 0; + amatchAddWord(pCur, pWord->rCost+p->rDel, pWord->nMatch+nNextIn, + zBuf, ""); + } + + for(pRule=p->pRule; pRule; pRule=pRule->pNext){ + if( pRule->iLang!=pCur->iLang ) continue; + if( strncmp(pRule->zFrom, pCur->zInput+pWord->nMatch, pRule->nFrom)==0 ){ + amatchAddWord(pCur, pWord->rCost+pRule->rCost, + pWord->nMatch+pRule->nFrom, pWord->zWord+2, pRule->zTo); + } + } + }while( !isMatch ); + pCur->pCurrent = pWord; + sqlite3_free(zBuf); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any amatchColumn, amatchRowid, or amatchEof call. +*/ +static int amatchFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + amatch_cursor *pCur = (amatch_cursor *)pVtabCursor; + const char *zWord = "*"; + int idx; + + amatchClearCursor(pCur); + idx = 0; + if( idxNum & 1 ){ + zWord = (const char*)sqlite3_value_text(argv[0]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iLang = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + pCur->zInput = sqlite3_mprintf("%s", zWord); + if( pCur->zInput==0 ) return SQLITE_NOMEM; + amatchAddWord(pCur, 0, 0, "", ""); + amatchNext(pVtabCursor); + + return SQLITE_OK; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int amatchColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + amatch_cursor *pCur = (amatch_cursor*)cur; + switch( i ){ + case AMATCH_COL_WORD: { + sqlite3_result_text(ctx, pCur->pCurrent->zWord+2, -1, SQLITE_STATIC); + break; + } + case AMATCH_COL_DISTANCE: { + sqlite3_result_int(ctx, pCur->pCurrent->rCost); + break; + } + case AMATCH_COL_LANGUAGE: { + sqlite3_result_int(ctx, pCur->iLang); + break; + } + case AMATCH_COL_NWORD: { + sqlite3_result_int(ctx, pCur->nWord); + break; + } + default: { + sqlite3_result_null(ctx); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int amatchRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + amatch_cursor *pCur = (amatch_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int amatchEof(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) language == $language +** +** The distance< and distance<= are both treated as distance<=. +** The query plan number is a bit vector: +** +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. +*/ +static int amatchBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int iPlan = 0; + int iDistTerm = -1; + int iLangTerm = -1; + int i; + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 2)==0 + && pConstraint->iColumn==1 + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) + ){ + iPlan |= 2; + iDistTerm = i; + } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iLangTerm = i; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==1 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = (double)10000; + + return SQLITE_OK; +} + +/* +** The xUpdate() method. +** +** This implementation disallows DELETE and UPDATE. The only thing +** allowed is INSERT into the "command" column. +*/ +static int amatchUpdate( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + amatch_vtab *p = (amatch_vtab*)pVTab; + const unsigned char *zCmd; + (void)pRowid; + if( argc==1 ){ + pVTab->zErrMsg = sqlite3_mprintf("DELETE from %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pVTab->zErrMsg = sqlite3_mprintf("UPDATE of %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[2+AMATCH_COL_WORD])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_DISTANCE])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_LANGUAGE])!=SQLITE_NULL + ){ + pVTab->zErrMsg = sqlite3_mprintf( + "INSERT INTO %s allowed for column [command] only", p->zSelf); + return SQLITE_ERROR; + } + zCmd = sqlite3_value_text(argv[2+AMATCH_COL_COMMAND]); + if( zCmd==0 ) return SQLITE_OK; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "approximate_match". +*/ +static sqlite3_module amatchModule = { + 0, /* iVersion */ + amatchConnect, /* xCreate */ + amatchConnect, /* xConnect */ + amatchBestIndex, /* xBestIndex */ + amatchDisconnect, /* xDisconnect */ + amatchDisconnect, /* xDestroy */ + amatchOpen, /* xOpen - open a cursor */ + amatchClose, /* xClose - close a cursor */ + amatchFilter, /* xFilter - configure scan constraints */ + amatchNext, /* xNext - advance a cursor */ + amatchEof, /* xEof - check for end of scan */ + amatchColumn, /* xColumn - read data */ + amatchRowid, /* xRowid - read data */ + amatchUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Register the amatch virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_amatch_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Not used */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "approximate_match", &amatchModule, 0); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + return rc; +} diff --git a/ext/misc/anycollseq.c b/ext/misc/anycollseq.c new file mode 100644 index 0000000000000000000000000000000000000000..27b7049d5fececb4b7413fc076a06b422212cbb9 --- /dev/null +++ b/ext/misc/anycollseq.c @@ -0,0 +1,58 @@ +/* +** 2017-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a run-time loadable extension to SQLite that +** registers a sqlite3_collation_needed() callback to register a fake +** collating function for any unknown collating sequence. The fake +** collating function works like BINARY. +** +** This extension can be used to load schemas that contain one or more +** unknown collating sequences. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +static int anyCollFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1 +#include + +/* The append mark at the end of the database is: +** +** Start-Of-SQLite3-NNNNNNNN +** 123456789 123456789 12345 +** +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is +** the offset to page 1, and also the length of the prefix content. +*/ +#define APND_MARK_PREFIX "Start-Of-SQLite3-" +#define APND_MARK_PREFIX_SZ 17 +#define APND_MARK_FOS_SZ 8 +#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ) + +/* +** Maximum size of the combined prefix + database + append-mark. This +** must be less than 0x40000000 to avoid locking issues on Windows. +*/ +#define APND_MAX_SIZE (0x40000000) + +/* +** Try to align the database to an even multiple of APND_ROUNDUP bytes. +*/ +#ifndef APND_ROUNDUP +#define APND_ROUNDUP 4096 +#endif +#define APND_ALIGN_MASK ((sqlite3_int64)(APND_ROUNDUP-1)) +#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK) + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs ApndVfs; +typedef struct ApndFile ApndFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) + +/* An open appendvfs file +** +** An instance of this structure describes the appended database file. +** A separate sqlite3_file object is always appended. The appended +** sqlite3_file object (which can be accessed using ORIGFILE()) describes +** the entire file, including the prefix, the database, and the +** append-mark. +** +** The structure of an AppendVFS database is like this: +** +** +-------------+---------+----------+-------------+ +** | prefix-file | padding | database | append-mark | +** +-------------+---------+----------+-------------+ +** ^ ^ +** | | +** iPgOne iMark +** +** +** "prefix file" - file onto which the database has been appended. +** "padding" - zero or more bytes inserted so that "database" +** starts on an APND_ROUNDUP boundary +** "database" - The SQLite database file +** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates +** the offset from the start of prefix-file to the start +** of "database". +** +** The size of the database is iMark - iPgOne. +** +** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value +** of iPgOne stored as a big-ending 64-bit integer. +** +** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). +** Or, iMark is -1 to indicate that it has not yet been written. +*/ +struct ApndFile { + sqlite3_file base; /* Subclass. MUST BE FIRST! */ + sqlite3_int64 iPgOne; /* Offset to the start of the database */ + sqlite3_int64 iMark; /* Offset of the append mark. -1 if unwritten */ + /* Always followed by another sqlite3_file that describes the whole file */ +}; + +/* +** Methods for ApndFile +*/ +static int apndClose(sqlite3_file*); +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int apndTruncate(sqlite3_file*, sqlite3_int64 size); +static int apndSync(sqlite3_file*, int flags); +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int apndLock(sqlite3_file*, int); +static int apndUnlock(sqlite3_file*, int); +static int apndCheckReservedLock(sqlite3_file*, int *pResOut); +static int apndFileControl(sqlite3_file*, int op, void *pArg); +static int apndSectorSize(sqlite3_file*); +static int apndDeviceCharacteristics(sqlite3_file*); +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int apndShmLock(sqlite3_file*, int offset, int n, int flags); +static void apndShmBarrier(sqlite3_file*); +static int apndShmUnmap(sqlite3_file*, int deleteFlag); +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for ApndVfs +*/ +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void apndDlClose(sqlite3_vfs*, void*); +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int apndSleep(sqlite3_vfs*, int microseconds); +static int apndCurrentTime(sqlite3_vfs*, double*); +static int apndGetLastError(sqlite3_vfs*, int, char *); +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs apnd_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "apndvfs", /* zName */ + 0, /* pAppData (set when registered) */ + apndOpen, /* xOpen */ + apndDelete, /* xDelete */ + apndAccess, /* xAccess */ + apndFullPathname, /* xFullPathname */ + apndDlOpen, /* xDlOpen */ + apndDlError, /* xDlError */ + apndDlSym, /* xDlSym */ + apndDlClose, /* xDlClose */ + apndRandomness, /* xRandomness */ + apndSleep, /* xSleep */ + apndCurrentTime, /* xCurrentTime */ + apndGetLastError, /* xGetLastError */ + apndCurrentTimeInt64, /* xCurrentTimeInt64 */ + apndSetSystemCall, /* xSetSystemCall */ + apndGetSystemCall, /* xGetSystemCall */ + apndNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods apnd_io_methods = { + 3, /* iVersion */ + apndClose, /* xClose */ + apndRead, /* xRead */ + apndWrite, /* xWrite */ + apndTruncate, /* xTruncate */ + apndSync, /* xSync */ + apndFileSize, /* xFileSize */ + apndLock, /* xLock */ + apndUnlock, /* xUnlock */ + apndCheckReservedLock, /* xCheckReservedLock */ + apndFileControl, /* xFileControl */ + apndSectorSize, /* xSectorSize */ + apndDeviceCharacteristics, /* xDeviceCharacteristics */ + apndShmMap, /* xShmMap */ + apndShmLock, /* xShmLock */ + apndShmBarrier, /* xShmBarrier */ + apndShmUnmap, /* xShmUnmap */ + apndFetch, /* xFetch */ + apndUnfetch /* xUnfetch */ +}; + +/* +** Close an apnd-file. +*/ +static int apndClose(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from an apnd-file. +*/ +static int apndRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Add the append-mark onto what should become the end of the file. +* If and only if this succeeds, internal ApndFile.iMark is updated. +* Parameter iWriteEnd is the appendvfs-relative offset of the new mark. +*/ +static int apndWriteMark( + ApndFile *paf, + sqlite3_file *pFile, + sqlite_int64 iWriteEnd +){ + sqlite_int64 iPgOne = paf->iPgOne; + unsigned char a[APND_MARK_SIZE]; + int i = APND_MARK_FOS_SZ; + int rc; + assert(pFile == ORIGFILE(paf)); + memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); + while( --i >= 0 ){ + a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); + iPgOne >>= 8; + } + iWriteEnd += paf->iPgOne; + if( SQLITE_OK==(rc = pFile->pMethods->xWrite + (pFile, a, APND_MARK_SIZE, iWriteEnd)) ){ + paf->iMark = iWriteEnd; + } + return rc; +} + +/* +** Write data to an apnd-file. +*/ +static int apndWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + sqlite_int64 iWriteEnd = iOfst + iAmt; + if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; + pFile = ORIGFILE(pFile); + /* If append-mark is absent or will be overwritten, write it. */ + if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ + int rc = apndWriteMark(paf, pFile, iWriteEnd); + if( SQLITE_OK!=rc ) return rc; + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Truncate an apnd-file. +*/ +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + /* The append mark goes out first so truncate failure does not lose it. */ + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR; + /* Truncate underlying file just past append mark */ + return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); +} + +/* +** Sync an apnd-file. +*/ +static int apndSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of an apnd-file. +** If the append mark is not yet there, the file-size is 0. +*/ +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ApndFile *paf = (ApndFile *)pFile; + *pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0; + return SQLITE_OK; +} + +/* +** Lock an apnd-file. +*/ +static int apndLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock an apnd-file. +*/ +static int apndUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on an apnd-file. +*/ +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on an apnd-file. +*/ +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ + ApndFile *paf = (ApndFile *)pFile; + int rc; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_SIZE_HINT ) *(sqlite3_int64*)pArg += paf->iPgOne; + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", paf->iPgOne,*(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an apnd-file. +*/ +static int apndSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by an apnd-file. +*/ +static int apndDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int apndShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void apndShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int apndFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + ApndFile *p = (ApndFile *)pFile; + if( p->iMark < 0 || iOfst+iAmt > p->iMark ){ + return SQLITE_IOERR; /* Cannot read what is not yet there. */ + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); +} + +/* +** Try to read the append-mark off the end of a file. Return the +** start of the appended database if the append-mark is present. +** If there is no valid append-mark, return -1; +** +** An append-mark is only valid if the NNNNNNNN start-of-database offset +** indicates that the appended database contains at least one page. The +** start-of-database value must be a multiple of 512. +*/ +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc, i; + sqlite3_int64 iMark; + int msbs = 8 * (APND_MARK_FOS_SZ-1); + unsigned char a[APND_MARK_SIZE]; + + if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1; + rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); + if( rc ) return -1; + if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs; + for(i=1; i<8; i++){ + msbs -= 8; + iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]< (sz - APND_MARK_SIZE - 512) ) return -1; + if( iMark & 0x1ff ) return -1; + return iMark; +} + +static const char apvfsSqliteHdr[] = "SQLite format 3"; +/* +** Check to see if the file is an appendvfs SQLite database file. +** Return true iff it is such. Parameter sz is the file's size. +*/ +static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc; + char zHdr[16]; + sqlite3_int64 iMark = apndReadMark(sz, pFile); + if( iMark>=0 ){ + /* If file has the correct end-marker, the expected odd size, and the + ** SQLite DB type marker where the end-marker puts it, then it + ** is an appendvfs database. + */ + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); + if( SQLITE_OK==rc + && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff) == APND_MARK_SIZE + && sz>=512+APND_MARK_SIZE + ){ + return 1; /* It's an appendvfs database */ + } + } + return 0; +} + +/* +** Check to see if the file is an ordinary SQLite database file. +** Return true iff so. Parameter sz is the file's size. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + char zHdr[16]; + if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */ + || (sz & 0x1ff) != 0 + || SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0) + || memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 + ){ + return 0; + }else{ + return 1; + } +} + +/* +** Open an apnd file handle. +*/ +static int apndOpen( + sqlite3_vfs *pApndVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + ApndFile *pApndFile = (ApndFile*)pFile; + sqlite3_file *pBaseFile = ORIGFILE(pFile); + sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs); + int rc; + sqlite3_int64 sz = 0; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + /* The appendvfs is not to be used for transient or temporary databases. + ** Just use the base VFS open to initialize the given file object and + ** open the underlying file. (Appendvfs is then unused for this file.) + */ + return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); + } + memset(pApndFile, 0, sizeof(ApndFile)); + pFile->pMethods = &apnd_io_methods; + pApndFile->iMark = -1; /* Append mark not yet written */ + + rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); + if( rc==SQLITE_OK ){ + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); + if( rc ){ + pBaseFile->pMethods->xClose(pBaseFile); + } + } + if( rc ){ + pFile->pMethods = 0; + return rc; + } + if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ + /* The file being opened appears to be just an ordinary DB. Copy + ** the base dispatch-table so this instance mimics the base VFS. + */ + memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); + return SQLITE_OK; + } + pApndFile->iPgOne = apndReadMark(sz, pFile); + if( pApndFile->iPgOne>=0 ){ + pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */ + return SQLITE_OK; + } + if( (flags & SQLITE_OPEN_CREATE)==0 ){ + pBaseFile->pMethods->xClose(pBaseFile); + rc = SQLITE_CANTOPEN; + pFile->pMethods = 0; + }else{ + /* Round newly added appendvfs location to #define'd page boundary. + ** Note that nothing has yet been written to the underlying file. + ** The append mark will be written along with first content write. + ** Until then, paf->iMark value indicates it is not yet written. + */ + pApndFile->iPgOne = APND_START_ROUNDUP(sz); + } + return rc; +} + +/* +** Delete an apnd file. +** For an appendvfs, this could mean delete the appendvfs portion, +** leaving the appendee as it was before it gained an appendvfs. +** For now, this code deletes the underlying file too. +*/ +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int apndAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int apndFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int apndSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr apndGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_appendvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + (void)db; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + apnd_vfs.iVersion = pOrig->iVersion; + apnd_vfs.pAppData = pOrig; + apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); + rc = sqlite3_vfs_register(&apnd_vfs, 0); +#ifdef APPENDVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/base64.c b/ext/misc/base64.c new file mode 100644 index 0000000000000000000000000000000000000000..bc7a976abcf905b555fee14a73ae1b738a140bdb --- /dev/null +++ b/ext/misc/base64.c @@ -0,0 +1,292 @@ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters and to +** terminate any encoded blob having non-zero length. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes matching that of USASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +#include "sqlite3ext.h" + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; + } + pOut += nbo; + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c new file mode 100644 index 0000000000000000000000000000000000000000..e7ef0a04c97538bbc66faf56e3094804f7023dfd --- /dev/null +++ b/ext/misc/base85.c @@ -0,0 +1,450 @@ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit USASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +# include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +/* Classify c according to interval within USASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static u8 base85DigitValue( char c ){ + u8 dv = (u8)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + +static char * skipNonB85( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. + * Do not use the macro form with argument expression having a side-effect.*/ +#if 0 +static char base85Numeral( u8 b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} +#else +# define base85Numeral( dn )\ + ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*'))) +#endif + +static char *putcs(char *pc, char *s){ + char c; + while( (c = *s++)!=0 ) *pc++ = c; + return pc; +} + +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + while( nbIn >= 4 ){ + int nco = 5; + unsigned long qbv = (((unsigned long)pIn[0])<<24) | + (pIn[1]<<16) | (pIn[2]<<8) | pIn[3]; + while( nco > 0 ){ + unsigned nqv = (unsigned)(qbv/85UL); + unsigned char dv = qbv - 85UL*nqv; + qbv = nqv; + pOut[--nco] = base85Numeral(dv); + } + nbIn -= 4; + pIn += 4; + pOut += 5; + if( pSep && (nCol += 5)>=B85_DARK_MAX ){ + pOut = putcs(pOut, pSep); + nCol = 0; + } + } + if( nbIn > 0 ){ + int nco = nbIn + 1; + unsigned long qv = *pIn++; + int nbe = 1; + while( nbe++ < nbIn ){ + qv = (qv<<8) | *pIn++; + } + nCol += nco; + while( nco > 0 ){ + u8 dv = (u8)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += (nbIn+1); + } + if( pSep && nCol>0 ) pOut = putcs(pOut, pSep); + *pOut = 0; + return pOut; +} + +/* Decode base85 text into a byte buffer. */ +static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + if( nbo==0 ) break; + while( nti>0 ){ + char c = *pIn++; + u8 cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + (c - cdo); + --nti; + } + nbo -= nti; /* Adjust for early (non-digit) end of group. */ + switch( nbo ){ + case 4: + *pOut++ = (qv >> 24)&0xff; + case 3: + *pOut++ = (qv >> 16)&0xff; + case 2: + *pOut++ = (qv >> 8)&0xff; + case 1: + *pOut++ = qv&0xff; + case 0: + break; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); + return; + } +} +# endif + +/* This function does the work for the SQLite base85(x) UDF. */ +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + u8 bBuf[4*(B85_DARK_MAX/5)]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000000000000000000000000000000000000..0dcde543566f4b595984360c080f4c399a3ce55d --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,89 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +** +** Example usages: +** +** - Build a runtime-loadable extension from SQLite checkout directory: +** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c +** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll +** +** - Incorporate as built-in in sqlite3 shell: +** *Nix, OSX with gcc on a like platform: +** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** make sqlite3 "OPTS=$mop1 $mop2" +** Win32 with Microsoft toolset on Windows: +** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** set mops="OPTS=%mop1% %mop2%" +** nmake -f Makefile.msc sqlite3.exe %mops% +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +typedef unsigned char u8; +#define U8_TYPEDEF + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + int rc1; + int rc2; + + init_api_ptr(pApi); + rc1 = BASE64_INIT(db); + rc2 = BASE85_INIT(db); + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/blobio.c b/ext/misc/blobio.c new file mode 100644 index 0000000000000000000000000000000000000000..3a1ee8465c9323e590585b46775514af7c4ee7cf --- /dev/null +++ b/ext/misc/blobio.c @@ -0,0 +1,152 @@ +/* +** 2019-03-30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** An SQL function that uses the incremental BLOB I/O mechanism of SQLite +** to read or write part of a blob. This is intended for debugging use +** in the CLI. +** +** readblob(SCHEMA,TABLE,COLUMN,ROWID,OFFSET,N) +** +** Returns N bytes of the blob starting at OFFSET. +** +** writeblob(SCHEMA,TABLE,COLUMN,ROWID,OFFSET,NEWDATA) +** +** NEWDATA must be a blob. The content of NEWDATA overwrites the +** existing BLOB data at SCHEMA.TABLE.COLUMN for row ROWID beginning +** at OFFSET bytes into the blob. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +static void readblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_blob *pBlob = 0; + const char *zSchema; + const char *zTable; + const char *zColumn; + sqlite3_int64 iRowid; + int iOfst; + unsigned char *aData; + int nData; + sqlite3 *db; + int rc; + + zSchema = (const char*)sqlite3_value_text(argv[0]); + zTable = (const char*)sqlite3_value_text(argv[1]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad table name", -1); + return; + } + zColumn = (const char*)sqlite3_value_text(argv[2]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad column name", -1); + return; + } + iRowid = sqlite3_value_int64(argv[3]); + iOfst = sqlite3_value_int(argv[4]); + nData = sqlite3_value_int(argv[5]); + if( nData<=0 ) return; + aData = sqlite3_malloc64( nData+1 ); + if( aData==0 ){ + sqlite3_result_error_nomem(context); + return; + } + db = sqlite3_context_db_handle(context); + rc = sqlite3_blob_open(db, zSchema, zTable, zColumn, iRowid, 0, &pBlob); + if( rc ){ + sqlite3_free(aData); + sqlite3_result_error(context, "cannot open BLOB pointer", -1); + return; + } + rc = sqlite3_blob_read(pBlob, aData, nData, iOfst); + sqlite3_blob_close(pBlob); + if( rc ){ + sqlite3_free(aData); + sqlite3_result_error(context, "BLOB read failed", -1); + }else{ + sqlite3_result_blob(context, aData, nData, sqlite3_free); + } +} + +static void writeblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_blob *pBlob = 0; + const char *zSchema; + const char *zTable; + const char *zColumn; + sqlite3_int64 iRowid; + int iOfst; + unsigned char *aData; + int nData; + sqlite3 *db; + int rc; + + zSchema = (const char*)sqlite3_value_text(argv[0]); + zTable = (const char*)sqlite3_value_text(argv[1]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad table name", -1); + return; + } + zColumn = (const char*)sqlite3_value_text(argv[2]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad column name", -1); + return; + } + iRowid = sqlite3_value_int64(argv[3]); + iOfst = sqlite3_value_int(argv[4]); + if( sqlite3_value_type(argv[5])!=SQLITE_BLOB ){ + sqlite3_result_error(context, "6th argument must be a BLOB", -1); + return; + } + nData = sqlite3_value_bytes(argv[5]); + aData = (unsigned char *)sqlite3_value_blob(argv[5]); + db = sqlite3_context_db_handle(context); + rc = sqlite3_blob_open(db, zSchema, zTable, zColumn, iRowid, 1, &pBlob); + if( rc ){ + sqlite3_result_error(context, "cannot open BLOB pointer", -1); + return; + } + rc = sqlite3_blob_write(pBlob, aData, nData, iOfst); + sqlite3_blob_close(pBlob); + if( rc ){ + sqlite3_result_error(context, "BLOB write failed", -1); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_blobio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readblob", 6, SQLITE_UTF8, 0, + readblobFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writeblob", 6, SQLITE_UTF8, 0, + writeblobFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c new file mode 100644 index 0000000000000000000000000000000000000000..02f8c0319c6c7ab45b71780c7fd3ddc32cc415d8 --- /dev/null +++ b/ext/misc/btreeinfo.c @@ -0,0 +1,430 @@ +/* +** 2017-10-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_btreeinfo" virtual table. +** +** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual +** table that shows information about all btrees in an SQLite database file. +** The schema is like this: +** +** CREATE TABLE sqlite_btreeinfo( +** type TEXT, -- "table" or "index" +** name TEXT, -- Name of table or index for this btree. +** tbl_name TEXT, -- Associated table +** rootpage INT, -- The root page of the btree +** sql TEXT, -- SQL for this btree - from sqlite_schema +** hasRowid BOOLEAN, -- True if the btree has a rowid +** nEntry INT, -- Estimated number of entries +** nPage INT, -- Estimated number of pages +** depth INT, -- Depth of the btree +** szPage INT, -- Size of each page in bytes +** zSchema TEXT HIDDEN -- The schema to which this btree belongs +** ); +** +** The first 5 fields are taken directly from the sqlite_schema table. +** Considering only the first 5 fields, the only difference between +** this virtual table and the sqlite_schema table is that this virtual +** table omits all entries that have a 0 or NULL rowid - in other words +** it omits triggers and views. +** +** The value added by this table comes in the next 5 fields. +** +** Note that nEntry and nPage are *estimated*. They are computed doing +** a single search from the root to a leaf, counting the number of cells +** at each level, and assuming that unvisited pages have a similar number +** of cells. +** +** The sqlite_dbpage virtual table must be available for this virtual table +** to operate. +** +** USAGE EXAMPLES: +** +** Show the table btrees in a schema order with the tables with the most +** rows occuring first: +** +** SELECT name, nEntry +** FROM sqlite_btreeinfo +** WHERE type='table' +** ORDER BY nEntry DESC, name; +** +** Show the names of all WITHOUT ROWID tables: +** +** SELECT name FROM sqlite_btreeinfo +** WHERE type='table' AND NOT hasRowid; +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Columns available in this virtual table */ +#define BINFO_COLUMN_TYPE 0 +#define BINFO_COLUMN_NAME 1 +#define BINFO_COLUMN_TBL_NAME 2 +#define BINFO_COLUMN_ROOTPAGE 3 +#define BINFO_COLUMN_SQL 4 +#define BINFO_COLUMN_HASROWID 5 +#define BINFO_COLUMN_NENTRY 6 +#define BINFO_COLUMN_NPAGE 7 +#define BINFO_COLUMN_DEPTH 8 +#define BINFO_COLUMN_SZPAGE 9 +#define BINFO_COLUMN_SCHEMA 10 + +/* Forward declarations */ +typedef struct BinfoTable BinfoTable; +typedef struct BinfoCursor BinfoCursor; + +/* A cursor for the sqlite_btreeinfo table */ +struct BinfoCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* Query against sqlite_schema */ + int rc; /* Result of previous sqlite_step() call */ + int hasRowid; /* hasRowid value. Negative if unknown. */ + sqlite3_int64 nEntry; /* nEntry value */ + int nPage; /* nPage value */ + int depth; /* depth value */ + int szPage; /* size of a btree page. 0 if unknown */ + char *zSchema; /* Schema being interrogated */ +}; + +/* The sqlite_btreeinfo table */ +struct BinfoTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The databse connection */ +}; + +/* +** Connect to the sqlite_btreeinfo virtual table. +*/ +static int binfoConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + BinfoTable *pTab = 0; + int rc = SQLITE_OK; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(\n" + " type TEXT,\n" + " name TEXT,\n" + " tbl_name TEXT,\n" + " rootpage INT,\n" + " sql TEXT,\n" + " hasRowid BOOLEAN,\n" + " nEntry INT,\n" + " nPage INT,\n" + " depth INT,\n" + " szPage INT,\n" + " zSchema TEXT HIDDEN\n" + ")"); + if( rc==SQLITE_OK ){ + pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable)); + if( pTab==0 ) rc = SQLITE_NOMEM; + } + assert( rc==SQLITE_OK || pTab==0 ); + if( pTab ){ + pTab->db = db; + } + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a btreeinfo virtual table. +*/ +static int binfoDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 Use "main" for the schema +** 1 Schema identified by parameter ?1 +*/ +static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + pIdxInfo->estimatedCost = 10000.0; /* Cost estimate */ + pIdxInfo->estimatedRows = 100; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable + && p->iColumn==BINFO_COLUMN_SCHEMA + && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pIdxInfo->estimatedCost = 1000.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + } + return SQLITE_OK; +} + +/* +** Open a new btreeinfo cursor. +*/ +static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + BinfoCursor *pCsr; + + pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(BinfoCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a btreeinfo cursor. +*/ +static int binfoClose(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->zSchema); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a btreeinfo cursor to the next entry in the file. +*/ +static int binfoNext(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + pCsr->rc = sqlite3_step(pCsr->pStmt); + pCsr->hasRowid = -1; + return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int binfoEof(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + return pCsr->rc!=SQLITE_ROW; +} + +/* Position a cursor back to the beginning. +*/ +static int binfoFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + BinfoTable *pTab = (BinfoTable *)pCursor->pVtab; + char *zSql; + int rc; + + sqlite3_free(pCsr->zSchema); + if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + }else{ + pCsr->zSchema = sqlite3_mprintf("main"); + } + zSql = sqlite3_mprintf( + "SELECT 0, 'table','sqlite_schema','sqlite_schema',1,NULL " + "UNION ALL " + "SELECT rowid, type, name, tbl_name, rootpage, sql" + " FROM \"%w\".sqlite_schema WHERE rootpage>=1", + pCsr->zSchema); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + pCsr->hasRowid = -1; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + rc = binfoNext(pCursor); + } + return rc; +} + +/* Decode big-endian integers */ +static unsigned int get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static unsigned int get_uint32(unsigned char *a){ + return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; +} + +/* Examine the b-tree rooted at pgno and estimate its size. +** Return non-zero if anything goes wrong. +*/ +static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){ + sqlite3_int64 nEntry = 1; + int nPage = 1; + unsigned char *aData; + sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; + int pgsz = 0; + int nCell; + int iCell; + + rc = sqlite3_prepare_v2(db, + "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1, + &pStmt, 0); + if( rc ) return rc; + pCsr->depth = 1; + while(1){ + sqlite3_bind_int(pStmt, 1, pgno); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + rc = SQLITE_ERROR; + break; + } + pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0); + aData = (unsigned char*)sqlite3_column_blob(pStmt, 0); + if( aData==0 ){ + rc = SQLITE_NOMEM; + break; + } + if( pgno==1 ){ + aData += 100; + pgsz -= 100; + } + pCsr->hasRowid = aData[0]!=2 && aData[0]!=10; + nCell = get_uint16(aData+3); + nEntry *= (nCell+1); + if( aData[0]==10 || aData[0]==13 ) break; + nPage *= (nCell+1); + if( nCell<=1 ){ + pgno = get_uint32(aData+8); + }else{ + iCell = get_uint16(aData+12+2*(nCell/2)); + if( pgno==1 ) iCell -= 100; + if( iCell<=12 || iCell>=pgsz-4 ){ + rc = SQLITE_CORRUPT; + break; + } + pgno = get_uint32(aData+iCell); + } + pCsr->depth++; + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + pCsr->nPage = nPage; + pCsr->nEntry = nEntry; + if( rc==SQLITE_ROW ) rc = SQLITE_OK; + return rc; +} + +/* Return a column for the sqlite_btreeinfo table */ +static int binfoColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){ + int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1); + sqlite3 *db = sqlite3_context_db_handle(ctx); + int rc = binfoCompute(db, pgno, pCsr); + if( rc ){ + pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + return SQLITE_ERROR; + } + } + switch( i ){ + case BINFO_COLUMN_NAME: + case BINFO_COLUMN_TYPE: + case BINFO_COLUMN_TBL_NAME: + case BINFO_COLUMN_ROOTPAGE: + case BINFO_COLUMN_SQL: { + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1)); + break; + } + case BINFO_COLUMN_HASROWID: { + sqlite3_result_int(ctx, pCsr->hasRowid); + break; + } + case BINFO_COLUMN_NENTRY: { + sqlite3_result_int64(ctx, pCsr->nEntry); + break; + } + case BINFO_COLUMN_NPAGE: { + sqlite3_result_int(ctx, pCsr->nPage); + break; + } + case BINFO_COLUMN_DEPTH: { + sqlite3_result_int(ctx, pCsr->depth); + break; + } + case BINFO_COLUMN_SCHEMA: { + sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_btreeinfo table */ +static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "sqlite_btreeinfo" virtual table module +*/ +int sqlite3BinfoRegister(sqlite3 *db){ + static sqlite3_module binfo_module = { + 0, /* iVersion */ + 0, /* xCreate */ + binfoConnect, /* xConnect */ + binfoBestIndex, /* xBestIndex */ + binfoDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + binfoOpen, /* xOpen - open a cursor */ + binfoClose, /* xClose - close a cursor */ + binfoFilter, /* xFilter - configure scan constraints */ + binfoNext, /* xNext - advance a cursor */ + binfoEof, /* xEof - check for end of scan */ + binfoColumn, /* xColumn - read data */ + binfoRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_btreeinfo_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3BinfoRegister(db); +} diff --git a/ext/misc/carray.c b/ext/misc/carray.c new file mode 100644 index 0000000000000000000000000000000000000000..b1caa98c3fa899a472bc0952eb7341b5fc04dfad --- /dev/null +++ b/ext/misc/carray.c @@ -0,0 +1,561 @@ +/* +** 2016-06-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function that +** returns the values in a C-language array. +** Examples: +** +** SELECT * FROM carray($ptr,5) +** +** The query above returns 5 integers contained in a C-language array +** at the address $ptr. $ptr is a pointer to the array of integers. +** The pointer value must be assigned to $ptr using the +** sqlite3_bind_pointer() interface with a pointer type of "carray". +** For example: +** +** static int aX[] = { 53, 9, 17, 2231, 4, 99 }; +** int i = sqlite3_bind_parameter_index(pStmt, "$ptr"); +** sqlite3_bind_pointer(pStmt, i, aX, "carray", 0); +** +** There is an optional third parameter to determine the datatype of +** the C-language array. Allowed values of the third parameter are +** 'int32', 'int64', 'double', 'char*', 'struct iovec'. Example: +** +** SELECT * FROM carray($ptr,10,'char*'); +** +** The default value of the third parameter is 'int32'. +** +** HOW IT WORKS +** +** The carray "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE carray( +** value, +** pointer HIDDEN, +** count HIDDEN, +** ctype TEXT HIDDEN +** ); +** +** If the hidden columns "pointer" and "count" are unconstrained, then +** the virtual table has no rows. Otherwise, the virtual table interprets +** the integer value of "pointer" as a pointer to the array and "count" +** as the number of elements in the array. The virtual table steps through +** the array, element by element. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#ifdef _WIN32 + struct iovec { + void *iov_base; + size_t iov_len; + }; +#else +# include +#endif + +/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). +** Must exactly match the definitions in carray.h. +*/ +#ifndef CARRAY_INT32 +# define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +# define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +# define CARRAY_DOUBLE 2 /* Data is doubles */ +# define CARRAY_TEXT 3 /* Data is char* */ +# define CARRAY_BLOB 4 /* Data is struct iovec* */ +#endif + +#ifndef SQLITE_API +# ifdef _WIN32 +# define SQLITE_API __declspec(dllexport) +# else +# define SQLITE_API +# endif +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Names of allowed datatypes +*/ +static const char *azType[] = { "int32", "int64", "double", "char*", + "struct iovec" }; + +/* +** Structure used to hold the sqlite3_carray_bind() information +*/ +typedef struct carray_bind carray_bind; +struct carray_bind { + void *aData; /* The data */ + int nData; /* Number of elements */ + int mFlags; /* Control flags */ + void (*xDel)(void*); /* Destructor for aData */ +}; + + +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct carray_cursor carray_cursor; +struct carray_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + void *pPtr; /* Pointer to the array of values */ + sqlite3_int64 iCnt; /* Number of integers in the array */ + unsigned char eType; /* One of the CARRAY_type values */ +}; + +/* +** The carrayConnect() method is invoked to create a new +** carray_vtab that describes the carray virtual table. +** +** Think of this routine as the constructor for carray_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the carray_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against carray will look like. +*/ +static int carrayConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define CARRAY_COLUMN_VALUE 0 +#define CARRAY_COLUMN_POINTER 1 +#define CARRAY_COLUMN_COUNT 2 +#define CARRAY_COLUMN_CTYPE 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for carray_cursor objects. +*/ +static int carrayDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new carray_cursor object. +*/ +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + carray_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a carray_cursor. +*/ +static int carrayClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a carray_cursor to its next row of output. +*/ +static int carrayNext(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the carray_cursor +** is currently pointing. +*/ +static int carrayColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + carray_cursor *pCur = (carray_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case CARRAY_COLUMN_POINTER: return SQLITE_OK; + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; + case CARRAY_COLUMN_CTYPE: { + sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC); + return SQLITE_OK; + } + default: { + switch( pCur->eType ){ + case CARRAY_INT32: { + int *p = (int*)pCur->pPtr; + sqlite3_result_int(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_INT64: { + sqlite3_int64 *p = (sqlite3_int64*)pCur->pPtr; + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_DOUBLE: { + double *p = (double*)pCur->pPtr; + sqlite3_result_double(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_TEXT: { + const char **p = (const char**)pCur->pPtr; + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); + return SQLITE_OK; + } + case CARRAY_BLOB: { + const struct iovec *p = (struct iovec*)pCur->pPtr; + sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base, + (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); + return SQLITE_OK; + } + } + } + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + carray_cursor *pCur = (carray_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int carrayEof(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + return pCur->iRowid>pCur->iCnt; +} + +/* +** This method is called to "rewind" the carray_cursor object back +** to the first row of output. +*/ +static int carrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + carray_cursor *pCur = (carray_cursor *)pVtabCursor; + pCur->pPtr = 0; + pCur->iCnt = 0; + switch( idxNum ){ + case 1: { + carray_bind *pBind = sqlite3_value_pointer(argv[0], "carray-bind"); + if( pBind==0 ) break; + pCur->pPtr = pBind->aData; + pCur->iCnt = pBind->nData; + pCur->eType = pBind->mFlags & 0x07; + break; + } + case 2: + case 3: { + pCur->pPtr = sqlite3_value_pointer(argv[0], "carray"); + pCur->iCnt = pCur->pPtr ? sqlite3_value_int64(argv[1]) : 0; + if( idxNum<3 ){ + pCur->eType = CARRAY_INT32; + }else{ + unsigned char i; + const char *zType = (const char*)sqlite3_value_text(argv[2]); + for(i=0; i=sizeof(azType)/sizeof(azType[0]) ){ + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "unknown datatype: %Q", zType); + return SQLITE_ERROR; + }else{ + pCur->eType = i; + } + } + break; + } + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the carray virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** idxNum is: +** +** 1 If only the pointer= constraint exists. In this case, the +** parameter must be bound using sqlite3_carray_bind(). +** +** 2 if the pointer= and count= constraints exist. +** +** 3 if the ctype= constraint also exists. +** +** idxNum is 0 otherwise and carray becomes an empty table. +*/ +static int carrayBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case CARRAY_COLUMN_POINTER: + ptrIdx = i; + break; + case CARRAY_COLUMN_COUNT: + cntIdx = i; + break; + case CARRAY_COLUMN_CTYPE: + ctypeIdx = i; + break; + } + } + if( ptrIdx>=0 ){ + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 1; + if( cntIdx>=0 ){ + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; + pIdxInfo->idxNum = 2; + if( ctypeIdx>=0 ){ + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + } + }else{ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + pIdxInfo->idxNum = 0; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** carray virtual table. +*/ +static sqlite3_module carrayModule = { + 0, /* iVersion */ + 0, /* xCreate */ + carrayConnect, /* xConnect */ + carrayBestIndex, /* xBestIndex */ + carrayDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + carrayOpen, /* xOpen - open a cursor */ + carrayClose, /* xClose - close a cursor */ + carrayFilter, /* xFilter - configure scan constraints */ + carrayNext, /* xNext - advance a cursor */ + carrayEof, /* xEof - check for end of scan */ + carrayColumn, /* xColumn - read data */ + carrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadow */ + 0 /* xIntegrity */ +}; + +/* +** Destructor for the carray_bind object +*/ +static void carrayBindDel(void *pPtr){ + carray_bind *p = (carray_bind*)pPtr; + if( p->xDel!=SQLITE_STATIC ){ + p->xDel(p->aData); + } + sqlite3_free(p); +} + +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*) +){ + carray_bind *pNew; + int i; + pNew = sqlite3_malloc64(sizeof(*pNew)); + if( pNew==0 ){ + if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ + xDestroy(aData); + } + return SQLITE_NOMEM; + } + pNew->nData = nData; + pNew->mFlags = mFlags; + if( xDestroy==SQLITE_TRANSIENT ){ + sqlite3_int64 sz = nData; + switch( mFlags & 0x07 ){ + case CARRAY_INT32: sz *= 4; break; + case CARRAY_INT64: sz *= 8; break; + case CARRAY_DOUBLE: sz *= 8; break; + case CARRAY_TEXT: sz *= sizeof(char*); break; + case CARRAY_BLOB: sz *= sizeof(struct iovec); break; + } + if( (mFlags & 0x07)==CARRAY_TEXT ){ + for(i=0; iaData = sqlite3_malloc64( sz ); + if( pNew->aData==0 ){ + sqlite3_free(pNew); + return SQLITE_NOMEM; + } + if( (mFlags & 0x07)==CARRAY_TEXT ){ + char **az = (char**)pNew->aData; + char *z = (char*)&az[nData]; + for(i=0; iaData; + unsigned char *z = (unsigned char*)&p[nData]; + for(i=0; iaData, aData, sz); + } + pNew->xDel = sqlite3_free; + }else{ + pNew->aData = aData; + pNew->xDel = xDestroy; + } + return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); +} + + +/* +** For testing purpose in the TCL test harness, we need a method for +** setting the pointer value. The inttoptr(X) SQL function accomplishes +** this. Tcl script will bind an integer to X and the inttoptr() SQL +** function will use sqlite3_result_pointer() to convert that integer into +** a pointer. +** +** This is for testing on TCL only. +*/ +#ifdef SQLITE_TEST +static void inttoptrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + void *p; + sqlite3_int64 i64; + i64 = sqlite3_value_int64(argv[0]); + if( sizeof(i64)==sizeof(p) ){ + memcpy(&p, &i64, sizeof(p)); + }else{ + int i32 = i64 & 0xffffffff; + memcpy(&p, &i32, sizeof(p)); + } + sqlite3_result_pointer(context, p, "carray", 0); +} +#endif /* SQLITE_TEST */ + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +SQLITE_API int sqlite3_carray_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "carray", &carrayModule, 0); +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0, + inttoptrFunc, 0, 0); + } +#endif /* SQLITE_TEST */ +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + return rc; +} diff --git a/ext/misc/carray.h b/ext/misc/carray.h new file mode 100644 index 0000000000000000000000000000000000000000..ae0a9e8ab07095147542e918a4203e8af1817475 --- /dev/null +++ b/ext/misc/carray.h @@ -0,0 +1,50 @@ +/* +** 2020-11-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Interface definitions for the CARRAY table-valued function +** extension. +*/ + +#ifndef _CARRAY_H +#define _CARRAY_H + +#include "sqlite3.h" /* Required for error code definitions */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Use this interface to bind an array to the single-argument version +** of CARRAY(). +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructgor for aData*/ +); + +/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _CARRAY_H */ diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c new file mode 100644 index 0000000000000000000000000000000000000000..2d7f6584ea1b48740ec0ec4be012bcf6617ca632 --- /dev/null +++ b/ext/misc/cksumvfs.c @@ -0,0 +1,880 @@ +/* +** 2020-04-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a checksum on each page +** of an SQLite database file. When reading pages, the checksum is verified +** and an error is raised if the checksum is incorrect. +** +** COMPILING +** +** This extension requires SQLite 3.32.0 or later. It uses the +** sqlite3_database_file_object() interface which was added in +** version 3.32.0, so it will not link with an earlier version of +** SQLite. +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared cksumvfs.c -o cksumvfs.so +** (mac) clang -fPIC -dynamiclib cksumvfs.c -o cksumvfs.dylib +** (windows) cl cksumvfs.c -link -dll -out:cksumvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** If you want to statically link this extension with your product, +** then compile it like any other C-language module but add the +** "-DSQLITE_CKSUMVFS_STATIC" option so that this module knows that +** it is being statically linked rather than dynamically linked +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extension(db, "./cksumvfs"); +** sqlite3_close(db); +** +** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and +** statically linked against the application, initialize it using +** a single API call as follows: +** +** sqlite3_register_cksumvfs(); +** +** Cksumvfs is a VFS Shim. When loaded, "cksmvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, in +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that cksumvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a checksum) will operate normally. Databases with +** checksums will return an SQLITE_IOERR_DATA error if a page is +** encountered that contains an invalid checksum. +** +** Checksumming only works on databases that have a reserve-bytes +** value of exactly 8. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the checksum by +** default. To create a database that includes a checksum, change +** the reserve-bytes value to 8 by runing: +** +** int n = 8; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** If the database is in WAL mode, you should shutdown and +** reopen all database connections before continuing. +** +** From the CLI, use the ".filectrl reserve_bytes 8" command, +** followed by "VACUUM;". +** +** Note that SQLite allows the number of reserve-bytes to be +** increased but not decreased. So if a database file already +** has a reserve-bytes value greater than 8, there is no way to +** activate checksumming on that database, other than to dump +** and restore the database file. Note also that other extensions +** might also make use of the reserve-bytes. Checksumming will +** be incompatible with those other extensions. +** +** VERIFICATION OF CHECKSUMS +** +** If any checksum is incorrect, the "PRAGMA quick_check" command +** will find it. To verify that checksums are actually enabled +** and running, use the following query: +** +** SELECT count(*), verify_checksum(data) +** FROM sqlite_dbpage +** GROUP BY 2; +** +** There are three possible outputs form the verify_checksum() +** function: 1, 0, and NULL. 1 is returned if the checksum is +** correct. 0 is returned if the checksum is incorrect. NULL +** is returned if the page is unreadable. If checksumming is +** enabled, the read will fail if the checksum is wrong, so the +** usual result from verify_checksum() on a bad checksum is NULL. +** +** If everything is OK, the query above should return a single +** row where the second column is 1. Any other result indicates +** either that there is a checksum error, or checksum validation +** is disabled. +** +** CONTROLLING CHECKSUM VERIFICATION +** +** The cksumvfs extension implements a new PRAGMA statement that can +** be used to disable, re-enable, or query the status of checksum +** verification: +** +** PRAGMA checksum_verification; -- query status +** PRAGMA checksum_verification=OFF; -- disable verification +** PRAGMA checksum_verification=ON; -- re-enable verification +** +** The "checksum_verification" pragma will return "1" (true) or "0" +** (false) if checksum verification is enabled or disabled, respectively. +** "Verification" in this context means the feature that causes +** SQLITE_IOERR_DATA errors if a checksum mismatch is detected while +** reading. Checksums are always kept up-to-date as long as the +** reserve-bytes value of the database is 8, regardless of the setting +** of this pragma. Checksum verification can be disabled (for example) +** to do forensic analysis of a database that has previously reported +** a checksum error. +** +** The "checksum_verification" pragma will always respond with "0" if +** the database file does not have a reserve-bytes value of 8. The +** pragma will return no rows at all if the cksumvfs extension is +** not loaded. +** +** IMPLEMENTATION NOTES +** +** The checksum is stored in the last 8 bytes of each page. This +** module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 8. If +** the reserved-space value is not 8, this module is a no-op. +*/ +#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_CKSUMVFS_STATIC) +# define SQLITE_CKSUMVFS_STATIC +#endif +#ifdef SQLITE_CKSUMVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs CksmVfs; +typedef struct CksmFile CksmFile; + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_AMALGAMATION) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((CksmFile*)(p))+1)) + +/* An open file */ +struct CksmFile { + sqlite3_file base; /* IO methods */ + const char *zFName; /* Original name of the file */ + char computeCksm; /* True to compute checksums. + ** Always true if reserve size is 8. */ + char verifyCksm; /* True to verify checksums */ + char isWal; /* True if processing a WAL file */ + char inCkpt; /* Currently doing a checkpoint */ + CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ +}; + +/* +** Methods for CksmFile +*/ +static int cksmClose(sqlite3_file*); +static int cksmRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int cksmWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int cksmTruncate(sqlite3_file*, sqlite3_int64 size); +static int cksmSync(sqlite3_file*, int flags); +static int cksmFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int cksmLock(sqlite3_file*, int); +static int cksmUnlock(sqlite3_file*, int); +static int cksmCheckReservedLock(sqlite3_file*, int *pResOut); +static int cksmFileControl(sqlite3_file*, int op, void *pArg); +static int cksmSectorSize(sqlite3_file*); +static int cksmDeviceCharacteristics(sqlite3_file*); +static int cksmShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int cksmShmLock(sqlite3_file*, int offset, int n, int flags); +static void cksmShmBarrier(sqlite3_file*); +static int cksmShmUnmap(sqlite3_file*, int deleteFlag); +static int cksmFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int cksmUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for CksmVfs +*/ +static int cksmOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int cksmDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int cksmAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int cksmFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *cksmDlOpen(sqlite3_vfs*, const char *zFilename); +static void cksmDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void cksmDlClose(sqlite3_vfs*, void*); +static int cksmRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int cksmSleep(sqlite3_vfs*, int microseconds); +static int cksmCurrentTime(sqlite3_vfs*, double*); +static int cksmGetLastError(sqlite3_vfs*, int, char *); +static int cksmCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int cksmSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr cksmGetSystemCall(sqlite3_vfs*, const char *z); +static const char *cksmNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs cksm_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "cksmvfs", /* zName */ + 0, /* pAppData (set when registered) */ + cksmOpen, /* xOpen */ + cksmDelete, /* xDelete */ + cksmAccess, /* xAccess */ + cksmFullPathname, /* xFullPathname */ + cksmDlOpen, /* xDlOpen */ + cksmDlError, /* xDlError */ + cksmDlSym, /* xDlSym */ + cksmDlClose, /* xDlClose */ + cksmRandomness, /* xRandomness */ + cksmSleep, /* xSleep */ + cksmCurrentTime, /* xCurrentTime */ + cksmGetLastError, /* xGetLastError */ + cksmCurrentTimeInt64, /* xCurrentTimeInt64 */ + cksmSetSystemCall, /* xSetSystemCall */ + cksmGetSystemCall, /* xGetSystemCall */ + cksmNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods cksm_io_methods = { + 3, /* iVersion */ + cksmClose, /* xClose */ + cksmRead, /* xRead */ + cksmWrite, /* xWrite */ + cksmTruncate, /* xTruncate */ + cksmSync, /* xSync */ + cksmFileSize, /* xFileSize */ + cksmLock, /* xLock */ + cksmUnlock, /* xUnlock */ + cksmCheckReservedLock, /* xCheckReservedLock */ + cksmFileControl, /* xFileControl */ + cksmSectorSize, /* xSectorSize */ + cksmDeviceCharacteristics, /* xDeviceCharacteristics */ + cksmShmMap, /* xShmMap */ + cksmShmLock, /* xShmLock */ + cksmShmBarrier, /* xShmBarrier */ + cksmShmUnmap, /* xShmUnmap */ + cksmFetch, /* xFetch */ + cksmUnfetch /* xUnfetch */ +}; + +/* Do byte swapping on a unsigned 32-bit integer */ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* Compute a checksum on a buffer */ +static void cksmCompute( + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + u8 *aOut /* OUT: Final 8-byte checksum value output */ +){ + u32 s1 = 0, s2 = 0; + u32 *aData = (u32*)a; + u32 *aEnd = (u32*)&a[nByte]; + u32 x = 1; + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + + if( 1 == *(u8*)&x ){ + /* Little-endian */ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aData65536 || (nByte & (nByte-1))!=0 ) return; + cksmCompute(data, nByte-8, cksum); + sqlite3_result_int(context, memcmp(data+nByte-8,cksum,8)==0); +} + +#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME +/* +** SQL function: initialize_cksumvfs(SCHEMANAME) +** +** This SQL functions (whose name is actually determined at compile-time +** by the value of the SQLITE_CKSUMVFS_INIT_FUNCNAME macro) invokes: +** +** sqlite3_file_control(db, SCHEMANAME, SQLITE_FCNTL_RESERVE_BYTE, &n); +** +** In order to set the reserve bytes value to 8, so that cksumvfs will +** operation. This feature is provided (if and only if the +** SQLITE_CKSUMVFS_INIT_FUNCNAME compile-time option is set to a string +** which is the name of the SQL function) so as to provide the ability +** to invoke the file-control in programming languages that lack +** direct access to the sqlite3_file_control() interface (ex: Java). +** +** This interface is undocumented, apart from this comment. Usage +** example: +** +** 1. Compile with -DSQLITE_CKSUMVFS_INIT_FUNCNAME="ckvfs_init" +** 2. Run: "SELECT cksum_init('main'); VACUUM;" +*/ +static void cksmInitFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nByte = 8; + const char *zSchemaName = (const char*)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_file_control(db, zSchemaName, SQLITE_FCNTL_RESERVE_BYTES, &nByte); + /* Return NULL */ +} +#endif /* SQLITE_CKSUMBFS_INIT_FUNCNAME */ + +/* +** Close a cksm-file. +*/ +static int cksmClose(sqlite3_file *pFile){ + CksmFile *p = (CksmFile *)pFile; + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Set the computeCkSm and verifyCksm flags, if they need to be +** changed. +*/ +static void cksmSetFlags(CksmFile *p, int hasCorrectReserveSize){ + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + if( p->pPartner ){ + p->pPartner->verifyCksm = hasCorrectReserveSize; + p->pPartner->computeCksm = hasCorrectReserveSize; + } + } +} + +/* +** Read data from a cksm-file. +*/ +static int cksmRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + if( iOfst==0 && iAmt>=100 && ( + memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 + )){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + cksmSetFlags(p, hasCorrectReserveSize); + } + /* Verify the checksum if + ** (1) the size indicates that we are dealing with a complete + ** database page + ** (2) checksum verification is enabled + ** (3) we are not in the middle of checkpoint + */ + if( iAmt>=512 && (iAmt & (iAmt-1))==0 /* (1) */ + && p->verifyCksm /* (2) */ + && !p->inCkpt /* (3) */ + ){ + u8 cksum[8]; + cksmCompute((u8*)zBuf, iAmt-8, cksum); + if( memcmp((u8*)zBuf+iAmt-8, cksum, 8)!=0 ){ + sqlite3_log(SQLITE_IOERR_DATA, + "checksum fault offset %lld of \"%s\"", + iOfst, p->zFName); + rc = SQLITE_IOERR_DATA; + } + } + } + return rc; +} + +/* +** Write data to a cksm-file. +*/ +static int cksmWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + if( iOfst==0 && iAmt>=100 && ( + memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 + )){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + cksmSetFlags(p, hasCorrectReserveSize); + } + /* If the write size is appropriate for a database page and if + ** checksums where ever enabled, then it will be safe to compute + ** the checksums. The reserve byte size might have increased, but + ** it will never decrease. And because it cannot decrease, the + ** checksum will not overwrite anything. + */ + if( iAmt>=512 + && p->computeCksm + && !p->inCkpt + ){ + cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst); +} + +/* +** Truncate a cksm-file. +*/ +static int cksmTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a cksm-file. +*/ +static int cksmSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a cksm-file. +*/ +static int cksmFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a cksm-file. +*/ +static int cksmLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a cksm-file. +*/ +static int cksmUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a cksm-file. +*/ +static int cksmCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a cksm-file. +*/ +static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + CksmFile *p = (CksmFile*)pFile; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **azArg = (char**)pArg; + assert( azArg[1]!=0 ); + if( sqlite3_stricmp(azArg[1],"checksum_verification")==0 ){ + char *zArg = azArg[2]; + if( zArg!=0 ){ + if( (zArg[0]>='1' && zArg[0]<='9') + || sqlite3_strlike("enable%",zArg,0)==0 + || sqlite3_stricmp("yes",zArg)==0 + || sqlite3_stricmp("on",zArg)==0 + ){ + p->verifyCksm = p->computeCksm; + }else{ + p->verifyCksm = 0; + } + if( p->pPartner ) p->pPartner->verifyCksm = p->verifyCksm; + } + azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); + return SQLITE_OK; + }else if( p->computeCksm && azArg[2]!=0 + && sqlite3_stricmp(azArg[1], "page_size")==0 ){ + /* Do not allow page size changes on a checksum database */ + return SQLITE_OK; + } + }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){ + p->inCkpt = op==SQLITE_FCNTL_CKPT_START; + if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt; + }else if( op==SQLITE_FCNTL_CKSM_FILE ){ + /* This VFS needs to obtain a pointer to the corresponding database + ** file handle from within xOpen() calls to open wal files. To do this, + ** it uses the sqlite3_database_file_object() API to obtain a pointer + ** to the file-handle used by SQLite to access the db file. This is + ** fine if cksmvfs happens to be the top-level VFS, but not if there + ** are one or more wrapper VFS. To handle this case, this file-control + ** is used to extract the cksmvfs file-handle from any wrapper file + ** handle. */ + sqlite3_file **ppFile = (sqlite3_file**)pArg; + *ppFile = (sqlite3_file*)p; + return SQLITE_OK; + } + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("cksm/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for a cksm-file. +*/ +static int cksmSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a cksm-file. +*/ +static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int cksmShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int cksmShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void cksmShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int cksmShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int cksmFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + CksmFile *p = (CksmFile *)pFile; + if( p->computeCksm ){ + *pp = 0; + return SQLITE_OK; + } + pFile = ORIGFILE(pFile); + if( pFile->pMethods->iVersion>2 && pFile->pMethods->xFetch ){ + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); + } + *pp = 0; + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int cksmUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + if( pFile->pMethods->iVersion>2 && pFile->pMethods->xUnfetch ){ + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); + } + return SQLITE_OK; +} + +/* +** Open a cksm file handle. +*/ +static int cksmOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + CksmFile *p; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + p = (CksmFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + pFile->pMethods = &cksm_io_methods; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto cksm_open_done; + if( flags & SQLITE_OPEN_WAL ){ + sqlite3_file *pDb = sqlite3_database_file_object(zName); + rc = pDb->pMethods->xFileControl(pDb, SQLITE_FCNTL_CKSM_FILE, (void*)&pDb); + assert( rc==SQLITE_OK ); + p->pPartner = (CksmFile*)pDb; + assert( p->pPartner->pPartner==0 ); + p->pPartner->pPartner = p; + p->isWal = 1; + p->computeCksm = p->pPartner->computeCksm; + }else{ + p->isWal = 0; + p->computeCksm = 0; + } + p->zFName = zName; +cksm_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int cksmDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} +static int cksmAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int cksmFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *cksmDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void cksmDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void cksmDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int cksmRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int cksmSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int cksmCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int cksmGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int cksmCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + sqlite3_vfs *pOrig = ORIGVFS(pVfs); + int rc; + assert( pOrig->iVersion>=2 ); + if( pOrig->xCurrentTimeInt64 ){ + rc = pOrig->xCurrentTimeInt64(pOrig, p); + }else{ + double r; + rc = pOrig->xCurrentTime(pOrig, &r); + *p = (sqlite3_int64)(r*86400000.0); + } + return rc; +} +static int cksmSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr cksmGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *cksmNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + +/* Register the verify_checksum() SQL function. +*/ +static int cksmRegisterFunc( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + if( db==0 ) return SQLITE_OK; + rc = sqlite3_create_function(db, "verify_checksum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, cksmVerifyFunc, 0, 0); +#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME + (void)sqlite3_create_function(db, SQLITE_CKSUMVFS_INIT_FUNCNAME, 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, + 0, cksmInitFunc, 0, 0); +#endif + return rc; +} + +/* +** Register the cksum VFS as the default VFS for the system. +** Also make arrangements to automatically register the "verify_checksum()" +** SQL function on each new database connection. +*/ +static int cksmRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + if( sqlite3_vfs_find("cksmvfs")!=0 ) return SQLITE_OK; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + cksm_vfs.iVersion = pOrig->iVersion; + cksm_vfs.pAppData = pOrig; + cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); + rc = sqlite3_vfs_register(&cksm_vfs, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); + } + return rc; +} + +#if defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_cksumvfs(const char *NotUsed){ + (void)NotUsed; + return cksmRegisterVfs(); +} +int sqlite3_unregister_cksumvfs(void){ + if( sqlite3_vfs_find("cksmvfs") ){ + sqlite3_vfs_unregister(&cksm_vfs); + sqlite3_cancel_auto_extension((void(*)(void))cksmRegisterFunc); + } + return SQLITE_OK; +} +#endif /* defined(SQLITE_CKSUMVFS_STATIC */ + +#if !defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_cksumvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + rc = cksmRegisterFunc(db, 0, 0); + if( rc==SQLITE_OK ){ + rc = cksmRegisterVfs(); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ diff --git a/ext/misc/closure.c b/ext/misc/closure.c new file mode 100644 index 0000000000000000000000000000000000000000..79a5a21d1e6f43a6aa3b0c0fea9ecd114bba7e76 --- /dev/null +++ b/ext/misc/closure.c @@ -0,0 +1,966 @@ +/* +** 2013-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a virtual table that finds the transitive +** closure of a parent/child relationship in a real table. The virtual +** table is called "transitive_closure". +** +** A transitive_closure virtual table is created like this: +** +** CREATE VIRTUAL TABLE x USING transitive_closure( +** tablename=, -- T +** idcolumn=, -- X +** parentcolumn= -- P +** ); +** +** When it is created, the new transitive_closure table may be supplied +** with default values for the name of a table T and columns T.X and T.P. +** The T.X and T.P columns must contain integers. The ideal case is for +** T.X to be the INTEGER PRIMARY KEY. The T.P column should reference +** the T.X column. The row referenced by T.P is the parent of the current row. +** +** The tablename, idcolumn, and parentcolumn supplied by the CREATE VIRTUAL +** TABLE statement may be overridden in individual queries by including +** terms like tablename='newtable', idcolumn='id2', or +** parentcolumn='parent3' in the WHERE clause of the query. +** +** For efficiency, it is essential that there be an index on the P column: +** +** CREATE Tidx1 ON T(P) +** +** Suppose a specific instance of the closure table is as follows: +** +** CREATE VIRTUAL TABLE ct1 USING transitive_closure( +** tablename='group', +** idcolumn='groupId', +** parentcolumn='parentId' +** ); +** +** Such an instance of the transitive_closure virtual table would be +** appropriate for walking a tree defined using a table like this, for example: +** +** CREATE TABLE group( +** groupId INTEGER PRIMARY KEY, +** parentId INTEGER REFERENCES group +** ); +** CREATE INDEX group_idx1 ON group(parentId); +** +** The group table above would presumably have other application-specific +** fields. The key point here is that rows of the group table form a +** tree. The purpose of the ct1 virtual table is to easily extract +** branches of that tree. +** +** Once it has been created, the ct1 virtual table can be queried +** as follows: +** +** SELECT * FROM element +** WHERE element.groupId IN (SELECT id FROM ct1 WHERE root=?1); +** +** The above query will return all elements that are part of group ?1 +** or children of group ?1 or grand-children of ?1 and so forth for all +** descendents of group ?1. The same query can be formulated as a join: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1; +** +** The depth of the transitive_closure (the number of generations of +** parent/child relations to follow) can be limited by setting "depth" +** column in the WHERE clause. So, for example, the following query +** finds only children and grandchildren but no further descendents: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.depth<=2; +** +** The "ct1.depth<=2" term could be a strict equality "ct1.depth=2" in +** order to find only the grandchildren of ?1, not ?1 itself or the +** children of ?1. +** +** The root=?1 term must be supplied in WHERE clause or else the query +** of the ct1 virtual table will return an empty set. The tablename, +** idcolumn, and parentcolumn attributes can be overridden in the WHERE +** clause if desired. So, for example, the ct1 table could be repurposed +** to find ancestors rather than descendents by inverting the roles of +** the idcolumn and parentcolumn: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.idcolumn='parentId' +** AND ct1.parentcolumn='groupId'; +** +** Multiple calls to ct1 could be combined. For example, the following +** query finds all elements that "cousins" of groupId ?1. That is to say +** elements where the groupId is a grandchild of the grandparent of ?1. +** (This definition of "cousins" also includes siblings and self.) +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupId=ct1.id +** AND ct1.depth=2 +** AND ct1.root IN (SELECT id FROM ct1 +** WHERE root=?1 +** AND depth=2 +** AND idcolumn='parentId' +** AND parentcolumn='groupId'); +** +** In our example, the group.groupId column is unique and thus the +** subquery will return exactly one row. For that reason, the IN +** operator could be replaced by "=" to get the same result. But +** in the general case where the idcolumn is not unique, an IN operator +** would be required for this kind of query. +** +** Note that because the tablename, idcolumn, and parentcolumn can +** all be specified in the query, it is possible for an application +** to define a single transitive_closure virtual table for use on lots +** of different hierarchy tables. One might say: +** +** CREATE VIRTUAL TABLE temp.closure USING transitive_closure; +** +** As each database connection is being opened. Then the application +** would always have a "closure" virtual table handy to use for querying. +** +** SELECT element.* FROM element, closure +** WHERE element.groupid=ct1.id +** AND closure.root=?1 +** AND closure.tablename='group' +** AND closure.idname='groupId' +** AND closure.parentname='parentId'; +** +** See the documentation at http://www.sqlite.org/loadext.html for information +** on how to compile and use loadable extensions such as this one. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct closure_vtab closure_vtab; +typedef struct closure_cursor closure_cursor; +typedef struct closure_queue closure_queue; +typedef struct closure_avl closure_avl; + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct closure_avl { + sqlite3_int64 id; /* Id of this entry in the table */ + int iGeneration; /* Which generation is this entry part of */ + closure_avl *pList; /* A linked list of nodes */ + closure_avl *pBefore; /* Other elements less than id */ + closure_avl *pAfter; /* Other elements greater than id */ + closure_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the closure_avl.height and closure_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void closureAvlRecomputeHeight(closure_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static closure_avl *closureAvlRotateBefore(closure_avl *pP){ + closure_avl *pB = pP->pBefore; + closure_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static closure_avl *closureAvlRotateAfter(closure_avl *pP){ + closure_avl *pA = pP->pAfter; + closure_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static closure_avl **closureAvlFromPtr(closure_avl *p, closure_avl **pp){ + closure_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static closure_avl *closureAvlBalance(closure_avl *p){ + closure_avl *pTop = p; + closure_avl **pp; + while( p ){ + closureAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + closure_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = closureAvlRotateAfter(pB); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + closure_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = closureAvlRotateBefore(pA); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with id. Return a pointer +** to the entry or return NULL. +*/ +static closure_avl *closureAvlSearch(closure_avl *p, sqlite3_int64 id){ + while( p && id!=p->id ){ + p = (idid) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static closure_avl *closureAvlFirst(closure_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +/* Return the node with the next larger key after p. +*/ +closure_avl *closureAvlNext(closure_avl *p){ + closure_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = closureAvlFirst(p->pAfter); + } + return p; +} + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static closure_avl *closureAvlInsert( + closure_avl **ppHead, /* Head of the tree */ + closure_avl *pNew /* New node to be inserted */ +){ + closure_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + if( pNew->idid ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( pNew->id>p->id ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = closureAvlBalance(p); + return 0; +} + +/* Walk the tree can call xDestroy on each node +*/ +static void closureAvlDestroy(closure_avl *p, void (*xDestroy)(closure_avl*)){ + if( p ){ + closureAvlDestroy(p->pBefore, xDestroy); + closureAvlDestroy(p->pAfter, xDestroy); + xDestroy(p); + } +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + +/* +** A closure virtual-table object +*/ +struct closure_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + sqlite3 *db; /* The database connection */ + int nCursor; /* Number of pending cursors */ +}; + +/* A closure cursor object */ +struct closure_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + closure_vtab *pVtab; /* The virtual table this cursor belongs to */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + closure_avl *pCurrent; /* Current element of output */ + closure_avl *pClosure; /* The complete closure tree */ +}; + +/* A queue of AVL nodes */ +struct closure_queue { + closure_avl *pFirst; /* Oldest node on the queue */ + closure_avl *pLast; /* Youngest node on the queue */ +}; + +/* +** Add a node to the end of the queue +*/ +static void queuePush(closure_queue *pQueue, closure_avl *pNode){ + pNode->pList = 0; + if( pQueue->pLast ){ + pQueue->pLast->pList = pNode; + }else{ + pQueue->pFirst = pNode; + } + pQueue->pLast = pNode; +} + +/* +** Extract the oldest element (the front element) from the queue. +*/ +static closure_avl *queuePull(closure_queue *pQueue){ + closure_avl *p = pQueue->pFirst; + if( p ){ + pQueue->pFirst = p->pList; + if( pQueue->pFirst==0 ) pQueue->pLast = 0; + } + return p; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *closureDequote(const char *zIn){ + sqlite3_int64 nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc64(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, (size_t)(nIn+1)); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInzDb); + sqlite3_free(p->zSelf); + sqlite3_free(p->zTableName); + sqlite3_free(p->zIdColumn); + sqlite3_free(p->zParentColumn); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the closure module. +*/ +static int closureDisconnect(sqlite3_vtab *pVtab){ + closure_vtab *p = (closure_vtab*)pVtab; + assert( p->nCursor==0 ); + closureFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *closureValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("transitive_closure") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int closureConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + closure_vtab *pNew = 0; /* New virtual table */ + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto closureConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto closureConnectError; + for(i=3; izTableName); + pNew->zTableName = closureDequote(zVal); + if( pNew->zTableName==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("idcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zIdColumn); + pNew->zIdColumn = closureDequote(zVal); + if( pNew->zIdColumn==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("parentcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zParentColumn); + pNew->zParentColumn = closureDequote(zVal); + if( pNew->zParentColumn==0 ) goto closureConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + closureFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(id,depth,root HIDDEN,tablename HIDDEN," + "idcolumn HIDDEN,parentcolumn HIDDEN)" + ); +#define CLOSURE_COL_ID 0 +#define CLOSURE_COL_DEPTH 1 +#define CLOSURE_COL_ROOT 2 +#define CLOSURE_COL_TABLENAME 3 +#define CLOSURE_COL_IDCOLUMN 4 +#define CLOSURE_COL_PARENTCOLUMN 5 + if( rc!=SQLITE_OK ){ + closureFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +closureConnectError: + closureFree(pNew); + return rc; +} + +/* +** Open a new closure cursor. +*/ +static int closureOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + closure_vtab *p = (closure_vtab*)pVTab; + closure_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void closureClearCursor(closure_cursor *pCur){ + closureAvlDestroy(pCur->pClosure, (void(*)(closure_avl*))sqlite3_free); + sqlite3_free(pCur->zTableName); + sqlite3_free(pCur->zIdColumn); + sqlite3_free(pCur->zParentColumn); + pCur->zTableName = 0; + pCur->zIdColumn = 0; + pCur->zParentColumn = 0; + pCur->pCurrent = 0; + pCur->pClosure = 0; +} + +/* +** Close a closure cursor. +*/ +static int closureClose(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor *)cur; + closureClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Advance a cursor to its next row of output +*/ +static int closureNext(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + pCur->pCurrent = closureAvlNext(pCur->pCurrent); + return SQLITE_OK; +} + +/* +** Allocate and insert a node +*/ +static int closureInsertNode( + closure_queue *pQueue, /* Add new node to this queue */ + closure_cursor *pCur, /* The cursor into which to add the node */ + sqlite3_int64 id, /* The node ID */ + int iGeneration /* The generation number for this node */ +){ + closure_avl *pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->id = id; + pNew->iGeneration = iGeneration; + closureAvlInsert(&pCur->pClosure, pNew); + queuePush(pQueue, pNew); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any closureColumn, closureRowid, or closureEof call. +** +** This routine actually computes the closure. +** +** See the comment at the beginning of closureBestIndex() for a +** description of the meaning of idxNum. The idxStr parameter is +** not used. +*/ +static int closureFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + closure_cursor *pCur = (closure_cursor *)pVtabCursor; + closure_vtab *pVtab = pCur->pVtab; + sqlite3_int64 iRoot; + int mxGen = 999999999; + char *zSql; + sqlite3_stmt *pStmt; + closure_avl *pAvl; + int rc = SQLITE_OK; + const char *zTableName = pVtab->zTableName; + const char *zIdColumn = pVtab->zIdColumn; + const char *zParentColumn = pVtab->zParentColumn; + closure_queue sQueue; + + (void)idxStr; /* Unused parameter */ + (void)argc; /* Unused parameter */ + closureClearCursor(pCur); + memset(&sQueue, 0, sizeof(sQueue)); + if( (idxNum & 1)==0 ){ + /* No root=$root in the WHERE clause. Return an empty set */ + return SQLITE_OK; + } + iRoot = sqlite3_value_int64(argv[0]); + if( (idxNum & 0x000f0)!=0 ){ + mxGen = sqlite3_value_int(argv[(idxNum>>4)&0x0f]); + if( (idxNum & 0x00002)!=0 ) mxGen--; + } + if( (idxNum & 0x00f00)!=0 ){ + zTableName = (const char*)sqlite3_value_text(argv[(idxNum>>8)&0x0f]); + pCur->zTableName = sqlite3_mprintf("%s", zTableName); + } + if( (idxNum & 0x0f000)!=0 ){ + zIdColumn = (const char*)sqlite3_value_text(argv[(idxNum>>12)&0x0f]); + pCur->zIdColumn = sqlite3_mprintf("%s", zIdColumn); + } + if( (idxNum & 0x0f0000)!=0 ){ + zParentColumn = (const char*)sqlite3_value_text(argv[(idxNum>>16)&0x0f]); + pCur->zParentColumn = sqlite3_mprintf("%s", zParentColumn); + } + + zSql = sqlite3_mprintf( + "SELECT \"%w\".\"%w\" FROM \"%w\" WHERE \"%w\".\"%w\"=?1", + zTableName, zIdColumn, zTableName, zTableName, zParentColumn); + if( zSql==0 ){ + return SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_free(pVtab->base.zErrMsg); + pVtab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pVtab->db)); + return rc; + } + } + if( rc==SQLITE_OK ){ + rc = closureInsertNode(&sQueue, pCur, iRoot, 0); + } + while( (pAvl = queuePull(&sQueue))!=0 ){ + if( pAvl->iGeneration>=mxGen ) continue; + sqlite3_bind_int64(pStmt, 1, pAvl->id); + while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + if( sqlite3_column_type(pStmt,0)==SQLITE_INTEGER ){ + sqlite3_int64 iNew = sqlite3_column_int64(pStmt, 0); + if( closureAvlSearch(pCur->pClosure, iNew)==0 ){ + rc = closureInsertNode(&sQueue, pCur, iNew, pAvl->iGeneration+1); + } + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + pCur->pCurrent = closureAvlFirst(pCur->pClosure); + } + + return rc; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int closureColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + closure_cursor *pCur = (closure_cursor*)cur; + switch( i ){ + case CLOSURE_COL_ID: { + sqlite3_result_int64(ctx, pCur->pCurrent->id); + break; + } + case CLOSURE_COL_DEPTH: { + sqlite3_result_int(ctx, pCur->pCurrent->iGeneration); + break; + } + case CLOSURE_COL_ROOT: { + sqlite3_result_null(ctx); + break; + } + case CLOSURE_COL_TABLENAME: { + sqlite3_result_text(ctx, + pCur->zTableName ? pCur->zTableName : pCur->pVtab->zTableName, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_IDCOLUMN: { + sqlite3_result_text(ctx, + pCur->zIdColumn ? pCur->zIdColumn : pCur->pVtab->zIdColumn, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_PARENTCOLUMN: { + sqlite3_result_text(ctx, + pCur->zParentColumn ? pCur->zParentColumn : pCur->pVtab->zParentColumn, + -1, SQLITE_TRANSIENT); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. For the closure table, this is the same as the "id" column. +*/ +static int closureRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + closure_cursor *pCur = (closure_cursor*)cur; + *pRowid = pCur->pCurrent->id; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int closureEof(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) root = $root +** (B1) depth < $depth +** (B2) depth <= $depth +** (B3) depth = $depth +** (C) tablename = $tablename +** (D) idcolumn = $idcolumn +** (E) parentcolumn = $parentcolumn +** +** +** +** idxNum meaning +** ---------- ------------------------------------------------------ +** 0x00000001 Term of the form (A) found +** 0x00000002 The term of bit-2 is like (B1) +** 0x000000f0 Index in filter.argv[] of $depth. 0 if not used. +** 0x00000f00 Index in filter.argv[] of $tablename. 0 if not used. +** 0x0000f000 Index in filter.argv[] of $idcolumn. 0 if not used +** 0x000f0000 Index in filter.argv[] of $parentcolumn. 0 if not used. +** +** There must be a term of type (A). If there is not, then the index type +** is 0 and the query will return an empty set. +*/ +static int closureBestIndex( + sqlite3_vtab *pTab, /* The virtual table */ + sqlite3_index_info *pIdxInfo /* Information about the query */ +){ + int iPlan = 0; + int i; + int idx = 1; + const struct sqlite3_index_constraint *pConstraint; + closure_vtab *pVtab = (closure_vtab*)pTab; + double rCost = 10000000.0; + + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==CLOSURE_COL_ROOT + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 100.0; + } + if( (iPlan & 0x0000f0)==0 + && pConstraint->iColumn==CLOSURE_COL_DEPTH + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ) + ){ + iPlan |= idx<<4; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002; + rCost /= 5.0; + } + if( (iPlan & 0x000f00)==0 + && pConstraint->iColumn==CLOSURE_COL_TABLENAME + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<8; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 5.0; + } + if( (iPlan & 0x00f000)==0 + && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<12; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x0f0000)==0 + && pConstraint->iColumn==CLOSURE_COL_PARENTCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<16; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + } + if( (pVtab->zTableName==0 && (iPlan & 0x000f00)==0) + || (pVtab->zIdColumn==0 && (iPlan & 0x00f000)==0) + || (pVtab->zParentColumn==0 && (iPlan & 0x0f0000)==0) + ){ + /* All of tablename, idcolumn, and parentcolumn must be specified + ** in either the CREATE VIRTUAL TABLE or in the WHERE clause constraints + ** or else the result is an empty set. */ + iPlan = 0; + } + if( (iPlan&1)==0 ){ + /* If there is no usable "root=?" term, then set the index-type to 0. + ** Also clear any argvIndex variables already set. This is necessary + ** to prevent the core from throwing an "xBestIndex malfunction error" + ** error (because the argvIndex values are not contiguously assigned + ** starting from 1). */ + rCost *= 1e30; + for(i=0; inConstraint; i++, pConstraint++){ + pIdxInfo->aConstraintUsage[i].argvIndex = 0; + } + iPlan = 0; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = rCost; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "transitive_closure". +*/ +static sqlite3_module closureModule = { + 0, /* iVersion */ + closureConnect, /* xCreate */ + closureConnect, /* xConnect */ + closureBestIndex, /* xBestIndex */ + closureDisconnect, /* xDisconnect */ + closureDisconnect, /* xDestroy */ + closureOpen, /* xOpen - open a cursor */ + closureClose, /* xClose - close a cursor */ + closureFilter, /* xFilter - configure scan constraints */ + closureNext, /* xNext - advance a cursor */ + closureEof, /* xEof - check for end of scan */ + closureColumn, /* xColumn - read data */ + closureRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Register the closure virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_closure_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "transitive_closure", &closureModule, 0); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + return rc; +} diff --git a/ext/misc/completion.c b/ext/misc/completion.c new file mode 100644 index 0000000000000000000000000000000000000000..54abc0ae154722629668a04f0b4fcba2b963ea2a --- /dev/null +++ b/ext/misc/completion.c @@ -0,0 +1,502 @@ +/* +** 2017-07-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an eponymous virtual table that returns suggested +** completions for a partial SQL input. +** +** Suggested usage: +** +** SELECT DISTINCT candidate COLLATE nocase +** FROM completion($prefix,$wholeline) +** ORDER BY 1; +** +** The two query parameters are optional. $prefix is the text of the +** current word being typed and that is to be completed. $wholeline is +** the complete input line, used for context. +** +** The raw completion() table might return the same candidate multiple +** times, for example if the same column name is used to two or more +** tables. And the candidates are returned in an arbitrary order. Hence, +** the DISTINCT and ORDER BY are recommended. +** +** This virtual table operates at the speed of human typing, and so there +** is no attempt to make it fast. Even a slow implementation will be much +** faster than any human can type. +** +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* completion_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a completion virtual table +*/ +typedef struct completion_vtab completion_vtab; +struct completion_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this completion vtab */ +}; + +/* completion_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct completion_cursor completion_cursor; +struct completion_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + int nPrefix, nLine; /* Number of bytes in zPrefix and zLine */ + char *zPrefix; /* The prefix for the word we want to complete */ + char *zLine; /* The whole that we want to complete */ + const char *zCurrentRow; /* Current output row */ + int szRow; /* Length of the zCurrentRow string */ + sqlite3_stmt *pStmt; /* Current statement */ + sqlite3_int64 iRowid; /* The rowid */ + int ePhase; /* Current phase */ + int j; /* inter-phase counter */ +}; + +/* Values for ePhase: +*/ +#define COMPLETION_FIRST_PHASE 1 +#define COMPLETION_KEYWORDS 1 +#define COMPLETION_PRAGMAS 2 +#define COMPLETION_FUNCTIONS 3 +#define COMPLETION_COLLATIONS 4 +#define COMPLETION_INDEXES 5 +#define COMPLETION_TRIGGERS 6 +#define COMPLETION_DATABASES 7 +#define COMPLETION_TABLES 8 /* Also VIEWs and TRIGGERs */ +#define COMPLETION_COLUMNS 9 +#define COMPLETION_MODULES 10 +#define COMPLETION_EOF 11 + +/* +** The completionConnect() method is invoked to create a new +** completion_vtab that describes the completion virtual table. +** +** Think of this routine as the constructor for completion_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the completion_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against completion will look like. +*/ +static int completionConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + completion_vtab *pNew; + int rc; + + (void)(pAux); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + (void)(argv); /* Unused parameter */ + (void)(pzErr); /* Unused parameter */ + +/* Column numbers */ +#define COMPLETION_COLUMN_CANDIDATE 0 /* Suggested completion of the input */ +#define COMPLETION_COLUMN_PREFIX 1 /* Prefix of the word to be completed */ +#define COMPLETION_COLUMN_WHOLELINE 2 /* Entire line seen so far */ +#define COMPLETION_COLUMN_PHASE 3 /* ePhase - used for debugging only */ + + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(" + " candidate TEXT," + " prefix TEXT HIDDEN," + " wholeline TEXT HIDDEN," + " phase INT HIDDEN" /* Used for debugging only */ + ")"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for completion_cursor objects. +*/ +static int completionDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new completion_cursor object. +*/ +static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + completion_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((completion_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset the completion_cursor. +*/ +static void completionCursorReset(completion_cursor *pCur){ + sqlite3_free(pCur->zPrefix); pCur->zPrefix = 0; pCur->nPrefix = 0; + sqlite3_free(pCur->zLine); pCur->zLine = 0; pCur->nLine = 0; + sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; + pCur->j = 0; +} + +/* +** Destructor for a completion_cursor. +*/ +static int completionClose(sqlite3_vtab_cursor *cur){ + completionCursorReset((completion_cursor*)cur); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Advance a completion_cursor to its next row of output. +** +** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object +** record the current state of the scan. This routine sets ->zCurrentRow +** to the current row of output and then returns. If no more rows remain, +** then ->ePhase is set to COMPLETION_EOF which will signal the virtual +** table that has reached the end of its scan. +** +** The current implementation just lists potential identifiers and +** keywords and filters them by zPrefix. Future enhancements should +** take zLine into account to try to restrict the set of identifiers and +** keywords based on what would be legal at the current point of input. +*/ +static int completionNext(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + int eNextPhase = 0; /* Next phase to try if current phase reaches end */ + int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + pCur->iRowid++; + while( pCur->ePhase!=COMPLETION_EOF ){ + switch( pCur->ePhase ){ + case COMPLETION_KEYWORDS: { + if( pCur->j >= sqlite3_keyword_count() ){ + pCur->zCurrentRow = 0; + pCur->ePhase = COMPLETION_DATABASES; + }else{ + sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow); + } + iCol = -1; + break; + } + case COMPLETION_DATABASES: { + if( pCur->pStmt==0 ){ + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, + &pCur->pStmt, 0); + } + iCol = 1; + eNextPhase = COMPLETION_TABLES; + break; + } + case COMPLETION_TABLES: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT name FROM \"%w\".sqlite_schema", + zSql, zSep, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_COLUMNS; + break; + } + case COMPLETION_COLUMNS: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" + " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" + " WHERE sm.type='table'", + zSql, zSep, zDb, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_EOF; + break; + } + } + if( iCol<0 ){ + /* This case is when the phase presets zCurrentRow */ + if( pCur->zCurrentRow==0 ) continue; + }else{ + if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){ + /* Extract the next row of content */ + pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol); + pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); + }else{ + /* When all rows are finished, advance to the next phase */ + sqlite3_finalize(pCur->pStmt); + pCur->pStmt = 0; + pCur->ePhase = eNextPhase; + continue; + } + } + if( pCur->nPrefix==0 ) break; + if( pCur->nPrefix<=pCur->szRow + && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 + ){ + break; + } + } + + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the completion_cursor +** is currently pointing. +*/ +static int completionColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + completion_cursor *pCur = (completion_cursor*)cur; + switch( i ){ + case COMPLETION_COLUMN_CANDIDATE: { + sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PREFIX: { + sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_WHOLELINE: { + sqlite3_result_text(ctx, pCur->zLine, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PHASE: { + sqlite3_result_int(ctx, pCur->ePhase); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int completionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + completion_cursor *pCur = (completion_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int completionEof(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + return pCur->ePhase >= COMPLETION_EOF; +} + +/* +** This method is called to "rewind" the completion_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to completionColumn() or completionRowid() or +** completionEof(). +*/ +static int completionFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + completion_cursor *pCur = (completion_cursor *)pVtabCursor; + int iArg = 0; + (void)(idxStr); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + completionCursorReset(pCur); + if( idxNum & 1 ){ + pCur->nPrefix = sqlite3_value_bytes(argv[iArg]); + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + iArg = 1; + } + if( idxNum & 2 ){ + pCur->nLine = sqlite3_value_bytes(argv[iArg]); + if( pCur->nLine>0 ){ + pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zLine==0 ) return SQLITE_NOMEM; + } + } + if( pCur->zLine!=0 && pCur->zPrefix==0 ){ + int i = pCur->nLine; + while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){ + i--; + } + pCur->nPrefix = pCur->nLine - i; + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + } + pCur->iRowid = 0; + pCur->ePhase = COMPLETION_FIRST_PHASE; + return completionNext(pVtabCursor); +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the completion virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** There are two hidden parameters that act as arguments to the table-valued +** function: "prefix" and "wholeline". Bit 0 of idxNum is set if "prefix" +** is available and bit 1 is set if "wholeline" is available. +*/ +static int completionBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int prefixIdx = -1; /* Index of the start= constraint, or -1 if none */ + int wholelineIdx = -1; /* Index of the stop= constraint, or -1 if none */ + int nArg = 0; /* Number of arguments that completeFilter() expects */ + const struct sqlite3_index_constraint *pConstraint; + + (void)(tab); /* Unused parameter */ + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case COMPLETION_COLUMN_PREFIX: + prefixIdx = i; + idxNum |= 1; + break; + case COMPLETION_COLUMN_WHOLELINE: + wholelineIdx = i; + idxNum |= 2; + break; + } + } + if( prefixIdx>=0 ){ + pIdxInfo->aConstraintUsage[prefixIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[prefixIdx].omit = 1; + } + if( wholelineIdx>=0 ){ + pIdxInfo->aConstraintUsage[wholelineIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[wholelineIdx].omit = 1; + } + pIdxInfo->idxNum = idxNum; + pIdxInfo->estimatedCost = (double)5000 - 1000*nArg; + pIdxInfo->estimatedRows = 500 - 100*nArg; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** completion virtual table. +*/ +static sqlite3_module completionModule = { + 0, /* iVersion */ + 0, /* xCreate */ + completionConnect, /* xConnect */ + completionBestIndex, /* xBestIndex */ + completionDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + completionOpen, /* xOpen - open a cursor */ + completionClose, /* xClose - close a cursor */ + completionFilter, /* xFilter - configure scan constraints */ + completionNext, /* xNext - advance a cursor */ + completionEof, /* xEof - check for end of scan */ + completionColumn, /* xColumn - read data */ + completionRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3CompletionVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "completion", &completionModule, 0); +#endif + return rc; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_completion_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)(pzErrMsg); /* Unused parameter */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3CompletionVtabInit(db); +#endif + return rc; +} diff --git a/ext/misc/compress.c b/ext/misc/compress.c new file mode 100644 index 0000000000000000000000000000000000000000..6b034eb45fde4ae073efd5a1786e0fab253510ff --- /dev/null +++ b/ext/misc/compress.c @@ -0,0 +1,131 @@ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL compression functions +** compress() and uncompress() using ZLIB. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Implementation of the "compress(X)" SQL function. The input X is +** compressed using zLib and the output is returned. +** +** The output is a BLOB that begins with a variable-length integer that +** is the input size in bytes (the size of X before compression). The +** variable-length integer is implemented as 1 to 5 bytes. There are +** seven bits per integer stored in the lower seven bits of each byte. +** More significant bits occur first. The most significant bit (0x80) +** is a flag to indicate the end of the integer. +** +** This function, SQLAR, and ZIP all use the same "deflate" compression +** algorithm, but each is subtly different: +** +** * ZIP uses raw deflate. +** +** * SQLAR uses the "zlib format" which is raw deflate with a two-byte +** algorithm-identification header and a four-byte checksum at the end. +** +** * This utility uses the "zlib format" like SQLAR, but adds the variable- +** length integer uncompressed size value at the beginning. +** +** This function might be extended in the future to support compression +** formats other than deflate, by providing a different algorithm-id +** mark following the variable-length integer size parameter. +*/ +static void compressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pIn; + unsigned char *pOut; + unsigned int nIn; + unsigned long int nOut; + unsigned char x[8]; + int rc; + int i, j; + + pIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + nOut = 13 + nIn + (nIn+999)/1000; + pOut = sqlite3_malloc( nOut+5 ); + for(i=4; i>=0; i--){ + x[i] = (nIn >> (7*(4-i)))&0x7f; + } + for(i=0; i<4 && x[i]==0; i++){} + for(j=0; i<=4; i++, j++) pOut[j] = x[i]; + pOut[j-1] |= 0x80; + rc = compress(&pOut[j], &nOut, pIn, nIn); + if( rc==Z_OK ){ + sqlite3_result_blob(context, pOut, nOut+j, sqlite3_free); + }else{ + sqlite3_free(pOut); + } +} + +/* +** Implementation of the "uncompress(X)" SQL function. The argument X +** is a blob which was obtained from compress(Y). The output will be +** the value Y. +*/ +static void uncompressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pIn; + unsigned char *pOut; + unsigned int nIn; + unsigned long int nOut; + int rc; + unsigned int i; + + pIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + nOut = 0; + for(i=0; i +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define CSV_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define CSV_NOINLINE __declspec(noinline) +#else +# define CSV_NOINLINE +#endif + + +/* Max size of the error message in a CsvReader */ +#define CSV_MXERR 200 + +/* Size of the CsvReader input buffer */ +#define CSV_INBUFSZ 1024 + +/* A context object used when read a CSV file. */ +typedef struct CsvReader CsvReader; +struct CsvReader { + FILE *in; /* Read the CSV text from this input stream */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + int bNotFirst; /* True if prior text has been seen */ + int cTerm; /* Character that terminated the most recent field */ + size_t iIn; /* Next unread character in the input buffer */ + size_t nIn; /* Number of characters in the input buffer */ + char *zIn; /* The input buffer */ + char zErr[CSV_MXERR]; /* Error message */ +}; + +/* Initialize a CsvReader object */ +static void csv_reader_init(CsvReader *p){ + p->in = 0; + p->z = 0; + p->n = 0; + p->nAlloc = 0; + p->nLine = 0; + p->bNotFirst = 0; + p->nIn = 0; + p->zIn = 0; + p->zErr[0] = 0; +} + +/* Close and reset a CsvReader object */ +static void csv_reader_reset(CsvReader *p){ + if( p->in ){ + fclose(p->in); + sqlite3_free(p->zIn); + } + sqlite3_free(p->z); + csv_reader_init(p); +} + +/* Report an error on a CsvReader */ +static void csv_errmsg(CsvReader *p, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap); + va_end(ap); +} + +/* Open the file associated with a CsvReader +** Return the number of errors. +*/ +static int csv_reader_open( + CsvReader *p, /* The reader to open */ + const char *zFilename, /* Read from this filename */ + const char *zData /* ... or use this data */ +){ + if( zFilename ){ + p->zIn = sqlite3_malloc( CSV_INBUFSZ ); + if( p->zIn==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + p->in = fopen(zFilename, "rb"); + if( p->in==0 ){ + sqlite3_free(p->zIn); + csv_reader_reset(p); + csv_errmsg(p, "cannot open '%s' for reading", zFilename); + return 1; + } + }else{ + assert( p->in==0 ); + p->zIn = (char*)zData; + p->nIn = strlen(zData); + } + return 0; +} + +/* The input buffer has overflowed. Refill the input buffer, then +** return the next character +*/ +static CSV_NOINLINE int csv_getc_refill(CsvReader *p){ + size_t got; + + assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */ + assert( p->in!=0 ); /* Only called if reading froma file */ + + got = fread(p->zIn, 1, CSV_INBUFSZ, p->in); + if( got==0 ) return EOF; + p->nIn = got; + p->iIn = 1; + return p->zIn[0]; +} + +/* Return the next character of input. Return EOF at end of input. */ +static int csv_getc(CsvReader *p){ + if( p->iIn >= p->nIn ){ + if( p->in!=0 ) return csv_getc_refill(p); + return EOF; + } + return ((unsigned char*)p->zIn)[p->iIn++]; +} + +/* Increase the size of p->z and append character c to the end. +** Return 0 on success and non-zero if there is an OOM error */ +static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ + char *zNew; + int nNew = p->nAlloc*2 + 100; + zNew = sqlite3_realloc64(p->z, nNew); + if( zNew ){ + p->z = zNew; + p->nAlloc = nNew; + p->z[p->n++] = c; + return 0; + }else{ + csv_errmsg(p, "out of memory"); + return 1; + } +} + +/* Append a single character to the CsvReader.z[] array. +** Return 0 on success and non-zero if there is an OOM error */ +static int csv_append(CsvReader *p, char c){ + if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c); + p->z[p->n++] = c; + return 0; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** +** Return 0 at EOF or on OOM. On EOF, the p->cTerm character will have +** been set to EOF. +*/ +static char *csv_read_one_field(CsvReader *p){ + int c; + p->n = 0; + c = csv_getc(p); + if( c==EOF ){ + p->cTerm = EOF; + return 0; + } + if( c=='"' ){ + int pc, ppc; + int startLine = p->nLine; + pc = ppc = 0; + while( 1 ){ + c = csv_getc(p); + if( c<='"' || pc=='"' ){ + if( c=='\n' ) p->nLine++; + if( c=='"' ){ + if( pc=='"' ){ + pc = 0; + continue; + } + } + if( (c==',' && pc=='"') + || (c=='\n' && pc=='"') + || (c=='\n' && pc=='\r' && ppc=='"') + || (c==EOF && pc=='"') + ){ + do{ p->n--; }while( p->z[p->n]!='"' ); + p->cTerm = (char)c; + break; + } + if( pc=='"' && c!='\r' ){ + csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"'); + break; + } + if( c==EOF ){ + csv_errmsg(p, "line %d: unterminated %c-quoted field\n", + startLine, '"'); + p->cTerm = (char)c; + break; + } + } + if( csv_append(p, (char)c) ) return 0; + ppc = pc; + pc = c; + } + }else{ + /* If this is the first field being parsed and it begins with the + ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ + if( (c&0xff)==0xef && p->bNotFirst==0 ){ + csv_append(p, (char)c); + c = csv_getc(p); + if( (c&0xff)==0xbb ){ + csv_append(p, (char)c); + c = csv_getc(p); + if( (c&0xff)==0xbf ){ + p->bNotFirst = 1; + p->n = 0; + return csv_read_one_field(p); + } + } + } + while( c>',' || (c!=EOF && c!=',' && c!='\n') ){ + if( csv_append(p, (char)c) ) return 0; + c = csv_getc(p); + } + if( c=='\n' ){ + p->nLine++; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; + } + p->cTerm = (char)c; + } + assert( p->z==0 || p->nnAlloc ); + if( p->z ) p->z[p->n] = 0; + p->bNotFirst = 1; + return p->z; +} + + +/* Forward references to the various virtual table methods implemented +** in this file. */ +static int csvtabCreate(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabConnect(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*); +static int csvtabDisconnect(sqlite3_vtab*); +static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); +static int csvtabClose(sqlite3_vtab_cursor*); +static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); +static int csvtabNext(sqlite3_vtab_cursor*); +static int csvtabEof(sqlite3_vtab_cursor*); +static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); +static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); + +/* An instance of the CSV virtual table */ +typedef struct CsvTable { + sqlite3_vtab base; /* Base class. Must be first */ + char *zFilename; /* Name of the CSV file */ + char *zData; /* Raw CSV data in lieu of zFilename */ + long iStart; /* Offset to start of data in zFilename */ + int nCol; /* Number of columns in the CSV file */ + unsigned int tstFlags; /* Bit values used for testing */ +} CsvTable; + +/* Allowed values for tstFlags */ +#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/ + +/* A cursor for the CSV virtual table */ +typedef struct CsvCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + CsvReader rdr; /* The CsvReader object */ + char **azVal; /* Value of the current row */ + int *aLen; /* Length of each entry */ + sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */ +} CsvCursor; + +/* Transfer error message text from a reader into a CsvTable */ +static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){ + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr); +} + +/* +** This method is the destructor fo a CsvTable object. +*/ +static int csvtabDisconnect(sqlite3_vtab *pVtab){ + CsvTable *p = (CsvTable*)pVtab; + sqlite3_free(p->zFilename); + sqlite3_free(p->zData); + sqlite3_free(p); + return SQLITE_OK; +} + +/* Skip leading whitespace. Return a pointer to the first non-whitespace +** character, or to the zero terminator if the string has only whitespace */ +static const char *csv_skip_whitespace(const char *z){ + while( isspace((unsigned char)z[0]) ) z++; + return z; +} + +/* Remove trailing whitespace from the end of string z[] */ +static void csv_trim_whitespace(char *z){ + size_t n = strlen(z); + while( n>0 && isspace((unsigned char)z[n]) ) n--; + z[n] = 0; +} + +/* Dequote the string */ +static void csv_dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; izErr. If there are no errors, p->zErr[0]==0. +*/ +static int csv_string_parameter( + CsvReader *p, /* Leave the error message here, if there is one */ + const char *zParam, /* Parameter we are checking for */ + const char *zArg, /* Raw text of the virtual table argment */ + char **pzVal /* Write the dequoted string value here */ +){ + const char *zValue; + zValue = csv_parameter(zParam,(int)strlen(zParam),zArg); + if( zValue==0 ) return 0; + p->zErr[0] = 0; + if( *pzVal ){ + csv_errmsg(p, "more than one '%s' parameter", zParam); + return 1; + } + *pzVal = sqlite3_mprintf("%s", zValue); + if( *pzVal==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + csv_trim_whitespace(*pzVal); + csv_dequote(*pzVal); + return 1; +} + + +/* Return 0 if the argument is false and 1 if it is true. Return -1 if +** we cannot really tell. +*/ +static int csv_boolean(const char *z){ + if( sqlite3_stricmp("yes",z)==0 + || sqlite3_stricmp("on",z)==0 + || sqlite3_stricmp("true",z)==0 + || (z[0]=='1' && z[1]==0) + ){ + return 1; + } + if( sqlite3_stricmp("no",z)==0 + || sqlite3_stricmp("off",z)==0 + || sqlite3_stricmp("false",z)==0 + || (z[0]=='0' && z[1]==0) + ){ + return 0; + } + return -1; +} + +/* Check to see if the string is of the form: "TAG = BOOLEAN" or just "TAG". +** If it is, set *pValue to be the value of the boolean ("true" if there is +** not "= BOOLEAN" component) and return non-zero. If the input string +** does not begin with TAG, return zero. +*/ +static int csv_boolean_parameter( + const char *zTag, /* Tag we are looking for */ + int nTag, /* Size of the tag in bytes */ + const char *z, /* Input parameter */ + int *pValue /* Write boolean value here */ +){ + int b; + z = csv_skip_whitespace(z); + if( strncmp(zTag, z, nTag)!=0 ) return 0; + z = csv_skip_whitespace(z + nTag); + if( z[0]==0 ){ + *pValue = 1; + return 1; + } + if( z[0]!='=' ) return 0; + z = csv_skip_whitespace(z+1); + b = csv_boolean(z); + if( b>=0 ){ + *pValue = b; + return 1; + } + return 0; +} + +/* +** Parameters: +** filename=FILENAME Name of file containing CSV content +** data=TEXT Direct CSV content. +** schema=SCHEMA Alternative CSV schema. +** header=YES|NO First row of CSV defines the names of +** columns if "yes". Default "no". +** columns=N Assume the CSV file contains N columns. +** +** Only available if compiled with SQLITE_TEST: +** +** testflags=N Bitmask of test flags. Optional +** +** If schema= is omitted, then the columns are named "c0", "c1", "c2", +** and so forth. If columns=N is omitted, then the file is opened and +** the number of columns in the first row is counted to determine the +** column count. If header=YES, then the first row is skipped. +*/ +static int csvtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + CsvTable *pNew = 0; /* The CsvTable object to construct */ + int bHeader = -1; /* header= flags. -1 means not seen yet */ + int rc = SQLITE_OK; /* Result code from this routine */ + int i, j; /* Loop counters */ +#ifdef SQLITE_TEST + int tstFlags = 0; /* Value for testflags=N parameter */ +#endif + int b; /* Value of a boolean parameter */ + int nCol = -99; /* Value of the columns= parameter */ + CsvReader sRdr; /* A CSV file reader used to store an error + ** message and/or to count the number of columns */ + static const char *azParam[] = { + "filename", "data", "schema", + }; + char *azPValue[3]; /* Parameter values */ +# define CSV_FILENAME (azPValue[0]) +# define CSV_DATA (azPValue[1]) +# define CSV_SCHEMA (azPValue[2]) + + + assert( sizeof(azPValue)==sizeof(azParam) ); + memset(&sRdr, 0, sizeof(sRdr)); + memset(azPValue, 0, sizeof(azPValue)); + for(i=3; i=0 ){ + csv_errmsg(&sRdr, "more than one 'header' parameter"); + goto csvtab_connect_error; + } + bHeader = b; + }else +#ifdef SQLITE_TEST + if( (zValue = csv_parameter("testflags",9,z))!=0 ){ + tstFlags = (unsigned int)atoi(zValue); + }else +#endif + if( (zValue = csv_parameter("columns",7,z))!=0 ){ + if( nCol>0 ){ + csv_errmsg(&sRdr, "more than one 'columns' parameter"); + goto csvtab_connect_error; + } + nCol = atoi(zValue); + if( nCol<=0 ){ + csv_errmsg(&sRdr, "column= value must be positive"); + goto csvtab_connect_error; + } + }else + { + csv_errmsg(&sRdr, "bad parameter: '%s'", z); + goto csvtab_connect_error; + } + } + if( (CSV_FILENAME==0)==(CSV_DATA==0) ){ + csv_errmsg(&sRdr, "must specify either filename= or data= but not both"); + goto csvtab_connect_error; + } + + if( (nCol<=0 || bHeader==1) + && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) + ){ + goto csvtab_connect_error; + } + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) goto csvtab_connect_oom; + memset(pNew, 0, sizeof(*pNew)); + if( CSV_SCHEMA==0 ){ + sqlite3_str *pStr = sqlite3_str_new(0); + char *zSep = ""; + int iCol = 0; + sqlite3_str_appendf(pStr, "CREATE TABLE x("); + if( nCol<0 && bHeader<1 ){ + nCol = 0; + do{ + csv_read_one_field(&sRdr); + nCol++; + }while( sRdr.cTerm==',' ); + } + if( nCol>0 && bHeader<1 ){ + for(iCol=0; iCol0 && iColnCol = nCol; + sqlite3_str_appendf(pStr, ")"); + CSV_SCHEMA = sqlite3_str_finish(pStr); + if( CSV_SCHEMA==0 ) goto csvtab_connect_oom; + }else if( nCol<0 ){ + do{ + csv_read_one_field(&sRdr); + pNew->nCol++; + }while( sRdr.cTerm==',' ); + }else{ + pNew->nCol = nCol; + } + pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0; + pNew->zData = CSV_DATA; CSV_DATA = 0; +#ifdef SQLITE_TEST + pNew->tstFlags = tstFlags; +#endif + if( bHeader!=1 ){ + pNew->iStart = 0; + }else if( pNew->zData ){ + pNew->iStart = (int)sRdr.iIn; + }else{ + pNew->iStart = (int)(ftell(sRdr.in) - sRdr.nIn + sRdr.iIn); + } + csv_reader_reset(&sRdr); + rc = sqlite3_declare_vtab(db, CSV_SCHEMA); + if( rc ){ + csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db)); + goto csvtab_connect_error; + } + for(i=0; ibase); + for(i=0; ibase.pVtab; + int i; + for(i=0; inCol; i++){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + } +} + +/* +** The xConnect and xCreate methods do the same thing, but they must be +** different so that the virtual table is not an eponymous virtual table. +*/ +static int csvtabCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Destructor for a CsvCursor. +*/ +static int csvtabClose(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + csvtabCursorRowReset(pCur); + csv_reader_reset(&pCur->rdr); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Constructor for a new CsvTable cursor object. +*/ +static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + CsvTable *pTab = (CsvTable*)p; + CsvCursor *pCur; + size_t nByte; + nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol; + pCur = sqlite3_malloc64( nByte ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, nByte); + pCur->azVal = (char**)&pCur[1]; + pCur->aLen = (int*)&pCur->azVal[pTab->nCol]; + *ppCursor = &pCur->base; + if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){ + csv_xfer_error(pTab, &pCur->rdr); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + + +/* +** Advance a CsvCursor to its next row of input. +** Set the EOF marker if we reach the end of input. +*/ +static int csvtabNext(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + int i = 0; + char *z; + do{ + z = csv_read_one_field(&pCur->rdr); + if( z==0 ){ + break; + } + if( inCol ){ + if( pCur->aLen[i] < pCur->rdr.n+1 ){ + char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1); + if( zNew==0 ){ + csv_errmsg(&pCur->rdr, "out of memory"); + csv_xfer_error(pTab, &pCur->rdr); + break; + } + pCur->azVal[i] = zNew; + pCur->aLen[i] = pCur->rdr.n+1; + } + memcpy(pCur->azVal[i], z, pCur->rdr.n+1); + i++; + } + }while( pCur->rdr.cTerm==',' ); + if( z==0 && i==0 ){ + pCur->iRowid = -1; + }else{ + pCur->iRowid++; + while( inCol ){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + i++; + } + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the CsvCursor +** is currently pointing. +*/ +static int csvtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + if( i>=0 && inCol && pCur->azVal[i]!=0 ){ + sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. +*/ +static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + CsvCursor *pCur = (CsvCursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int csvtabEof(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + return pCur->iRowid<0; +} + +/* +** Only a full table scan is supported. So xFilter simply rewinds to +** the beginning. +*/ +static int csvtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + CsvCursor *pCur = (CsvCursor*)pVtabCursor; + CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab; + pCur->iRowid = 0; + + /* Ensure the field buffer is always allocated. Otherwise, if the + ** first field is zero bytes in size, this may be mistaken for an OOM + ** error in csvtabNext(). */ + if( csv_append(&pCur->rdr, 0) ) return SQLITE_NOMEM; + + if( pCur->rdr.in==0 ){ + assert( pCur->rdr.zIn==pTab->zData ); + assert( pTab->iStart>=0 ); + assert( (size_t)pTab->iStart<=pCur->rdr.nIn ); + pCur->rdr.iIn = pTab->iStart; + }else{ + fseek(pCur->rdr.in, pTab->iStart, SEEK_SET); + pCur->rdr.iIn = 0; + pCur->rdr.nIn = 0; + } + return csvtabNext(pVtabCursor); +} + +/* +** Only a forward full table scan is supported. xBestIndex is mostly +** a no-op. If CSVTEST_FIDX is set, then the presence of equality +** constraints lowers the estimated cost, which is fiction, but is useful +** for testing certain kinds of virtual table behavior. +*/ +static int csvtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + pIdxInfo->estimatedCost = 1000000; +#ifdef SQLITE_TEST + if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){ + /* The usual (and sensible) case is to always do a full table scan. + ** The code in this branch only runs when testflags=1. This code + ** generates an artifical and unrealistic plan which is useful + ** for testing virtual table logic but is not helpful to real applications. + ** + ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual + ** table (even though it is not) and the cost of running the virtual table + ** is reduced from 1 million to just 10. The constraints are *not* marked + ** as omittable, however, so the query planner should still generate a + ** plan that gives a correct answer, even if they plan is not optimal. + */ + int i; + int nConst = 0; + for(i=0; inConstraint; i++){ + unsigned char op; + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + op = pIdxInfo->aConstraint[i].op; + if( op==SQLITE_INDEX_CONSTRAINT_EQ + || op==SQLITE_INDEX_CONSTRAINT_LIKE + || op==SQLITE_INDEX_CONSTRAINT_GLOB + ){ + pIdxInfo->estimatedCost = 10; + pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1; + nConst++; + } + } + } +#endif + return SQLITE_OK; +} + + +static sqlite3_module CsvModule = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#ifdef SQLITE_TEST +/* +** For virtual table testing, make a version of the CSV virtual table +** available that has an xUpdate function. But the xUpdate always returns +** SQLITE_READONLY since the CSV file is not really writable. +*/ +static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){ + return SQLITE_READONLY; +} +static sqlite3_module CsvModuleFauxWrite = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + csvtabUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; +#endif /* SQLITE_TEST */ + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. The new +** CSV virtual table module is registered with the calling database +** connection. +*/ +int sqlite3_csv_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "csv", &CsvModule, 0); +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0); + } +#endif + return rc; +#else + return SQLITE_OK; +#endif +} diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c new file mode 100644 index 0000000000000000000000000000000000000000..ecf7d810d514833823227136af6bd8bfe0781093 --- /dev/null +++ b/ext/misc/dbdump.c @@ -0,0 +1,724 @@ +/* +** 2016-03-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a C-language subroutine that converts the content +** of an SQLite database into UTF-8 text SQL statements that can be used +** to exactly recreate the original database. ROWID values are preserved. +** +** A prototype of the implemented subroutine is this: +** +** int sqlite3_db_dump( +** sqlite3 *db, +** const char *zSchema, +** const char *zTable, +** void (*xCallback)(void*, const char*), +** void *pArg +** ); +** +** The db parameter is the database connection. zSchema is the schema within +** that database which is to be dumped. Usually the zSchema is "main" but +** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then +** only the content of that one table is dumped. If zTable is NULL, then all +** tables are dumped. +** +** The generate text is passed to xCallback() in multiple calls. The second +** argument to xCallback() is a copy of the pArg parameter. The first +** argument is some of the output text that this routine generates. The +** signature to xCallback() is designed to make it compatible with fputs(). +** +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error +** code if it encounters a problem. +** +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine +** is included so that this routine becomes a command-line utility. The +** command-line utility takes two or three arguments which are the name +** of the database file, the schema, and optionally the table, forming the +** first three arguments of a single call to the library routine. +*/ +#include "sqlite3.h" +#include +#include +#include + +/* +** The state of the dump process. +*/ +typedef struct DState DState; +struct DState { + sqlite3 *db; /* The database connection */ + int nErr; /* Number of errors seen so far */ + int rc; /* Error code */ + int writableSchema; /* True if in writable_schema mode */ + int (*xCallback)(const char*,void*); /* Send output here */ + void *pArg; /* Argument to xCallback() */ +}; + +/* +** A variable length string to which one can append text. +*/ +typedef struct DText DText; +struct DText { + char *z; /* The text */ + int n; /* Number of bytes of content in z[] */ + int nAlloc; /* Number of bytes allocated to z[] */ +}; + +/* +** Initialize and destroy a DText object +*/ +static void initText(DText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(DText *p){ + sqlite3_free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(DText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = (int)(strlen(zAppend) & 0x3fffffff); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + len + 20; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + freeText(p); + return; + } + p->z = zNew; + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} + + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(DState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt = 0; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = 1; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + if( zSql==0 ) return 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + char **azNew; + nAlloc = nAlloc*2 + nCol + 10; + azNew = sqlite3_realloc64(azCol, nAlloc*sizeof(azCol[0])); + if( azNew==0 ) goto col_oom; + azCol = azNew; + azCol[0] = 0; + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( azCol[nCol]==0 ) goto col_oom; + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + if( zSql==0 ) goto col_oom; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; + +col_oom: + sqlite3_finalize(pStmt); + freeColumnList(azCol); + p->nErr++; + p->rc = SQLITE_NOMEM; + return 0; +} + +/* +** Send mprintf-formatted content to the output callback. +*/ +static void output_formatted(DState *p, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + p->xCallback(z, p->pArg); + sqlite3_free(z); +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *unused_string( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** Additionallly , escape the "\n" and "\r" characters so that they do not +** get corrupted by end-of-line translation facilities in some operating +** systems. +*/ +static void output_quoted_escaped_string(DState *p, const char *z){ + int i; + char c; + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ + output_formatted(p,"'%s'",z); + }else{ + const char *zNL = 0; + const char *zCR = 0; + int nNL = 0; + int nCR = 0; + char zBuf1[20], zBuf2[20]; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) nNL++; + if( z[i]=='\r' ) nCR++; + } + if( nNL ){ + p->xCallback("replace(", p->pArg); + zNL = unused_string(z, "\\n", "\\012", zBuf1); + } + if( nCR ){ + p->xCallback("replace(", p->pArg); + zCR = unused_string(z, "\\r", "\\015", zBuf2); + } + p->xCallback("'", p->pArg); + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + output_formatted(p, "%.*s", i, z); + z += i; + } + if( c=='\'' ){ + p->xCallback("'", p->pArg); + continue; + } + if( c==0 ){ + break; + } + z++; + if( c=='\n' ){ + p->xCallback(zNL, p->pArg); + continue; + } + p->xCallback(zCR, p->pArg); + } + p->xCallback("'", p->pArg); + if( nCR ){ + output_formatted(p, ",'%s',char(13))", zCR); + } + if( nNL ){ + output_formatted(p, ",'%s',char(10))", zNL); + } + } +} + +/* +** This is an sqlite3_exec callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + DState *p = (DState*)pArg; + sqlite3_stmt *pStmt; + + (void)azCol; + if( nArg!=3 ) return 1; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + + if( strcmp(zTable, "sqlite_sequence")==0 ){ + p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ + p->xCallback("ANALYZE sqlite_schema;\n", p->pArg); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + if( !p->writableSchema ){ + p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg); + p->writableSchema = 1; + } + output_formatted(p, + "INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + return 0; + }else{ + if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){ + p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg); + p->xCallback(zSql+13, p->pArg); + }else{ + p->xCallback(zSql, p->pArg); + } + p->xCallback(";\n", p->pArg); + } + + if( strcmp(zType, "table")==0 ){ + DText sSelect; + DText sTable; + char **azTCol; + int i; + int nCol; + + azTCol = tableColumnList(p, zTable); + if( azTCol==0 ) return 0; + + initText(&sTable); + appendText(&sTable, "INSERT INTO ", 0); + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + appendText(&sTable, zTable, quoteChar(zTable)); + + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azTCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azTCol[0], 0); + for(i=1; azTCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azTCol[i], quoteChar(azTCol[i])); + } + appendText(&sTable, ")", 0); + } + appendText(&sTable, " VALUES(", 0); + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azTCol[0] ){ + appendText(&sSelect, azTCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azTCol[i]; i++){ + appendText(&sSelect, azTCol[i], quoteChar(azTCol[i])); + if( azTCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + nCol = i; + if( azTCol[0]==0 ) nCol--; + freeColumnList(azTCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + p->nErr++; + if( p->rc==SQLITE_OK ) p->rc = rc; + }else{ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->xCallback(sTable.z, p->pArg); + for(i=0; ixCallback(",", p->pArg); + switch( sqlite3_column_type(pStmt,i) ){ + case SQLITE_INTEGER: { + output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i)); + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_column_double(pStmt,i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + p->xCallback("1e999", p->pArg); + }else if( ur==0xfff0000000000000LL ){ + p->xCallback("-1e999", p->pArg); + }else{ + output_formatted(p, "%!.20g", r); + } + break; + } + case SQLITE_NULL: { + p->xCallback("NULL", p->pArg); + break; + } + case SQLITE_TEXT: { + output_quoted_escaped_string(p, + (const char*)sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_BLOB: { + int nByte = sqlite3_column_bytes(pStmt,i); + unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); + int j; + p->xCallback("x'", p->pArg); + for(j=0; j>4)&15]; + zWord[1] = "0123456789abcdef"[a[j]&15]; + zWord[2] = 0; + p->xCallback(zWord, p->pArg); + } + p->xCallback("'", p->pArg); + break; + } + } + } + p->xCallback(");\n", p->pArg); + } + } + sqlite3_finalize(pStmt); + freeText(&sTable); + freeText(&sSelect); + } + return 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static void output_sql_from_query( + DState *p, /* Query context */ + const char *zSelect, /* SELECT statement to extract content */ + ... +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + char *zSql; + va_list ap; + va_start(ap, zSelect); + zSql = sqlite3_vmprintf(zSelect, ap); + va_end(ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + p->nErr++; + return; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK || !pSelect ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + p->nErr++; + return; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + p->xCallback(z, p->pArg); + for(i=1; ixCallback(",", p->pArg); + p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + p->xCallback("\n;\n", p->pArg); + }else{ + p->xCallback(";\n", p->pArg); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static void run_schema_dump_query( + DState *p, + const char *zQuery, + ... +){ + char *zErr = 0; + char *z; + va_list ap; + va_start(ap, zQuery); + z = sqlite3_vmprintf(zQuery, ap); + va_end(ap); + sqlite3_exec(p->db, z, dump_callback, p, &zErr); + sqlite3_free(z); + if( zErr ){ + output_formatted(p, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + p->nErr++; + zErr = 0; + } +} + +/* +** Convert an SQLite database into SQL statements that will recreate that +** database. +*/ +int sqlite3_db_dump( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which schema to dump. Usually "main". */ + const char *zTable, /* Which table to dump. NULL means everything. */ + int (*xCallback)(const char*,void*), /* Output sent to this callback */ + void *pArg /* Second argument of the callback */ +){ + DState x; + memset(&x, 0, sizeof(x)); + x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + if( x.rc ) return x.rc; + x.db = db; + x.xCallback = xCallback; + x.pArg = pArg; + xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg); + if( zTable==0 ){ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_schema " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'", + zSchema + ); + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_schema " + "WHERE name=='sqlite_sequence'", zSchema + ); + output_sql_from_query(&x, + "SELECT sql FROM sqlite_schema " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); + }else{ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_schema " + "WHERE tbl_name=%Q COLLATE nocase AND type=='table'" + " AND sql NOT NULL", + zSchema, zTable + ); + output_sql_from_query(&x, + "SELECT sql FROM \"%w\".sqlite_schema " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name=%Q COLLATE nocase", + zSchema, zTable + ); + } + if( x.writableSchema ){ + xCallback("PRAGMA writable_schema=OFF;\n", pArg); + } + xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg); + sqlite3_exec(db, "COMMIT", 0, 0, 0); + return x.rc; +} + + + +/* The generic subroutine is above. The code the follows implements +** the command-line interface. +*/ +#ifdef DBDUMP_STANDALONE +#include + +/* +** Command-line interface +*/ +int main(int argc, char **argv){ + sqlite3 *db; + const char *zDb; + const char *zSchema; + const char *zTable = 0; + int rc; + + if( argc<2 || argc>4 ){ + fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]); + return 1; + } + zDb = argv[1]; + zSchema = argc>=3 ? argv[2] : "main"; + zTable = argc==4 ? argv[3] : 0; + + rc = sqlite3_open(zDb, &db); + if( rc ){ + fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + rc = sqlite3_db_dump(db, zSchema, zTable, + (int(*)(const char*,void*))fputs, (void*)stdout); + if( rc ){ + fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc); + } + sqlite3_close(db); + return rc!=SQLITE_OK; +} +#endif /* DBDUMP_STANDALONE */ diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c new file mode 100644 index 0000000000000000000000000000000000000000..9365ae68b9f3aaa15752429056829ff2487cbe77 --- /dev/null +++ b/ext/misc/decimal.c @@ -0,0 +1,884 @@ +/* +** 2020-06-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Routines to implement arbitrary-precision decimal math. +** +** The focus here is on simplicity and correctness, not performance. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + + +/* A decimal object */ +typedef struct Decimal Decimal; +struct Decimal { + char sign; /* 0 for positive, 1 for negative */ + char oom; /* True if an OOM is encountered */ + char isNull; /* True if holds a NULL rather than a number */ + char isInit; /* True upon initialization */ + int nDigit; /* Total number of digits */ + int nFrac; /* Number of digits to the right of the decimal point */ + signed char *a; /* Array of digits. Most significant first. */ +}; + +/* +** Release memory held by a Decimal, but do not free the object itself. +*/ +static void decimal_clear(Decimal *p){ + sqlite3_free(p->a); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. +*/ +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; + int iExp = 0; + + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) goto new_from_text_failed; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_from_text_failed; + for(i=0; isspace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + return p; + +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; ioom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc( p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** Make the given Decimal the result in an format similar to '%+#e'. +** In other words, show exponential notation with leading and trailing +** zeros omitted. +*/ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ + char *z; /* The output buffer */ + int i; /* Loop counter */ + int nZero; /* Number of leading zeros */ + int nDigit; /* Number of digits not counting trailing zeros */ + int nFrac; /* Digits to the right of the decimal point */ + int exp; /* Exponent value */ + signed char zero; /* Zero value */ + signed char *a; /* Array of digits */ + + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc( nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(const Decimal *pA, const Decimal *pB){ + int nASig, nBSig, rc, n; + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + const Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 1); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 1); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + p->a = sqlite3_realloc64(p->a, nDigit+1); + if( p->a==0 ){ + p->oom = 1; + return; + } + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. A := A + B. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + UNUSED_PARAMETER(argc); + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pB ){ + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + } + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate funcion: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + decimalMul(pA, pB); + if( pA->oom ){ + goto mul_end; + } + decimal_result(context, pA); + +mul_end: + decimal_free(pA); + decimal_free(pB); +} + +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA); + decimal_free(pA); + } +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + int iArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_window_function(db, "decimal_sum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, + decimalSumStep, decimalSumFinalize, + decimalSumValue, decimalSumInverse, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_collation(db, "decimal", SQLITE_UTF8, + 0, decimalCollFunc); + } + return rc; +} diff --git a/ext/misc/eval.c b/ext/misc/eval.c new file mode 100644 index 0000000000000000000000000000000000000000..d3849d65871b1fbc15dfca1cba1bfdd63ed14238 --- /dev/null +++ b/ext/misc/eval.c @@ -0,0 +1,125 @@ +/* +** 2014-11-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL function eval() which runs +** SQL statements recursively. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Structure used to accumulate the output +*/ +struct EvalResult { + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ +}; + +/* +** Callback from sqlite_exec() for the eval() function. +*/ +static int callback(void *pCtx, int argc, char **argv, char **colnames){ + struct EvalResult *p = (struct EvalResult*)pCtx; + int i; + if( argv==0 ) return 0; + for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc64(p->z, p->nAlloc) : 0; + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + return 1; + } + p->z = zNew; + } + if( p->nUsed>0 ){ + memcpy(&p->z[p->nUsed], p->zSep, p->szSep); + p->nUsed += p->szSep; + } + memcpy(&p->z[p->nUsed], z, sz); + p->nUsed += sz; + } + return 0; +} + +/* +** Implementation of the eval(X) and eval(X,Y) SQL functions. +** +** Evaluate the SQL text in X. Return the results, using string +** Y as the separator. If Y is omitted, use a single space character. +*/ +static void sqlEvalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSql; + sqlite3 *db; + char *zErr = 0; + int rc; + struct EvalResult x; + + memset(&x, 0, sizeof(x)); + x.zSep = " "; + zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ) return; + if( argc>1 ){ + x.zSep = (const char*)sqlite3_value_text(argv[1]); + if( x.zSep==0 ) return; + } + x.szSep = (int)strlen(x.zSep); + db = sqlite3_context_db_handle(context); + rc = sqlite3_exec(db, zSql, callback, &x, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else if( x.zSep==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(x.z); + }else{ + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_eval_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "eval", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + sqlEvalFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "eval", 2, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + sqlEvalFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/explain.c b/ext/misc/explain.c new file mode 100644 index 0000000000000000000000000000000000000000..726af76b9660707ebbae9b1b5b1f33c603d9c05c --- /dev/null +++ b/ext/misc/explain.c @@ -0,0 +1,323 @@ +/* +** 2018-09-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates an eponymous virtual table that returns the +** EXPLAIN output from an SQL statement. +** +** Usage example: +** +** .load ./explain +** SELECT p2 FROM explain('SELECT * FROM sqlite_schema') +** WHERE opcode='OpenRead'; +** +** This module was originally written to help simplify SQLite testing, +** by providing an easier means of verifying certain patterns in the +** generated bytecode. +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* explain_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a explain virtual table +*/ +typedef struct explain_vtab explain_vtab; +struct explain_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this explain vtab */ +}; + +/* explain_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result from an EXPLAIN operation. +*/ +typedef struct explain_cursor explain_cursor; +struct explain_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + char *zSql; /* Value for the EXPLN_COLUMN_SQL column */ + sqlite3_stmt *pExplain; /* Statement being explained */ + int rc; /* Result of last sqlite3_step() on pExplain */ +}; + +/* +** The explainConnect() method is invoked to create a new +** explain_vtab that describes the explain virtual table. +** +** Think of this routine as the constructor for explain_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the explain_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against explain will look like. +*/ +static int explainConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + explain_vtab *pNew; + int rc; + +/* Column numbers */ +#define EXPLN_COLUMN_ADDR 0 /* Instruction address */ +#define EXPLN_COLUMN_OPCODE 1 /* Opcode */ +#define EXPLN_COLUMN_P1 2 /* Operand 1 */ +#define EXPLN_COLUMN_P2 3 /* Operand 2 */ +#define EXPLN_COLUMN_P3 4 /* Operand 3 */ +#define EXPLN_COLUMN_P4 5 /* Operand 4 */ +#define EXPLN_COLUMN_P5 6 /* Operand 5 */ +#define EXPLN_COLUMN_COMMENT 7 /* Comment */ +#define EXPLN_COLUMN_SQL 8 /* SQL that is being explained */ + + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for explain_cursor objects. +*/ +static int explainDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new explain_cursor object. +*/ +static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + explain_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((explain_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a explain_cursor. +*/ +static int explainClose(sqlite3_vtab_cursor *cur){ + explain_cursor *pCur = (explain_cursor*)cur; + sqlite3_finalize(pCur->pExplain); + sqlite3_free(pCur->zSql); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a explain_cursor to its next row of output. +*/ +static int explainNext(sqlite3_vtab_cursor *cur){ + explain_cursor *pCur = (explain_cursor*)cur; + pCur->rc = sqlite3_step(pCur->pExplain); + if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the explain_cursor +** is currently pointing. +*/ +static int explainColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + explain_cursor *pCur = (explain_cursor*)cur; + if( i==EXPLN_COLUMN_SQL ){ + sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i)); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + explain_cursor *pCur = (explain_cursor*)cur; + *pRowid = sqlite3_column_int64(pCur->pExplain, 0); + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int explainEof(sqlite3_vtab_cursor *cur){ + explain_cursor *pCur = (explain_cursor*)cur; + return pCur->rc!=SQLITE_ROW; +} + +/* +** This method is called to "rewind" the explain_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to explainColumn() or explainRowid() or +** explainEof(). +** +** The argv[0] is the SQL statement that is to be explained. +*/ +static int explainFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + explain_cursor *pCur = (explain_cursor *)pVtabCursor; + char *zSql = 0; + int rc; + sqlite3_finalize(pCur->pExplain); + pCur->pExplain = 0; + if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){ + pCur->rc = SQLITE_DONE; + return SQLITE_OK; + } + sqlite3_free(pCur->zSql); + pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + if( pCur->zSql ){ + zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql); + } + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0); + sqlite3_free(zSql); + } + if( rc ){ + sqlite3_finalize(pCur->pExplain); + pCur->pExplain = 0; + sqlite3_free(pCur->zSql); + pCur->zSql = 0; + }else{ + pCur->rc = sqlite3_step(pCur->pExplain); + rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc; + } + return rc; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the explain virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int explainBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop counter */ + int idx = -1; /* Index of a usable == constraint against SQL */ + int unusable = 0; /* True if there are unusable constraints on SQL */ + + pIdxInfo->estimatedRows = 500; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->iColumn!=EXPLN_COLUMN_SQL ) continue; + if( !p->usable ){ + unusable = 1; + }else if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idx = i; + } + } + if( idx>=0 ){ + /* There exists a usable == constraint against the SQL column */ + pIdxInfo->estimatedCost = 10.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[idx].argvIndex = 1; + pIdxInfo->aConstraintUsage[idx].omit = 1; + }else if( unusable ){ + /* There are unusable constraints against the SQL column. Do not allow + ** this plan to continue forward. */ + return SQLITE_CONSTRAINT; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** explain virtual table. +*/ +static sqlite3_module explainModule = { + 0, /* iVersion */ + 0, /* xCreate */ + explainConnect, /* xConnect */ + explainBestIndex, /* xBestIndex */ + explainDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + explainOpen, /* xOpen - open a cursor */ + explainClose, /* xClose - close a cursor */ + explainFilter, /* xFilter - configure scan constraints */ + explainNext, /* xNext - advance a cursor */ + explainEof, /* xEof - check for end of scan */ + explainColumn, /* xColumn - read data */ + explainRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3ExplainVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "explain", &explainModule, 0); +#endif + return rc; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_explain_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3ExplainVtabInit(db); +#endif + return rc; +} diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c new file mode 100644 index 0000000000000000000000000000000000000000..483ef0187f9d59d54b9b0c78af4c1eadc9a39a6d --- /dev/null +++ b/ext/misc/fileio.c @@ -0,0 +1,1046 @@ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL functions readfile() and +** writefile(), and eponymous virtual type "fsdir". +** +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): +** +** If neither of the optional arguments is present, then this UDF +** function writes blob DATA to file FILE. If successful, the number +** of bytes written is returned. If an error occurs, NULL is returned. +** +** If the first option argument - MODE - is present, then it must +** be passed an integer value that corresponds to a POSIX mode +** value (file type + permissions, as returned in the stat.st_mode +** field by the stat() system call). Three types of files may +** be written/created: +** +** regular files: (mode & 0170000)==0100000 +** symbolic links: (mode & 0170000)==0120000 +** directories: (mode & 0170000)==0040000 +** +** For a directory, the DATA is ignored. For a symbolic link, it is +** interpreted as text and used as the target of the link. For a +** regular file, it is interpreted as a blob and written into the +** named file. Regardless of the type of file, its permissions are +** set to (mode & 0777) before returning. +** +** If the optional MTIME argument is present, then it is interpreted +** as an integer - the number of seconds since the unix epoch. The +** modification-time of the target file is set to this value before +** returning. +** +** If five or more arguments are passed to this function and an +** error is encountered, an exception is raised. +** +** READFILE(FILE): +** +** Read and return the contents of file FILE (type blob) from disk. +** +** FSDIR: +** +** Used as follows: +** +** SELECT * FROM fsdir($path [, $dir]); +** +** Parameter $path is an absolute or relative pathname. If the file that it +** refers to does not exist, it is an error. If the path refers to a regular +** file or symbolic link, it returns a single row. Or, if the path refers +** to a directory, it returns one row for the directory, and one row for each +** file within the hierarchy rooted at $path. +** +** Each row has the following columns: +** +** name: Path to file or directory (text value). +** mode: Value of stat.st_mode for directory entry (an integer). +** mtime: Value of stat.st_mtime for directory entry (an integer). +** data: For a regular file, a blob containing the file data. For a +** symlink, a text value containing the text of the link. For a +** directory, NULL. +** +** If a non-NULL value is specified for the optional $dir parameter and +** $path is a relative path, then $path is interpreted relative to $dir. +** And the paths returned in the "name" column of the table are also +** relative to directory $dir. +** +** Notes on building this extension for Windows: +** Unless linked statically with the SQLite library, a preprocessor +** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone +** DLL form of this extension for WIN32. See its use below for details. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#include +#include +#include +#if !defined(_WIN32) && !defined(WIN32) +# include +# include +# include +# include +#else +# include "windows.h" +# include +# include +# include "test_windirent.h" +# define dirent DIRENT +# ifndef chmod +# define chmod _chmod +# endif +# ifndef stat +# define stat _stat +# endif +# define mkdir(path,mode) _mkdir(path) +# define lstat(path,buf) stat(path,buf) +#endif +#include +#include + +/* When used as part of the CLI, the sqlite3_stdio.h module will have +** been included before this one. In that case use the sqlite3_stdio.h +** #defines. If not, create our own for fopen(). +*/ +#ifndef _SQLITE3_STDIO_H_ +# define sqlite3_fopen fopen +#endif + +/* +** Structure of the fsdir() table-valued function +*/ + /* 0 1 2 3 4 5 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" +#define FSDIR_COLUMN_NAME 0 /* Name of the file */ +#define FSDIR_COLUMN_MODE 1 /* Access mode */ +#define FSDIR_COLUMN_MTIME 2 /* Last modification time */ +#define FSDIR_COLUMN_DATA 3 /* File content */ +#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ + + +/* +** Set the result stored by context ctx to a blob containing the +** contents of file zName. Or, leave the result unchanged (NULL) +** if the file does not exist or is unreadable. +** +** If the file exceeds the SQLite blob size limit, through an +** SQLITE_TOOBIG error. +** +** Throw an SQLITE_IOERR if there are difficulties pulling the file +** off of disk. +*/ +static void readFileContents(sqlite3_context *ctx, const char *zName){ + FILE *in; + sqlite3_int64 nIn; + void *pBuf; + sqlite3 *db; + int mxBlob; + + in = sqlite3_fopen(zName, "rb"); + if( in==0 ){ + /* File does not exist or is unreadable. Leave the result set to NULL. */ + return; + } + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + db = sqlite3_context_db_handle(ctx); + mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + if( nIn>mxBlob ){ + sqlite3_result_error_code(ctx, SQLITE_TOOBIG); + fclose(in); + return; + } + pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); + if( pBuf==0 ){ + sqlite3_result_error_nomem(ctx); + fclose(in); + return; + } + if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ + sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_result_error_code(ctx, SQLITE_IOERR); + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + readFileContents(context, zName); +} + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +#if defined(_WIN32) +/* +** This function is designed to convert a Win32 FILETIME structure into the +** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). +*/ +static sqlite3_uint64 fileTimeToUnixTime( + LPFILETIME pFileTime +){ + SYSTEMTIME epochSystemTime; + ULARGE_INTEGER epochIntervals; + FILETIME epochFileTime; + ULARGE_INTEGER fileIntervals; + + memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); + epochSystemTime.wYear = 1970; + epochSystemTime.wMonth = 1; + epochSystemTime.wDay = 1; + SystemTimeToFileTime(&epochSystemTime, &epochFileTime); + epochIntervals.LowPart = epochFileTime.dwLowDateTime; + epochIntervals.HighPart = epochFileTime.dwHighDateTime; + + fileIntervals.LowPart = pFileTime->dwLowDateTime; + fileIntervals.HighPart = pFileTime->dwHighDateTime; + + return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; +} + + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +# /* To allow a standalone DLL, use this next replacement function: */ +# undef sqlite3_win32_utf8_to_unicode +# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 +# +LPWSTR utf8_to_utf16(const char *z){ + int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); + LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); + if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) + return rv; + sqlite3_free(rv); + return 0; +} +#endif + +/* +** This function attempts to normalize the time values found in the stat() +** buffer to UTC. This is necessary on Win32, where the runtime library +** appears to return these values as local times. +*/ +static void statTimesToUtc( + const char *zPath, + struct stat *pStatBuf +){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); + if( zUnicodeName ){ + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(zUnicodeName, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + sqlite3_free(zUnicodeName); + } +} +#endif + +/* +** This function is used in place of stat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls stat(). +*/ +static int fileStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = stat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return stat(zPath, pStatBuf); +#endif +} + +/* +** This function is used in place of lstat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls lstat(). +*/ +static int fileLinkStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = lstat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return lstat(zPath, pStatBuf); +#endif +} + +/* +** Argument zFile is the name of a file that will be created and/or written +** by SQL function writefile(). This function ensures that the directory +** zFile will be written to exists, creating it if required. The permissions +** for any path components created by this function are set in accordance +** with the current umask. +** +** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise, +** SQLITE_OK is returned if the directory is successfully created, or +** SQLITE_ERROR otherwise. +*/ +static int makeDirectory( + const char *zFile +){ + char *zCopy = sqlite3_mprintf("%s", zFile); + int rc = SQLITE_OK; + + if( zCopy==0 ){ + rc = SQLITE_NOMEM; + }else{ + int nCopy = (int)strlen(zCopy); + int i = 1; + + while( rc==SQLITE_OK ){ + struct stat sStat; + int rc2; + + for(; zCopy[i]!='/' && i=0 ){ +#if defined(_WIN32) +#if !SQLITE_OS_WINRT + /* Windows */ + FILETIME lastAccess; + FILETIME lastWrite; + SYSTEMTIME currentTime; + LONGLONG intervals; + HANDLE hFile; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + + GetSystemTime(¤tTime); + SystemTimeToFileTime(¤tTime, &lastAccess); + intervals = Int32x32To64(mtime, 10000000) + 116444736000000000; + lastWrite.dwLowDateTime = (DWORD)intervals; + lastWrite.dwHighDateTime = intervals >> 32; + zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile); + if( zUnicodeName==0 ){ + return 1; + } + hFile = CreateFileW( + zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL + ); + sqlite3_free(zUnicodeName); + if( hFile!=INVALID_HANDLE_VALUE ){ + BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite); + CloseHandle(hFile); + return !bResult; + }else{ + return 1; + } +#endif +#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ + /* Recent unix */ + struct timespec times[2]; + times[0].tv_nsec = times[1].tv_nsec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){ + return 1; + } +#else + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } + } +#endif + } + + return 0; +} + +/* +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function. +** Refer to header comments at the top of this file for details. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zFile; + mode_t mode = 0; + int res; + sqlite3_int64 mtime = -1; + + if( argc<2 || argc>4 ){ + sqlite3_result_error(context, + "wrong number of arguments to function writefile()", -1 + ); + return; + } + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + if( argc>=3 ){ + mode = (mode_t)sqlite3_value_int(argv[2]); + } + if( argc==4 ){ + mtime = sqlite3_value_int64(argv[3]); + } + + res = writeFile(context, zFile, argv[1], mode, mtime); + if( res==1 && errno==ENOENT ){ + if( makeDirectory(zFile)==SQLITE_OK ){ + res = writeFile(context, zFile, argv[1], mode, mtime); + } + } + + if( argc>2 && res!=0 ){ + if( S_ISLNK(mode) ){ + ctxErrorMsg(context, "failed to create symlink: %s", zFile); + }else if( S_ISDIR(mode) ){ + ctxErrorMsg(context, "failed to create directory: %s", zFile); + }else{ + ctxErrorMsg(context, "failed to write file: %s", zFile); + } + } +} + +/* +** SQL function: lsmode(MODE) +** +** Given a numberic st_mode from stat(), convert it into a human-readable +** text string in the style of "ls -l". +*/ +static void lsModeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int iMode = sqlite3_value_int(argv[0]); + char z[16]; + (void)argc; + if( S_ISLNK(iMode) ){ + z[0] = 'l'; + }else if( S_ISREG(iMode) ){ + z[0] = '-'; + }else if( S_ISDIR(iMode) ){ + z[0] = 'd'; + }else{ + z[0] = '?'; + } + for(i=0; i<3; i++){ + int m = (iMode >> ((2-i)*3)); + char *a = &z[1 + i*3]; + a[0] = (m & 0x4) ? 'r' : '-'; + a[1] = (m & 0x2) ? 'w' : '-'; + a[2] = (m & 0x1) ? 'x' : '-'; + } + z[10] = '\0'; + sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT); +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Cursor type for recursively iterating through a directory structure. +*/ +typedef struct fsdir_cursor fsdir_cursor; +typedef struct FsdirLevel FsdirLevel; + +struct FsdirLevel { + DIR *pDir; /* From opendir() */ + char *zDir; /* Name of directory (nul-terminated) */ +}; + +struct fsdir_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + + int nLvl; /* Number of entries in aLvl[] array */ + int iLvl; /* Index of current entry */ + FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ + + const char *zBase; + int nBase; + + struct stat sStat; /* Current lstat() results */ + char *zPath; /* Path to current entry */ + sqlite3_int64 iRowid; /* Current rowid */ +}; + +typedef struct fsdir_tab fsdir_tab; +struct fsdir_tab { + sqlite3_vtab base; /* Base class - must be first */ +}; + +/* +** Construct a new fsdir virtual table object. +*/ +static int fsdirConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fsdir_tab *pNew = 0; + int rc; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; + rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + } + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** This method is the destructor for fsdir vtab objects. +*/ +static int fsdirDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new fsdir_cursor object. +*/ +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + fsdir_cursor *pCur; + (void)p; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->iLvl = -1; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset a cursor back to the state it was in when first returned +** by fsdirOpen(). +*/ +static void fsdirResetCursor(fsdir_cursor *pCur){ + int i; + for(i=0; i<=pCur->iLvl; i++){ + FsdirLevel *pLvl = &pCur->aLvl[i]; + if( pLvl->pDir ) closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + } + sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); + pCur->aLvl = 0; + pCur->zPath = 0; + pCur->zBase = 0; + pCur->nBase = 0; + pCur->nLvl = 0; + pCur->iLvl = -1; + pCur->iRowid = 1; +} + +/* +** Destructor for an fsdir_cursor. +*/ +static int fsdirClose(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + + fsdirResetCursor(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Set the error message for the virtual table associated with cursor +** pCur to the results of vprintf(zFmt, ...). +*/ +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + + +/* +** Advance an fsdir_cursor to its next row of output. +*/ +static int fsdirNext(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + mode_t m = pCur->sStat.st_mode; + + pCur->iRowid++; + if( S_ISDIR(m) ){ + /* Descend into this directory */ + int iNew = pCur->iLvl + 1; + FsdirLevel *pLvl; + if( iNew>=pCur->nLvl ){ + int nNew = iNew+1; + sqlite3_int64 nByte = nNew*sizeof(FsdirLevel); + FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc64(pCur->aLvl, nByte); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl)); + pCur->aLvl = aNew; + pCur->nLvl = nNew; + } + pCur->iLvl = iNew; + pLvl = &pCur->aLvl[iNew]; + + pLvl->zDir = pCur->zPath; + pCur->zPath = 0; + pLvl->pDir = opendir(pLvl->zDir); + if( pLvl->pDir==0 ){ + fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + return SQLITE_ERROR; + } + } + + while( pCur->iLvl>=0 ){ + FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl]; + struct dirent *pEntry = readdir(pLvl->pDir); + if( pEntry ){ + if( pEntry->d_name[0]=='.' ){ + if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue; + if( pEntry->d_name[1]=='\0' ) continue; + } + sqlite3_free(pCur->zPath); + pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + pLvl->pDir = 0; + pLvl->zDir = 0; + pCur->iLvl--; + } + + /* EOF */ + sqlite3_free(pCur->zPath); + pCur->zPath = 0; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int fsdirColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + switch( i ){ + case FSDIR_COLUMN_NAME: { + sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT); + break; + } + + case FSDIR_COLUMN_MODE: + sqlite3_result_int64(ctx, pCur->sStat.st_mode); + break; + + case FSDIR_COLUMN_MTIME: + sqlite3_result_int64(ctx, pCur->sStat.st_mtime); + break; + + case FSDIR_COLUMN_DATA: { + mode_t m = pCur->sStat.st_mode; + if( S_ISDIR(m) ){ + sqlite3_result_null(ctx); +#if !defined(_WIN32) && !defined(WIN32) + }else if( S_ISLNK(m) ){ + char aStatic[64]; + char *aBuf = aStatic; + sqlite3_int64 nBuf = 64; + int n; + + while( 1 ){ + n = readlink(pCur->zPath, aBuf, nBuf); + if( nzPath); + } + } + case FSDIR_COLUMN_PATH: + default: { + /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. + ** always return their values as NULL */ + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. +*/ +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fsdirEof(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + return (pCur->zPath==0); +} + +/* +** xFilter callback. +** +** idxNum==1 PATH parameter only +** idxNum==2 Both PATH and DIR supplied +*/ +static int fsdirFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + const char *zDir = 0; + fsdir_cursor *pCur = (fsdir_cursor*)cur; + (void)idxStr; + fsdirResetCursor(pCur); + + if( idxNum==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); + return SQLITE_ERROR; + } + + assert( argc==idxNum && (argc==1 || argc==2) ); + zDir = (const char*)sqlite3_value_text(argv[0]); + if( zDir==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); + return SQLITE_ERROR; + } + if( argc==2 ){ + pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + } + if( pCur->zBase ){ + pCur->nBase = (int)strlen(pCur->zBase)+1; + pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir); + }else{ + pCur->zPath = sqlite3_mprintf("%s", zDir); + } + + if( pCur->zPath==0 ){ + return SQLITE_NOMEM; + } + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by values of idxNum: +** +** (1) The path value is supplied by argv[0] +** (2) Path is in argv[0] and dir is in argv[1] +*/ +static int fsdirBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ + int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int seenPath = 0; /* True if an unusable PATH= constraint is seen */ + int seenDir = 0; /* True if an unusable DIR= constraint is seen */ + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; + } + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; + } + } + } + if( seenPath || seenDir ){ + /* If input parameters are unusable, disallow this plan */ + return SQLITE_CONSTRAINT; + } + + if( idxPath<0 ){ + pIdxInfo->idxNum = 0; + /* The pIdxInfo->estimatedCost should have been initialized to a huge + ** number. Leave it unchanged. */ + pIdxInfo->estimatedRows = 0x7fffffff; + }else{ + pIdxInfo->aConstraintUsage[idxPath].omit = 1; + pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + if( idxDir>=0 ){ + pIdxInfo->aConstraintUsage[idxDir].omit = 1; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; + pIdxInfo->idxNum = 2; + pIdxInfo->estimatedCost = 10.0; + }else{ + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 100.0; + } + } + + return SQLITE_OK; +} + +/* +** Register the "fsdir" virtual table. +*/ +static int fsdirRegister(sqlite3 *db){ + static sqlite3_module fsdirModule = { + 0, /* iVersion */ + 0, /* xCreate */ + fsdirConnect, /* xConnect */ + fsdirBestIndex, /* xBestIndex */ + fsdirDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + fsdirOpen, /* xOpen - open a cursor */ + fsdirClose, /* xClose - close a cursor */ + fsdirFilter, /* xFilter - configure scan constraints */ + fsdirNext, /* xNext - advance a cursor */ + fsdirEof, /* xEof - check for end of scan */ + fsdirColumn, /* xColumn - read data */ + fsdirRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + + int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define fsdirRegister(x) SQLITE_OK +#endif + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fileio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readfile", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + readfileFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writefile", -1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + writefileFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0, + lsModeFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = fsdirRegister(db); + } + return rc; +} + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +/* To allow a standalone DLL, make test_windirent.c use the same + * redefined SQLite API calls as the above extension code does. + * Just pull in this .c to accomplish this. As a beneficial side + * effect, this extension becomes a single translation unit. */ +# include "test_windirent.c" +#endif diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c new file mode 100644 index 0000000000000000000000000000000000000000..e638737d2b2dfba7734caf053b06e7c1aaef4f61 --- /dev/null +++ b/ext/misc/fossildelta.c @@ -0,0 +1,1093 @@ +/* +** 2019-02-19 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the delta functions used by the RBU +** extension. Three scalar functions and one table-valued function are +** implemented here: +** +** delta_apply(X,D) -- apply delta D to file X and return the result +** delta_create(X,Y) -- compute and return a delta that carries X into Y +** delta_output_size(D) -- blob size in bytes output from applying delta D +** delta_parse(D) -- returns rows describing delta D +** +** The delta format is the Fossil delta format, described in a comment +** on the delete_create() function implementation below, and also at +** +** https://www.fossil-scm.org/fossil/doc/trunk/www/delta_format.wiki +** +** This delta format is used by the RBU extension, which is the main +** reason that these routines are included in the extension library. +** RBU does not use this extension directly. Rather, this extension is +** provided as a convenience to developers who want to analyze RBU files +** that contain deltas. +*/ +#include +#include +#include +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +/* +** The "u32" type must be an unsigned 32-bit integer. Adjust this +*/ +typedef unsigned int u32; + +/* +** Must be a 16-bit value +*/ +typedef short int s16; +typedef unsigned short int u16; + +#endif /* SQLITE_AMALGAMATION */ + + +/* +** The width of a hash window in bytes. The algorithm only works if this +** is a power of 2. +*/ +#define NHASH 16 + +/* +** The current state of the rolling hash. +** +** z[] holds the values that have been hashed. z[] is a circular buffer. +** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of +** the window. +** +** Hash.a is the sum of all elements of hash.z[]. Hash.b is a weighted +** sum. Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1. +** (Each index for z[] should be module NHASH, of course. The %NHASH operator +** is omitted in the prior expression for brevity.) +*/ +typedef struct hash hash; +struct hash { + u16 a, b; /* Hash values */ + u16 i; /* Start of the hash window */ + char z[NHASH]; /* The values that have been hashed */ +}; + +/* +** Initialize the rolling hash using the first NHASH characters of z[] +*/ +static void hash_init(hash *pHash, const char *z){ + u16 a, b, i; + a = b = z[0]; + for(i=1; iz, z, NHASH); + pHash->a = a & 0xffff; + pHash->b = b & 0xffff; + pHash->i = 0; +} + +/* +** Advance the rolling hash by a single character "c" +*/ +static void hash_next(hash *pHash, int c){ + u16 old = pHash->z[pHash->i]; + pHash->z[pHash->i] = c; + pHash->i = (pHash->i+1)&(NHASH-1); + pHash->a = pHash->a - old + c; + pHash->b = pHash->b - NHASH*old + pHash->a; +} + +/* +** Return a 32-bit hash value +*/ +static u32 hash_32bit(hash *pHash){ + return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16); +} + +/* +** Compute a hash on NHASH bytes. +** +** This routine is intended to be equivalent to: +** hash h; +** hash_init(&h, zInput); +** return hash_32bit(&h); +*/ +static u32 hash_once(const char *z){ + u16 a, b, i; + a = b = z[0]; + for(i=1; i0; i++, v>>=6){ + zBuf[i] = zDigits[v&0x3f]; + } + for(j=i-1; j>=0; j--){ + *(*pz)++ = zBuf[j]; + } +} + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int deltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Return the number digits in the base-64 representation of a positive integer +*/ +static int digit_count(int v){ + unsigned int i, x; + for(i=1, x=64; v>=x; i++, x <<= 6){} + return i; +} + +#ifdef __GNUC__ +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif + +/* +** Compute a 32-bit big-endian checksum on the N-byte buffer. If the +** buffer is not a multiple of 4 bytes length, compute the sum that would +** have occurred if the buffer was padded with zeros to the next multiple +** of four bytes. +*/ +static unsigned int checksum(const char *zIn, size_t N){ + static const int byteOrderTest = 1; + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zEnd = (const unsigned char*)&zIn[N&~3]; + unsigned sum = 0; + assert( (z - (const unsigned char*)0)%4==0 ); /* Four-byte alignment */ + if( 0==*(char*)&byteOrderTest ){ + /* This is a big-endian machine */ + while( z=4003000 + while( z=1300 + while( z= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum += z[3]; + z += 4; + N -= 4; + } + sum += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); +#endif + } + switch(N&3){ + case 3: sum += (z[2] << 8); + case 2: sum += (z[1] << 16); + case 1: sum += (z[0] << 24); + default: ; + } + return sum; +} + +/* +** Create a new delta. +** +** The delta is written into a preallocated buffer, zDelta, which +** should be at least 60 bytes longer than the target file, zOut. +** The delta string will be NUL-terminated, but it might also contain +** embedded NUL characters if either the zSrc or zOut files are +** binary. This function returns the length of the delta string +** in bytes, excluding the final NUL terminator character. +** +** Output Format: +** +** The delta begins with a base64 number followed by a newline. This +** number is the number of bytes in the TARGET file. Thus, given a +** delta file z, a program can compute the size of the output file +** simply by reading the first line and decoding the base-64 number +** found there. The delta_output_size() routine does exactly this. +** +** After the initial size number, the delta consists of a series of +** literal text segments and commands to copy from the SOURCE file. +** A copy command looks like this: +** +** NNN@MMM, +** +** where NNN is the number of bytes to be copied and MMM is the offset +** into the source file of the first byte (both base-64). If NNN is 0 +** it means copy the rest of the input file. Literal text is like this: +** +** NNN:TTTTT +** +** where NNN is the number of bytes of text (base-64) and TTTTT is the text. +** +** The last term is of the form +** +** NNN; +** +** In this case, NNN is a 32-bit bigendian checksum of the output file +** that can be used to verify that the delta applied correctly. All +** numbers are in base-64. +** +** Pure text files generate a pure text delta. Binary files generate a +** delta that may contain some binary data. +** +** Algorithm: +** +** The encoder first builds a hash table to help it find matching +** patterns in the source file. 16-byte chunks of the source file +** sampled at evenly spaced intervals are used to populate the hash +** table. +** +** Next we begin scanning the target file using a sliding 16-byte +** window. The hash of the 16-byte window in the target is used to +** search for a matching section in the source file. When a match +** is found, a copy command is added to the delta. An effort is +** made to extend the matching section to regions that come before +** and after the 16-byte hash window. A copy command is only issued +** if the result would use less space that just quoting the text +** literally. Literal text is added to the delta for sections that +** do not match or which can not be encoded efficiently using copy +** commands. +*/ +static int delta_create( + const char *zSrc, /* The source or pattern file */ + unsigned int lenSrc, /* Length of the source file */ + const char *zOut, /* The target file */ + unsigned int lenOut, /* Length of the target file */ + char *zDelta /* Write the delta into this buffer */ +){ + int i, base; + char *zOrigDelta = zDelta; + hash h; + int nHash; /* Number of hash table entries */ + int *landmark; /* Primary hash table */ + int *collide; /* Collision chain */ + int lastRead = -1; /* Last byte of zSrc read by a COPY command */ + + /* Add the target file size to the beginning of the delta + */ + putInt(lenOut, &zDelta); + *(zDelta++) = '\n'; + + /* If the source file is very small, it means that we have no + ** chance of ever doing a copy command. Just output a single + ** literal segment for the entire target and exit. + */ + if( lenSrc<=NHASH ){ + putInt(lenOut, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, zOut, lenOut); + zDelta += lenOut; + putInt(checksum(zOut, lenOut), &zDelta); + *(zDelta++) = ';'; + return zDelta - zOrigDelta; + } + + /* Compute the hash table used to locate matching sections in the + ** source file. + */ + nHash = lenSrc/NHASH; + collide = sqlite3_malloc64( (sqlite3_int64)nHash*2*sizeof(int) ); + memset(collide, -1, nHash*2*sizeof(int)); + landmark = &collide[nHash]; + for(i=0; i=0 && (limit--)>0 ){ + /* + ** The hash window has identified a potential match against + ** landmark block iBlock. But we need to investigate further. + ** + ** Look for a region in zOut that matches zSrc. Anchor the search + ** at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to + ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen]. + ** + ** Set cnt equal to the length of the match and set ofst so that + ** zSrc[ofst] is the first element of the match. litsz is the number + ** of characters between zOut[base] and the beginning of the match. + ** sz will be the overhead (in bytes) needed to encode the copy + ** command. Only generate copy command if the overhead of the + ** copy command is less than the amount of literal text to be copied. + */ + int cnt, ofst, litsz; + int j, k, x, y; + int sz; + int limitX; + + /* Beginning at iSrc, match forwards as far as we can. j counts + ** the number of characters that match */ + iSrc = iBlock*NHASH; + y = base+i; + limitX = ( lenSrc-iSrc <= lenOut-y ) ? lenSrc : iSrc + lenOut - y; + for(x=iSrc; x=sz && cnt>bestCnt ){ + /* Remember this match only if it is the best so far and it + ** does not increase the file size */ + bestCnt = cnt; + bestOfst = iSrc-k; + bestLitsz = litsz; + } + + /* Check the next matching block */ + iBlock = collide[iBlock]; + } + + /* We have a copy command that does not cause the delta to be larger + ** than a literal insert. So add the copy command to the delta. + */ + if( bestCnt>0 ){ + if( bestLitsz>0 ){ + /* Add an insert command before the copy */ + putInt(bestLitsz,&zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], bestLitsz); + zDelta += bestLitsz; + base += bestLitsz; + } + base += bestCnt; + putInt(bestCnt, &zDelta); + *(zDelta++) = '@'; + putInt(bestOfst, &zDelta); + *(zDelta++) = ','; + if( bestOfst + bestCnt -1 > lastRead ){ + lastRead = bestOfst + bestCnt - 1; + } + bestCnt = 0; + break; + } + + /* If we reach this point, it means no match is found so far */ + if( base+i+NHASH>=lenOut ){ + /* We have reached the end of the file and have not found any + ** matches. Do an "insert" for everything that does not match */ + putInt(lenOut-base, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], lenOut-base); + zDelta += lenOut-base; + base = lenOut; + break; + } + + /* Advance the hash by one character. Keep looking for a match */ + hash_next(&h, zOut[base+i+NHASH]); + i++; + } + } + /* Output a final "insert" record to get all the text at the end of + ** the file that does not match anything in the source file. + */ + if( base0 ){ + unsigned int cnt, ofst; + cnt = deltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = deltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( ofst+cnt > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifdef FOSSIL_ENABLE_DELTA_CKSUM_TEST + if( cnt!=checksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +/* +** SQL functions: delta_create(X,Y) +** +** Return a delta for carrying X into Y. +*/ +static void deltaCreateFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aOrig; int nOrig; /* old blob */ + const char *aNew; int nNew; /* new blob */ + char *aOut; int nOut; /* output delta */ + + assert( argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nNew = sqlite3_value_bytes(argv[1]); + aNew = (const char*)sqlite3_value_blob(argv[1]); + aOut = sqlite3_malloc64(nNew+70); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut = delta_create(aOrig, nOrig, aNew, nNew, aOut); + if( nOut<0 ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "cannot create fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + +/* +** SQL functions: delta_apply(X,D) +** +** Return the result of applying delta D to input X. +*/ +static void deltaApplyFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aOrig; int nOrig; /* The X input */ + const char *aDelta; int nDelta; /* The input delta (D) */ + char *aOut; int nOut, nOut2; /* The output */ + + assert( argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = delta_output_size(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + aOut = sqlite3_malloc64((sqlite3_int64)nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = delta_apply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + +/* +** SQL functions: delta_output_size(D) +** +** Return the size of the output that results from applying delta D. +*/ +static void deltaOutputSizeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; int nDelta; /* The input delta (D) */ + int nOut; /* Size of output */ + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + nDelta = sqlite3_value_bytes(argv[0]); + aDelta = (const char*)sqlite3_value_blob(argv[0]); + + /* Figure out the size of the output */ + nOut = delta_output_size(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + }else{ + sqlite3_result_int(context, nOut); + } +} + +/***************************************************************************** +** Table-valued SQL function: delta_parse(DELTA) +** +** Schema: +** +** CREATE TABLE delta_parse( +** op TEXT, +** a1 INT, +** a2 ANY, +** delta HIDDEN BLOB +** ); +** +** Given an input DELTA, this function parses the delta and returns +** rows for each entry in the delta. The op column has one of the +** values SIZE, COPY, INSERT, CHECKSUM, ERROR. +** +** Assuming no errors, the first row has op='SIZE'. a1 is the size of +** the output in bytes and a2 is NULL. +** +** After the initial SIZE row, there are zero or more 'COPY' and/or 'INSERT' +** rows. A COPY row means content is copied from the source into the +** output. Column a1 is the number of bytes to copy and a2 is the offset +** into source from which to begin copying. An INSERT row means to +** insert text into the output stream. Column a1 is the number of bytes +** to insert and column is a BLOB that contains the text to be inserted. +** +** The last row of a well-formed delta will have an op value of 'CHECKSUM'. +** The a1 column will be the value of the checksum and a2 will be NULL. +** +** If the input delta is not well-formed, then a row with an op value +** of 'ERROR' is returned. The a1 value of the ERROR row is the offset +** into the delta where the error was encountered and a2 is NULL. +*/ +typedef struct deltaparsevtab_vtab deltaparsevtab_vtab; +typedef struct deltaparsevtab_cursor deltaparsevtab_cursor; +struct deltaparsevtab_vtab { + sqlite3_vtab base; /* Base class - must be first */ + /* No additional information needed */ +}; +struct deltaparsevtab_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + char *aDelta; /* The delta being parsed */ + int nDelta; /* Number of bytes in the delta */ + int iCursor; /* Current cursor location */ + int eOp; /* Name of current operator */ + unsigned int a1, a2; /* Arguments to current operator */ + int iNext; /* Next cursor value */ +}; + +/* Operator names: +*/ +static const char *azOp[] = { + "SIZE", "COPY", "INSERT", "CHECKSUM", "ERROR", "EOF" +}; +#define DELTAPARSE_OP_SIZE 0 +#define DELTAPARSE_OP_COPY 1 +#define DELTAPARSE_OP_INSERT 2 +#define DELTAPARSE_OP_CHECKSUM 3 +#define DELTAPARSE_OP_ERROR 4 +#define DELTAPARSE_OP_EOF 5 + +/* +** The deltaparsevtabConnect() method is invoked to create a new +** deltaparse virtual table. +** +** Think of this routine as the constructor for deltaparsevtab_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the deltaparsevtab_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against the virtual table will look like. +*/ +static int deltaparsevtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + deltaparsevtab_vtab *pNew; + int rc; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(op,a1,a2,delta HIDDEN)" + ); + /* For convenience, define symbolic names for the index to each column. */ +#define DELTAPARSEVTAB_OP 0 +#define DELTAPARSEVTAB_A1 1 +#define DELTAPARSEVTAB_A2 2 +#define DELTAPARSEVTAB_DELTA 3 + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc64( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for deltaparsevtab_vtab objects. +*/ +static int deltaparsevtabDisconnect(sqlite3_vtab *pVtab){ + deltaparsevtab_vtab *p = (deltaparsevtab_vtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new deltaparsevtab_cursor object. +*/ +static int deltaparsevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + deltaparsevtab_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a deltaparsevtab_cursor. +*/ +static int deltaparsevtabClose(sqlite3_vtab_cursor *cur){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + sqlite3_free(pCur->aDelta); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a deltaparsevtab_cursor to its next row of output. +*/ +static int deltaparsevtabNext(sqlite3_vtab_cursor *cur){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + const char *z; + int i = 0; + + pCur->iCursor = pCur->iNext; + z = pCur->aDelta + pCur->iCursor; + pCur->a1 = deltaGetInt(&z, &i); + switch( z[0] ){ + case '@': { + z++; + pCur->a2 = deltaGetInt(&z, &i); + pCur->eOp = DELTAPARSE_OP_COPY; + pCur->iNext = (int)(&z[1] - pCur->aDelta); + break; + } + case ':': { + z++; + pCur->a2 = (unsigned int)(z - pCur->aDelta); + pCur->eOp = DELTAPARSE_OP_INSERT; + pCur->iNext = (int)(&z[pCur->a1] - pCur->aDelta); + break; + } + case ';': { + pCur->eOp = DELTAPARSE_OP_CHECKSUM; + pCur->iNext = pCur->nDelta; + break; + } + default: { + if( pCur->iNext==pCur->nDelta ){ + pCur->eOp = DELTAPARSE_OP_EOF; + }else{ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->iNext = pCur->nDelta; + } + break; + } + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the deltaparsevtab_cursor +** is currently pointing. +*/ +static int deltaparsevtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + switch( i ){ + case DELTAPARSEVTAB_OP: { + sqlite3_result_text(ctx, azOp[pCur->eOp], -1, SQLITE_STATIC); + break; + } + case DELTAPARSEVTAB_A1: { + sqlite3_result_int(ctx, pCur->a1); + break; + } + case DELTAPARSEVTAB_A2: { + if( pCur->eOp==DELTAPARSE_OP_COPY ){ + sqlite3_result_int(ctx, pCur->a2); + }else if( pCur->eOp==DELTAPARSE_OP_INSERT ){ + sqlite3_result_blob(ctx, pCur->aDelta+pCur->a2, pCur->a1, + SQLITE_TRANSIENT); + } + break; + } + case DELTAPARSEVTAB_DELTA: { + sqlite3_result_blob(ctx, pCur->aDelta, pCur->nDelta, SQLITE_TRANSIENT); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int deltaparsevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + *pRowid = pCur->iCursor; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int deltaparsevtabEof(sqlite3_vtab_cursor *cur){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + return pCur->eOp==DELTAPARSE_OP_EOF; +} + +/* +** This method is called to "rewind" the deltaparsevtab_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to deltaparsevtabColumn() or deltaparsevtabRowid() or +** deltaparsevtabEof(). +*/ +static int deltaparsevtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor *)pVtabCursor; + const char *a; + int i = 0; + pCur->eOp = DELTAPARSE_OP_ERROR; + if( idxNum!=1 ){ + return SQLITE_OK; + } + pCur->nDelta = sqlite3_value_bytes(argv[0]); + a = (const char*)sqlite3_value_blob(argv[0]); + if( pCur->nDelta==0 || a==0 ){ + return SQLITE_OK; + } + pCur->aDelta = sqlite3_malloc64( pCur->nDelta+1 ); + if( pCur->aDelta==0 ){ + pCur->nDelta = 0; + return SQLITE_NOMEM; + } + memcpy(pCur->aDelta, a, pCur->nDelta); + pCur->aDelta[pCur->nDelta] = 0; + a = pCur->aDelta; + pCur->eOp = DELTAPARSE_OP_SIZE; + pCur->a1 = deltaGetInt(&a, &i); + if( a[0]!='\n' ){ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->a1 = pCur->a2 = 0; + pCur->iNext = pCur->nDelta; + return SQLITE_OK; + } + a++; + pCur->iNext = (unsigned int)(a - pCur->aDelta); + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int deltaparsevtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + for(i=0; inConstraint; i++){ + if( pIdxInfo->aConstraint[i].iColumn != DELTAPARSEVTAB_DELTA ) continue; + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 10; + pIdxInfo->idxNum = 1; + return SQLITE_OK; + } + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)0x7fffffff; + pIdxInfo->estimatedRows = 0x7fffffff; + return SQLITE_CONSTRAINT; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module deltaparsevtabModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ deltaparsevtabConnect, + /* xBestIndex */ deltaparsevtabBestIndex, + /* xDisconnect */ deltaparsevtabDisconnect, + /* xDestroy */ 0, + /* xOpen */ deltaparsevtabOpen, + /* xClose */ deltaparsevtabClose, + /* xFilter */ deltaparsevtabFilter, + /* xNext */ deltaparsevtabNext, + /* xEof */ deltaparsevtabEof, + /* xColumn */ deltaparsevtabColumn, + /* xRowid */ deltaparsevtabRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 +}; + + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fossildelta_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const int enc = SQLITE_UTF8|SQLITE_INNOCUOUS; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "delta_create", 2, enc, 0, + deltaCreateFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "delta_apply", 2, enc, 0, + deltaApplyFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "delta_output_size", 1, enc, 0, + deltaOutputSizeFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "delta_parse", &deltaparsevtabModule, 0); + } + return rc; +} diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c new file mode 100644 index 0000000000000000000000000000000000000000..92b7c0dae0fea5b3548e6b4aea84763180a46f40 --- /dev/null +++ b/ext/misc/fuzzer.c @@ -0,0 +1,1192 @@ +/* +** 2011 March 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Code for a demonstration virtual table that generates variations +** on an input word at increasing edit distances from the original. +** +** A fuzzer virtual table is created like this: +** +** CREATE VIRTUAL TABLE f USING fuzzer(); +** +** When it is created, the new fuzzer table must be supplied with the +** name of a "fuzzer data table", which must reside in the same database +** file as the new fuzzer table. The fuzzer data table contains the various +** transformations and their costs that the fuzzer logic uses to generate +** variations. +** +** The fuzzer data table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. +** +** Each row in the fuzzer data table represents a single character +** transformation. The left most column of the row (column 0) contains an +** integer value - the identifier of the ruleset to which the transformation +** rule belongs (see "MULTIPLE RULE SETS" below). The second column of the +** row (column 0) contains the input character or characters. The third +** column contains the output character or characters. And the fourth column +** contains the integer cost of making the transformation. For example: +** +** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); +** +** The first row inserted into the fuzzer data table by the SQL script +** above indicates that the cost of inserting a letter 'a' is 100. (All +** costs are integers. We recommend that costs be scaled so that the +** average cost is around 100.) The second INSERT statement creates a rule +** saying that the cost of deleting a single letter 'b' is 87. The third +** and fourth INSERT statements mean that the cost of transforming a +** single letter "o" into the two-letter sequence "oe" is 38 and that the +** cost of transforming "oe" back into "o" is 40. +** +** The contents of the fuzzer data table are loaded into main memory when +** a fuzzer table is first created, and may be internally reloaded by the +** system at any subsequent time. Therefore, the fuzzer data table should be +** populated before the fuzzer table is created and not modified thereafter. +** If you do need to modify the contents of the fuzzer data table, it is +** recommended that the associated fuzzer table be dropped, the fuzzer data +** table edited, and the fuzzer table recreated within a single transaction. +** Alternatively, the fuzzer data table can be edited then the database +** connection can be closed and reopened. +** +** Once it has been created, the fuzzer table can be queried as follows: +** +** SELECT word, distance FROM f +** WHERE word MATCH 'abcdefg' +** AND distance<200; +** +** This first query outputs the string "abcdefg" and all strings that +** can be derived from that string by appling the specified transformations. +** The strings are output together with their total transformation cost +** (called "distance") and appear in order of increasing cost. No string +** is output more than once. If there are multiple ways to transform the +** target string into the output string then the lowest cost transform is +** the one that is returned. In the example, the search is limited to +** strings with a total distance of less than 200. +** +** The fuzzer is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a fuzzer table will throw an error. +** +** It is important to put some kind of a limit on the fuzzer output. This +** can be either in the form of a LIMIT clause at the end of the query, +** or better, a "distance +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct fuzzer_vtab fuzzer_vtab; +typedef struct fuzzer_cursor fuzzer_cursor; +typedef struct fuzzer_rule fuzzer_rule; +typedef struct fuzzer_seen fuzzer_seen; +typedef struct fuzzer_stem fuzzer_stem; + +/* +** Various types. +** +** fuzzer_cost is the "cost" of an edit operation. +** +** fuzzer_len is the length of a matching string. +** +** fuzzer_ruleid is an ruleset identifier. +*/ +typedef int fuzzer_cost; +typedef signed char fuzzer_len; +typedef int fuzzer_ruleid; + +/* +** Limits +*/ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a rule string */ +#define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ +#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ +#define FUZZER_MX_OUTPUT_LENGTH 100 /* Maximum length of an output string */ + + +/* +** Each transformation rule is stored as an instance of this object. +** All rules are kept on a linked list sorted by rCost. +*/ +struct fuzzer_rule { + fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from */ + fuzzer_cost rCost; /* Cost of this transformation */ + fuzzer_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + fuzzer_ruleid iRuleset; /* The rule set to which this rule belongs */ + char zTo[4]; /* Transform to (extra space appended) */ +}; + +/* +** A stem object is used to generate variants. It is also used to record +** previously generated outputs. +** +** Every stem is added to a hash table as it is output. Generation of +** duplicate stems is suppressed. +** +** Active stems (those that might generate new outputs) are kepts on a linked +** list sorted by increasing cost. The cost is the sum of rBaseCost and +** pRule->rCost. +*/ +struct fuzzer_stem { + char *zBasis; /* Word being fuzzed */ + const fuzzer_rule *pRule; /* Current rule to apply */ + fuzzer_stem *pNext; /* Next stem in rCost order */ + fuzzer_stem *pHash; /* Next stem with same hash on zBasis */ + fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ + fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ + fuzzer_len nBasis; /* Length of the zBasis string */ + fuzzer_len n; /* Apply pRule at this character offset */ +}; + +/* +** A fuzzer virtual-table object +*/ +struct fuzzer_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zClassName; /* Name of this class. Default: "fuzzer" */ + fuzzer_rule *pRule; /* All active rules in this fuzzer */ + int nCursor; /* Number of active cursors */ +}; + +#define FUZZER_HASH 4001 /* Hash table size */ +#define FUZZER_NQUEUE 20 /* Number of slots on the stem queue */ + +/* A fuzzer cursor object */ +struct fuzzer_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid of the current word */ + fuzzer_vtab *pVtab; /* The virtual table this cursor belongs to */ + fuzzer_cost rLimit; /* Maximum cost of any term */ + fuzzer_stem *pStem; /* Stem with smallest rCostX */ + fuzzer_stem *pDone; /* Stems already processed to completion */ + fuzzer_stem *aQueue[FUZZER_NQUEUE]; /* Queue of stems with higher rCostX */ + int mxQueue; /* Largest used index in aQueue[] */ + char *zBuf; /* Temporary use buffer */ + int nBuf; /* Bytes allocated for zBuf */ + int nStem; /* Number of stems allocated */ + int iRuleset; /* Only process rules from this ruleset */ + fuzzer_rule nullRule; /* Null rule used first */ + fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ +}; + +/* +** The two input rule lists are both sorted in order of increasing +** cost. Merge them together into a single list, sorted by cost, and +** return a pointer to the head of that list. +*/ +static fuzzer_rule *fuzzerMergeRules(fuzzer_rule *pA, fuzzer_rule *pB){ + fuzzer_rule head; + fuzzer_rule *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCost<=pB->rCost ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Statement pStmt currently points to a row in the fuzzer data table. This +** function allocates and populates a fuzzer_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int fuzzerLoadOneRule( + fuzzer_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + fuzzer_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iRuleset = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + int nCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + fuzzer_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + *ppRule = 0; + return SQLITE_OK; + } + + if( nCost<=0 || nCost>FUZZER_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, FUZZER_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, FUZZER_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ + *pzErr = sqlite3_mprintf("%s: ruleset must be between 0 and %d", + p->zClassName, FUZZER_MX_RULEID + ); + rc = SQLITE_ERROR; + }else{ + + pRule = sqlite3_malloc64( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = pRule->zTo; + pRule->zFrom += nTo + 1; + pRule->nFrom = (fuzzer_len)nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = (fuzzer_len)nTo; + pRule->rCost = nCost; + pRule->iRuleset = (int)iRuleset; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Load the content of the fuzzer data table into memory. +*/ +static int fuzzerLoadRules( + sqlite3 *db, /* Database handle */ + fuzzer_vtab *p, /* Virtual fuzzer table to configure */ + const char *zDb, /* Database containing rules data */ + const char *zData, /* Table containing rules data */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + fuzzer_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zData); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, zData, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + fuzzer_rule *pRule = 0; + rc = fuzzerLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets fuzzer_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + fuzzer_rule *pX; + fuzzer_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = fuzzerMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *fuzzerDequote(const char *zIn){ + sqlite3_int64 nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc64(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, (size_t)(nIn+1)); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInnCursor==0 ); + while( p->pRule ){ + fuzzer_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** xConnect/xCreate method for the fuzzer module. Arguments are: +** +** argv[0] -> module name ("fuzzer") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> fuzzer rule table name +*/ +static int fuzzerConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + fuzzer_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", zModule + ); + rc = SQLITE_ERROR; + }else{ + sqlite3_int64 nModule; /* Length of zModule, in bytes */ + + nModule = strlen(zModule); + pNew = sqlite3_malloc64( sizeof(*pNew) + nModule + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zTab; /* Dequoted name of fuzzer data table */ + + memset(pNew, 0, sizeof(*pNew)); + pNew->zClassName = (char*)&pNew[1]; + memcpy(pNew->zClassName, zModule, (size_t)(nModule+1)); + + zTab = fuzzerDequote(argv[3]); + if( zTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = fuzzerLoadRules(db, pNew, zDb, zTab, pzErr); + sqlite3_free(zTab); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,ruleset)"); + } + if( rc!=SQLITE_OK ){ + fuzzerDisconnect((sqlite3_vtab *)pNew); + pNew = 0; + }else{ + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + } + } + + *ppVtab = (sqlite3_vtab *)pNew; + return rc; +} + +/* +** Open a new fuzzer cursor. +*/ +static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + fuzzer_vtab *p = (fuzzer_vtab*)pVTab; + fuzzer_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free all stems in a list. +*/ +static void fuzzerClearStemList(fuzzer_stem *pStem){ + while( pStem ){ + fuzzer_stem *pNext = pStem->pNext; + sqlite3_free(pStem); + pStem = pNext; + } +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void fuzzerClearCursor(fuzzer_cursor *pCur, int clearHash){ + int i; + fuzzerClearStemList(pCur->pStem); + fuzzerClearStemList(pCur->pDone); + for(i=0; iaQueue[i]); + pCur->rLimit = (fuzzer_cost)0; + if( clearHash && pCur->nStem ){ + pCur->mxQueue = 0; + pCur->pStem = 0; + pCur->pDone = 0; + memset(pCur->aQueue, 0, sizeof(pCur->aQueue)); + memset(pCur->apHash, 0, sizeof(pCur->apHash)); + } + pCur->nStem = 0; +} + +/* +** Close a fuzzer cursor. +*/ +static int fuzzerClose(sqlite3_vtab_cursor *cur){ + fuzzer_cursor *pCur = (fuzzer_cursor *)cur; + fuzzerClearCursor(pCur, 0); + sqlite3_free(pCur->zBuf); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Compute the current output term for a fuzzer_stem. +*/ +static int fuzzerRender( + fuzzer_stem *pStem, /* The stem to be rendered */ + char **pzBuf, /* Write results into this buffer. realloc if needed */ + int *pnBuf /* Size of the buffer */ +){ + const fuzzer_rule *pRule = pStem->pRule; + int n; /* Size of output term without nul-term */ + char *z; /* Buffer to assemble output term in */ + + n = pStem->nBasis + pRule->nTo - pRule->nFrom; + if( (*pnBuf)n; + z = *pzBuf; + if( n<0 ){ + memcpy(z, pStem->zBasis, pStem->nBasis+1); + }else{ + memcpy(z, pStem->zBasis, n); + memcpy(&z[n], pRule->zTo, pRule->nTo); + memcpy(&z[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], + pStem->nBasis-n-pRule->nFrom+1); + } + + assert( z[pStem->nBasis + pRule->nTo - pRule->nFrom]==0 ); + return SQLITE_OK; +} + +/* +** Compute a hash on zBasis. +*/ +static unsigned int fuzzerHash(const char *z){ + unsigned int h = 0; + while( *z ){ h = (h<<3) ^ (h>>29) ^ *(z++); } + return h % FUZZER_HASH; +} + +/* +** Current cost of a stem +*/ +static fuzzer_cost fuzzerCost(fuzzer_stem *pStem){ + return pStem->rCostX = pStem->rBaseCost + pStem->pRule->rCost; +} + +#if 0 +/* +** Print a description of a fuzzer_stem on stderr. +*/ +static void fuzzerStemPrint( + const char *zPrefix, + fuzzer_stem *pStem, + const char *zSuffix +){ + if( pStem->n<0 ){ + fprintf(stderr, "%s[%s](%d)-->self%s", + zPrefix, + pStem->zBasis, pStem->rBaseCost, + zSuffix + ); + }else{ + char *zBuf = 0; + int nBuf = 0; + if( fuzzerRender(pStem, &zBuf, &nBuf)!=SQLITE_OK ) return; + fprintf(stderr, "%s[%s](%d)-->{%s}(%d)%s", + zPrefix, + pStem->zBasis, pStem->rBaseCost, zBuf, pStem->, + zSuffix + ); + sqlite3_free(zBuf); + } +} +#endif + +/* +** Return 1 if the string to which the cursor is point has already +** been emitted. Return 0 if not. Return -1 on a memory allocation +** failures. +*/ +static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ + unsigned int h; + fuzzer_stem *pLookup; + + if( fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ + return -1; + } + h = fuzzerHash(pCur->zBuf); + pLookup = pCur->apHash[h]; + while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ + pLookup = pLookup->pHash; + } + return pLookup!=0; +} + +/* +** If argument pRule is NULL, this function returns false. +** +** Otherwise, it returns true if rule pRule should be skipped. A rule +** should be skipped if it does not belong to rule-set iRuleset, or if +** applying it to stem pStem would create a string longer than +** FUZZER_MX_OUTPUT_LENGTH bytes. +*/ +static int fuzzerSkipRule( + const fuzzer_rule *pRule, /* Determine whether or not to skip this */ + fuzzer_stem *pStem, /* Stem rule may be applied to */ + int iRuleset /* Rule-set used by the current query */ +){ + return pRule && ( + (pRule->iRuleset!=iRuleset) + || (pStem->nBasis + pRule->nTo - pRule->nFrom)>FUZZER_MX_OUTPUT_LENGTH + ); +} + +/* +** Advance a fuzzer_stem to its next value. Return 0 if there are +** no more values that can be generated by this fuzzer_stem. Return +** -1 on a memory allocation failure. +*/ +static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ + const fuzzer_rule *pRule; + while( (pRule = pStem->pRule)!=0 ){ + assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); + while( pStem->n < pStem->nBasis - pRule->nFrom ){ + pStem->n++; + if( pRule->nFrom==0 + || memcmp(&pStem->zBasis[pStem->n], pRule->zFrom, pRule->nFrom)==0 + ){ + /* Found a rewrite case. Make sure it is not a duplicate */ + int rc = fuzzerSeen(pCur, pStem); + if( rc<0 ) return -1; + if( rc==0 ){ + fuzzerCost(pStem); + return 1; + } + } + } + pStem->n = -1; + do{ + pRule = pRule->pNext; + }while( fuzzerSkipRule(pRule, pStem, pCur->iRuleset) ); + pStem->pRule = pRule; + if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; + } + return 0; +} + +/* +** The two input stem lists are both sorted in order of increasing +** rCostX. Merge them together into a single list, sorted by rCostX, and +** return a pointer to the head of that new list. +*/ +static fuzzer_stem *fuzzerMergeStems(fuzzer_stem *pA, fuzzer_stem *pB){ + fuzzer_stem head; + fuzzer_stem *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCostX<=pB->rCostX ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Load pCur->pStem with the lowest-cost stem. Return a pointer +** to the lowest-cost stem. +*/ +static fuzzer_stem *fuzzerLowestCostStem(fuzzer_cursor *pCur){ + fuzzer_stem *pBest, *pX; + int iBest; + int i; + + if( pCur->pStem==0 ){ + iBest = -1; + pBest = 0; + for(i=0; i<=pCur->mxQueue; i++){ + pX = pCur->aQueue[i]; + if( pX==0 ) continue; + if( pBest==0 || pBest->rCostX>pX->rCostX ){ + pBest = pX; + iBest = i; + } + } + if( pBest ){ + pCur->aQueue[iBest] = pBest->pNext; + pBest->pNext = 0; + pCur->pStem = pBest; + } + } + return pCur->pStem; +} + +/* +** Insert pNew into queue of pending stems. Then find the stem +** with the lowest rCostX and move it into pCur->pStem. +** list. The insert is done such the pNew is in the correct order +** according to fuzzer_stem.zBaseCost+fuzzer_stem.pRule->rCost. +*/ +static fuzzer_stem *fuzzerInsert(fuzzer_cursor *pCur, fuzzer_stem *pNew){ + fuzzer_stem *pX; + int i; + + /* If pCur->pStem exists and is greater than pNew, then make pNew + ** the new pCur->pStem and insert the old pCur->pStem instead. + */ + if( (pX = pCur->pStem)!=0 && pX->rCostX>pNew->rCostX ){ + pNew->pNext = 0; + pCur->pStem = pNew; + pNew = pX; + } + + /* Insert the new value */ + pNew->pNext = 0; + pX = pNew; + for(i=0; i<=pCur->mxQueue; i++){ + if( pCur->aQueue[i] ){ + pX = fuzzerMergeStems(pX, pCur->aQueue[i]); + pCur->aQueue[i] = 0; + }else{ + pCur->aQueue[i] = pX; + break; + } + } + if( i>pCur->mxQueue ){ + if( imxQueue = i; + pCur->aQueue[i] = pX; + }else{ + assert( pCur->mxQueue==FUZZER_NQUEUE-1 ); + pX = fuzzerMergeStems(pX, pCur->aQueue[FUZZER_NQUEUE-1]); + pCur->aQueue[FUZZER_NQUEUE-1] = pX; + } + } + + return fuzzerLowestCostStem(pCur); +} + +/* +** Allocate a new fuzzer_stem. Add it to the hash table but do not +** link it into either the pCur->pStem or pCur->pDone lists. +*/ +static fuzzer_stem *fuzzerNewStem( + fuzzer_cursor *pCur, + const char *zWord, + fuzzer_cost rBaseCost +){ + fuzzer_stem *pNew; + fuzzer_rule *pRule; + unsigned int h; + + pNew = sqlite3_malloc64( sizeof(*pNew) + strlen(zWord) + 1 ); + if( pNew==0 ) return 0; + memset(pNew, 0, sizeof(*pNew)); + pNew->zBasis = (char*)&pNew[1]; + pNew->nBasis = (fuzzer_len)strlen(zWord); + memcpy(pNew->zBasis, zWord, pNew->nBasis+1); + pRule = pCur->pVtab->pRule; + while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ + pRule = pRule->pNext; + } + pNew->pRule = pRule; + pNew->n = -1; + pNew->rBaseCost = pNew->rCostX = rBaseCost; + h = fuzzerHash(pNew->zBasis); + pNew->pHash = pCur->apHash[h]; + pCur->apHash[h] = pNew; + pCur->nStem++; + return pNew; +} + + +/* +** Advance a cursor to its next row of output +*/ +static int fuzzerNext(sqlite3_vtab_cursor *cur){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + int rc; + fuzzer_stem *pStem, *pNew; + + pCur->iRowid++; + + /* Use the element the cursor is currently point to to create + ** a new stem and insert the new stem into the priority queue. + */ + pStem = pCur->pStem; + if( pStem->rCostX>0 ){ + rc = fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + pNew = fuzzerNewStem(pCur, pCur->zBuf, pStem->rCostX); + if( pNew ){ + if( fuzzerAdvance(pCur, pNew)==0 ){ + pNew->pNext = pCur->pDone; + pCur->pDone = pNew; + }else{ + if( fuzzerInsert(pCur, pNew)==pNew ){ + return SQLITE_OK; + } + } + }else{ + return SQLITE_NOMEM; + } + } + + /* Adjust the priority queue so that the first element of the + ** stem list is the next lowest cost word. + */ + while( (pStem = pCur->pStem)!=0 ){ + int res = fuzzerAdvance(pCur, pStem); + if( res<0 ){ + return SQLITE_NOMEM; + }else if( res>0 ){ + pCur->pStem = 0; + pStem = fuzzerInsert(pCur, pStem); + if( (rc = fuzzerSeen(pCur, pStem))!=0 ){ + if( rc<0 ) return SQLITE_NOMEM; + continue; + } + return SQLITE_OK; /* New word found */ + } + pCur->pStem = 0; + pStem->pNext = pCur->pDone; + pCur->pDone = pStem; + if( fuzzerLowestCostStem(pCur) ){ + rc = fuzzerSeen(pCur, pCur->pStem); + if( rc<0 ) return SQLITE_NOMEM; + if( rc==0 ){ + return SQLITE_OK; + } + } + } + + /* Reach this point only if queue has been exhausted and there is + ** nothing left to be output. */ + pCur->rLimit = (fuzzer_cost)0; + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any fuzzerColumn, fuzzerRowid, or fuzzerEof call. +*/ +static int fuzzerFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; + const char *zWord = ""; + fuzzer_stem *pStem; + int idx; + + fuzzerClearCursor(pCur, 1); + pCur->rLimit = 2147483647; + idx = 0; + if( idxNum & 1 ){ + zWord = (const char*)sqlite3_value_text(argv[0]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + pCur->nullRule.pNext = pCur->pVtab->pRule; + pCur->nullRule.rCost = 0; + pCur->nullRule.nFrom = 0; + pCur->nullRule.nTo = 0; + pCur->nullRule.zFrom = ""; + pCur->iRowid = 1; + assert( pCur->pStem==0 ); + + /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this + ** query will return zero rows. */ + if( (int)strlen(zWord)pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); + if( pStem==0 ) return SQLITE_NOMEM; + pStem->pRule = &pCur->nullRule; + pStem->n = pStem->nBasis; + }else{ + pCur->rLimit = 0; + } + + return SQLITE_OK; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int fuzzerColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + if( i==0 ){ + /* the "word" column */ + if( fuzzerRender(pCur->pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ + return SQLITE_NOMEM; + } + sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT); + }else if( i==1 ){ + /* the "distance" column */ + sqlite3_result_int(ctx, pCur->pStem->rCostX); + }else{ + /* All other columns are NULL */ + sqlite3_result_null(ctx); + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int fuzzerRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** When the fuzzer_cursor.rLimit value is 0 or less, that is a signal +** that the cursor has nothing more to output. +*/ +static int fuzzerEof(sqlite3_vtab_cursor *cur){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + return pCur->rLimit<=(fuzzer_cost)0; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) ruleid == $ruleid +** +** The distance< and distance<= are both treated as distance<=. +** The query plan number is a bit vector: +** +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. +*/ +static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int iPlan = 0; + int iDistTerm = -1; + int iRulesetTerm = -1; + int i; + int seenMatch = 0; + const struct sqlite3_index_constraint *pConstraint; + double rCost = 1e12; + + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + seenMatch = 1; + } + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 1e6; + } + if( (iPlan & 2)==0 + && pConstraint->iColumn==1 + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) + ){ + iPlan |= 2; + iDistTerm = i; + rCost /= 10.0; + } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iRulesetTerm = i; + rCost /= 10.0; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iRulesetTerm].argvIndex = idx; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==1 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + if( seenMatch && (iPlan&1)==0 ) rCost = 1e99; + pIdxInfo->estimatedCost = rCost; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "fuzzer". +*/ +static sqlite3_module fuzzerModule = { + 0, /* iVersion */ + fuzzerConnect, + fuzzerConnect, + fuzzerBestIndex, + fuzzerDisconnect, + fuzzerDisconnect, + fuzzerOpen, /* xOpen - open a cursor */ + fuzzerClose, /* xClose - close a cursor */ + fuzzerFilter, /* xFilter - configure scan constraints */ + fuzzerNext, /* xNext - advance a cursor */ + fuzzerEof, /* xEof - check for end of scan */ + fuzzerColumn, /* xColumn - read data */ + fuzzerRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fuzzer_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); +#endif + return rc; +} diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c new file mode 100644 index 0000000000000000000000000000000000000000..99489fe9caebe4138e8c365fca2bb66ee764978b --- /dev/null +++ b/ext/misc/ieee754.c @@ -0,0 +1,324 @@ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(2,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('mininum positive nornal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + if( 1-e >= 64 ){ + m = 0; + }else{ + m >>= 1-e; + } + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_inc", 2, 0, ieee754inc }, + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* memstat_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a memstat virtual table +*/ +typedef struct memstat_vtab memstat_vtab; +struct memstat_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this memstat vtab */ +}; + +/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct memstat_cursor memstat_cursor; +struct memstat_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + int iRowid; /* Current row in aMemstatColumn[] */ + int iDb; /* Which schema we are looking at */ + int nDb; /* Number of schemas */ + char **azDb; /* Names of all schemas */ + sqlite3_int64 aVal[2]; /* Result values */ +}; + +/* +** The memstatConnect() method is invoked to create a new +** memstat_vtab that describes the memstat virtual table. +** +** Think of this routine as the constructor for memstat_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the memstat_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against memstat will look like. +*/ +static int memstatConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + memstat_vtab *pNew; + int rc; + +/* Column numbers */ +#define MSV_COLUMN_NAME 0 /* Name of quantity being measured */ +#define MSV_COLUMN_SCHEMA 1 /* schema name */ +#define MSV_COLUMN_VALUE 2 /* Current value */ +#define MSV_COLUMN_HIWTR 3 /* Highwater mark */ + + rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for memstat_cursor objects. +*/ +static int memstatDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new memstat_cursor object. +*/ +static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + memstat_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((memstat_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Clear all the schema names from a cursor +*/ +static void memstatClearSchema(memstat_cursor *pCur){ + int i; + if( pCur->azDb==0 ) return; + for(i=0; inDb; i++){ + sqlite3_free(pCur->azDb[i]); + } + sqlite3_free(pCur->azDb); + pCur->azDb = 0; + pCur->nDb = 0; +} + +/* +** Fill in the azDb[] array for the cursor. +*/ +static int memstatFindSchemas(memstat_cursor *pCur){ + sqlite3_stmt *pStmt = 0; + int rc; + if( pCur->nDb ) return SQLITE_OK; + rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + sqlite3_finalize(pStmt); + return rc; + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char **az, *z; + az = sqlite3_realloc64(pCur->azDb, sizeof(char*)*(pCur->nDb+1)); + if( az==0 ){ + memstatClearSchema(pCur); + return SQLITE_NOMEM; + } + pCur->azDb = az; + z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( z==0 ){ + memstatClearSchema(pCur); + return SQLITE_NOMEM; + } + pCur->azDb[pCur->nDb] = z; + pCur->nDb++; + } + sqlite3_finalize(pStmt); + return SQLITE_OK; +} + + +/* +** Destructor for a memstat_cursor. +*/ +static int memstatClose(sqlite3_vtab_cursor *cur){ + memstat_cursor *pCur = (memstat_cursor*)cur; + memstatClearSchema(pCur); + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Allowed values for aMemstatColumn[].eType +*/ +#define MSV_GSTAT 0 /* sqlite3_status64() information */ +#define MSV_DB 1 /* sqlite3_db_status() information */ +#define MSV_ZIPVFS 2 /* ZIPVFS file-control with 64-bit return */ + +/* +** An array of quantities that can be measured and reported by +** this virtual table +*/ +static const struct MemstatColumns { + const char *zName; /* Symbolic name */ + unsigned char eType; /* Type of interface */ + unsigned char mNull; /* Bitmask of which columns are NULL */ + /* 2: dbname, 4: current, 8: hiwtr */ + int eOp; /* Opcode */ +} aMemstatColumn[] = { + {"MEMORY_USED", MSV_GSTAT, 2, SQLITE_STATUS_MEMORY_USED }, + {"MALLOC_SIZE", MSV_GSTAT, 6, SQLITE_STATUS_MALLOC_SIZE }, + {"MALLOC_COUNT", MSV_GSTAT, 2, SQLITE_STATUS_MALLOC_COUNT }, + {"PAGECACHE_USED", MSV_GSTAT, 2, SQLITE_STATUS_PAGECACHE_USED }, + {"PAGECACHE_OVERFLOW", MSV_GSTAT, 2, SQLITE_STATUS_PAGECACHE_OVERFLOW }, + {"PAGECACHE_SIZE", MSV_GSTAT, 6, SQLITE_STATUS_PAGECACHE_SIZE }, + {"PARSER_STACK", MSV_GSTAT, 6, SQLITE_STATUS_PARSER_STACK }, + {"DB_LOOKASIDE_USED", MSV_DB, 2, SQLITE_DBSTATUS_LOOKASIDE_USED }, + {"DB_LOOKASIDE_HIT", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_HIT }, + {"DB_LOOKASIDE_MISS_SIZE", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE}, + {"DB_LOOKASIDE_MISS_FULL", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL}, + {"DB_CACHE_USED", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_USED }, +#if SQLITE_VERSION_NUMBER >= 3140000 + {"DB_CACHE_USED_SHARED", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_USED_SHARED }, +#endif + {"DB_SCHEMA_USED", MSV_DB, 10, SQLITE_DBSTATUS_SCHEMA_USED }, + {"DB_STMT_USED", MSV_DB, 10, SQLITE_DBSTATUS_STMT_USED }, + {"DB_CACHE_HIT", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_HIT }, + {"DB_CACHE_MISS", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_MISS }, + {"DB_CACHE_WRITE", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_WRITE }, +#if SQLITE_VERSION_NUMBER >= 3230000 + {"DB_CACHE_SPILL", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_SPILL }, +#endif + {"DB_DEFERRED_FKS", MSV_DB, 10, SQLITE_DBSTATUS_DEFERRED_FKS }, +#ifdef SQLITE_ENABLE_ZIPVFS + {"ZIPVFS_CACHE_USED", MSV_ZIPVFS, 8, 231454 }, + {"ZIPVFS_CACHE_HIT", MSV_ZIPVFS, 8, 231455 }, + {"ZIPVFS_CACHE_MISS", MSV_ZIPVFS, 8, 231456 }, + {"ZIPVFS_CACHE_WRITE", MSV_ZIPVFS, 8, 231457 }, + {"ZIPVFS_DIRECT_READ", MSV_ZIPVFS, 8, 231458 }, + {"ZIPVFS_DIRECT_BYTES", MSV_ZIPVFS, 8, 231459 }, +#endif /* SQLITE_ENABLE_ZIPVFS */ +}; +#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0])) + +/* +** Advance a memstat_cursor to its next row of output. +*/ +static int memstatNext(sqlite3_vtab_cursor *cur){ + memstat_cursor *pCur = (memstat_cursor*)cur; + int i; + assert( pCur->iRowid<=MSV_NROW ); + while(1){ + i = (int)pCur->iRowid - 1; + if( i<0 || (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){ + pCur->iRowid++; + if( pCur->iRowid>MSV_NROW ) return SQLITE_OK; /* End of the table */ + pCur->iDb = 0; + i++; + } + pCur->aVal[0] = 0; + pCur->aVal[1] = 0; + switch( aMemstatColumn[i].eType ){ + case MSV_GSTAT: { + if( sqlite3_libversion_number()>=3010000 ){ + sqlite3_status64(aMemstatColumn[i].eOp, + &pCur->aVal[0], &pCur->aVal[1],0); + }else{ + int xCur, xHiwtr; + sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0); + pCur->aVal[0] = xCur; + pCur->aVal[1] = xHiwtr; + } + break; + } + case MSV_DB: { + int xCur, xHiwtr; + sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0); + pCur->aVal[0] = xCur; + pCur->aVal[1] = xHiwtr; + break; + } + case MSV_ZIPVFS: { + int rc; + rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb], + aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]); + if( rc!=SQLITE_OK ) continue; + break; + } + } + break; + } + return SQLITE_OK; +} + + +/* +** Return values of columns for the row at which the memstat_cursor +** is currently pointing. +*/ +static int memstatColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int iCol /* Which column to return */ +){ + memstat_cursor *pCur = (memstat_cursor*)cur; + int i; + assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW ); + i = (int)pCur->iRowid - 1; + if( (aMemstatColumn[i].mNull & (1<azDb[pCur->iDb], -1, 0); + break; + } + case MSV_COLUMN_VALUE: { + sqlite3_result_int64(ctx, pCur->aVal[0]); + break; + } + case MSV_COLUMN_HIWTR: { + sqlite3_result_int64(ctx, pCur->aVal[1]); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + memstat_cursor *pCur = (memstat_cursor*)cur; + *pRowid = pCur->iRowid*1000 + pCur->iDb; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int memstatEof(sqlite3_vtab_cursor *cur){ + memstat_cursor *pCur = (memstat_cursor*)cur; + return pCur->iRowid>MSV_NROW; +} + +/* +** This method is called to "rewind" the memstat_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to memstatColumn() or memstatRowid() or +** memstatEof(). +*/ +static int memstatFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + memstat_cursor *pCur = (memstat_cursor *)pVtabCursor; + int rc = memstatFindSchemas(pCur); + if( rc ) return rc; + pCur->iRowid = 0; + pCur->iDb = 0; + return memstatNext(pVtabCursor); +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the memstat virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int memstatBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + pIdxInfo->estimatedCost = (double)500; + pIdxInfo->estimatedRows = 500; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** memstat virtual table. +*/ +static sqlite3_module memstatModule = { + 0, /* iVersion */ + 0, /* xCreate */ + memstatConnect, /* xConnect */ + memstatBestIndex, /* xBestIndex */ + memstatDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + memstatOpen, /* xOpen - open a cursor */ + memstatClose, /* xClose - close a cursor */ + memstatFilter, /* xFilter - configure scan constraints */ + memstatNext, /* xNext - advance a cursor */ + memstatEof, /* xEof - check for end of scan */ + memstatColumn, /* xColumn - read data */ + memstatRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3MemstatVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0); +#endif + return rc; +} + +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_memstat_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3MemstatVtabInit(db); +#endif + return rc; +} +#endif /* SQLITE_CORE */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */ diff --git a/ext/misc/memtrace.c b/ext/misc/memtrace.c new file mode 100644 index 0000000000000000000000000000000000000000..19ff4c4170550efba6c7fb787654750ab2b14523 --- /dev/null +++ b/ext/misc/memtrace.c @@ -0,0 +1,108 @@ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; + +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} + +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } + } + memtraceOut = 0; + return rc; +} diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c new file mode 100644 index 0000000000000000000000000000000000000000..83fc9468e69ae654e93ef839d86164f9992e44a1 --- /dev/null +++ b/ext/misc/memvfs.c @@ -0,0 +1,575 @@ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an in-memory VFS implementation. The application supplies +** a chunk of memory to hold the database file. +** +** Because there is place to store a rollback or wal journal, the database +** must use one of journal_mode=MEMORY or journal_mode=NONE. +** +** USAGE: +** +** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db, +** SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI, +** "memvfs"); +** +** These are the query parameters: +** +** ptr= The address of the memory buffer that holds the database. +** +** sz= The current size the database file +** +** maxsz= The maximum size of the database. In other words, the +** amount of space allocated for the ptr= buffer. +** +** freeonclose= If true, then sqlite3_free() is called on the ptr= +** value when the connection closes. +** +** The ptr= and sz= query parameters are required. If maxsz= is omitted, +** then it defaults to the sz= value. Parameter values can be in either +** decimal or hexadecimal. The filename in the URI is ignored. +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szMax; /* Space allocated to aData */ + unsigned char *aData; /* content of the file */ + int bFreeOnClose; /* Invoke sqlite3_free() on aData at close */ +}; + +/* +** Methods for MemFile +*/ +static int memClose(sqlite3_file*); +static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memTruncate(sqlite3_file*, sqlite3_int64 size); +static int memSync(sqlite3_file*, int flags); +static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memLock(sqlite3_file*, int); +static int memUnlock(sqlite3_file*, int); +static int memCheckReservedLock(sqlite3_file*, int *pResOut); +static int memFileControl(sqlite3_file*, int op, void *pArg); +static int memSectorSize(sqlite3_file*); +static int memDeviceCharacteristics(sqlite3_file*); +static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int memShmLock(sqlite3_file*, int offset, int n, int flags); +static void memShmBarrier(sqlite3_file*); +static int memShmUnmap(sqlite3_file*, int deleteFlag); +static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memDlOpen(sqlite3_vfs*, const char *zFilename); +static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memDlClose(sqlite3_vfs*, void*); +static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memSleep(sqlite3_vfs*, int microseconds); +static int memCurrentTime(sqlite3_vfs*, double*); +static int memGetLastError(sqlite3_vfs*, int, char *); +static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs mem_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memvfs", /* zName */ + 0, /* pAppData (set when registered) */ + memOpen, /* xOpen */ + memDelete, /* xDelete */ + memAccess, /* xAccess */ + memFullPathname, /* xFullPathname */ + memDlOpen, /* xDlOpen */ + memDlError, /* xDlError */ + memDlSym, /* xDlSym */ + memDlClose, /* xDlClose */ + memRandomness, /* xRandomness */ + memSleep, /* xSleep */ + memCurrentTime, /* xCurrentTime */ + memGetLastError, /* xGetLastError */ + memCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods mem_io_methods = { + 3, /* iVersion */ + memClose, /* xClose */ + memRead, /* xRead */ + memWrite, /* xWrite */ + memTruncate, /* xTruncate */ + memSync, /* xSync */ + memFileSize, /* xFileSize */ + memLock, /* xLock */ + memUnlock, /* xUnlock */ + memCheckReservedLock, /* xCheckReservedLock */ + memFileControl, /* xFileControl */ + memSectorSize, /* xSectorSize */ + memDeviceCharacteristics, /* xDeviceCharacteristics */ + memShmMap, /* xShmMap */ + memShmLock, /* xShmLock */ + memShmBarrier, /* xShmBarrier */ + memShmUnmap, /* xShmUnmap */ + memFetch, /* xFetch */ + memUnfetch /* xUnfetch */ +}; + + + +/* +** Close an mem-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. +*/ +static int memClose(sqlite3_file *pFile){ + MemFile *p = (MemFile *)pFile; + if( p->bFreeOnClose ) sqlite3_free(p->aData); + return SQLITE_OK; +} + +/* +** Read data from an mem-file. +*/ +static int memRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + memcpy(zBuf, p->aData+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Write data to an mem-file. +*/ +static int memWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + if( iOfst+iAmt>p->sz ){ + if( iOfst+iAmt>p->szMax ) return SQLITE_FULL; + if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); + p->sz = iOfst+iAmt; + } + memcpy(p->aData+iOfst, z, iAmt); + return SQLITE_OK; +} + +/* +** Truncate an mem-file. +*/ +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ + MemFile *p = (MemFile *)pFile; + if( size>p->sz ){ + if( size>p->szMax ) return SQLITE_FULL; + memset(p->aData+p->sz, 0, size-p->sz); + } + p->sz = size; + return SQLITE_OK; +} + +/* +** Sync an mem-file. +*/ +static int memSync(sqlite3_file *pFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an mem-file. +*/ +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an mem-file. +*/ +static int memLock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Unlock an mem-file. +*/ +static int memUnlock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an mem-file. +*/ +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an mem-file. +*/ +static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return the sector-size in bytes for an mem-file. +*/ +static int memSectorSize(sqlite3_file *pFile){ + return 1024; +} + +/* +** Return the device characteristic flags supported by an mem-file. +*/ +static int memDeviceCharacteristics(sqlite3_file *pFile){ + return SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL; +} + +/* Create a shared memory file mapping */ +static int memShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + return SQLITE_IOERR_SHMMAP; +} + +/* Perform locking on a shared-memory segment */ +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + return SQLITE_IOERR_SHMLOCK; +} + +/* Memory barrier operation on shared memory */ +static void memShmBarrier(sqlite3_file *pFile){ + return; +} + +/* Unmap a shared memory segment */ +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int memFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemFile *p = (MemFile *)pFile; + *pp = (void*)(p->aData + iOfst); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + MemFile *p = (MemFile*)pFile; + memset(p, 0, sizeof(*p)); + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; + p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); + if( p->aData==0 ) return SQLITE_CANTOPEN; + p->sz = sqlite3_uri_int64(zName,"sz",0); + if( p->sz<0 ) return SQLITE_CANTOPEN; + p->szMax = sqlite3_uri_int64(zName,"max",p->sz); + if( p->szMaxsz ) return SQLITE_CANTOPEN; + p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0); + pFile->pMethods = &mem_io_methods; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int memAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} + +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +#ifdef MEMVFS_TEST +/* +** memvfs_from_file(FILENAME, MAXSIZE) +** +** This an SQL function used to help in testing the memvfs VFS. The +** function reads the content of a file into memory and then returns +** a URI that can be handed to ATTACH to attach the memory buffer as +** a database. Example: +** +** ATTACH memvfs_from_file('test.db',1048576) AS inmem; +** +** The optional MAXSIZE argument gives the size of the memory allocation +** used to hold the database. If omitted, it defaults to the size of the +** file on disk. +*/ +#include +static void memvfsFromFileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *p; + sqlite3_int64 sz; + sqlite3_int64 szMax; + FILE *in; + const char *zFilename = (const char*)sqlite3_value_text(argv[0]); + char *zUri; + + if( zFilename==0 ) return; + in = fopen(zFilename, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + szMax = sz = ftell(in); + rewind(in); + if( argc>=2 ){ + szMax = sqlite3_value_int64(argv[1]); + if( szMaxzName,"memvfs")!=0 ) return; + rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); + if( rc ) return; + fwrite(p->aData, 1, (size_t)p->sz, out); + fclose(out); +} +#endif /* MEMVFS_TEST */ + +#ifdef MEMVFS_TEST +/* Called for each new database connection */ +static int memvfsRegister( + sqlite3 *db, + char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0, + memvfsFromFileFunc, 0, 0); + sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0, + memvfsFromFileFunc, 0, 0); + sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0, + memvfsToFileFunc, 0, 0); + return SQLITE_OK; +} +#endif /* MEMVFS_TEST */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_memvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + mem_vfs.pAppData = sqlite3_vfs_find(0); + if( mem_vfs.pAppData==0 ) return SQLITE_ERROR; + mem_vfs.szOsFile = sizeof(MemFile); + rc = sqlite3_vfs_register(&mem_vfs, 1); +#ifdef MEMVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); + } + if( rc==SQLITE_OK ){ + rc = memvfsRegister(db, pzErrMsg, pApi); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/mmapwarm.c b/ext/misc/mmapwarm.c new file mode 100644 index 0000000000000000000000000000000000000000..851f8b0eb7a39a9e77b0a6e0ddac805980685bfd --- /dev/null +++ b/ext/misc/mmapwarm.c @@ -0,0 +1,108 @@ +/* +** 2017-09-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + +#include "sqlite3.h" + + +/* +** This function is used to touch each page of a mapping of a memory +** mapped SQLite database. Assuming that the system has sufficient free +** memory and supports sufficiently large mappings, this causes the OS +** to cache the entire database in main memory, making subsequent +** database accesses faster. +** +** If the second parameter to this function is not NULL, it is the name of +** the specific database to operate on (i.e. "main" or the name of an +** attached database). +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if the file is not memory-mapped, or if +** the mapping does not span the entire file. If an error does occur, a +** transaction may be left open on the database file. +** +** It is illegal to call this function when the database handle has an +** open transaction. SQLITE_MISUSE is returned in this case. +*/ +int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; + char *zSql = 0; + int pgsz = 0; + unsigned int nTotal = 0; + + if( 0==sqlite3_get_autocommit(db) ) return SQLITE_MISUSE; + + /* Open a read-only transaction on the file in question */ + zSql = sqlite3_mprintf("BEGIN; SELECT * FROM %s%q%ssqlite_schema", + (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "") + ); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + + /* Find the SQLite page size of the file */ + if( rc==SQLITE_OK ){ + zSql = sqlite3_mprintf("PRAGMA %s%q%spage_size", + (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "") + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pPgsz = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pPgsz, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + if( sqlite3_step(pPgsz)==SQLITE_ROW ){ + pgsz = sqlite3_column_int(pPgsz, 0); + } + rc = sqlite3_finalize(pPgsz); + } + if( rc==SQLITE_OK && pgsz==0 ){ + rc = SQLITE_ERROR; + } + } + } + + /* Touch each mmap'd page of the file */ + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_file *pFd = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFd); + if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){ + sqlite3_int64 iPg = 1; + sqlite3_io_methods const *p = pFd->pMethods; + while( 1 ){ + unsigned char *pMap; + rc = p->xFetch(pFd, pgsz*iPg, pgsz, (void**)&pMap); + if( rc!=SQLITE_OK || pMap==0 ) break; + + nTotal += (unsigned int)pMap[0]; + nTotal += (unsigned int)pMap[pgsz-1]; + + rc = p->xUnfetch(pFd, pgsz*iPg, (void*)pMap); + if( rc!=SQLITE_OK ) break; + iPg++; + } + sqlite3_log(SQLITE_OK, + "sqlite3_mmap_warm_cache: Warmed up %d pages of %s", iPg==1?0:iPg, + sqlite3_db_filename(db, zDb) + ); + } + + rc2 = sqlite3_exec(db, "END", 0, 0, 0); + if( rc==SQLITE_OK ) rc = rc2; + } + + (void)nTotal; + return rc; +} diff --git a/ext/misc/nextchar.c b/ext/misc/nextchar.c new file mode 100644 index 0000000000000000000000000000000000000000..60fa3db94dc0530f9b1379a7ec589e37a4877faa --- /dev/null +++ b/ext/misc/nextchar.c @@ -0,0 +1,314 @@ +/* +** 2013-02-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the next_char(A,T,F,W,C) SQL function. +** +** The next_char(A,T,F,W,C) function finds all valid "next" characters for +** string A given the vocabulary in T.F. If the W value exists and is a +** non-empty string, then it is an SQL expression that limits the entries +** in T.F that will be considered. If C exists and is a non-empty string, +** then it is the name of the collating sequence to use for comparison. If +** +** Only the first three arguments are required. If the C parameter is +** omitted or is NULL or is an empty string, then the default collating +** sequence of T.F is used for comparision. If the W parameter is omitted +** or is NULL or is an empty string, then no filtering of the output is +** done. +** +** The T.F column should be indexed using collation C or else this routine +** will be quite slow. +** +** For example, suppose an application has a dictionary like this: +** +** CREATE TABLE dictionary(word TEXT UNIQUE); +** +** Further suppose that for user keypad entry, it is desired to disable +** (gray out) keys that are not valid as the next character. If the +** the user has previously entered (say) 'cha' then to find all allowed +** next characters (and thereby determine when keys should not be grayed +** out) run the following query: +** +** SELECT next_char('cha','dictionary','word'); +** +** IMPLEMENTATION NOTES: +** +** The next_char function is implemented using recursive SQL that makes +** use of the table name and column name as part of a query. If either +** the table name or column name are keywords or contain special characters, +** then they should be escaped. For example: +** +** SELECT next_char('cha','[dictionary]','[word]'); +** +** This also means that the table name can be a subquery: +** +** SELECT next_char('cha','(SELECT word AS w FROM dictionary)','w'); +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** A structure to hold context of the next_char() computation across +** nested function calls. +*/ +typedef struct nextCharContext nextCharContext; +struct nextCharContext { + sqlite3 *db; /* Database connection */ + sqlite3_stmt *pStmt; /* Prepared statement used to query */ + const unsigned char *zPrefix; /* Prefix to scan */ + int nPrefix; /* Size of zPrefix in bytes */ + int nAlloc; /* Space allocated to aResult */ + int nUsed; /* Space used in aResult */ + unsigned int *aResult; /* Array of next characters */ + int mallocFailed; /* True if malloc fails */ + int otherError; /* True for any other failure */ +}; + +/* +** Append a result character if the character is not already in the +** result. +*/ +static void nextCharAppend(nextCharContext *p, unsigned c){ + int i; + for(i=0; inUsed; i++){ + if( p->aResult[i]==c ) return; + } + if( p->nUsed+1 > p->nAlloc ){ + unsigned int *aNew; + int n = p->nAlloc*2 + 30; + aNew = sqlite3_realloc64(p->aResult, n*sizeof(unsigned int)); + if( aNew==0 ){ + p->mallocFailed = 1; + return; + }else{ + p->aResult = aNew; + p->nAlloc = n; + } + } + p->aResult[p->nUsed++] = c; +} + +/* +** Write a character into z[] as UTF8. Return the number of bytes needed +** to hold the character +*/ +static int writeUtf8(unsigned char *z, unsigned c){ + if( c<0x00080 ){ + z[0] = (unsigned char)(c&0xff); + return 1; + } + if( c<0x00800 ){ + z[0] = 0xC0 + (unsigned char)((c>>6)&0x1F); + z[1] = 0x80 + (unsigned char)(c & 0x3F); + return 2; + } + if( c<0x10000 ){ + z[0] = 0xE0 + (unsigned char)((c>>12)&0x0F); + z[1] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[2] = 0x80 + (unsigned char)(c & 0x3F); + return 3; + } + z[0] = 0xF0 + (unsigned char)((c>>18) & 0x07); + z[1] = 0x80 + (unsigned char)((c>>12) & 0x3F); + z[2] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[3] = 0x80 + (unsigned char)(c & 0x3F); + return 4; +} + +/* +** Read a UTF8 character out of z[] and write it into *pOut. Return +** the number of bytes in z[] that were used to construct the character. +*/ +static int readUtf8(const unsigned char *z, unsigned *pOut){ + static const unsigned char validBits[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, + }; + unsigned c = z[0]; + if( c<0xc0 ){ + *pOut = c; + return 1; + }else{ + int n = 1; + c = validBits[c-0xc0]; + while( (z[n] & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & z[n++]); + } + if( c<0x80 || (c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ + c = 0xFFFD; + } + *pOut = c; + return n; + } +} + +/* +** The nextCharContext structure has been set up. Add all "next" characters +** to the result set. +*/ +static void findNextChars(nextCharContext *p){ + unsigned cPrev = 0; + unsigned char zPrev[8]; + int n, rc; + + for(;;){ + sqlite3_bind_text(p->pStmt, 1, (char*)p->zPrefix, p->nPrefix, + SQLITE_STATIC); + n = writeUtf8(zPrev, cPrev+1); + sqlite3_bind_text(p->pStmt, 2, (char*)zPrev, n, SQLITE_STATIC); + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_DONE ){ + sqlite3_reset(p->pStmt); + return; + }else if( rc!=SQLITE_ROW ){ + p->otherError = rc; + return; + }else{ + const unsigned char *zOut = sqlite3_column_text(p->pStmt, 0); + unsigned cNext; + n = readUtf8(zOut+p->nPrefix, &cNext); + sqlite3_reset(p->pStmt); + nextCharAppend(p, cNext); + cPrev = cNext; + if( p->mallocFailed ) return; + } + } +} + + +/* +** next_character(A,T,F,W) +** +** Return a string composted of all next possible characters after +** A for elements of T.F. If W is supplied, then it is an SQL expression +** that limits the elements in T.F that are considered. +*/ +static void nextCharFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + nextCharContext c; + const unsigned char *zTable = sqlite3_value_text(argv[1]); + const unsigned char *zField = sqlite3_value_text(argv[2]); + const unsigned char *zWhere; + const unsigned char *zCollName; + char *zWhereClause = 0; + char *zColl = 0; + char *zSql; + int rc; + + memset(&c, 0, sizeof(c)); + c.db = sqlite3_context_db_handle(context); + c.zPrefix = sqlite3_value_text(argv[0]); + c.nPrefix = sqlite3_value_bytes(argv[0]); + if( zTable==0 || zField==0 || c.zPrefix==0 ) return; + if( argc>=4 + && (zWhere = sqlite3_value_text(argv[3]))!=0 + && zWhere[0]!=0 + ){ + zWhereClause = sqlite3_mprintf("AND (%s)", zWhere); + if( zWhereClause==0 ){ + sqlite3_result_error_nomem(context); + return; + } + }else{ + zWhereClause = ""; + } + if( argc>=5 + && (zCollName = sqlite3_value_text(argv[4]))!=0 + && zCollName[0]!=0 + ){ + zColl = sqlite3_mprintf("collate \"%w\"", zCollName); + if( zColl==0 ){ + sqlite3_result_error_nomem(context); + if( zWhereClause[0] ) sqlite3_free(zWhereClause); + return; + } + }else{ + zColl = ""; + } + zSql = sqlite3_mprintf( + "SELECT %s FROM %s" + " WHERE %s>=(?1 || ?2) %s" + " AND %s<=(?1 || char(1114111)) %s" /* 1114111 == 0x10ffff */ + " %s" + " ORDER BY 1 %s ASC LIMIT 1", + zField, zTable, zField, zColl, zField, zColl, zWhereClause, zColl + ); + if( zWhereClause[0] ) sqlite3_free(zWhereClause); + if( zColl[0] ) sqlite3_free(zColl); + if( zSql==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + rc = sqlite3_prepare_v2(c.db, zSql, -1, &c.pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_result_error(context, sqlite3_errmsg(c.db), -1); + return; + } + findNextChars(&c); + if( c.mallocFailed ){ + sqlite3_result_error_nomem(context); + }else{ + unsigned char *pRes; + pRes = sqlite3_malloc64( c.nUsed*4 + 1 ); + if( pRes==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + int n = 0; + for(i=0; i +#include + +/* +** Implementation of the noop() function. +** +** The function returns its argument, unchanged. +*/ +static void noopfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + sqlite3_result_value(context, argv[0]); +} + +/* +** Implementation of the multitype_text() function. +** +** The function returns its argument. The result will always have a +** TEXT value. But if the original input is numeric, it will also +** have that numeric value. +*/ +static void multitypeTextFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + (void)argc; + (void)sqlite3_value_text(argv[0]); + sqlite3_result_value(context, argv[0]); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_noop_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "noop", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "noop_i", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_INNOCUOUS, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "noop_do", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_DIRECTONLY, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "noop_nd", 1, + SQLITE_UTF8, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "multitype_text", 1, + SQLITE_UTF8, + 0, multitypeTextFunc, 0, 0); + return rc; +} diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c new file mode 100644 index 0000000000000000000000000000000000000000..08d7733b96090aab0c3d871cf36d7312a6bce2e9 --- /dev/null +++ b/ext/misc/normalize.c @@ -0,0 +1,716 @@ +/* +** 2018-01-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the sqlite3_normalize() function. +** +** char *sqlite3_normalize(const char *zSql); +** +** This function takes an SQL string as input and returns a "normalized" +** version of that string in memory obtained from sqlite3_malloc64(). The +** caller is responsible for ensuring that the returned memory is freed. +** +** If a memory allocation error occurs, this routine returns NULL. +** +** The normalization consists of the following transformations: +** +** (1) Convert every literal (string, blob literal, numeric constant, +** or "NULL" constant) into a ? +** +** (2) Remove all superfluous whitespace, including comments. Change +** all required whitespace to a single space character. +** +** (3) Lowercase all ASCII characters. +** +** (4) If an IN or NOT IN operator is followed by a list of 1 or more +** values, convert that list into "(?,?,?)". +** +** The purpose of normalization is two-fold: +** +** (1) Sanitize queries by removing potentially private or sensitive +** information contained in literals. +** +** (2) Identify structurally identical queries by comparing their +** normalized forms. +** +** Command-Line Utility +** -------------------- +** +** This file also contains code for a command-line utility that converts +** SQL queries in text files into their normalized forms. To build the +** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI +** and link it against the SQLite library. +*/ +#include +#include + +/* +** Implementation note: +** +** Much of the tokenizer logic is copied out of the tokenize.c source file +** of SQLite. That logic could be simplified for this particular application, +** but that would impose a risk of introducing subtle errors. It is best to +** keep the code as close to the original as possible. +** +** The tokenize code is in sync with the SQLite core as of 2018-01-08. +** Any future changes to the core tokenizer might require corresponding +** adjustments to the tokenizer logic in this module. +*/ + + +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +}; + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +** +** SQLite only considers US-ASCII (or EBCDIC) characters. We do not +** handle case conversions for the UTF character set since the tables +** involved are nearly as big or bigger than SQLite itself. +*/ +static const unsigned char sqlite3UpperToLower[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255 +}; + +/* +** The following 256 byte lookup table is used to support SQLites built-in +** equivalents to the following standard library functions: +** +** isspace() 0x01 +** isalpha() 0x02 +** isdigit() 0x04 +** isalnum() 0x06 +** isxdigit() 0x08 +** toupper() 0x20 +** SQLite identifier character 0x40 +** Quote character 0x80 +** +** Bit 0x20 is set if the mapped character requires translation to upper +** case. i.e. if the character is a lower-case ASCII character. +** If x is a lower-case ASCII character, then its upper-case equivalent +** is (x - 0x20). Therefore toupper() can be implemented as: +** +** (x & ~(map[x]&0x20)) +** +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] +** array. tolower() is used more often than toupper() by SQLite. +** +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +*/ +static const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ + 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; +#define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20)) +#define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01) +#define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06) +#define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02) +#define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) +#define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) +#define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +#define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) + +/* +** Ignore testcase() macros +*/ +#define testcase(X) + +/* +** Token values +*/ +#define TK_SPACE 0 +#define TK_NAME 1 +#define TK_LITERAL 2 +#define TK_PUNCT 3 +#define TK_ERROR 4 + +#define TK_MINUS TK_PUNCT +#define TK_LP TK_PUNCT +#define TK_RP TK_PUNCT +#define TK_SEMI TK_PUNCT +#define TK_PLUS TK_PUNCT +#define TK_STAR TK_PUNCT +#define TK_SLASH TK_PUNCT +#define TK_REM TK_PUNCT +#define TK_EQ TK_PUNCT +#define TK_LE TK_PUNCT +#define TK_NE TK_PUNCT +#define TK_LSHIFT TK_PUNCT +#define TK_LT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_RSHIFT TK_PUNCT +#define TK_GT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_BITOR TK_PUNCT +#define TK_CONCAT TK_PUNCT +#define TK_COMMA TK_PUNCT +#define TK_BITAND TK_PUNCT +#define TK_BITNOT TK_PUNCT +#define TK_STRING TK_LITERAL +#define TK_ID TK_NAME +#define TK_ILLEGAL TK_ERROR +#define TK_DOT TK_PUNCT +#define TK_INTEGER TK_LITERAL +#define TK_FLOAT TK_LITERAL +#define TK_VARIABLE TK_LITERAL +#define TK_BLOB TK_LITERAL + +/* Disable nuisence warnings about case fall-through */ +#if !defined(deliberate_fall_through) && defined(__GCC__) && __GCC__>=7 +# define deliberate_fall_through __attribute__((fallthrough)); +#else +# define deliberate_fall_through +#endif + +/* +** Return the length (in bytes) of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +static int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case CC_MINUS: { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case CC_LP: { + *tokenType = TK_LP; + return 1; + } + case CC_RP: { + *tokenType = TK_RP; + return 1; + } + case CC_SEMI: { + *tokenType = TK_SEMI; + return 1; + } + case CC_PLUS: { + *tokenType = TK_PLUS; + return 1; + } + case CC_STAR: { + *tokenType = TK_STAR; + return 1; + } + case CC_SLASH: { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; + return i; + } + case CC_PERCENT: { + *tokenType = TK_REM; + return 1; + } + case CC_EQ: { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case CC_LT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case CC_GT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case CC_BANG: { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 1; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case CC_PIPE: { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case CC_COMMA: { + *tokenType = TK_COMMA; + return 1; + } + case CC_AND: { + *tokenType = TK_BITAND; + return 1; + } + case CC_TILDA: { + *tokenType = TK_BITNOT; + return 1; + } + case CC_QUOTE: { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case CC_DOT: { + if( !sqlite3Isdigit(z[1]) ){ + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + /* no break */ deliberate_fall_through + } + case CC_DIGIT: { + *tokenType = TK_INTEGER; + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } + for(i=0; sqlite3Isdigit(z[i]); i++){} + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case CC_QUOTE2: { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case CC_VARNUM: { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case CC_DOLLAR: + case CC_VARALPHA: { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return i; + } + case CC_X: { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + /* no break */ deliberate_fall_through + } + case CC_ID: { + i = 1; + break; + } + default: { + *tokenType = TK_ILLEGAL; + return 1; + } + } + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; +} + +char *sqlite3_normalize(const char *zSql){ + char *z; /* The output string */ + sqlite3_int64 nZ; /* Size of the output string in bytes */ + sqlite3_int64 nSql; /* Size of the input string in bytes */ + int i; /* Next character to read from zSql[] */ + int j; /* Next slot to fill in on z[] */ + int tokenType; /* Type of the next token */ + int n; /* Size of the next token */ + int k; /* Loop counter */ + + nSql = strlen(zSql); + nZ = nSql; + z = sqlite3_malloc64( nZ+2 ); + if( z==0 ) return 0; + for(i=j=0; zSql[i]; i += n){ + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_ERROR: { + sqlite3_free(z); + return 0; + } + case TK_LITERAL: { + z[j++] = '?'; + break; + } + case TK_PUNCT: + case TK_NAME: { + if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){ + if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3])) + || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4])) + ){ + /* NULL is a keyword in this case, not a literal value */ + }else{ + /* Here the NULL is a literal value */ + z[j++] = '?'; + break; + } + } + if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' '; + for(k=0; k0 && z[j-1]==' ' ){ j--; } + if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; } + z[j] = 0; + + /* Make a second pass converting "in(...)" where the "..." is not a + ** SELECT statement into "in(?,?,?)" */ + for(i=0; i5 ){ + memmove(z+n+5, z+n+k, j-(n+k)); + } + j = j-k+5; + z[j] = 0; + memcpy(z+n, "?,?,?", 5); + } + return z; +} + +/* +** For testing purposes, or to build a stand-alone SQL normalizer program, +** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link +** it against any SQLite library. The resulting command-line program will +** run sqlite3_normalize() over the text of all files named on the command- +** line and show the result on standard output. +*/ +#ifdef SQLITE_NORMALIZE_CLI +#include +#include + +/* +** Break zIn up into separate SQL statements and run sqlite3_normalize() +** on each one. Print the result of each run. +*/ +static void normalizeFile(char *zIn){ + int i; + if( zIn==0 ) return; + for(i=0; zIn[i]; i++){ + char cSaved; + if( zIn[i]!=';' ) continue; + cSaved = zIn[i+1]; + zIn[i+1] = 0; + if( sqlite3_complete(zIn) ){ + char *zOut = sqlite3_normalize(zIn); + if( zOut ){ + printf("%s\n", zOut); + sqlite3_free(zOut); + }else{ + fprintf(stderr, "ERROR: %s\n", zIn); + } + zIn[i+1] = cSaved; + zIn += i+1; + i = -1; + }else{ + zIn[i+1] = cSaved; + } + } +} + +/* +** The main routine for "sql_normalize". Read files named on the +** command-line and run the text of each through sqlite3_normalize(). +*/ +int main(int argc, char **argv){ + int i; + FILE *in; + char *zBuf = 0; + sqlite3_int64 sz, got; + + for(i=1; i +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg +){ + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); + } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); + } + pcacheBase.xUnpin(p, pPg, bDiscard); +} +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } + } + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } + } + pcachetraceOut = 0; + return rc; +} diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c new file mode 100644 index 0000000000000000000000000000000000000000..06865185df506010960f66d2248e6dbf8f4f043d --- /dev/null +++ b/ext/misc/percentile.c @@ -0,0 +1,503 @@ +/* +** 2013-05-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the percentile(Y,P) SQL function +** and similar as described below: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cont(Y,P) function is equivalent to +** percentile(Y,P/100.0). In other words, the fraction value in +** the second argument is in the range of 0 to 1 instead of 0 to 100. +** +** (15) A separate percentile_disc(Y,P) function is like +** percentile_cont(Y,P) except that instead of returning the weighted +** average of the nearest two input values, it returns the next lower +** value. So the percentile_disc(Y,P) will always return a value +** that was one of the inputs. +** +** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and +** percentile_disc(Y,P) can be used as window functions. +** +** Differences from standard SQL: +** +** * The percentile_cont(X,P) function is equivalent to the following in +** standard SQL: +** +** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) +** +** The SQLite syntax is much more compact. The standard SQL syntax +** is also supported if SQLite is compiled with the +** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. +** +** * No median(X) function exists in the SQL standard. App developers +** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". +** +** * No percentile(Y,P) function exists in the SQL standard. Instead of +** percential(Y,P), developers must write this: +** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that +** the fraction parameter to percentile() goes from 0 to 100 whereas +** the fraction parameter in SQL standard percentile_cont() goes from +** 0 to 1. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of these routines work +** by accumulating all values in an array of doubles, then sorting +** that array using quicksort before computing the answer. Thus +** the runtime is O(NlogN) where N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can imagine alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. +*/ +#if defined(SQLITE3_H) + /* no-op */ +#elif defined(SQLITE_STATIC_PERCENTILE) +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include +#include + +/* The following object is the group context for a single percentile() +** aggregate. Remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + unsigned nAlloc; /* Number of slots allocated for a[] */ + unsigned nUsed; /* Number of slots actually used in a[] */ + char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ + char bPctValid; /* True if rPct is valid */ + double rPct; /* Fraction. 0.0 to 1.0 */ + double *a; /* Array of Y values */ +}; + +/* Details of each function in the percentile family */ +typedef struct PercentileFunc PercentileFunc; +struct PercentileFunc { + const char *zName; /* Function name */ + char nArg; /* Number of arguments */ + char mxFrac; /* Maximum value of the "fraction" input */ + char bDiscrete; /* True for percentile_disc() */ +}; +static const PercentileFunc aPercentFunc[] = { + { "median", 1, 1, 0 }, + { "percentile", 2, 100, 0 }, + { "percentile_cont", 2, 1, 0 }, + { "percentile_disc", 2, 1, 1 }, +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int percentIsInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less. +*/ +static int percentSameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static int percentBinarySearch(Percentile *p, double y, int bExact){ + int iFirst = 0; /* First element of search range */ + int iLast = p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + int iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( xy ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + +/* +** Generate an error for a percentile function. +** +** The error format string must have exactly one occurrance of "%%s()" +** (with two '%' characters). That substring will be replaced by the name +** of the function. +*/ +static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + char *zMsg1; + char *zMsg2; + va_list ap; + + va_start(ap, zFormat); + zMsg1 = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0; + sqlite3_result_error(pCtx, zMsg2, -1); + sqlite3_free(zMsg1); + sqlite3_free(zMsg2); +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 0.5; + }else{ + /* Requirement 3: P must be a number between 0 and 100 */ + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac; + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 + ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not between 0.0 and %.1f", + (double)pFunc->mxFrac); + return; + } + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( !p->bPctValid ){ + p->rPct = rPct; + p->bPctValid = 1; + }else if( !percentSameValue(p->rPct,rPct) ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not the same for all input rows"); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + percentError(pCtx, "input to %%s() is not numeric"); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + percentError(pCtx, "Inf input to %%s()"); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + unsigned n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc64(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; + p->bSorted = 1; + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + int i; + i = percentBinarySearch(p, y, 0); + if( i<(int)p->nUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; + p->bSorted = 0; + } +} + +/* +** Interchange two doubles. +*/ +#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} + +/* +** Sort an array of doubles. +** +** Algorithm: quicksort +** +** This is implemented separately rather than using the qsort() routine +** from the standard library because: +** +** (1) To avoid a dependency on qsort() +** (2) To avoid the function call to the comparison routine for each +** comparison. +*/ +static void percentSort(double *a, unsigned int n){ + int iLt; /* Entries before a[iLt] are less than rPivot */ + int iGt; /* Entries at or after a[iGt] are greater than rPivot */ + int i; /* Loop counter */ + double rPivot; /* The pivot value */ + + assert( n>=2 ); + if( a[0]>a[n-1] ){ + SWAP_DOUBLE(a[0],a[n-1]) + } + if( n==2 ) return; + iGt = n-1; + i = n/2; + if( a[0]>a[i] ){ + SWAP_DOUBLE(a[0],a[i]) + }else if( a[i]>a[iGt] ){ + SWAP_DOUBLE(a[i],a[iGt]) + } + if( n==3 ) return; + rPivot = a[i]; + iLt = i = 1; + do{ + if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + SWAP_DOUBLE(a[i],a[iGt]) + }else{ + i++; + } + }while( i=2 ) percentSort(a, iLt); + if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; ibSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( i<(int)p->nUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. +*/ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ + Percentile *p; + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + unsigned i1, i2; + double v1, v2; + double ix, vx; + p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); + if( p==0 ) return; + if( p->a==0 ) return; + if( p->nUsed ){ + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + ix = p->rPct*(p->nUsed-1); + i1 = (unsigned)ix; + if( pFunc->bDiscrete ){ + vx = p->a[i1]; + }else{ + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + } + sqlite3_result_double(pCtx, vx); + } + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; + } +} +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); +} + +#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE) +__declspec(dllexport) +#endif +int sqlite3_percentile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + unsigned int i; +#ifdef SQLITE3EXT_H + SQLITE_EXTENSION_INIT2(pApi); +#else + (void)pApi; /* Unused parameter */ +#endif + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i +#include + +/* prefixes_vtab is a subclass of sqlite3_vtab which is +** underlying representation of the virtual table +*/ +typedef struct prefixes_vtab prefixes_vtab; +struct prefixes_vtab { + sqlite3_vtab base; /* Base class - must be first */ + /* No additional fields are necessary */ +}; + +/* prefixes_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct prefixes_cursor prefixes_cursor; +struct prefixes_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + char *zStr; /* Original string to be prefixed */ + int nStr; /* Length of the string in bytes */ +}; + +/* +** The prefixesConnect() method is invoked to create a new +** template virtual table. +** +** Think of this routine as the constructor for prefixes_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the prefixes_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against the virtual table will look like. +*/ +static int prefixesConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + prefixes_vtab *pNew; + int rc; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE prefixes(prefix TEXT, original_string TEXT HIDDEN)" + ); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for prefixes_vtab objects. +*/ +static int prefixesDisconnect(sqlite3_vtab *pVtab){ + prefixes_vtab *p = (prefixes_vtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new prefixes_cursor object. +*/ +static int prefixesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + prefixes_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a prefixes_cursor. +*/ +static int prefixesClose(sqlite3_vtab_cursor *cur){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + sqlite3_free(pCur->zStr); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a prefixes_cursor to its next row of output. +*/ +static int prefixesNext(sqlite3_vtab_cursor *cur){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the prefixes_cursor +** is currently pointing. +*/ +static int prefixesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + switch( i ){ + case 0: + sqlite3_result_text(ctx, pCur->zStr, pCur->nStr - (int)pCur->iRowid, + 0); + break; + default: + sqlite3_result_text(ctx, pCur->zStr, pCur->nStr, 0); + break; + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int prefixesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int prefixesEof(sqlite3_vtab_cursor *cur){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + return pCur->iRowid>pCur->nStr; +} + +/* +** This method is called to "rewind" the prefixes_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to prefixesColumn() or prefixesRowid() or +** prefixesEof(). +*/ +static int prefixesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + prefixes_cursor *pCur = (prefixes_cursor *)pVtabCursor; + sqlite3_free(pCur->zStr); + if( argc>0 ){ + pCur->zStr = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + pCur->nStr = pCur->zStr ? (int)strlen(pCur->zStr) : 0; + }else{ + pCur->zStr = 0; + pCur->nStr = 0; + } + pCur->iRowid = 0; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int prefixesBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + /* Search for a usable equality constraint against column 1 + ** (original_string) and use it if at all possible */ + int i; + const struct sqlite3_index_constraint *p; + + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->iColumn!=1 ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( !p->usable ) continue; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->estimatedCost = (double)10; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->estimatedCost = (double)1000000000; + pIdxInfo->estimatedRows = 1000000000; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module prefixesModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ prefixesConnect, + /* xBestIndex */ prefixesBestIndex, + /* xDisconnect */ prefixesDisconnect, + /* xDestroy */ 0, + /* xOpen */ prefixesOpen, + /* xClose */ prefixesClose, + /* xFilter */ prefixesFilter, + /* xNext */ prefixesNext, + /* xEof */ prefixesEof, + /* xColumn */ prefixesColumn, + /* xRowid */ prefixesRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 +}; + +/* +** This is a copy of the SQLITE_SKIP_UTF8(zIn) macro in sqliteInt.h. +** +** Assuming zIn points to the first byte of a UTF-8 character, +** advance zIn to point to the first byte of the next UTF-8 character. +*/ +#define PREFIX_SKIP_UTF8(zIn) { \ + if( (*(zIn++))>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + +/* +** Implementation of function prefix_length(). This function accepts two +** strings as arguments and returns the length in characters (not bytes), +** of the longest prefix shared by the two strings. For example: +** +** prefix_length('abcdxxx', 'abcyy') == 3 +** prefix_length('abcdxxx', 'bcyyy') == 0 +** prefix_length('abcdxxx', 'ab') == 2 +** prefix_length('ab', 'abcd') == 2 +** +** This function assumes the input is well-formed utf-8. If it is not, +** it is possible for this function to return -1. +*/ +static void prefixLengthFunc( + sqlite3_context *ctx, + int nVal, + sqlite3_value **apVal +){ + int nByte; /* Number of bytes to compare */ + int nRet = 0; /* Return value */ + const unsigned char *zL = sqlite3_value_text(apVal[0]); + const unsigned char *zR = sqlite3_value_text(apVal[1]); + int nL = sqlite3_value_bytes(apVal[0]); + int nR = sqlite3_value_bytes(apVal[1]); + int i; + + nByte = (nL > nR ? nL : nR); + for(i=0; i