From 8fcbdc63aaa456ae2d603c6a2ea0d4066b749765 Mon Sep 17 00:00:00 2001 From: travelliu Date: Mon, 20 Jun 2022 11:56:11 +0800 Subject: [PATCH] feat(mtk):release v2.3.4 --- product/en/docs-mtk/v2.0/mtk-config.md | 2385 ++++++++++++++++----- product/en/docs-mtk/v2.0/release-notes.md | 45 + product/zh/docs-mtk/v2.0/mtk-config.md | 55 +- product/zh/docs-mtk/v2.0/release-notes.md | 45 + 4 files changed, 1939 insertions(+), 591 deletions(-) diff --git a/product/en/docs-mtk/v2.0/mtk-config.md b/product/en/docs-mtk/v2.0/mtk-config.md index 09bcf945..60ce93bb 100644 --- a/product/en/docs-mtk/v2.0/mtk-config.md +++ b/product/en/docs-mtk/v2.0/mtk-config.md @@ -1,66 +1,1190 @@ --- -title: Configuration File -summary: Configuration File -author: Liu Xu -date: 2021-03-04 +title: MTK Configuration File Description +summary: Configuration File Description +author: mtk +date: 2022-06-19 18:23:32 --- -# Configuration File +# MTK Configuration File Description + +## Config + +Config MTK迁移配置文件. + +| Field | Type | Description | +|-------|------|-------------| +| source |Option |Source Database configuration information. | +| target |Option |Target Database configuration information. | +| limit |Limit |Concurrency configuration. | +| object |Object |Object definition. | +| dataOnly |bool |Whether only data is migrated. | +| schemaOnly |bool |Whether only the data structure is migrated. | +| disableTableDataComp |bool |Disabling table data comparison. | +| disableCollStatistics |bool |Disabling collection statistics. | +| reportFile |string |Migration report directory. | +| debug |bool |debug. | +| preRun |bool |preRun. | +| test |bool |test. | +| disableIgnoreCase |bool |Disables ignoring case queries. | +| disableSelectPart |bool |Disable the select by partition. | +| disableFKCons |bool |Disable table foreign key sync. | +| disableSyncIdxAfterData |bool | disable the synchronization of table data and create this table index immediately| +| disablePrintMigDataProgress |bool | disable printing migration data progress | + +**Example**: -An MTK configuration file mainly includes the following content: +```json +{ + "source": { + "type": "oracle", + "connect": { + "host": "127.0.0.1", + "user": "system", + "port": 1521, + "password": "******", + "dbName": "orcl" + } + }, + "target": { + "type": "opengauss", + "connect": { + "version": "3.0.0", + "host": "127.0.0.1", + "user": "gaussdb", + "port": 26000, + "password": "******", + "dbName": "postgres" + }, + "parameter": { + "parallelInsert": 1, + "dropExistingObject": false, + "truncTable": false, + "caseSensitive": 0, + "colKeyWords": {}, + "objKeyWords": {}, + "quoteMark": false, + "path": "./data", + "schemaPath": "", + "dataPath": "", + "fileType": "", + "fileSize": "", + "csvHeader": false, + "csvNullValue": "", + "csvFieldDelimiter": ",", + "csvOptionallyEnclosed": "\"", + "excludeSysTable": [], + "remapSchema": {}, + "remapTable": {}, + "remapTablespace": {}, + "enableSyncTabTbsPro": false, + "enableSyncCompTabPro": false, + "timeFormat": "HH:MI:SS", + "dateFormat": "YYYY-MM-DD", + "dateTimeFormat": "YYYY-MM-DD HH24:MI:SS", + "noSupportPartTabToNormalTab": false, + "ignoreDB2PartInclusive": false, + "igNotSupportIntervalPart": false, + "igErrorData": false, + "ignoreTabPartition": false, + "autoAddMaxvaluePart": false, + "autoAddMySQLAutoIncr": false, + "autoAddMySQLAutoIncrTabList": null, + "ignoreNotSupportDefault": false, + "replaceZeroDate": "", + "virtualColToNormalCol": false, + "virtualColConv": {}, + "mySQLSkipErrorDateTimeData": false, + "ignoreTableDDLCompErr": false, + "convertPackageMethod": "", + "enableOgBlobClob": false, + "enableConvertSrid": false, + "defaultSrid": "4326", + "seqLastNumAddNum": 0, + "skipColumnType": {}, + "skipColumnName": {}, + "templateSeqName": "", + "charAppendEmptyString": false, + "tableOptions": null, + "indexOptions": null + } + }, + "limit": { + "parallel": 2, + "fetchSize": 1000, + "batchSize": 1000, + "bufferSize": 8, + "cpBufferSize": 8, + "oracleSelectParallel": 2, + "channelCacheNum": 10000, + "limit": 0 + }, + "object": { + "tables": [ + "MTK.TAB1", + "MTK.TAB111%", + "MTK1.*" + ], + "schemas": [ + "SCHEMA1", + "SCHEMA2" + ], + "excludeTable": { + "SCHEMA1": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ], + "SCHEMA2": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ] + }, + "tableSplit": { + "SCHEMA1": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + }, + "SCHEMA2": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + } + } + }, + "dataOnly": false, + "schemaOnly": false, + "disableTableDataComp": false, + "disableCollStatistics": false, + "reportFile": "./report/", + "debug": false, + "preRun": false, + "test": false, + "disableIgnoreCase": false, + "disableSelectPart": false, + "disableFKCons": false, + "disableSyncIdxAfterData": false, + "disablePrintMigDataProgress": false +} +``` + +### source + +**Type**: Option + +**Desc**: Source Database configuration information. + +**Example**: + +source database config example + +```json +{ + "source": { + "type": "oracle", + "connect": { + "host": "127.0.0.1", + "user": "system", + "port": 1521, + "password": "******", + "dbName": "orcl" + } + } +} +``` + +### target + +**Type**: Option + +**Desc**: Target Database configuration information. + +**Example**: + +target database config example + +```json +{ + "target": { + "type": "opengauss", + "connect": { + "version": "3.0.0", + "host": "127.0.0.1", + "user": "gaussdb", + "port": 26000, + "password": "******", + "dbName": "postgres" + }, + "parameter": { + "parallelInsert": 1, + "dropExistingObject": false, + "truncTable": false, + "caseSensitive": 0, + "colKeyWords": {}, + "objKeyWords": {}, + "quoteMark": false, + "path": "./data", + "schemaPath": "", + "dataPath": "", + "fileType": "", + "fileSize": "", + "csvHeader": false, + "csvNullValue": "", + "csvFieldDelimiter": ",", + "csvOptionallyEnclosed": "\"", + "excludeSysTable": [], + "remapSchema": {}, + "remapTable": {}, + "remapTablespace": {}, + "enableSyncTabTbsPro": false, + "enableSyncCompTabPro": false, + "timeFormat": "HH:MI:SS", + "dateFormat": "YYYY-MM-DD", + "dateTimeFormat": "YYYY-MM-DD HH24:MI:SS", + "noSupportPartTabToNormalTab": false, + "ignoreDB2PartInclusive": false, + "igNotSupportIntervalPart": false, + "igErrorData": false, + "ignoreTabPartition": false, + "autoAddMaxvaluePart": false, + "autoAddMySQLAutoIncr": false, + "autoAddMySQLAutoIncrTabList": null, + "ignoreNotSupportDefault": false, + "replaceZeroDate": "", + "virtualColToNormalCol": false, + "virtualColConv": {}, + "mySQLSkipErrorDateTimeData": false, + "ignoreTableDDLCompErr": false, + "convertPackageMethod": "", + "enableOgBlobClob": false, + "enableConvertSrid": false, + "defaultSrid": "4326", + "seqLastNumAddNum": 0, + "skipColumnType": {}, + "skipColumnName": {}, + "templateSeqName": "", + "charAppendEmptyString": false, + "tableOptions": null, + "indexOptions": null + } + } +} +``` + +### limit + +**Type**: Limit + +**Desc**: Concurrency configuration. + +**Example**: + +limit example + +```json +{ + "limit": { + "parallel": 2, + "fetchSize": 1000, + "batchSize": 1000, + "bufferSize": 8, + "cpBufferSize": 8, + "oracleSelectParallel": 2, + "channelCacheNum": 10000, + "limit": 0 + } +} +``` + +### object + +**Type**: Object + +**Desc**: Object definition. + +**Example**: + +object example + +```json +{ + "object": { + "tables": [ + "MTK.TAB1", + "MTK.TAB111%", + "MTK1.*" + ], + "schemas": [ + "SCHEMA1", + "SCHEMA2" + ], + "excludeTable": { + "SCHEMA1": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ], + "SCHEMA2": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ] + }, + "tableSplit": { + "SCHEMA1": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + }, + "SCHEMA2": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + } + } + } +} +``` + +### dataOnly + +**Type**: bool + +**Desc**: Indicates whether to migrate only data. + +**Default**: `false` + +**Option**: + +- true +- false + +### schemaOnly + +**Type**: bool + +**Desc**: Indicates whether to migration only the data structure. + +**Default**: `false` + +**Option**: + +- true +- false + +### disableTableDataComp + +**Type**: bool + +**Desc**: Disables the table data comparison function. + +After data migration is complete, MTK will count and compare the number of rows in the source and target databases. + +**Default**: `false` + +**Option**: + +- true +- false + +### disableCollStatistics + +**Type**: bool + +**Desc**: Disables the collection statistics function. + +After data migration is complete, MTK will collect statistics information from the target database. + +**Default**: `false` + +**Option**: + +- true +- false + +### reportFile + +**Type**: string + +**Desc**: Indicates the migration report directory. + +For MTK of a version earlier than v0.0.18, the migration report is shown in HTML. + +For MTK v0.0.18 or later, the migration report is stored in a directory. + +### debug + +**Type**: bool + +**Desc**: Indicates whether to enable the log debug mode. + +**Default**: `false` + +**Option**: + +- true +- false + +### preRun + +**Type**: bool + +**Desc**: pre Run. + +**Default**: `false` + +**Option**: + +- true +- false + +### test + +**Type**: bool + +**Desc**: Try migration. +Migration parameter `limit.limit` row data. In this mode, the data can be submitted without any data, and the data will be automatically rolled back. +Just to test if the data is inserted normally + +**Default**: `false` + +**Option**: + +- true +- false + +### disableIgnoreCase + +**Type**: bool + +**Desc**: Disables ignoring case queries. + +**Default**: `false` + +**Option**: + +- true +- false + +### disableSelectPart + +**Type**: bool + +**Desc**: Disable the partition table migration data by partition feature. + +**Default**: `false` + +**Option**: + +- true +- false + +### disableFKCons + +**Type**: bool + +**Desc**: Disables synchronization of table foreign key constraints. + +**Default**: `false` + +**Option**: + +- true +- false + +### disableSyncIdxAfterData + +**Type**: bool + +**Desc**: disable the synchronization of table data and create this table index immediately + +**Default**: `false` + +**Option**: + +- true +- false + +### disablePrintMigDataProgress + +**Type**: bool + +**Desc**: disable printing migration data progress + +**Default**: `false` + +**Option**: + +- true +- false + +## Object + +Defines migration objects +迁移对象定义 + +- Case-sensitive +- Migration user, database, and schema +- schema = mysql database +- schema = oracle user +- schema = postgres schema +- schema = db2 schema + +> Parameters **schemas** and **tables** are mutually exclusive. +> If both parameters are configured, the error "mtk-1002 schema and table cannot exist together" will occur during migration. +> +> If no migration object is defined, the error "not define Migrate object" will occur during migration. + +| Field | Type | Description | +|-------|------|-------------| +| tables |[]string |Defines which tables are to be migrated. | +| schemas |[]string |Defines schema during migration | +| excludeTable |map[string][]string |Defines which tables are excluded. | +| tableSplit |map[string]map[string][]string |Defines table granularity parallelism. | + +Appears in: + +- Config.object + +**Example**: + +object example + +```json +{ + "tables": [ + "MTK.TAB1", + "MTK.TAB111%", + "MTK1.*" + ], + "schemas": [ + "SCHEMA1", + "SCHEMA2" + ], + "excludeTable": { + "SCHEMA1": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ], + "SCHEMA2": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ] + }, + "tableSplit": { + "SCHEMA1": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + }, + "SCHEMA2": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + } + } +} +``` + +### tables + +**Type**: []string + +**Desc**: Defines which tables are to be migrated. + +> when running a subcommand migration, such as `sync-function` and configured with tables (`MTK.XXX`) is the 'XXX' function under Migrating Schema 'MTK'. +> Except for sequence. + +**Option**: + +- Full table name +- Support for schema.* +- Support for schema.TAB% + +**Example**: + +tables example + +```json +{ + "tables": [ + "MTK.TAB1", + "MTK.TAB111%", + "MTK1.*" + ] +} +``` + +### schemas + +**Type**: []string + +**Desc**: Defines schema during migration + +**Example**: + +schema example + +```json +{ + "schemas": [ + "SCHEMA1", + "SCHEMA2" + ] +} +``` + +### excludeTable + +**Type**: map[string][]string + +**Desc**: Defines which tables are excluded.. + +**Example**: + +exclude Table example + +```json +{ + "excludeTable": { + "SCHEMA1": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ], + "SCHEMA2": [ + "TABLE_SKIP1", + "TABLE_SKIP1" + ] + } +} +``` + +### tableSplit + +**Type**: map[string]map[string][]string + +**Desc**: Defines table granularity parallelism, which can be automatically generated by using [mtk show-table-split](mtk_show-table-split). + +Defines table migration query conditions. + +**Example**: + +tableSplit example + +```json +{ + "tableSplit": { + "SCHEMA1": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + }, + "SCHEMA2": { + "TAB_1": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ], + "TAB_2": [ + "ID \u003c 10000 ", + "ID \u003c 20000 AND ID \u003e=10000", + "ID \u003e= 90000" + ] + } + } +} +``` + +## Limit + +Limit concurrency configuration + +| Field | Type | Description | +|-------|------|-------------| +| parallel |int | Degree of parallelism. | +| fetchSize |int |Specifies the number of rows to fetch at a time. | +| batchSize |int |Specify the size of batch insert or copy and Commit. | +| bufferSize |int |Specifies the buffer size (in MB) when querying or inserting in batches. | +| cpBufferSize |int |Defines the buffer size (in MB) used in the Copy command. | +| oracleSelectParallel |int |Add hint `/* +parallel(t,n) */` | +| limit |int64 |Defines how many rows to migrate per table. | + +Appears in: + +- Config.limit + +**Example**: + +limit example + +```json +{ + "parallel": 2, + "fetchSize": 1000, + "batchSize": 1000, + "bufferSize": 8, + "cpBufferSize": 8, + "oracleSelectParallel": 2, + "channelCacheNum": 10000, + "limit": 0 +} +``` + +### parallel + +**Type**: int + +**Desc**: Indicates the degree of parallelism. + +Create tables, constraints, etc. in parallel.. + +**Default**: 1 + +### fetchSize + +**Type**: int + +**Desc**: Specifies the row size for one row fetch. + +Valid values ​​are 1-50000 + +Oracle support. + +**Default**: 1000 + +### batchSize + +**Type**: int + +**Desc**: Specify the size of batch insert or Copy and make Commit. + +Versions after 2.3.4 began to support batch Commit. The previous version only interacted with data in batches without committing + +If the submission fails the program will exit and log the error data to the error file. + +If you configure [igErrorData](#igerrordara) the program will not exit and will log the error data to the error file and continue processing the data. + +Valid values ​​are 1-50000 + +Support Oracle, PostgreSQL, openGauss, MySQL. + +Batch commit supports PostgreSQL/MogDB + +**Default**: 1000 + +### bufferSize + +**Type**: int + +**Desc**: Specifies the cache size (in MB) when querying or bulk inserting. + +Valid values ​​are 1-1024 + +Support PostgreSQL, openGauss, MySQL. + +**Default**: 8 MB + +### cpBufferSize + +**Type**: int + +**Desc**: Defines the cache size (in MB) used in the Copy command. + +Valid values ​​are 1-1024 + +Support PostgreSQL, openGauss, MySQL + +**Default**: 8 MB + +### oracleSelectParallel + +**Type**: int + +**Desc**: Add hint `/* +parallel(t,n) */` to Oracle query statement + +### limit + +**Type**: int64 + +**Desc**: Define how many rows to migrate per table. + +Defining this parameter no longer does table concurrent migrations. + +## Connect + +Connect database connection information + +| Field | Type | Description | +|-------|------|-------------| +| version |string |Database version. | +| vendor |string |Database issuer. | +| host |string |Database host | +| user |string |Database user | +| port |int |Database port | +| password |Password |Database User Password | +| dbName |string |Database name | +| dsn |string |User-specified connection string. | +| timeout |Duration |Connection timeout. | +| charset |string |Database character set. | +| datCompatibility |string |For openGauss database compatibility mode. | +| sqlMode |string |For MySQL database `sql_mode`. | +| clientCharset |string |It is used for database encoding conversion scenarios, and generally does not need to be set. | + +Appears in: + +- Option.connect + +### version + +**Type**: string + +**Desc**: Database version. + +No need to specify. Automatic query when connecting to database + +Migrating into a file requires manually specifying the version + +**Example**: + +version example + +```json +{ + "version": "2.1.0" +} +``` + +### vendor + +**Type**: string + +**Desc**: Database publisher. + +No need to specify. Automatic query when connecting to database + +**Example**: + +vendor example + +```json +{ + "vendor": "MySQL" +} +``` + +### host + +**Type**: string + +**Desc**: database host + +**Option**: + +- ip +- 域名 + +**Example**: + +host example + +```json +{ + "host": "127.0.0.1" +} +``` + +### user + +**Type**: string + +**Desc**: 数据库用户 + +**Example**: + +host example + +```json +{ + "user": "system" +} +``` + +### port + +**Type**: int + +**Desc**: database port + +**Example**: + +port example + +```json +{ + "port": 1521 +} +``` + +### password + +**Type**: Password + +**Desc**: database user password + +### dbName + +**Type**: string + +**Desc**: Name database + +**Example**: + +dbName example + +```json +{ + "dbName": "orcl" +} +``` + +### dsn + +**Type**: string + +**Desc**: User-specified connection string. + +No need to specify by default. Reserve functions for special scenarios + +### timeout + +**Type**: Duration + +**Desc**: connection timeout. + +No configuration required + +**Default**: 30s + +### charset + +**Type**: string + +**Desc**: database character set. + +No configuration by default, connect to database query + +**Option**: + +- gbk +- utf8 + +**Example**: + +charset example ```json { - "source": { - }, - "target": { - }, - "limit": { - }, - "object": { - }, - "dataOnly": false, - "schemaOnly": false, - "reportFile": "./mtk_report", - "debug": false + "charset": "gbk" +} +``` + +### datCompatibility + +**Type**: string + +**Desc**: Compatibility mode for the openGauss database. + +By default, no configuration is required, and the database query is connected. + +Migrating into a file requires configuration + +**Option**: + +- A +- B +- PG + +**Example**: + +charset example + +```json +{ + "datCompatibility": "A" } ``` -- source Database configuration information -- target Database configuration information -- limit [Concurrency configuration](#limit) -- object [Object definition](#object) -- dataOnly [Whether only data is migrated](#dataOnly) -- schemaOnly [Whether only the data structure is migrated](#schemaOnly) -- disableTableDataComp [Disabling table data comparison](#disableTableDataComp) -- disableCollStatistics [Disabling collection statistics](#disableCollStatistics) -- reportFile [Migration report directory](#reportFile) -- debug [debug](#debug) +### sqlMode + +**Type**: string + +**Desc**: `sql_mode` for MySQL databases. + +By default, no configuration is required, and the database query is connected. + +Migrating into a file requires configuration + +### clientCharset + +**Type**: string + +**Desc**: It is used for database encoding conversion scenarios, and generally does not need to be set. + +Such as Oracle ZHS16GBK encoding migrated to openGauss UTF8. Encountered `ORA-29275: partial multibyte character`. + +## Option + +Option database configuration + +| Field | Type | Description | +|-------|------|-------------| +| type |string |Database type, case insensitive | +| connect |Connect |Database connection information | +| parameter |Parameter |Parameter configuration | + +Appears in: + +- Config.source +- Config.target + +**Example**: -## Database Configuration Parameters +source database config example ```json { "type": "oracle", "connect": { - "host": "127.0.0.1", - "user": "system", - "port": 1521, - "password": "oracle", - "dbName": "orcl" + "host": "127.0.0.1", + "user": "system", + "port": 1521, + "password": "******", + "dbName": "orcl" + } +} +``` + +target database config example + +```json +{ + "type": "opengauss", + "connect": { + "version": "3.0.0", + "host": "127.0.0.1", + "user": "gaussdb", + "port": 26000, + "password": "******", + "dbName": "postgres" }, "parameter": { + "parallelInsert": 1, + "dropExistingObject": false, + "truncTable": false, + "caseSensitive": 0, + "colKeyWords": {}, + "objKeyWords": {}, + "quoteMark": false, + "path": "./data", + "schemaPath": "", + "dataPath": "", + "fileType": "", + "fileSize": "", + "csvHeader": false, + "csvNullValue": "", + "csvFieldDelimiter": ",", + "csvOptionallyEnclosed": "\"", + "excludeSysTable": [], + "remapSchema": {}, + "remapTable": {}, + "remapTablespace": {}, + "enableSyncTabTbsPro": false, + "enableSyncCompTabPro": false, + "timeFormat": "HH:MI:SS", + "dateFormat": "YYYY-MM-DD", + "dateTimeFormat": "YYYY-MM-DD HH24:MI:SS", + "noSupportPartTabToNormalTab": false, + "ignoreDB2PartInclusive": false, + "igNotSupportIntervalPart": false, + "igErrorData": false, + "ignoreTabPartition": false, + "autoAddMaxvaluePart": false, + "autoAddMySQLAutoIncr": false, + "autoAddMySQLAutoIncrTabList": null, + "ignoreNotSupportDefault": false, + "replaceZeroDate": "", + "virtualColToNormalCol": false, + "virtualColConv": {}, + "mySQLSkipErrorDateTimeData": false, + "ignoreTableDDLCompErr": false, + "convertPackageMethod": "", + "enableOgBlobClob": false, + "enableConvertSrid": false, + "defaultSrid": "4326", + "seqLastNumAddNum": 0, + "skipColumnType": {}, + "skipColumnName": {}, + "templateSeqName": "", + "charAppendEmptyString": false, + "tableOptions": null, + "indexOptions": null } } ``` -- type [Database type](#type) -- connect [Connection information](#connect) -- parameter [Parameter](##parameter) - ### type -The database type is case-insensitive, which includes: +**Type**: string + +**Desc**: database type, case insensitive + +**Option**: - MySQL - Oracle @@ -69,103 +1193,159 @@ The database type is case-insensitive, which includes: - MogDB - DB2 - sqlServer -- file, which is only used in the target database and for exporting data to a file. +- file +- informix -### connect +**Example**: + +type example ```json { - "host": "127.0.0.1", - "user": "system", - "port": 1521, - "password": "oracle", - "dbName": "orcl" + "type": "Oracle" } ``` -| Parameter | Description | -| :-------- | :----------------------------------------------------------- | -| version | Specifies the database version, which does not need to be specified and can be automatically queried during database connection. | -| host | Specifies the database host. The value can be the IP address, name, and domain name of a host. | -| user | Specifies the database user who must have permission to query data dictionaries and data. | -| port | Specifies the database port. | -| password | Specifies the password of a database user. | -| dbName | Specifies the name of a database. | -| dsn | Specifies the complete string for connecting a database, which is used in certain scenarios. | +### connect + +**Type**: Connect + +**Desc**: Database connection information + +### parameter + +**Type**: Parameter + +**Desc**: parameter configuration + +## Parameter + +Parameter parameter configuration + +| Field | Type | Description | +|-------|------|-------------| +| enableTableParallelQuery |int |How to query table information. | +| parallelInsert |int |Parallel Insert Parallelism. | +| dropExistingObject |bool |Drops the object if it already exists in the target database. | +| truncTable |bool | Prevent data conflict use when only migrating data. | +| caseSensitive |int |Case conversion for the generated object name. | +| colKeyWords |map[string]int |Column name keywords. | +| objKeyWords |map[string]int | Reserved object keywords. | +| quoteMark |bool |Whether double quotes or backticks are used for the generated object name. | +| path |string |When exporting data as a file, specify the export directory. | +| schemaPath |string |sql script directory | +| fileType |string |When exporting data as a file, the file type | +| fileSize |string | The size of a single file when data is exported as a file. No limit if not specified. | +| sqlDBType |string |When exporting data to a file, specify the syntax of which database to generate. | +| csvHeader |bool |When the export file is csv, specify whether to include the csv file header. Not exported by default. | +| csvNullValue |string |csv null value display | +| csvFieldDelimiter |string |csv default delimiter. | +| csvOptionallyEnclosed |string |Data wrapper. | +| excludeSysTable |[]string |Exclude system object definitions. If not configured, use the default value | +| remapSchema |map[string]string |Whether to rename the database schema during migration. | +| remapTable |map[string]string |Whether to rename the table name during migration | +| remapTablespace |map[string]string |Whether to rename the tablespace name during migration. | +| enableSyncTabTbsPro |bool | Whether to configure the generated DDL to include tablespace syntax. Not generated by default | +| enableSyncCompTabPro |bool | Whether to configure the generated DDL to include table compression attribute syntax. Not generated by default. | +| timeFormat |string |Define the time format | +| dateFormat |string |Define date format | +| dateTimeFormat |string |Define the full format of the time | +| noSupportPartTabToNormalTab |bool |Convert a partition table not supported by the target to a normal table | +| ignoreDB2PartInclusive |bool |Whether to ignore the inclusion attribute of DB2 partition key value ENDING | +| igNotSupportIntervalPart |bool |Support ignore part of unsupported `intervalpart` attribute | +| igErrorData |bool |Ignore failed insert data and log to error file. | +| ignoreTabPartition |bool |Support migration to target database ignore partition syntax, now only support migration to MySQL. | +| autoAddMaxvaluePart |bool |The parameter `autoAddMaxvaluePart` allows a `maxvalue` partition to be added automatically to a partitioned table that does not already have a `maxvalue` partition. | +| autoAddMySQLAutoIncr |bool |Migrating to MySQL automatically adds MySQL `AUTO_INCREMENT` columns | +| autoAddMySQLAutoIncrTabList |[]string |Use with the `autoAddMySQLAutoIncr` parameter. | +| ignoreNotSupportDefault |bool | The support ignore part does not support column default values. Such as Oracle's `sys_guid`. MTK will embed a whitelist to ignore | +| replaceZeroDate |string |openGauss does not support the time format of 0000-00-00, the default replacement is 1970-01-01, this parameter can define the default replacement value, such as 2021-01-01 | +| virtualColToNormalCol |bool |Whether to convert the virtual column of the source library to the normal column of the target library. | +| virtualColConv |map[string]string |Virtual column expression conversion function | +| mySQLToOgCharExpression |string |mysql migrates to openGauss to calculate column length expressions for char/varchar columns. By default, no calculation is performed. For example, `*3` means the original length is `*3` | +| mySQLSkipErrorDateTimeData |bool | Automatically skip mysql error time. | +| ignoreTableDDLCompErr |bool |Ignore table structure comparison errors | +| convertPackageMethod |string |How to migrate Oracle Packages to openGauss/MogDB | +| enableOgBlobClob |bool |Blobs and Clobs migrated from other databases are openGauss Blobs and Clobs. | +| enableConvertSrid |bool |Enable Convert Srid | +| defaultSrid |string |PostGis Default Srid | +| seqLastNumAddNum |int | How much to increase when synchronizing the sequence last value. | +| skipColumnType |map[string]int |Skip a data type column when migrating | +| skipColumnName |map[string]int |Skip a column when migrating | +| templateSeqName |string |MySQL auto-incrementing column to sequence, template of sequence name | +| charAppendEmptyString |bool |For ORA-29275: partial multibyte character error. In oracle query is concatenated string | +| tableOptions |map[string]string |Configure the option to add table attributes when creating a table. Currently only openGauss/MogDB is supported. | +| indexOptions |map[string]string |The same as the tableOptions option configuration, it only takes effect for the index. The function development has not been completed. | + +Appears in: + +- Option.parameter + +### enableTableParallelQuery + +**Type**: int + +**Desc**: Ways to look up table information. + +Test feature. No modification required + +**Option**: + +- 0 is queried at one time, and then created concurrently +- 1 Parallel query, first query the basic information of the table, then concurrent query + target creation -### clientCharset +### parallelInsert -string +**Type**: int -Supported after `v2.2.2` version. +**Desc**: Parallel insert parallelism. -For database encoding conversion scenarios, it is generally not required to be set. -For example, Oracle ZHS16GBK encoding was migrated to openGauss UTF8. Encountered `ORA-29275: partial multibyte character` +Test feature. No modification required -### Parameter +**Default**: `false` -### dropExistingObject +**Option**: -Deletes the existing object. If there is a certain risk, manually delete the existing object. +- true +- false -Default value: `false` +### dropExistingObject -### truncTable +**Type**: bool -Prevents data conflict when only data is migrated. +**Desc**: If the object already exists in the target database, delete the object. -Default value: `true` +Generally used when data is reinitialized. Please use it with caution. -### quoteMark +**Default**: `false` -Use double or backquote inclusion for object names in SQL statements +**Option**: -```sql --- MySQl -`a1` --- Oracle/Postgres/openGauss/MogDB -"a1" -``` +- true +- false -默认值: `false` +### truncTable -Example: +**Type**: bool -```sql --- quoteMark: false -CREATE TABLE mtk.t_pri ( -id DECIMAL(38) NOT NULL, -id2 DECIMAL(38) NOT NULL, -id3 DECIMAL(38), -id4 DECIMAL(38), -id5 DECIMAL(38) -); +**Desc**: Prevent data conflict usage when only migrating data. --- quoteMark: true -CREATE TABLE "mtk"."t_pri" ( -"id" DECIMAL(38) NOT NULL, -"id2" DECIMAL(38) NOT NULL, -"id3" DECIMAL(38), -"id4" DECIMAL(38), -"id5" DECIMAL(38) -); -``` +**Default**: `false` -Can be used with the parameter [caseSensitive](#casesensitive) for object name case conversion. -Except for special keywords, special keywords use [objKeyWords](#objkeywords) and [colKeyWords](#colkeywords) +**Option**: -|quoteMark|caseSensitive|describe| -|---------|-------------|---------------------------------------| -|false|0|create statement without double or backquotes,
the case of object names is determined by the database default configuration| -|false|1|create statement without double or backquotes,
the case of object names is determined by the database default configuration| -|false|2|create statement without double or backquotes,
the case of object names is determined by the database default configuration| -|true |0|create statement with double or backquotes,
the object name is the same as the original| -|true |1|create statement with double or backquoted,
the object names are all lowercase| -|true |2|create statement with double or backquotes,
the object names are all uppercase| +- true +- false ### caseSensitive -Used for case conversion for the names of the generated objects. +**Type**: int + +**Desc**: Case conversion for generated object names. + +**Default**: 0 + +**Option**: - 0 indicates that no operation is needed. - 1 indicates lowercase. @@ -174,72 +1354,147 @@ Used for case conversion for the names of the generated objects. ### colKeyWords -Indicates the keyword of a column name. +**Type**: map[string]int -During creation in the target database, "" or '' will be automatically added and the keyword will be operated with case conversion automatically. +**Desc**: Column name keyword. + +When the target is created, "" will be automatically added or the keyword will be case-converted -Format: Column name : Case identification +**Option**: - 1 indicates lowercase. - 2 indicates uppercase. - Others indicate that the keyword is kept unchanged. +**Example**: + +Column KeyWords example + ```json -"colKeyWords": { - "STREAM": 1, - "TID": 1 +{ + "colKeyWords": { + "STREAM": 1, + "TID": 1 + } } ``` ### objKeyWords -Reserves the object keyword. +**Type**: map[string]int + +**Desc**: Reserves the object keyword. During creation in the target database, "" or '' will be automatically added and the keyword will be operated with case conversion automatically. -Format: Column name : Case identification +**Option**: - 1 indicates lowercase. - 2 indicates uppercase. - Others indicate that the keyword is kept unchanged. -```json -"objKeyWords": { - "STREAM": 1, - "TID": 1 -} -``` - -### path +**Example**: -Indicates the directory of a file to which data is to be exported. +Column KeyWords example ```json { - "type": "file", - "parameter": { - "fileType": "csv", - "path": "./data", - "fileSize": "5000MiB", - "sqlDBType": "opengauss", - "csvHeader": true, - "csvFieldDelimiter": ",", - "csvOptionallyEnclosed": "\"", + "objKeyWords": { + "STREAM": 1, + "TID": 1 } } +``` + +### quoteMark + +**Type**: bool + +**Desc**: Use double or backquote inclusion for object names in SQL statements. + +```sql +-- MySQl +`a1` +-- Oracle/PostgreSQL/openGauss/MogDB +"a1" +``` + +```sql +-- quoteMark: false +CREATE TABLE mtk.t_pri ( + id DECIMAL(38) NOT NULL, + id2 DECIMAL(38) NOT NULL, + id3 DECIMAL(38), + id4 DECIMAL(38), + id5 DECIMAL(38) +); +-- quoteMark: true +CREATE TABLE "mtk"."t_pri" ( + "id" DECIMAL(38) NOT NULL, + "id2" DECIMAL(38) NOT NULL, + "id3" DECIMAL(38), + "id4" DECIMAL(38), + "id5" DECIMAL(38) +); ``` +Can be used with the parameter [caseSensitive](#casesensitive) for object name case conversion. +Except for special keywords, special keywords use [objKeyWords](#objkeywords) and [colKeyWords](#colkeywords) + +|quoteMark|caseSensitive|describe| +|---------|-------------|---------------------------------------| +|false|0|create statement without double or backquotes,
the case of object names is determined by the database default configuration| +|false|1|create statement without double or backquotes,
the case of object names is determined by the database default configuration| +|false|2|create statement without double or backquotes,
the case of object names is determined by the database default configuration| +|true |0|create statement with double or backquotes,
the object name is the same as the original| +|true |1|create statement with double or backquoted,
the object names are all lowercase| +|true |2|create statement with double or backquotes,
the object names are all uppercase| + +**Default**: `false` + +**Option**: + +- true +- false + +### path + +**Type**: string + +**Desc**: Indicates the directory of a file to which data is to be exported. + +During data migration, the error data will be recorded in this directory to the `err_data_` directory下 + +**Default**: `./data` + +**Option**: + +- true +- false + +### schemaPath + +**Type**: string + +**Desc**: sql script directory + ### fileType -Indicates the type of a file when data is to be exported to a file. The supported file types include: +**Type**: string + +**Desc**: Indicates the type of a file when data is to be exported to a file. The supported file types include: + +**Option**: - csv - sql ### fileSize -Indicates the size of a file when data is to be exported to a file. If this parameter is not configured, the file size is not limited. +**Type**: string + +**Desc**: Indicates the size of a file when data is to be exported to a file. If this parameter is not configured, the file size is not limited. The supported formats of the file size includes: @@ -254,175 +1509,204 @@ The supported formats of the file size includes: - TB - PB -KB == 1000 +KB == 1000 KiB == 1024 +**Example**: + +fileSize example + +```json +{ + "fileSize": "2048MiB" +} +``` + ### sqlDBType -Indicates the data insert syntax in the target database when data is to be exported to a file. +**Type**: string + +**Desc**: Indicates the data insert syntax in the target database when data is to be exported to a file. For example, if data is migrated from Oracle to MySQL, the MySQL data insert syntax is generated by default. ### csvHeader -Indicates whether the CSV file header is included when data is to be exported to a CSV file. The CSV header is not exported by default. +**Type**: bool -Default value: `false` +**Desc**: Indicates whether the CSV file header is included when data is to be exported to a CSV file. + +The CSV header is not exported by default.。 + +**Default**: `false` + +### csvNullValue + +**Type**: string + +**Desc**: csv empty value display ### csvFieldDelimiter -Indicates the default CSV delimiter. +**Type**: string + +**Desc**: Indicates the default CSV delimiter.。 -Default value: `,` +**Default**: `,` ### csvOptionallyEnclosed -Indicates the data wrapper. +**Type**: string -Default value: `"` +**Desc**: Indicates the data wrapper. -### parallelInsert +**Default**: `"` + +### excludeSysTable -Indicates the degree of parallelism, which is used for testing the feature and does not need to be configured. +**Type**: []string -### timeFormat +**Desc**: Excludes the system object definition. If this parameter is not configured, the default value applies. -Indicates the time format. +**Example**: -Default value: `HH:MI:SS` +DB2 Exclude System Table example -### dateFormat +```json +{ + "excludeSysTable": [ + "EXPLAIN_ACTUALS", + "ADVISE_TABLE", + "ADVISE_PARTITION", + "ADVISE_MQT", + "ADVISE_WORKLOAD", + "ADVISE_INDEX", + "ADVISE_INSTANCE", + "OBJECT_METRICS", + "EXPLAIN_DIAGNOSTIC_DATA", + "EXPLAIN_DIAGNOSTIC", + "EXPLAIN_STREAM", + "EXPLAIN_PREDICATE", + "EXPLAIN_OPERATOR", + "EXPLAIN_OBJECT", + "EXPLAIN_ARGUMENT", + "EXPLAIN_STATEMENT", + "EXPLAIN_INSTANCE" + ] +} +``` -Indicates the date format. +### remapSchema -Default value: `YYYY-MM-DD` +**Type**: map[string]string -### dateTimeFormat +**Desc**: Indicates whether to modify the name of `schema` during migration. -Indicates the full time format. +Format: Name of the original solution : Name of the new solution -Default value: `YYYY-MM-DD HH24:MI:SS` +For example, `MTK1` is changed to `MTK1_NEW`, and `SOE` is changed to `SOE_NEW`. -### noSupportPartTabToNormalTab +**Example**: -- Indicates whether to convert a partition table that is not supported by the target database into a common table. -- Indicates whether to automatically remove the property, of a subpartition table, which is not supported by the target database. +remap schema examples -The List and Hash partitioned tables in Oracle are common tables in MogDB. +```json +{ + "remapSchema": { + "MTK1": "MTK1_NEW", + "SOE": "SOE_NEW" + } +} +``` -The composite range-list partitioned table in Oracle is a range partitioned table in MogDB. The subpartition tables will be automatically removed. +### remapTable -### mySQLToOgCharExpression +**Type**: map[string]string -The length of fields in the original database is automatically multiplied by 3. This parameter is valid for only the varchar/char field in the source database of MySQL. +**Desc**: Indicates whether to modify the table name during migration -This parameter is already not supported. +**Deprecated**: This parameter is already not supported -### mySQLSkipErrorDateTimeData +### remapTablespace -Automatically skips the wrong time of MySQL. +**Type**: map[string]string -For example, datetime type: `0000-00-00 13:14:15` +**Desc**: Indicates whether to modify the name of the tablespace during migration. -### autoAddMySQLAutoIncr +This parameter is used together with [enableSyncTabTbsPro](#enableSyncTabTbsPro). -Supported after `v2.1.0` version +For example, `USERSPACE1` is changed to `TBS01`. -Migrate to MySQL to automatically add MySQL `AUTO_INCREMENT` column. +**Example**: -If the value is true, when the table does not have a `AUTO_INCREMENT` primary key column,
-automatically add the `AUTO_INCREMENT` column and change the original primary key to a unique key. +remap tablespace example -```sql -PK_ID BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT NOT NULL [INVISIBLE] +```json +{ + "remapTablespace": { + "USERSPACE1": "TBS01" + } +} ``` -- Prior to MySQL 8.0.23 column are `visible` -- MySQL 8.0.23 column are `invisible column` - -> MySQL supports invisible columns as of MySQL 8.0.23.
-> An invisible column is normally hidden to queries, but can be accessed if explicitly referenced.
-> Prior to MySQL 8.0.23, all columns are visible. +### enableSyncTabTbsPro -Used in conjunction with parameter [ignoreTabPartition](#ignoretabpartition). +**Type**: bool -Default: `false` +**Desc**: Indicates whether to generate a DDL with the tablespace. -Example: +**Default**: `false` -- DB2 To MySQL +### enableSyncCompTabPro - - DB2 +**Type**: bool - ```sql - CREATE TABLE MTK1.TAB_PK_TEST ( - COL1 VARCHAR(10) NOT NULL, - ID BIGINT NOT NULL, - COL2 VARCHAR(10), - COL3 VARCHAR(10) - ) IN USERSPACE1 ORGANIZE BY ROW - ``` +**Desc**: Indicates whether to generate a DDL with the table compression. - - MySQL +**Default**: `false` - ```sql - CREATE TABLE db2_mtk.tab_pk_test ( - -- add AUTO_INCREMENT PRIMARY KEY - pk_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT NOT NULL INVISIBLE, - col1 VARCHAR(10) NOT NULL, - id BIGINT NOT NULL, - col2 VARCHAR(10), - col3 VARCHAR(10), - -- primary key to a unique key - UNIQUE KEY ( col1 ) - ) - ``` +### timeFormat -### autoAddMySQLAutoIncrTabList +**Type**: string -[]string +**Desc**: define time format -Supported after `v2.2.0` version +**Default**: `HH:MI:SS` -Used with the `AutoAddMySQLAutoIncr` parameter. +### dateFormat -Configure which tables automatically increment the `auto_incr` column. Ignored if the table already has a self-incrementing column. The default is empty to represent all tables. +**Type**: string -Parameters are not case-sensitive +**Desc**: define date format -Default: null +**Default**: `YYYY-MM-DD` -Examples: +### dateTimeFormat -```json -"autoAddMySQLAutoIncrTabList":[ - "TABLE_1", - "Schema1.TABLE_1" -] -``` +**Type**: string -- "TABLE_1" +**Desc**: define dateTime format - Allows all tables in the object named TABLE_1 to add `auto_incr` column +**Default**: `YYYY-MM-DD HH24:MI:SS` -- "Schema1.TABLE_1" +### noSupportPartTabToNormalTab - Allows the object to add `auto_incr` column to TABLE_1 tables under Schema1 +**Type**: bool -### ignoreTableDDLCompErr +**Desc**: Convert a partition table that is not supported by the target to a normal table -Indicates whether to ignore the table structure comparison error. +Automatically remove subpartition table attributes that are not supported by the target -Default value: `false` +Oracle's List partition and Hash partition table are ordinary tables in MogDB. -The table structure comparison function is already abandoned. +Oracle's Range List composite partition table is a Range partition table in MogDB, and the sub-partition table is automatically removed ### ignoreDB2PartInclusive -Indicates whether to ignore the `INCLUSIVE` property of the key value `ENDING` in the DB2 partition. +**Type**: bool -Default value: `false` +**Desc**: Indicates whether to ignore the `INCLUSIVE` property of the key value `ENDING` in the DB2 partition. For example, in the table creation syntax of the DB2 partition, if `ENDING` contains the definition of the `INCLUSIVE` property, the DB2 partition will contain `20180101`. @@ -439,30 +1723,40 @@ CREATE TABLE MTK1.TABLE_TEST_HAOTD ) PARTITION BY RANGE(DATADATE) ( PART "DATAMIN" STARTING(MINVALUE) ENDING('20180101') INCLUSIVE , PART "P20180101" STARTING('20180101') ENDING('20180102') INCLUSIVE -); +) ``` -### ignoreNotSupportDefault +**Default**: `false` + +### igNotSupportIntervalPart -Supports ignoring of some default values of some unsupported columns. For example, the `sys_guid`. MTK of Oracle is embedded with a whitelist that will be ignored. +**Type**: bool -Default value: `false` +**Desc**: Supports ignoring of some unsupported interval partition properties. -### igNotSupportIntervalPart +**Default**: `false` + +### igErrorData + +**Type**: bool + +**Desc**:Ignores the data that failed to insert and logs to a file. + +Only openGauss/MogDB. -Supports ignoring of some unsupported interval partition properties. +**Add**: v2.2.3 -Default value: `false` +**Default**: `false` ### ignoreTabPartition -Supports migration to the target database and ignoring of partition syntax. Currently, this parameter supports only migration to MySQL. +**Type**: bool -Default value: `false` +**Desc**: Supports migration to the target database and ignoring of partition syntax. -For example, the Oracle partitioned table becomes a non-partitioned table after being migrated to MySQL. +Currently, this parameter supports only migration to MySQL. -Oracle: +Oracle ```sql CREATE TABLE "MTK"."TAB_PART_LIST" ( @@ -470,7 +1764,7 @@ CREATE TABLE "MTK"."TAB_PART_LIST" ( "DEPTNAME" VARCHAR2(20 BYTE), "QUARTERLY_SALES" NUMBER(10,2), "STATE" VARCHAR2(2 BYTE) -)PARTITION BY LIST ("STATE") +) PARTITION BY LIST ("STATE") ( PARTITION "Q1_NORTHWEST" VALUES ('OR', 'WA') TABLESPACE "USERS", PARTITION "Q1_SOUTHWEST" VALUES ('AZ', 'CA', 'NM') TABLESPACE "USERS", @@ -481,7 +1775,7 @@ CREATE TABLE "MTK"."TAB_PART_LIST" ( ) ``` -MySQL: +MySQL ```sql CREATE TABLE mtk.tab_part_list ( @@ -492,20 +1786,15 @@ CREATE TABLE mtk.tab_part_list ( ) ``` -### igErrorData - -bool - -Supported after `v2.2.2` version. - -Ignores the data that failed to insert and logs to a file. -Only openGauss/MogDB versions after 2.1.0 are supported. +**Default**: `false` ### autoAddMaxvaluePart -Supported after `v0.0.35` version. +**Type**: bool -The parameter `autoAddMaxvaluePart` allows to automatically add a `maxvalue` partition to a partition table where no `maxvalue` partition exists. For example, DB2 partitioned tables are migrated to openGauss. +**Desc**: The parameter `autoAddMaxvaluePart` allows to automatically add a `maxvalue` partition to a partition table where no `maxvalue` partition exists. + +For example, DB2 partitioned tables are migrated to openGauss. DB2 allows defining `minvalue` partitions, which openGauss does not support. @@ -517,17 +1806,17 @@ DB2 does not define the `maxvalue` partition. ```sql CREATE TABLE MTK1.PART_TAB_TEST02 ( - ID INTEGER NOT NULL, - SALES_PERSON VARCHAR(50), - REGION VARCHAR(50), - SALES_DATE DATE + ID INTEGER NOT NULL, + SALES_PERSON VARCHAR(50), + REGION VARCHAR(50), + SALES_DATE DATE ) PARTITION BY RANGE(SALES_DATE) ( - PART PJAN STARTING('2017-01-01') ENDING('2017-03-31') INCLUSIVE IN USERSPACE1, - PART PFEB STARTING('2017-04-01') ENDING('2017-07-31') INCLUSIVE IN USERSPACE1, - PART PMAR STARTING('2017-08-01') ENDING('2017-12-31') INCLUSIVE IN USERSPACE1, - PART PAPR STARTING('2018-01-01') ENDING('2018-12-31') INCLUSIVE IN USERSPACE1 + PART PJAN STARTING('2017-01-01') ENDING('2017-03-31') INCLUSIVE IN USERSPACE1, + PART PFEB STARTING('2017-04-01') ENDING('2017-07-31') INCLUSIVE IN USERSPACE1, + PART PMAR STARTING('2017-08-01') ENDING('2017-12-31') INCLUSIVE IN USERSPACE1, + PART PAPR STARTING('2018-01-01') ENDING('2018-12-31') INCLUSIVE IN USERSPACE1 ) ORGANIZE BY ROW ``` @@ -535,10 +1824,10 @@ openGauss ```sql CREATE TABLE DB2_MTK.PART_TAB_TEST02 ( - ID INTEGER NOT NULL, - SALES_PERSON VARCHAR(50), - REGION VARCHAR(50), - SALES_DATE DATE + ID INTEGER NOT NULL, + SALES_PERSON VARCHAR(50), + REGION VARCHAR(50), + SALES_DATE DATE ) PARTITION BY RANGE (SALES_DATE) ( PARTITION PJAN VALUES LESS THAN('2017-03-31'), @@ -549,493 +1838,421 @@ CREATE TABLE DB2_MTK.PART_TAB_TEST02 ( ) ``` -### virtualColToNormalCol +**Add**: v0.0.36 -Indicates whether to convert virtual columns in the source database into normal columns in the target database. +### autoAddMySQLAutoIncr -Default value: `false` +**Type**: bool -### virtualColConv +**Desc**: Migrate to MySQL to automatically add MySQL `AUTO_INCREMENT` column. -Indicates the virtual column expression conversion function. +If the value is true, when the table does not have a `AUTO_INCREMENT` primary key column,
+automatically add the `AUTO_INCREMENT` column and change the original primary key to a unique key. -```json -"virtualColConv": { - "TRUNC_TIMESTAMP(SNAPTIME,'HH')+ (MINUTE(SNAPTIME)/10*10 +10) MINUTES": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", - "TRUNC_TIMESTAMP(SNAPTIME,'HH')+ (MINUTE(SNAPTIME)/10*10 +10) minutes": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", - "TRUNC_TIMESTAMP(SNAPTIME, 'HH') + (MINUTE(SNAPTIME) / 10 * 10 + 10) MINUTES": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", - "TRUNC_TIMESTAMP(SNAPTIME,'HH') + (MINUTE(SNAPTIME) / 10 * 10 + 10) minutes": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", - "LEFT(HOST,POSSTR(HOST,':')-1)": "SPLIT_PART(HOST,':',1)" -} +```sql +PK_ID BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT NOT NULL [INVISIBLE] ``` -### replaceZeroDate - -openGauss does not support the time format 0000-00-00, which is replaced with 1970-01-01 by default. This parameter can be used for defining the default value with which 0000-00-00 is replaced, such as 2021-01-01. +- Prior to MySQL 8.0.23 column are `visible` +- MySQL 8.0.23 column are `invisible column` -### remapTable +> MySQL supports invisible columns as of MySQL 8.0.23.
+> An invisible column is normally hidden to queries, but can be accessed if explicitly referenced.
+> Prior to MySQL 8.0.23, all columns are visible. -This parameter is already not supported. +Used with [ignoreTabPartition](#ignoretabpartition) parameter. -Indicates whether to modify the table name during migration. +- Mig DB2 to MySQL -### remapSchema + - DB2 -Indicates whether to modify the name of `schema` during migration. + ```sql + CREATE TABLE MTK1.TAB_PK_TEST ( + COL1 VARCHAR(10) NOT NULL, + ID BIGINT NOT NULL, + COL2 VARCHAR(10), + COL3 VARCHAR(10) + ) IN USERSPACE1 ORGANIZE BY ROW + ``` -Format: Name of the original solution : Name of the new solution + - MySQL -For example, `MTK1` is changed to `MTK1_NEW`, and `SOE` is changed to `SOE_NEW`. + ```sql + CREATE TABLE db2_mtk.tab_pk_test ( + -- auto increment primary key column + pk_id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT NOT NULL INVISIBLE, + col1 VARCHAR(10) NOT NULL, + id BIGINT NOT NULL, + col2 VARCHAR(10), + col3 VARCHAR(10), + -- The original primary key becomes the unique key + UNIQUE KEY ( col1 ) + ) + ``` -```json -"remapSchema":{ - "MTK1":"MTK1_NEW", - "SOE":"SOE_NEW" -} -``` +**Default**: `false` -### remapTablespace +**Add**: v2.1.1 -Indicates whether to modify the name of the tablespace during migration. +### autoAddMySQLAutoIncrTabList -This parameter is used together with [enableSyncTabTbsPro](#enableSyncTabTbsPro). +**Type**: []string -Format: Name of the original tablespace : Name of the new tablespace +**Desc**: Used with the `AutoAddMySQLAutoIncr` parameter. -For example, `USERSPACE1` is changed to `TBS01`. +Configure which tables automatically increment the `auto_incr` column. Ignored if the table already has a self-incrementing column. The default is empty to represent all tables. -```json -"remapTablespace":{ - "USERSPACE1":"TBS01", -} -``` +Parameters are not case-sensitive -### enableSyncTabTbsPro +- "TABLE_1" -Indicates whether to generate a DDL with the tablespace. + Allows all tables in the object named TABLE_1 to add `auto_incr` column -Default value: `false` +- "Schema1.TABLE_1" -### enableSyncCompTabPro + Allows the object to add `auto_incr` column to TABLE_1 tables under Schema1 -Indicates whether to generate a DDL with the table compression. +**Default**: null -Default value: `false` +**Example**: -### excludeSysTable +autoAddMySQLAutoIncrTabList examples -Excludes the system object definition. If this parameter is not configured, the default value applies. +```json +{ + "autoAddMySQLAutoIncrTabList": [ + "TABLE_1", + "Schema1.TABLE_1" + ] +} +``` -- DB2 - - "EXPLAIN_ACTUALS", - - "ADVISE_TABLE", - - "ADVISE_PARTITION", - - "ADVISE_MQT", - - "ADVISE_WORKLOAD", - - "ADVISE_INDEX", - - "ADVISE_INSTANCE", - - "OBJECT_METRICS", - - "EXPLAIN_DIAGNOSTIC_DATA", - - "EXPLAIN_DIAGNOSTIC", - - "EXPLAIN_STREAM", - - "EXPLAIN_PREDICATE", - - "EXPLAIN_OPERATOR", - - "EXPLAIN_OBJECT", - - "EXPLAIN_ARGUMENT", - - "EXPLAIN_STATEMENT", - - "EXPLAIN_INSTANCE", +**Add**: v2.2.1 -### enableOgBlobClob +### ignoreNotSupportDefault -Supported after `v2.0.0` version. +**Type**: bool -Migrate other databases 'blob', 'blob' to openGaus 'blob', 'Clob'.. +**Desc**: Supports ignoring of some default values of some unsupported columns. For example, the `sys_guid`. -Deafult convert to openGaus `bytea`、`text` +MTK of Oracle is embedded with a whitelist that will be ignored. -Default value: `false` +**Default**: `false` -### enableConvertSrid +### replaceZeroDate -Supported after `v2.0.0` version. +**Type**: string -This parameter allows you to control the automatically conversion of Oracle SRID to standard EPSG. +**Desc**: openGauss does not support the time format 0000-00-00, which is replaced with 1970-01-01 by default. -If the value is true, all SRID will be forced to `defaultSrid` . +This parameter can be used for defining the default value with which 0000-00-00 is replaced, such as 2021-01-01. -Default value: `false` +> When find a "zero" date: 0000-00-00 00:00:00 it is replaced by a NULL. +> +> This could be a problem if your column is defined with NOT NULL constraint. +> If you can not remove the constraint, +> +> use this directive to set an arbitral date that will be used instead. +> +> You can also use -INFINITY if you don't want to use a fake date. -### defaultSrid +### virtualColToNormalCol -Supported after `v2.0.0` version. +**Type**: bool -Use this parameter to override the default EPSG SRID to used: 4326. -Can be overwritten by `enableConvertSrid`, see [enableConvertSrid](#enableconvertsrid). +**Desc**: Indicates whether to convert virtual columns in the source database into normal columns in the target database. -Default value: 4326 +**Default**: `false` -### skipColumnType +**Option**: -Supported after `v2.1.3` version. +- true +- false -This parameter allows you to control skip a data type column during migration. +### virtualColConv -Format: "column type name": int +**Type**: map[string]string -int +**Desc**: Indicates the virtual column expression conversion function. -- 1 Do not create columns and do not migrate data -- 2 Create columns but do not migrate data +**Example**: -Examples: +virtualColConv example ```json -"skipColumnType":{ - "blob": 1, - "clob": 2 +{ + "virtualColConv": { + "LEFT(HOST,POSSTR(HOST,':')-1)": "SPLIT_PART(HOST,':',1)", + "TRUNC_TIMESTAMP(SNAPTIME, 'HH') + (MINUTE(SNAPTIME) / 10 * 10 + 10) MINUTES": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", + "TRUNC_TIMESTAMP(SNAPTIME,'HH') + (MINUTE(SNAPTIME) / 10 * 10 + 10) minutes": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", + "TRUNC_TIMESTAMP(SNAPTIME,'HH')+ (MINUTE(SNAPTIME)/10*10 +10) MINUTES": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'", + "TRUNC_TIMESTAMP(SNAPTIME,'HH')+ (MINUTE(SNAPTIME)/10*10 +10) minutes": "date_trunc('hour',snaptime) + (date_part('minute',snaptime) / 10 +1)::int * interval '10 min'" + } } ``` -### skipColumnName - -Supported after `v2.1.3` version. - -This parameter allows you to control skip a table column name during migration. - -Priority over `skipColumnType` +### mySQLToOgCharExpression -Format: "schema.table_name.column": int +**Type**: string -int +**Desc**:The length of fields in the original database is automatically multiplied by 3. -- 1 Do not create columns and do not migrate data -- 2 Create columns but do not migrate data +This parameter is valid for only the varchar/char field in the source database of MySQL -Examples: +**Deprecated**: mtk scales up the length based on the db type,column type, db compatibility mode -```json -"skipColumnName":{ - "schema1.tab_01.col1": 1, - "schema1.tab_01.col2": 2 -} -``` +### mySQLSkipErrorDateTimeData -### templateSeqName +**Type**: bool -Supported after `v2.1.5` version. +**Desc**: Automatically skips the wrong time of MySQL. -MySQL automatically increments columns convert sequences, templates for sequence names +For example, datetime type: `0000-00-00 13:14:15` -Default: `SEQ_{{.TabName}}_{{.ColName}}` +**Default**: `false` -Only the following two variables are supported +**Option**: -- {{.TabName}} 表名 -- {{.ColName}} 列名 +- true +- false -Examples: +### ignoreTableDDLCompErr -```json -"templateSeqName": "{{.TabName}}_{{.ColName}}_SEQ" -``` +**Type**: bool -### charAppendEmptyString +**Desc**: Indicates whether to ignore the table structure comparison error. -bool +**Default**: `false` -Supported after `v2.2.2` version. +**Deprecated**: Functional obsolescence -For ORA-29275: Partial multibyte character error. -When parameters are enabled, the Oracle query statement stitches the empty string. Primarily for the following types +**Option**: -- Char -- Character -- NChar -- Varchar -- NVarchar -- Varchar2 -- NVarchar2 +- true +- false -Examples: +### convertPackageMethod -```sql -select chr(195) from dual; -- ORA-29275 -select chr(195)||'' from dual; -- normal +**Type**: string --- Chinese "证券投" GBK encoding -select utl_raw.cast_to_varchar2(hextoraw('D6A4C8AFCDB6D6')) from dual; -- ORA-29275 -select utl_raw.cast_to_varchar2(hextoraw('D6A4C8AFCDB6D6'))||'' from dual; -- normal -``` +**Desc**: How to migrate Oracle Package to openGauss/MogDB -### convertPackageMethod +openGauss database version greater than or equal to 2.1.0 can be configured as `package` or `schema` -string +openGauss databases smaller than 2.1.0 can only be configured as `schema` -Supported after `v2.2.5` version. +**Default**: `schema` -How to migrate Oracle Package to openGauss/MogDB +**Option**: - schema - package -Default: `schema` +### enableOgBlobClob -openGauss database version greater than or equal to 2.1.0 can be configured as `package` or `schema` +**Type**: bool -openGauss databases smaller than 2.1.0 can only be configured as `schema` +**Desc**: Migrate other databases 'blob', 'blob' to openGaus 'blob', 'Clob'. -### tableOptions +**Default**: `false` -map[string]string +**Option**: -Supported after `v2.3.2` version +- true +- false -Configure the Add table attributes when creating a table option. For the time being, only openGauss/MogDB is supported. +### enableConvertSrid -- openGauss/MogDB `create table xxx() with (xx)`; +**Type**: bool -Parameters are not case-sensitive +**Desc**: Enable Convert Srid -Default value: Empty +**Default**: `false` -Examples: +**Option**: -```json -"tableOptions": { - "*": "orientation=row, compression=no", - "Schema1.*": "orientation=row, compression=no", - "Schema1.TABLE_1": "orientation=row, compression=no" -} -``` +- true +- false -- "*", // Migrate all tables in the object -- "Schema1.*" // Migrate all tables under Schema1 in the object -- "Schema1.TABLE_1" // only Schema1.TABLE_1 table add option +### defaultSrid -Option special value remove +**Type**: string -`compression=remove` The compression attribute is removed +**Desc**: PostGis Default Srid -### indexOptions +**Default**: `4326` -map[string]string +### seqLastNumAddNum -Supported after `v2.3.2` version +**Type**: int -As with the tableOptions option configuration, it only takes effect for indexes. +**Desc**: How much to increase when synchronizing sequenced the last value. -## limit +Only supports openGauss/MogDB targets -```json -"limit": { - "parallel": 2, - "fetchSize": 1000, - "batchSize": 1000, - "bufferSize": 4 - "cpBufferSize": 8, - "oracleSelectParallel": 2, -} -``` +**Default**: `0` -### parallel +### skipColumnType -Indicates the degree of parallelism. Create tables, constraints, etc. in parallel. +**Type**: map[string]int -### fetchSize +**Desc**: This parameter allows you to control skip a data type column during migration. -Indicates the fetch size (number of rows to be fetched from a result set every time) +Format: "column type name": int -This parameter can be used when a table contains several millions of rows and you need to avoid memory insufficiency. +**Default**: `0` -### batchSize +**Option**: -Indicates the batch size in batch insert. +- 1 Do not create columns and do not migrate data +- 2 Create columns but do not migrate data -The default value is 1000. If memory insufficiency occurs, lower the value. +**Example**: -### cpBufferSize +skipColumnType example -Indicates the size of data that can be submitted at a time for a database supporting the COPY function. +```json +{ + "skipColumnType": { + "blob": 1, + "clob": 2 + } +} +``` -### oracleSelectParallel +### skipColumnName -Adds hint `/* +parallel(t,n) */` to an Oracle query statement. +**Type**: map[string]int -## object +**Desc**: This parameter allows you to control skip a table column name during migration. -```json -"object": { - "tables": [], - "schemas": [ - "A!" - ], - "excludeTable": { - "MTK": [ - "MTK_TAB1", - "MTK_TAB2" - ], - "SYS": [ - "TAB$", - "OBJ$" - ] - }, - "tableSplit": { - "MTK": { - "TAB_1": [ - " rowid between 'AAAeoSAAEAAAACpAAA' and 'AAAeoSAAEAAAA4oEI/'", - " rowid between 'AAAeoSAAEAAAA4pAAA' and 'AAAeoSAAEAABVSoEI/'", - " rowid between 'AAAeoSAAEAABVSpAAA' and 'AAAeoSAAFAABOEoEI/'", - " rowid between 'AAAeoSAAFAABOEpAAA' and 'AAAeoSAAFAABOGIEI/'" - ], - "TAB_2": [ - "MOD(\"ID\",4)=0", - "MOD(\"ID\",4)=1", - "MOD(\"ID\",4)=2", - "MOD(\"ID\",4)=3" - ], - "TAB_3": [ - "MOD(`ID`,4)=0", - "MOD(`ID`,4)=1", - "MOD(`ID`,,4)=2", - "MOD(`ID`,4)=3" - ] - } - } - } -``` +Priority over `skipColumnType` -Defines migration objects. +Format: "schema.table_name.column": int -- Case-sensitive -- Migration user, database, and schema -- schema = mysql database -- schema = oracle user -- schema = postgres schema -- schema = db2 schema +**Default**: `0` -> Parameters **schemas** and **tables** are mutually exclusive. -> If both parameters are configured, the error "mtk-1002 schema and table cannot exist together" will occur during migration. -> -> If no migration object is defined, the error "not define Migrate object" will occur during migration. +**Option**: -### schema +- 1 Do not create columns and do not migrate data +- 2 Create columns but do not migrate data -Defines schema during migration. +**Example**: + +skipColumnName example ```json -"schemas": [ - "SOE", - "BT_X", - "CUST_X" -] +{ + "skipColumnName": { + "schema1.tab_01.col1": 1, + "schema1.tab_01.col2": 2 + } +} ``` -### tables +### templateSeqName -Defines which tables are to be migrated, including +**Type**: string -- Full table name -- Support for schema.* -- Support for schema.TAB% +**Desc**: MySQL automatically increments columns convert sequences, templates for sequence names -```json -"tables": [ - "MTK.TAB1" - "MTK.TAB111%" - "MTK1.*" -], -``` +Default: `SEQ_{{.TabName}}_{{.ColName}}` -> when running a subcommand migration, such as `sync-function` and configured with tables (`MTK.XXX`) is the 'XXX' function under Migrating Schema 'MTK'. -> Except for sequence. +Only the following two variables are supported -### excludeTable +仅支持以下两个变量 -Defines which tables are excluded. +- {{.TabName}} table name +- {{.ColName}} column name -```json -"excludeTable":{ - "MTK":[ - "TABLE_SKIP1", - "TABLE_SKIP2" - ] -} -``` +**Default**: SEQ_{{.TabName}}_{{.ColName}} -### tableSplit +**Example**: -- Defines table granularity parallelism, which can be automatically generated by using [mtk show-table-split](mtk_show-table-split). -- Defines table migration query conditions. +templateSeqName example ```json -"tableSplit":{ - "MTK":{ - "TABLE_4":[ - "ID < 10000 ", - "ID < 20000 AND ID >=10000", - "ID < 30000 AND ID >=20000", - "ID < 40000 AND ID >=30000", - "ID < 50000 AND ID >=40000", - "ID < 60000 AND ID >=50000", - "ID < 70000 AND ID >=60000", - "ID < 80000 AND ID >=70000", - "ID < 90000 AND ID >=80000", - "ID >= 90000" - ] - } +{ + "templateSeqName": "{{.TabName}}_{{.ColName}}_SEQ" } ``` -## dataOnly +**Add**: v2.1.6 + +### charAppendEmptyString + +**Type**: bool -Indicates whether to migrate only data. +**Desc**: For ORA-29275: Partial multibyte character error. -Default value: `false` +When parameters are enabled, the Oracle query statement stitches the empty string. Primarily for the following types -## schemaOnly +- Char +- Character +- NChar +- Varchar +- NVarchar +- Varchar2 +- NVarchar2 -Indicates whether to migration only the data structure. +```sql +select chr(195) from dual; -- ORA-29275 +select chr(195)||'' from dual; -- normal -Default value: `false` +-- Chinese "证券投" GBK encoding +select utl_raw.cast_to_varchar2(hextoraw('D6A4C8AFCDB6D6')) from dual; -- ORA-29275 +select utl_raw.cast_to_varchar2(hextoraw('D6A4C8AFCDB6D6'))||'' from dual; -- normal +``` -## reportFile +**Default**: `false` -Indicates the migration report directory. +**Option**: -For MTK of a version earlier than v0.0.18, the migration report is shown in HTML. +- true +- false -For MTK v0.0.18 or later, the migration report is stored in a directory. +### tableOptions -## debug +**Type**: map[string]string -Indicates whether to enable the log debug mode. +**Desc**: Configure the Add table attributes when creating a table option. For the time being, only openGauss/MogDB is supported. -Default value: `false` +- openGauss/MogDB `create table xxx() with (xx)`; -## disableTableDataComp +Parameters are not case-sensitive -Disables the table data comparison function. +- "*", // Migrate all tables in the object +- "Schema1.*" // Migrate all tables under Schema1 in the object +- "Schema1.TABLE_1" // only Schema1.TABLE_1 table add option -After data migration is complete, MTK will count and compare the number of rows in the source and target databases. +Option special value `remove` -Default value: `false` +`compression=remove` The compression attribute is removed -## disableCollStatistics +**Default**: null -Disables the collection statistics function. +**Example**: -After data migration is complete, MTK will collect statistics information from the target database. +tableOptions example -Default value: `false` +```json +{ + "tableOptions": { + "*": "orientation=row, compression=no", + "Schema1.*": "orientation=row, compression=no", + "Schema1.TABLE_1": "orientation=row, compression=no" + } +} +``` -## disableSelectPart +**Add**: v2.3.2 -Disable the partition table migration data by partition feature. +### indexOptions -默认值：`false` +**Type**: map[string]string -## disableFKCons +**Desc**: As with the tableOptions option configuration, it only takes effect for indexes. -Disables synchronization of table foreign key constraints. +**Default**: null -默认值：`false` +**Add**: v2.3.2 diff --git a/product/en/docs-mtk/v2.0/release-notes.md b/product/en/docs-mtk/v2.0/release-notes.md index 08cab1dc..8aeb353e 100644 --- a/product/en/docs-mtk/v2.0/release-notes.md +++ b/product/en/docs-mtk/v2.0/release-notes.md @@ -7,6 +7,51 @@ date: 2021-09-13 # Release Notes +## v2.3.4 + +2022-06-19 + +### MTK + +- [mtk_2.3.4_darwin_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_darwin_amd64.tar.gz) +- [mtk_2.3.4_darwin_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_darwin_amd64_db2.tar.gz) +- [mtk_2.3.4_darwin_arm64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_darwin_arm64.tar.gz) +- [mtk_2.3.4_linux_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_linux_amd64.tar.gz) +- [mtk_2.3.4_linux_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_linux_amd64_db2.tar.gz) +- [mtk_2.3.4_linux_arm64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_linux_arm64.tar.gz) +- [mtk_2.3.4_windows_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_windows_amd64.tar.gz) +- [mtk_2.3.4_windows_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_windows_amd64_db2.tar.gz) +- [mtk_checksums.txt](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_checksums.txt) + +### MTKD + +- [mtkd_2.3.4_linux_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtkd_2.3.4_linux_amd64.tar.gz) +- [mtkd_2.3.4_linux_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtkd_2.3.4_linux_amd64_db2.tar.gz) +- [mtkd_2.3.4_linux_arm64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtkd_2.3.4_linux_arm64.tar.gz) + +### Bug Fixes + +- **MySQL:** Migrate MySQL to openGauss to rewrite `if then` syntax issue +- **MySQL:** Migrating MySQL to openGauss B compatibility mode, `NULL` and empty strings, field length issues +- **Oracle:** Migrate Oracle to openGauss to rewrite `mod` function issue +- **Oracle:** Migrate Oracle to openGauss to remove extra brackets +- **Oracle:** no query table partition information in `dataOnly` mode +- **Oracle:** object name with $ problem, no need to add double quotes by default +- **mtk:** to troubleshoot table problems +- **openGauss:** ignore create plugin errors + +### Features + +- **Oracle:** declaration does not support syntax conversion in the case of functions containing functions +- **mtk:** Add parameter [disablePrintMigDataProgress](mtk-config#disableprintmigdataprogress) to disable printing migration data progress +- **mtk:** Add parameter [disableSyncIdxAfterData](mtk-config#disablesyncidxafterdata) to disable the synchronization of table data and create this table index immediately +- **mtk:** report add migration data query SQL +- **openGauss:** supports batch Commit. Task retry requires user configuration [truncTable](mtk-config#trunctable) + +### Performance Improvements + +- **mtk:** Optimize the performance of migrating data to openGauss. 2-3 times faster + ## v2.3.3 2022-06-08 diff --git a/product/zh/docs-mtk/v2.0/mtk-config.md b/product/zh/docs-mtk/v2.0/mtk-config.md index b591bb38..8aeeb47a 100644 --- a/product/zh/docs-mtk/v2.0/mtk-config.md +++ b/product/zh/docs-mtk/v2.0/mtk-config.md @@ -2,7 +2,7 @@ title: MTK Configuration File Description summary: Configuration File Description author: mtk -date: 2022-06-16 12:07:42 +date: 2022-06-19 18:23:32 --- # MTK Configuration File Description @@ -28,6 +28,8 @@ Config MTK迁移配置文件. | disableIgnoreCase |bool |关闭在源库查询忽略大小写功能. | | disableSelectPart |bool |关闭分区查询功能. | | disableFKCons |bool |禁用外键同步. | +| disableSyncIdxAfterData |bool |在一次性迁移模式下禁用在同步完单个表数据后立即创建这个表的索引. 改为全部完全部迁移表数据后,在进行同步索引 | +| disablePrintMigDataProgress |bool |关闭打印迁移表数据进度功能.打印进度影响迁移性能 | **示例**: @@ -173,7 +175,9 @@ Config MTK迁移配置文件. "test": false, "disableIgnoreCase": false, "disableSelectPart": false, - "disableFKCons": false + "disableFKCons": false, + "disableSyncIdxAfterData": false, + "disablePrintMigDataProgress": false } ``` @@ -509,6 +513,32 @@ v0.0.18后是一个目录 - true - false +### disableSyncIdxAfterData + +**类型**: bool + +**描述**: 在一次性迁移模式下禁用在同步完单个表数据后立即创建这个表的索引. 改为全部完全部迁移表数据后,在进行同步索引 + +**默认值**: `false` + +**可选值**: + +- true +- false + +### disablePrintMigDataProgress + +**类型**: bool + +**描述**: 关闭打印迁移表数据进度功能.打印进度影响迁移性能 + +**默认值**: `false` + +**可选值**: + +- true +- false + ## Object Object 迁移对象 @@ -714,7 +744,7 @@ Limit 并发配置 |-------|------|-------------| | parallel |int |并行度. | | fetchSize |int |指定一次行提取行数大小. | -| batchSize |int |指定批量插入或Copy条数大小. | +| batchSize |int |指定批量插入或Copy条数大小并进行Commit. | | bufferSize |int |指定查询或者批量插入时缓存大小(以MB为单位). | | cpBufferSize |int |定义Copy命令中使用的缓存大小(以MB为单位). | | oracleSelectParallel |int |Oracle查询语句中增加hint `/* +parallel(t,n) */` | @@ -767,11 +797,20 @@ limit example **类型**: int -**描述**: 指定批量插入或Copy条数大小. +**描述**: 指定批量插入或Copy条数大小并进行Commit. + +2.3.4 之后版本开始支持批量Commit. 之前版本只是批量与数据交互不进行提交 + +如果提交失败程序会推出并记录错误数据到错误文件. + +如果配置[igErrorData](#igerrordara)程序不会退出会记录错误数据到错误文件并继续处理数据. 有效值为 1-50000 + 支持Oracle、PostgreSQL、openGauss、MySQL. +批量提交支持 PostgreSQL/MogDB + **默认值**: 1000 ### bufferSize @@ -1208,7 +1247,7 @@ Parameter 参数配置 | noSupportPartTabToNormalTab |bool |把目标端不支持的分区表转换普通表 | | ignoreDB2PartInclusive |bool |是否忽略DB2分区键值ENDING的包含属性 | | igNotSupportIntervalPart |bool |支持忽略部分不支持的`interval分`区属性 | -| igErrorData |bool |忽略插入失败的数据并记录到文件`interval分`区属性 | +| igErrorData |bool |忽略插入失败的数据并记录到错误文件. | | ignoreTabPartition |bool |支持迁移到目标端数据库忽略分区语法, 现在只支持迁移到MySQL. | | autoAddMaxvaluePart |bool |参数`autoAddMaxvaluePart`允许不存在`maxvalue`分区的分区表自动添加`maxvalue`分区. | | autoAddMySQLAutoIncr |bool |迁移到MySQL自动添加MySQL `AUTO_INCREMENT` 列 | @@ -1695,9 +1734,11 @@ CREATE TABLE MTK1.TABLE_TEST_HAOTD **类型**: bool -**描述**: 忽略插入失败的数据并记录到错误文件 +**描述**: 忽略插入失败的数据并记录到错误文件. + +只支持openGauss/MogDB -只支持openGauss/MogDB 2.1.0 之后版本 +**新增于**: 2.2.3 **默认值**: `false` diff --git a/product/zh/docs-mtk/v2.0/release-notes.md b/product/zh/docs-mtk/v2.0/release-notes.md index 42cc2430..faded2e8 100644 --- a/product/zh/docs-mtk/v2.0/release-notes.md +++ b/product/zh/docs-mtk/v2.0/release-notes.md @@ -7,6 +7,51 @@ date: 2021-09-17 # 发布记录 +## v2.3.4 + +2022-06-19 + +### MTK + +- [mtk_2.3.4_darwin_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_darwin_amd64.tar.gz) +- [mtk_2.3.4_darwin_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_darwin_amd64_db2.tar.gz) +- [mtk_2.3.4_darwin_arm64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_darwin_arm64.tar.gz) +- [mtk_2.3.4_linux_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_linux_amd64.tar.gz) +- [mtk_2.3.4_linux_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_linux_amd64_db2.tar.gz) +- [mtk_2.3.4_linux_arm64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_linux_arm64.tar.gz) +- [mtk_2.3.4_windows_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_windows_amd64.tar.gz) +- [mtk_2.3.4_windows_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_2.3.4_windows_amd64_db2.tar.gz) +- [mtk_checksums.txt](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtk_checksums.txt) + +### MTKD + +- [mtkd_2.3.4_linux_amd64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtkd_2.3.4_linux_amd64.tar.gz) +- [mtkd_2.3.4_linux_amd64_db2.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtkd_2.3.4_linux_amd64_db2.tar.gz) +- [mtkd_2.3.4_linux_arm64.tar.gz](https://cdn-mogdb.enmotech.com/mtk/v2.3.4/mtkd_2.3.4_linux_arm64.tar.gz) + +### Bug Fixes + +- **MySQL:** 迁移 MySQL 到 openGauss 改写 `if then` 语法问题 +- **MySQL:** 迁移 MySQL 到 openGauss B 兼容模式,`NULL`和空字符串、字段长度问题 +- **Oracle:** 迁移 Oracle 到 openGauss 改写 `mod` 函数问题 +- **Oracle:** 迁移 Oracle 到 openGauss 移除多余括号问题 +- **Oracle:** `dataOnly` 模式下没有查询表分区信息 +- **Oracle:** 对象名带$问题,默认不需要增加双引号 +- **mtk:** 排除表问题 +- **openGauss:** 忽略创建插件错误 + +### Features + +- **Oracle:** 声明不支持函数里包含函数场景下语法转换 +- **mtk:** 添加参数 [disablePrintMigDataProgress](mtk-config#disableprintmigdataprogress) 禁用打印迁移数据进度 +- **mtk:** 添加参数 [disableSyncIdxAfterData](mtk-config#disablesyncidxafterdata) 禁用同步完表的数据立即创建此表索引 +- **mtk:** 报告添加迁移数据查询SQL +- **openGauss:** 支持分批次Commit. 任务重试需要用户配置 [truncTable](mtk-config#trunctable) + +### Performance Improvements + +- **mtk:** 优化迁移数据到openGauss性能. 提升2-3倍 + ## v2.3.3 2022-06-08 -- Gitee