From 09c4deb56f1840c68ab1e3d62d81949f65fbfcdf Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 21 Mar 2024 13:38:22 +0800 Subject: [PATCH 01/27] fix affectRows of executeBatch --- pom.xml | 28 +++++++++---------- .../data/database/api/impl/DataBaseSync.java | 6 ++-- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index 64fe81d..ef6a59a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,17 +10,17 @@ - maven_nexus @@ -116,16 +116,16 @@ - com.oracle.jdbc + com.oracle.database.jdbc ojdbc8 19.3.0.0 - com.ibm.db2.jcc - db2jcc4 - 10.1 + com.ibm.db2 + jcc + 11.1.4.4 runtime diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index f0cbdc2..7c46d0d 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -428,7 +428,8 @@ public class DataBaseSync implements DataSync { // 每10万行提交一次记录 int affectRows = 0; for (int i : pStemt.executeBatch()) { - affectRows += i; + if(i >= 0 || i == pStemt.SUCCESS_NO_INFO) + affectRows += i == pStemt.SUCCESS_NO_INFO ? 1 : i; } this.dbConn.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); @@ -439,7 +440,8 @@ public class DataBaseSync implements DataSync { // 处理剩余的记录 int affectRows = 0; for (int i : pStemt.executeBatch()) { - affectRows += i; + if(i >= 0 || i == pStemt.SUCCESS_NO_INFO) + affectRows += i == pStemt.SUCCESS_NO_INFO ? 1 : i; } this.dbConn.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); -- Gitee From 0206d6f25a2ccfe1476b8fea08bd0d08672f8521 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 29 Mar 2024 13:43:50 +0800 Subject: [PATCH 02/27] add orai18n.jar to resolve Non supported character ZHS16GBK --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index ef6a59a..02f0958 100644 --- a/pom.xml +++ b/pom.xml @@ -120,6 +120,12 @@ ojdbc8 19.3.0.0 + + + com.oracle.database.nls + orai18n + 19.3.0.0 + -- Gitee From 1624201085c9fe0355ff111bd3a15efe9b586690 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 29 Mar 2024 14:30:48 +0800 Subject: [PATCH 03/27] Fixed support for Oracle, such as "alter table" statement does not need "column" keyword, and added the same treatment for NUMERIC type as DECIMAL type. --- pom.xml | 10 +++++----- src/main/java/com/data/database/App.java | 16 ++++++++++++++-- .../com/data/database/api/impl/DataBaseSync.java | 14 +++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pom.xml b/pom.xml index 02f0958..c0a0a4d 100644 --- a/pom.xml +++ b/pom.xml @@ -121,11 +121,11 @@ 19.3.0.0 - - com.oracle.database.nls - orai18n - 19.3.0.0 - + + com.oracle.database.nls + orai18n + 19.3.0.0 + diff --git a/src/main/java/com/data/database/App.java b/src/main/java/com/data/database/App.java index 774f475..fcd4d3c 100644 --- a/src/main/java/com/data/database/App.java +++ b/src/main/java/com/data/database/App.java @@ -227,7 +227,7 @@ public final class App { } } - } else if (colType == Types.DECIMAL) {// 处理小数字段类型 + } else if (colType == Types.DECIMAL || colType == Types.NUMERIC) {// 处理小数字段类型 Integer fromColLength = (Integer) fromMap.get("COLUMN_SIZE"); Integer fromColDigitLength = (Integer) fromMap.get("DECIMAL_DIGITS"); Integer toColLength = (Integer) toMap.get("COLUMN_SIZE"); @@ -237,9 +237,12 @@ public final class App { if ("db2".equals(toType)) { sb.append(String.format("alter table %s.%s alter column %s set data type decimal(%d,%d);", toSchema, toTable, toColName, fromColLength, fromColDigitLength)); - } else if ("mysql".equals(toType) || "oracle".equals(toType)) { + } else if ("mysql".equals(toType)) { sb.append(String.format("alter table %s.%s modify column %s decimal(%d,%d);", toSchema, toTable, toColName, fromColLength, fromColDigitLength)); + } else if ("oracle".equals(toType)) { + sb.append(String.format("alter table %s.%s modify %s number(%d,%d);", toSchema, toTable, + toColName, fromColLength, fromColDigitLength)); } else if ("postgres".equals(toType)) { sb.append(String.format("alter table %s.%s alter column %s type decimal(%d,%d);", toSchema, toTable, toColName, fromColLength, fromColDigitLength)); @@ -274,6 +277,15 @@ public final class App { sb.append(String.format("alter table %s.%s add column %s decimal(%d,%d);", toSchema, toTable, fromColName, fromColLength, fromColDigitLength)); break; + case Types.NUMERIC: + if ("oracle".equals(toType)) { + sb.append(String.format("alter table %s.%s add %s number(%d,%d);", toSchema, toTable, + fromColName, fromColLength, fromColDigitLength)); + } else { + sb.append(String.format("alter table %s.%s add column %s number(%d,%d);", toSchema, toTable, + fromColName, fromColLength, fromColDigitLength)); + } + break; default: sb.append(String.format("alter table %s.%s add column %s %s;", toSchema, toTable, fromColName, typeName)); diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 7c46d0d..c9245cf 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -27,10 +27,6 @@ import org.apache.logging.log4j.LogManager; public class DataBaseSync implements DataSync { - /* - * 获取一个表的列名，用于 readData 时指定一个表的字段，之所以不使用 * 是因为源数据库表的字段有可能新增， - * 也可以用于表结构的自动同步：当目标表的字段数少于源表时，对目标表自动执行 alter table add columns xxx - */ protected final String jdbcDriver; protected final String dbUrl; // 数据库的用户名与密码，需要根据自己的设置 @@ -137,9 +133,9 @@ public class DataBaseSync implements DataSync { } public ResultSet getColMetaData(String schemaName, String tableName) throws SQLException { - // to do 如果源表与目标表不一致，则以愿表为准，修改目标表： + // to do 如果源表与目标表不一致，则以源表为准，修改目标表： // 用于 扩字段长度，增加字段，删除字段，修改字段类型。 - /* 由于 mysql postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ + /* 由于 mysql 、 postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ String tableCatalog = null; if (this.dbType.equals("mysql")) { @@ -169,6 +165,10 @@ public class DataBaseSync implements DataSync { } + /* + * 获取一个表的列名，用于 readData 时指定一个表的字段，之所以不使用 * 是因为源数据库表的字段有可能新增， + * 也可以用于表结构的自动同步：当目标表的字段数少于源表时，对目标表自动执行 alter table add columns xxx + */ public List getTableColumns(String schemaName, String tableName) throws SQLException { logger.info(String.format("get table columns from %s.%s ", schemaName, tableName)); List columnNames = new ArrayList<>(); @@ -262,7 +262,7 @@ public class DataBaseSync implements DataSync { Integer dataType = resultSet.getInt(5); String columnType = ""; if(this.dbType.equals(config.getDbType())){ - //说明原数据库与目标数据库一致，不需要任何字段类型转换 + //说明源数据库与目标数据库一致，不需要任何字段类型转换 columnType = resultSet.getString(6); } else{ columnType = convertColumnType(config.getDbType(), resultSet.getString(6)); -- Gitee From 33f05630a558b926c6856fd079dd910ce4d8282b Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 29 Mar 2024 15:20:33 +0800 Subject: [PATCH 04/27] Support for Oracle when adding fields --- src/main/java/com/data/database/App.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/data/database/App.java b/src/main/java/com/data/database/App.java index fcd4d3c..c1fdcff 100644 --- a/src/main/java/com/data/database/App.java +++ b/src/main/java/com/data/database/App.java @@ -270,8 +270,13 @@ public final class App { switch (colType) { case Types.CHAR: case Types.VARCHAR: - sb.append(String.format("alter table %s.%s add column %s varchar(%d);", toSchema, toTable, - fromColName, fromColLength * colSizeTimes.getTimes())); + if ("oracle".equals(toType)) { + sb.append(String.format("alter table %s.%s add %s varchar2(%d);", toSchema, toTable, + fromColName, fromColLength * colSizeTimes.getTimes())); + } else { + sb.append(String.format("alter table %s.%s add column %s varchar(%d);", toSchema, toTable, + fromColName, fromColLength * colSizeTimes.getTimes())); + } break; case Types.DECIMAL: sb.append(String.format("alter table %s.%s add column %s decimal(%d,%d);", toSchema, toTable, @@ -287,8 +292,13 @@ public final class App { } break; default: - sb.append(String.format("alter table %s.%s add column %s %s;", toSchema, toTable, fromColName, - typeName)); + if ("oracle".equals(toType)) { + sb.append(String.format("alter table %s.%s add %s %s;", toSchema, toTable, fromColName, + typeName)); + } else { + sb.append(String.format("alter table %s.%s add column %s %s;", toSchema, toTable, fromColName, + typeName)); + } } } -- Gitee From b7843ad6869ed86645e033508eea3be0dc88084a Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 10 Apr 2024 12:24:35 +0800 Subject: [PATCH 05/27] fix Column name pattern can not be NULL or empty on mysql-connector-java-6.x --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index c9245cf..5d35f6a 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -182,7 +182,7 @@ public class DataBaseSync implements DataSync { if (this.dbType.equals("mysql")) { tableCatalog = schemaName; } - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, null); + ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); while (resultSet.next()) { // for(int i =0 ;i < 24; i++){ // System.out.print(resultSet.getObject(i+1)+"\t"); -- Gitee From d0f455ebabc2636d389e99f6c6c8370121a8dc4a Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 10 Apr 2024 13:04:36 +0800 Subject: [PATCH 06/27] fix "Column name pattern can not be NULL or empty" the same as getTableColumns --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 5d35f6a..7c463bc 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -142,7 +142,7 @@ public class DataBaseSync implements DataSync { tableCatalog = schemaName; } DatabaseMetaData metaData = this.dbConn.getMetaData(); - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, null); + ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); return resultSet; } @@ -157,7 +157,7 @@ public class DataBaseSync implements DataSync { tableCatalog = schemaName; } DatabaseMetaData metaData = this.dbConn.getMetaData(); - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, null); + ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); while (resultSet.next()) { colTypes.add(resultSet.getInt(5)); } -- Gitee From 39a1e8048bb708276f58b7b5ae514f5ec4632fb1 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Apr 2024 13:31:18 +0800 Subject: [PATCH 07/27] Fix the commit mode to be compatible with accessing clickhouse with mysql api. --- .../java/com/data/database/api/impl/DataBaseSync.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 7c463bc..5f53da2 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -397,10 +397,12 @@ public class DataBaseSync implements DataSync { logger.info(String.format("clear data for %s is begin...", tbName)); logger.info("clearSql : " + clearSql); - this.dbConn.commit(); + int supportsTransactions = 0; + try {supportsTransactions = this.dbConn.getTransactionIsolation();} catch(SQLException e){logger.info("clearSql : " + e.getMessage()); supportsTransactions = Integer.parseInt(e.getMessage().replaceAll("[^0-9]", ""));} + if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); PreparedStatement pStemt = this.dbConn.prepareStatement(clearSql); pStemt.executeUpdate(); - this.dbConn.commit(); + if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); logger.info(String.format("clear data for %s is done", tbName)); List colTypes = this.getColumnTypes(schemaName, tableName); @@ -431,7 +433,7 @@ public class DataBaseSync implements DataSync { if(i >= 0 || i == pStemt.SUCCESS_NO_INFO) affectRows += i == pStemt.SUCCESS_NO_INFO ? 1 : i; } - this.dbConn.commit(); + if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); totalAffectRows += affectRows; rowCount = 0; @@ -443,7 +445,7 @@ public class DataBaseSync implements DataSync { if(i >= 0 || i == pStemt.SUCCESS_NO_INFO) affectRows += i == pStemt.SUCCESS_NO_INFO ? 1 : i; } - this.dbConn.commit(); + if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); totalAffectRows += affectRows; rowCount = 0; -- Gitee From 0686a939bf6efcd53e5dd4a8f5d80e7a36f64384 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Apr 2024 14:26:52 +0800 Subject: [PATCH 08/27] Correcting field type conversion, JDBC data type instead of db column type. --- .../data/database/api/impl/DataBaseSync.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 5f53da2..f93ca7e 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -24,6 +24,7 @@ import com.data.database.api.MyEnum.ColSizeTimes; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import java.lang.reflect.Field; public class DataBaseSync implements DataSync { @@ -117,6 +118,20 @@ public class DataBaseSync implements DataSync { } + public static String getJdbcTypeName(int jdbcType) { + Field[] fields = Types.class.getFields(); + for (Field field : fields) { + try { + if (field.getInt(null) == jdbcType) { + return field.getName(); + } + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + return "UNKNOWN"; + } + public String convertColumnType(String dbType, String colType) { if (dbType.equals("postgres")) { return postgresqlMap.getOrDefault(colType,colType); @@ -265,7 +280,7 @@ public class DataBaseSync implements DataSync { //说明源数据库与目标数据库一致，不需要任何字段类型转换 columnType = resultSet.getString(6); } else{ - columnType = convertColumnType(config.getDbType(), resultSet.getString(6)); + columnType = convertColumnType(config.getDbType(), getJdbcTypeName(dataType)); } Integer columnSize = resultSet.getInt(7); -- Gitee From 899b8c0d3285d8474fb85abbe892a44b8219aadb Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Apr 2024 14:40:19 +0800 Subject: [PATCH 09/27] Converting CLOB for mysql and postgresql --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index f93ca7e..2479ee2 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -74,6 +74,7 @@ public class DataBaseSync implements DataSync { //以下为 java.sql.types 到各数据库类型的映射关系，可以添加其他类型。 + mysqlMap.put("CLOB","LONGTEXT"); mysqlMap.put("BINARY","TINYBLOB"); mysqlMap.put("BIT",""); mysqlMap.put("LONGVARBINARY","MEDIUMBLOB"); @@ -103,6 +104,7 @@ public class DataBaseSync implements DataSync { oracleMap.put("VARBINARY","RAW"); + postgresqlMap.put("CLOB","TEXT"); postgresqlMap.put("BINARY",""); postgresqlMap.put("BIT",""); postgresqlMap.put("LONGVARBINARY",""); -- Gitee From 238884d72fa737cd4f2e9b8c3e2fe5dd4d7aa647 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 15 Apr 2024 17:57:42 +0800 Subject: [PATCH 10/27] Add support for clickhouse. --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index c0a0a4d..f3e1c7b 100644 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,12 @@ runtime + + com.clickhouse + clickhouse-jdbc + 0.5.0 + + org.postgresql postgresql -- Gitee From 9db36b79c43b4f29738b1b606729d50edca8518c Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 16 Apr 2024 17:43:57 +0800 Subject: [PATCH 11/27] columnNamePattern can not be null. --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 2479ee2..7eb74d8 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -263,7 +263,7 @@ public class DataBaseSync implements DataSync { table_remark = tbResultset.getString(5); } - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, null); + ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); int i = 0; ArrayList pks = new ArrayList(); -- Gitee From d795e59d6b0b08141bc3996d7ee5538b4dba6ff9 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 18 Apr 2024 12:15:00 +0800 Subject: [PATCH 12/27] reuse copyManager, don't defined it in the while loop --- .../com/data/database/api/impl/PostgresDataBaseSync.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java b/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java index 577aff6..3a77b2f 100644 --- a/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java @@ -64,6 +64,7 @@ public class PostgresDataBaseSync extends DataBaseSync { tbName,String.join(",",columnNames)); logger.info("copyFromSql = "+copyFromSql); long starttime = System.currentTimeMillis(); + CopyManager copyManager = new CopyManager((BaseConnection) this.dbConn); while (rs.next()) { for (int i = 0; i < numberOfCols - 1; i++) { String col = rs.getString(i+1); @@ -93,11 +94,10 @@ public class PostgresDataBaseSync extends DataBaseSync { rowCount++; if (rowCount >= bufferRows) { // 每10万行提交一次记录 - CopyManager copyManager = new CopyManager((BaseConnection) this.dbConn); + // CopyManager copyManager = new CopyManager((BaseConnection) this.dbConn); // What the funck. Why defined it in the while loop // long rowsInserted = copyManager.copyIn("copy " + tbName + " from stdin // delimiter e'\\x02' NULL as ''", - long rowsInserted = copyManager.copyIn(copyFromSql, - new BufferedReader(new StringReader(sBuilder.toString()))); + long rowsInserted = copyManager.copyIn(copyFromSql, new BufferedReader(new StringReader(sBuilder.toString()))); this.dbConn.commit(); // sBuilder.delete(0, sBuilder.length());//清空缓冲区 sBuilder.setLength(0);// 清空缓冲区 @@ -108,7 +108,7 @@ public class PostgresDataBaseSync extends DataBaseSync { } // 处理剩余的记录 - CopyManager copyManager = new CopyManager((BaseConnection) this.dbConn); + // CopyManager copyManager = new CopyManager((BaseConnection) this.dbConn); // Why not reuse it? long rowsInserted = copyManager.copyIn(copyFromSql, new BufferedReader(new StringReader(sBuilder.toString()))); this.dbConn.commit(); sBuilder.setLength(0);// 清空缓冲区 -- Gitee From 03af026cb4c0f974694baa079ee20dd06d1ab0f1 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 18 Apr 2024 13:34:00 +0800 Subject: [PATCH 13/27] Give a hint if the case sensitivity causes the list of fields to be empty. --- src/main/java/com/data/database/App.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/data/database/App.java b/src/main/java/com/data/database/App.java index c1fdcff..d449fd9 100644 --- a/src/main/java/com/data/database/App.java +++ b/src/main/java/com/data/database/App.java @@ -426,6 +426,9 @@ public final class App { toColumnNames = app.getToTableFields(); } else { toColumnNames = toDataBase.getTableColumns(app.getToSchema(),app.getToTable()); + if(toColumnNames.size() == 0) { + throw new RuntimeException(String.format("failed to get table columns from (%s)%s.%s. Please pay attention to the case sensitivity if the table exists or has just been created successfully.", app.getToDbId(), app.getToSchema(), app.getToTable())); + } } if (app.getFromTableFields().size() > 0) { -- Gitee From f980a9299447c6fd0d2ea3048e3b0e46ba259512 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 18 Apr 2024 13:50:02 +0800 Subject: [PATCH 14/27] fix the problem of there are not number datatype except Oracle --- src/main/java/com/data/database/App.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/data/database/App.java b/src/main/java/com/data/database/App.java index d449fd9..f49348b 100644 --- a/src/main/java/com/data/database/App.java +++ b/src/main/java/com/data/database/App.java @@ -287,7 +287,7 @@ public final class App { sb.append(String.format("alter table %s.%s add %s number(%d,%d);", toSchema, toTable, fromColName, fromColLength, fromColDigitLength)); } else { - sb.append(String.format("alter table %s.%s add column %s number(%d,%d);", toSchema, toTable, + sb.append(String.format("alter table %s.%s add column %s numeric(%d,%d);", toSchema, toTable, fromColName, fromColLength, fromColDigitLength)); } break; -- Gitee From 8352ac504c5a00e38c88ed6bcedc01a74662b198 Mon Sep 17 00:00:00 2001 From: Quake Date: Fri, 19 Apr 2024 09:48:54 +0800 Subject: [PATCH 15/27] Adjust the format --- src/main/java/com/data/database/App.java | 7 ++----- .../data/database/api/impl/DataBaseSync.java | 21 +++++++------------ .../api/impl/DataBaseSyncFactory.java | 2 +- .../database/api/impl/Db2DataBaseSync.java | 18 +++++++--------- .../database/api/impl/OracleDataBaseSync.java | 20 +++++++----------- .../api/impl/PostgresDataBaseSync.java | 6 ++---- 6 files changed, 28 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/data/database/App.java b/src/main/java/com/data/database/App.java index f49348b..cc3d6df 100644 --- a/src/main/java/com/data/database/App.java +++ b/src/main/java/com/data/database/App.java @@ -2,7 +2,7 @@ package com.data.database; import java.sql.*; -import com.data.database.utils.Tools; +// import com.data.database.utils.Tools; import java.io.IOException; import com.data.database.api.impl.DataBaseSyncFactory; @@ -57,7 +57,6 @@ public final class App { return this.toTable; } - public boolean getIsSyncTableDDL(){ return this.isSyncTableDDL; } @@ -112,7 +111,6 @@ public final class App { return false; } - this.fromDBId = argList.get(0); this.fromSchema = argList.get(1); this.fromTable = argList.get(2); @@ -163,8 +161,7 @@ public final class App { * @throws SQLException */ - public static String getAlterSqlFromMetaData(String toType, String toSchema, String toTable, ResultSet fromColumnMeta, - ResultSet toColumnMeta, ColSizeTimes colSizeTimes) throws SQLException { + public static String getAlterSqlFromMetaData(String toType, String toSchema, String toTable, ResultSet fromColumnMeta, ResultSet toColumnMeta, ColSizeTimes colSizeTimes) throws SQLException { StringBuilder sb = new StringBuilder(); HashMap> fromColMap = new HashMap>(); diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 7eb74d8..b5db7c0 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -4,7 +4,7 @@ import com.data.database.Config; import com.data.database.api.DataSync; import java.io.IOException; -import java.net.SecureCacheResponse; +// import java.net.SecureCacheResponse; import java.sql.Connection; import java.sql.SQLException; import java.sql.ResultSet; @@ -24,6 +24,7 @@ import com.data.database.api.MyEnum.ColSizeTimes; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; + import java.lang.reflect.Field; public class DataBaseSync implements DataSync { @@ -48,7 +49,6 @@ public class DataBaseSync implements DataSync { protected final HashMap sqlserverMap = new HashMap<>(); - public DataBaseSync(final String dbType, final String jdbcDriver, final String dbUrl, final String dbUser, final String dbPass, final Integer bufferRows) throws SQLException, ClassNotFoundException { this.dbType = dbType; @@ -91,7 +91,6 @@ public class DataBaseSync implements DataSync { db2Map.put("TEXT","CLOB"); db2Map.put("INT","INTEGER"); - oracleMap.put("BIGINT","NUMBER"); oracleMap.put("BINARY","RAW"); oracleMap.put("DECIMAL","NUMBER"); @@ -103,7 +102,6 @@ public class DataBaseSync implements DataSync { oracleMap.put("TINYINT","TINYINT"); oracleMap.put("VARBINARY","RAW"); - postgresqlMap.put("CLOB","TEXT"); postgresqlMap.put("BINARY",""); postgresqlMap.put("BIT",""); @@ -366,8 +364,7 @@ public class DataBaseSync implements DataSync { }; /* 获取一个表的数据，采用流式读取，可以提供 whereClause 表示增量读取 ，如果 whereClause 为空，表示全量读取 */ - public ResultSet readData(String schemaName, final String tableName, final List columnNames, - String whereClause) throws SQLException { + public ResultSet readData(String schemaName, final String tableName, final List columnNames, String whereClause) throws SQLException { this.dbConn.setAutoCommit(false); logger.info(String.format("read data from %s.%s ", schemaName, tableName)); final String columns = Joiner.on(", ").join(columnNames); @@ -387,8 +384,7 @@ public class DataBaseSync implements DataSync { }; /* 将 ResultSet 类型的数据定入一张表，写入成功返回 true,否则返回 false */ - public boolean writeData(String schemaName, final String tableName, final List columnNames, ResultSet rs, - final String whereClause) throws IOException, SQLException { + public boolean writeData(String schemaName, final String tableName, final List columnNames, ResultSet rs, final String whereClause) throws IOException, SQLException { this.dbConn.setAutoCommit(false); logger.info(String.format("begin insert into %s.%s...", schemaName, tableName)); String tbName = ""; @@ -447,8 +443,8 @@ public class DataBaseSync implements DataSync { // 每10万行提交一次记录 int affectRows = 0; for (int i : pStemt.executeBatch()) { - if(i >= 0 || i == pStemt.SUCCESS_NO_INFO) - affectRows += i == pStemt.SUCCESS_NO_INFO ? 1 : i; + if(i >= 0 || i == PreparedStatement.SUCCESS_NO_INFO) + affectRows += i == PreparedStatement.SUCCESS_NO_INFO ? 1 : i; } if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); @@ -459,8 +455,8 @@ public class DataBaseSync implements DataSync { // 处理剩余的记录 int affectRows = 0; for (int i : pStemt.executeBatch()) { - if(i >= 0 || i == pStemt.SUCCESS_NO_INFO) - affectRows += i == pStemt.SUCCESS_NO_INFO ? 1 : i; + if(i >= 0 || i == PreparedStatement.SUCCESS_NO_INFO) + affectRows += i == PreparedStatement.SUCCESS_NO_INFO ? 1 : i; } if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); @@ -474,7 +470,6 @@ public class DataBaseSync implements DataSync { /* 执行提供的 ddl 建表 */ public boolean createTable(String schemaName, String tableName, String ddl) throws SQLException { - this.dbConn.setAutoCommit(true); String newDDL = ""; if (schemaName == null || schemaName.length() == 0) { diff --git a/src/main/java/com/data/database/api/impl/DataBaseSyncFactory.java b/src/main/java/com/data/database/api/impl/DataBaseSyncFactory.java index aecd7d0..247c77d 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSyncFactory.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSyncFactory.java @@ -1,6 +1,6 @@ package com.data.database.api.impl; import com.data.database.Config; -import java.util.Map; +// import java.util.Map; import java.sql.SQLException; public class DataBaseSyncFactory { diff --git a/src/main/java/com/data/database/api/impl/Db2DataBaseSync.java b/src/main/java/com/data/database/api/impl/Db2DataBaseSync.java index 2e1d6a0..2926b4a 100644 --- a/src/main/java/com/data/database/api/impl/Db2DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/Db2DataBaseSync.java @@ -1,20 +1,17 @@ package com.data.database.api.impl; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.StringReader; -import java.sql.PreparedStatement; -import java.sql.ResultSet; +// import java.io.BufferedReader; +// import java.io.IOException; +// import java.io.StringReader; +// import java.sql.PreparedStatement; +// import java.sql.ResultSet; import java.sql.SQLException; -import java.util.List; - +// import java.util.List; public class Db2DataBaseSync extends DataBaseSync { //在此编写db2特性的代码逻辑 - - public Db2DataBaseSync(String dbType, String jdbcDriver, String dbUrl, String dbUser, String dbPass, - Integer bufferRows) throws SQLException, ClassNotFoundException { + public Db2DataBaseSync(String dbType, String jdbcDriver, String dbUrl, String dbUser, String dbPass, Integer bufferRows) throws SQLException, ClassNotFoundException { super(dbType, jdbcDriver, dbUrl, dbUser, dbPass, bufferRows); } @@ -26,5 +23,4 @@ public class Db2DataBaseSync extends DataBaseSync { // return true; // } - }; \ No newline at end of file diff --git a/src/main/java/com/data/database/api/impl/OracleDataBaseSync.java b/src/main/java/com/data/database/api/impl/OracleDataBaseSync.java index 1e05e9b..f3910cd 100644 --- a/src/main/java/com/data/database/api/impl/OracleDataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/OracleDataBaseSync.java @@ -1,24 +1,20 @@ package com.data.database.api.impl; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.StringReader; -import java.sql.PreparedStatement; -import java.sql.ResultSet; +// import java.io.BufferedReader; +// import java.io.IOException; +// import java.io.StringReader; +// import java.sql.PreparedStatement; +// import java.sql.ResultSet; import java.sql.SQLException; -import java.util.List; - +// import java.util.List; public class OracleDataBaseSync extends DataBaseSync { - //在此编写db2特性的代码逻辑 - + //在此编写Oracle特性的代码逻辑 - public OracleDataBaseSync(String dbType, String jdbcDriver, String dbUrl, String dbUser, String dbPass, - Integer bufferRows) throws SQLException, ClassNotFoundException { + public OracleDataBaseSync(String dbType, String jdbcDriver, String dbUrl, String dbUser, String dbPass, Integer bufferRows) throws SQLException, ClassNotFoundException { super(dbType, jdbcDriver, dbUrl, dbUser, dbPass, bufferRows); } - // @Override // public boolean writeData(String schemaName, final String tableName, List columnNames, ResultSet rs, // final String whereClause) { diff --git a/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java b/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java index 3a77b2f..a10ca2f 100644 --- a/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/PostgresDataBaseSync.java @@ -13,14 +13,12 @@ import org.postgresql.core.BaseConnection; public class PostgresDataBaseSync extends DataBaseSync { - public PostgresDataBaseSync(String dbType, String jdbcDriver, String dbUrl, String dbUser, String dbPass, - Integer bufferRows) throws SQLException, ClassNotFoundException { + public PostgresDataBaseSync(String dbType, String jdbcDriver, String dbUrl, String dbUser, String dbPass, Integer bufferRows) throws SQLException, ClassNotFoundException { super(dbType, jdbcDriver, dbUrl, dbUser, dbPass, bufferRows); } @Override - public boolean writeData(String schemaName, final String tableName, List columnNames, ResultSet rs, - final String whereClause) throws IOException, SQLException { + public boolean writeData(String schemaName, final String tableName, List columnNames, ResultSet rs, final String whereClause) throws IOException, SQLException { this.dbConn.setAutoCommit(false); logger.info(String.format("begin insert into %s.%s...", schemaName, tableName)); String tbName = ""; -- Gitee From d6b34ad649500bac605c49f8b5acde4dbe3d3a8a Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 19 Apr 2024 15:27:01 +0800 Subject: [PATCH 16/27] Adjust the format --- .gitignore | 3 ++- .../data/database/api/impl/DataBaseSync.java | 19 ++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index c455e6c..7078594 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ src/note.md *.xlsx *.swp .editorconfig -.travis.yml \ No newline at end of file +.travis.yml +.factorypath diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index b5db7c0..056dd57 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -49,8 +49,7 @@ public class DataBaseSync implements DataSync { protected final HashMap sqlserverMap = new HashMap<>(); - public DataBaseSync(final String dbType, final String jdbcDriver, final String dbUrl, final String dbUser, - final String dbPass, final Integer bufferRows) throws SQLException, ClassNotFoundException { + public DataBaseSync(final String dbType, final String jdbcDriver, final String dbUrl, final String dbUser, final String dbPass, final Integer bufferRows) throws SQLException, ClassNotFoundException { this.dbType = dbType; this.jdbcDriver = jdbcDriver; this.dbUrl = dbUrl; @@ -144,14 +143,12 @@ public class DataBaseSync implements DataSync { }else{ return colType; } - } + // to do 如果源表与目标表不一致，则以源表为准，修改目标表： + // 用于 扩字段长度，增加字段，删除字段，修改字段类型。 + /* 由于 mysql 、 postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ public ResultSet getColMetaData(String schemaName, String tableName) throws SQLException { - // to do 如果源表与目标表不一致，则以源表为准，修改目标表： - // 用于 扩字段长度，增加字段，删除字段，修改字段类型。 - /* 由于 mysql 、 postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ - String tableCatalog = null; if (this.dbType.equals("mysql")) { tableCatalog = schemaName; @@ -159,7 +156,6 @@ public class DataBaseSync implements DataSync { DatabaseMetaData metaData = this.dbConn.getMetaData(); ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); return resultSet; - } public List getColumnTypes(String schemaName, String tableName) throws SQLException { @@ -177,7 +173,6 @@ public class DataBaseSync implements DataSync { colTypes.add(resultSet.getInt(5)); } return colTypes; - } /* @@ -188,11 +183,9 @@ public class DataBaseSync implements DataSync { logger.info(String.format("get table columns from %s.%s ", schemaName, tableName)); List columnNames = new ArrayList<>(); DatabaseMetaData metaData = this.dbConn.getMetaData(); - if(schemaName.equals("null")){ schemaName = null; } - String tableCatalog = null; if (this.dbType.equals("mysql")) { tableCatalog = schemaName; @@ -206,7 +199,6 @@ public class DataBaseSync implements DataSync { columnNames.add(resultSet.getString(4)); } return columnNames; - }; /* 判断一个表是否存在 */ @@ -224,7 +216,6 @@ public class DataBaseSync implements DataSync { // schemaName = schemaName == null ? null : schemaName.toUpperCase(); // tableName = tableName == null ? null : tableName.toUpperCase(); // } - schemaName = schemaName.equals("null") ? null : schemaName; tableName = tableName.equals("null") ? null : tableName; ResultSet resultSet = metaData.getTables(null, schemaName, tableName, null); @@ -232,7 +223,6 @@ public class DataBaseSync implements DataSync { return true; } return false; - }; /* 获取一个表的 DDL 语句，用于在目标数据库创建表 lenSize 默认值 1 @@ -250,7 +240,6 @@ public class DataBaseSync implements DataSync { StringBuilder sb = new StringBuilder(1024); StringBuilder sb_remark = new StringBuilder(1024); String table_remark = ""; - String tableCatalog = null; if(schemaName.equals("null")){ -- Gitee From af45643002f2387b72bc69d973cb9f969ccfa96c Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 19 Apr 2024 15:42:21 +0800 Subject: [PATCH 17/27] Why not judge null? --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 056dd57..e922cf2 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -150,6 +150,9 @@ public class DataBaseSync implements DataSync { /* 由于 mysql 、 postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ public ResultSet getColMetaData(String schemaName, String tableName) throws SQLException { String tableCatalog = null; + if(schemaName.equals("null")){ + schemaName = null; + } if (this.dbType.equals("mysql")) { tableCatalog = schemaName; } -- Gitee From 9d47bebab7cc597418375d5507bacd8fc9f33581 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 19 Apr 2024 15:50:35 +0800 Subject: [PATCH 18/27] Why not reuse? --- .../data/database/api/impl/DataBaseSync.java | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index e922cf2..f7ae4b9 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -163,15 +163,7 @@ public class DataBaseSync implements DataSync { public List getColumnTypes(String schemaName, String tableName) throws SQLException { List colTypes = new ArrayList<>(); - String tableCatalog = null; - if(schemaName.equals("null")){ - schemaName = null; - } - if (this.dbType.equals("mysql")) { - tableCatalog = schemaName; - } - DatabaseMetaData metaData = this.dbConn.getMetaData(); - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); + ResultSet resultSet = this.getColMetaData(schemaName, tableName); while (resultSet.next()) { colTypes.add(resultSet.getInt(5)); } @@ -185,15 +177,7 @@ public class DataBaseSync implements DataSync { public List getTableColumns(String schemaName, String tableName) throws SQLException { logger.info(String.format("get table columns from %s.%s ", schemaName, tableName)); List columnNames = new ArrayList<>(); - DatabaseMetaData metaData = this.dbConn.getMetaData(); - if(schemaName.equals("null")){ - schemaName = null; - } - String tableCatalog = null; - if (this.dbType.equals("mysql")) { - tableCatalog = schemaName; - } - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); + ResultSet resultSet = this.getColMetaData(schemaName, tableName); while (resultSet.next()) { // for(int i =0 ;i < 24; i++){ // System.out.print(resultSet.getObject(i+1)+"\t"); -- Gitee From 3fa19515b93154581422d522ae121f8a03f25228 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 19 Apr 2024 16:13:12 +0800 Subject: [PATCH 19/27] simplify and reuse --- .../data/database/api/impl/DataBaseSync.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index f7ae4b9..86316d0 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -149,13 +149,8 @@ public class DataBaseSync implements DataSync { // 用于 扩字段长度，增加字段，删除字段，修改字段类型。 /* 由于 mysql 、 postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ public ResultSet getColMetaData(String schemaName, String tableName) throws SQLException { - String tableCatalog = null; - if(schemaName.equals("null")){ - schemaName = null; - } - if (this.dbType.equals("mysql")) { - tableCatalog = schemaName; - } + String tableCatalog = this.dbType.equals("mysql") ? schemaName : null; + schemaName = schemaName.equals("null") ? null : schemaName; DatabaseMetaData metaData = this.dbConn.getMetaData(); ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); return resultSet; @@ -227,17 +222,8 @@ public class DataBaseSync implements DataSync { StringBuilder sb = new StringBuilder(1024); StringBuilder sb_remark = new StringBuilder(1024); String table_remark = ""; - String tableCatalog = null; - - if(schemaName.equals("null")){ - schemaName = null; - } - ResultSet tbResultset = metaData.getTables(tableCatalog, schemaName, tableName, null); - if (tbResultset.next()) { - table_remark = tbResultset.getString(5); - } - - ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); + + ResultSet resultSet = this.getColMetaData(schemaName, tableName); int i = 0; ArrayList pks = new ArrayList(); -- Gitee From 2b4079eea3065a9a187ecc84d3970e979a8a80b9 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 19 Apr 2024 16:17:50 +0800 Subject: [PATCH 20/27] My misstake. That must be saved. --- .../java/com/data/database/api/impl/DataBaseSync.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 86316d0..eaef1f2 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -222,12 +222,16 @@ public class DataBaseSync implements DataSync { StringBuilder sb = new StringBuilder(1024); StringBuilder sb_remark = new StringBuilder(1024); String table_remark = ""; - + String tableCatalog = this.dbType.equals("mysql") ? schemaName : null; + ResultSet tbResultset = metaData.getTables(tableCatalog, schemaName, tableName, null); + if (tbResultset.next()) { + table_remark = tbResultset.getString(5); + } ResultSet resultSet = this.getColMetaData(schemaName, tableName); int i = 0; ArrayList pks = new ArrayList(); - ResultSet resultSetPKS = metaData.getPrimaryKeys(null, schemaName, tableName); + ResultSet resultSetPKS = metaData.getPrimaryKeys(tableCatalog, schemaName, tableName); while (resultSetPKS.next()) { pks.add(resultSetPKS.getString(4)); } -- Gitee From 8d0b0faf85e08247c2d9527476f871300f6d08c6 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 17 May 2024 15:29:09 +0800 Subject: [PATCH 21/27] TableCatalog needs to be specified. --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index eaef1f2..71d39b6 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -198,9 +198,10 @@ public class DataBaseSync implements DataSync { // schemaName = schemaName == null ? null : schemaName.toUpperCase(); // tableName = tableName == null ? null : tableName.toUpperCase(); // } + String tableCatalog = this.dbType.equals("mysql") ? schemaName : null; schemaName = schemaName.equals("null") ? null : schemaName; - tableName = tableName.equals("null") ? null : tableName; - ResultSet resultSet = metaData.getTables(null, schemaName, tableName, null); + // tableName = tableName.equals("null") ? null : tableName; // what the fuck? + ResultSet resultSet = metaData.getTables(tableCatalog, schemaName, tableName, null); if (resultSet.next()) { return true; } -- Gitee From e669142762046917ccfa45f9a0a000db91b31f09 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 14 Jun 2024 17:59:18 +0800 Subject: [PATCH 22/27] Compatible with clickhouse, because clickhouse does not support transactions. --- .../com/data/database/api/impl/DataBaseSync.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 71d39b6..04caf1f 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -379,10 +379,12 @@ public class DataBaseSync implements DataSync { logger.info("clearSql : " + clearSql); int supportsTransactions = 0; try {supportsTransactions = this.dbConn.getTransactionIsolation();} catch(SQLException e){logger.info("clearSql : " + e.getMessage()); supportsTransactions = Integer.parseInt(e.getMessage().replaceAll("[^0-9]", ""));} - if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); + logger.info("clearSql : supportsTransactions " + supportsTransactions); + logger.info("clearSql : getTransactionIsolation " + this.dbConn.getTransactionIsolation()); // what the fuck? why is the getTransactionIsolation return 1 on clickhouse + // if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); PreparedStatement pStemt = this.dbConn.prepareStatement(clearSql); pStemt.executeUpdate(); - if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); + if(supportsTransactions != Connection.TRANSACTION_NONE)this.commit(); logger.info(String.format("clear data for %s is done", tbName)); List colTypes = this.getColumnTypes(schemaName, tableName); @@ -413,7 +415,7 @@ public class DataBaseSync implements DataSync { if(i >= 0 || i == PreparedStatement.SUCCESS_NO_INFO) affectRows += i == PreparedStatement.SUCCESS_NO_INFO ? 1 : i; } - if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); + if(supportsTransactions != Connection.TRANSACTION_NONE)this.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); totalAffectRows += affectRows; rowCount = 0; @@ -425,7 +427,7 @@ public class DataBaseSync implements DataSync { if(i >= 0 || i == PreparedStatement.SUCCESS_NO_INFO) affectRows += i == PreparedStatement.SUCCESS_NO_INFO ? 1 : i; } - if(supportsTransactions != Connection.TRANSACTION_NONE)this.dbConn.commit(); + if(supportsTransactions != Connection.TRANSACTION_NONE)this.commit(); logger.info(String.format("rows insert into %s is %d", tbName, affectRows)); totalAffectRows += affectRows; rowCount = 0; @@ -465,4 +467,8 @@ public class DataBaseSync implements DataSync { return true; } + public boolean commit() throws SQLException { + try {this.dbConn.commit(); return true;} catch(SQLException e){logger.info("commit : " + e.getMessage()); return false;} + } + } \ No newline at end of file -- Gitee From 24d714594d26988abdedf74de90761c0d4313758 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 11 Jul 2024 10:33:39 +0800 Subject: [PATCH 23/27] Fix "Incorrect datetime value error" in MySQL --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 04caf1f..cdbadb2 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -402,6 +402,9 @@ public class DataBaseSync implements DataSync { if (colTypes.get(i) == Types.VARCHAR || colTypes.get(i) == Types.CHAR || colTypes.get(i) == Types.CLOB) { pStemt.setString(i + 1, Objects.toString(rs.getString(i + 1), "")); // 将 null 转化为空 + } else if (colTypes.get(i) == Types.TIMESTAMP + || colTypes.get(i) == Types.DATE) { + pStemt.setString(i + 1, rs.getString(i + 1)); // 不能将 null 转化为空 } else { pStemt.setObject(i + 1, rs.getObject(i + 1)); } -- Gitee From 1f1e3c664302e4286027e08f74208b2f6a2f191e Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 11 Jul 2024 14:07:16 +0800 Subject: [PATCH 24/27] Specifies MySQL --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index cdbadb2..fc4d1f7 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -402,8 +402,8 @@ public class DataBaseSync implements DataSync { if (colTypes.get(i) == Types.VARCHAR || colTypes.get(i) == Types.CHAR || colTypes.get(i) == Types.CLOB) { pStemt.setString(i + 1, Objects.toString(rs.getString(i + 1), "")); // 将 null 转化为空 - } else if (colTypes.get(i) == Types.TIMESTAMP - || colTypes.get(i) == Types.DATE) { + } else if (this.dbType.equals("mysql") && (colTypes.get(i) == Types.TIMESTAMP // specifies mysql + || colTypes.get(i) == Types.DATE)) { pStemt.setString(i + 1, rs.getString(i + 1)); // 不能将 null 转化为空 } else { pStemt.setObject(i + 1, rs.getObject(i + 1)); -- Gitee From e004b8cc48c105beb00f70c0083786f3e2bc99b0 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 12 Jul 2024 11:17:49 +0800 Subject: [PATCH 25/27] some 64bit clickhouse colType return 1111 cannot match jdbc type --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index fc4d1f7..45035b9 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -399,11 +399,14 @@ public class DataBaseSync implements DataSync { long starttime = System.currentTimeMillis(); while (rs.next()) { for (int i = 0; i < numberOfCols; i++) { + // logger.info(String.format("table:%s type:%s colname:%s coltype:%s timetype:%s", tbName, this.dbType, columnNames.get(i), colTypes.get(i), Types.TIMESTAMP)); if (colTypes.get(i) == Types.VARCHAR || colTypes.get(i) == Types.CHAR || colTypes.get(i) == Types.CLOB) { pStemt.setString(i + 1, Objects.toString(rs.getString(i + 1), "")); // 将 null 转化为空 } else if (this.dbType.equals("mysql") && (colTypes.get(i) == Types.TIMESTAMP // specifies mysql - || colTypes.get(i) == Types.DATE)) { + || colTypes.get(i) == Types.DATE + || columnNames.get(i).substring(columnNames.get(i).length() - 4).equalsIgnoreCase("TIME") // some 64bit clickhouse colType return 1111 cannot match jdbc type + || columnNames.get(i).substring(columnNames.get(i).length() - 4).equalsIgnoreCase("DATE"))) { pStemt.setString(i + 1, rs.getString(i + 1)); // 不能将 null 转化为空 } else { pStemt.setObject(i + 1, rs.getObject(i + 1)); -- Gitee From ff413e6380ec0934f4ede2c0a1655e26af8faa8e Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 12 Jul 2024 12:57:23 +0800 Subject: [PATCH 26/27] fix mistakes for last change --- src/main/java/com/data/database/api/impl/DataBaseSync.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 45035b9..18de9bb 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -405,8 +405,8 @@ public class DataBaseSync implements DataSync { pStemt.setString(i + 1, Objects.toString(rs.getString(i + 1), "")); // 将 null 转化为空 } else if (this.dbType.equals("mysql") && (colTypes.get(i) == Types.TIMESTAMP // specifies mysql || colTypes.get(i) == Types.DATE - || columnNames.get(i).substring(columnNames.get(i).length() - 4).equalsIgnoreCase("TIME") // some 64bit clickhouse colType return 1111 cannot match jdbc type - || columnNames.get(i).substring(columnNames.get(i).length() - 4).equalsIgnoreCase("DATE"))) { + || columnNames.get(i).length() >=4 && columnNames.get(i).substring(columnNames.get(i).length() - 4).equalsIgnoreCase("TIME") // some 64bit clickhouse colType return 1111 cannot match jdbc type + || columnNames.get(i).length() >=4 && columnNames.get(i).substring(columnNames.get(i).length() - 4).equalsIgnoreCase("DATE"))) { pStemt.setString(i + 1, rs.getString(i + 1)); // 不能将 null 转化为空 } else { pStemt.setObject(i + 1, rs.getObject(i + 1)); -- Gitee From 95d49c2394fc269f15cc3f62d382d37a68d3bf54 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 24 Jul 2024 13:10:47 +0800 Subject: [PATCH 27/27] Fix the duplication of column names caused by the same table name in different Catalogs. --- src/main/java/com/data/database/App.java | 2 ++ src/main/java/com/data/database/api/impl/DataBaseSync.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/data/database/App.java b/src/main/java/com/data/database/App.java index cc3d6df..ca40128 100644 --- a/src/main/java/com/data/database/App.java +++ b/src/main/java/com/data/database/App.java @@ -364,6 +364,8 @@ public final class App { toColumnNames = toDataBase.getTableColumns(app.getToSchema(), app.getToTable()); } + logger.info(String.format("ColumnNames (%s)%s.%s %s", app.getToDbId(), app.getToSchema(), app.getToTable(), toColumnNames)); + if (app.getFromTableFields().size() > 0) { // 如果指定源列，则使用源列，不指定则使用目标列 fromColumnNames = app.getFromTableFields(); diff --git a/src/main/java/com/data/database/api/impl/DataBaseSync.java b/src/main/java/com/data/database/api/impl/DataBaseSync.java index 18de9bb..93d16bd 100644 --- a/src/main/java/com/data/database/api/impl/DataBaseSync.java +++ b/src/main/java/com/data/database/api/impl/DataBaseSync.java @@ -149,7 +149,7 @@ public class DataBaseSync implements DataSync { // 用于 扩字段长度，增加字段，删除字段，修改字段类型。 /* 由于 mysql 、 postgres 数据库大小写是区分的，因此这里不做任何统一转换，自行控制输入参数大小写*/ public ResultSet getColMetaData(String schemaName, String tableName) throws SQLException { - String tableCatalog = this.dbType.equals("mysql") ? schemaName : null; + String tableCatalog = this.dbType.equals("mysql") || this.dbType.equals("clickhouse") ? schemaName : null; schemaName = schemaName.equals("null") ? null : schemaName; DatabaseMetaData metaData = this.dbConn.getMetaData(); ResultSet resultSet = metaData.getColumns(tableCatalog, schemaName, tableName, "%"); @@ -390,7 +390,7 @@ public class DataBaseSync implements DataSync { List colTypes = this.getColumnTypes(schemaName, tableName); final String insertSql = Tools.buildInsertSql(tbName, columnNames.stream().toArray(String[]::new)); - // System.out.println(insertSql); + logger.info("insertSql : " + insertSql); pStemt = this.dbConn.prepareStatement(insertSql); // final int numberOfCols = rs.getMetaData().getColumnCount(); final int numberOfCols = columnNames.size(); -- Gitee