diff --git a/.gitignore b/.gitignore index fbb1d7706cedbb5aed3072d6014eae3fc52448d5..96f47cedd5bddf1dc9d95217fe3241be1f6719bd 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,15 @@ Dockerfile_functest pkg/ test/ -tools/ +tools/intarkdb_server/ +tools/local-db/ +tools/local-server/ +tools/data-sync-agent/ +tools/gradle-7.2-all.zip +tools/pressure_test/ +tools/rust-gstor-multithreading/ +tools/rust-gstor/ +tools/rust-memory-allocato/ +tools/shell_tools/ +tools/wget-log diff --git a/CMakeLists.txt b/CMakeLists.txt index 28f0dde57fada416cf72388aa5dd90c880512c72..91297eb3d0cb7d2202ab4027f38aa5c72f4c1376 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,7 +425,7 @@ endif () # OPTION(TEST "option for test module" OFF) # message(STATUS "TEST = ${TEST}") # if (TEST) - add_subdirectory(test) +# add_subdirectory(test) # endif() install( diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..cf0ec86042a21c111540cc3976a5ef9e3b0a834a --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(intarkdb_cli) + +add_subdirectory(sqlite3-api-test) diff --git a/tools/data-sync-agent/.gitignore b/tools/data-sync-agent/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1f5d0169cbfa0ee143536f5e44d085e87be8b70c --- /dev/null +++ b/tools/data-sync-agent/.gitignore @@ -0,0 +1,60 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ +/.mvn/ +/mvnw +/mvnw.cmd + +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* diff --git a/tools/data-sync-agent/README.md b/tools/data-sync-agent/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a70d62dde50d709e66c4fcb7020259fcc807ddd6 --- /dev/null +++ b/tools/data-sync-agent/README.md @@ -0,0 +1,31 @@ +# data-sync-agent + +此工具用于将数据从IntarkDB同步到openGauss数据库中，支持时序数据表的全量同步和增量同步。 + +#### 一、工程说明 + +##### 1、编程语言：Java + +##### 2、编译工具：Maven + +#### 二、使用说明 + +##### 1、数据表准备 +云端数据库的数据表，第一列必须是node_id varchar，其余字段与边端IntarkDB数据表完全一致。 + +##### 2、配置文件修改 +修改src/main/resources文件夹下conf.yaml文件，其中 +- centerDb为云端openGauss数据库的对应配置信息，[ip, port, user, password]为openGauss数据库的ip地址、端口、用户名、密码， +- localDb为边端intarkDB数据库的对应配置信息，[ip, port, user, password, nodeId]为intarkDB数据库的ip地址、端口、用户名、密码、节点id +- syncTable为需要同步的数据表的配置信息，[tableName, syncColumn]为数据表名，数据表中的时间索引列 +- loopWait为任务队列处理线程空闲等待时长，单位为毫秒 + +##### 3、程序运行 +启动云端openGauss数据库 + +启动边端IntarkDB数据库的local-db程序 + +使用maven打包data-sync-agent，可以得到cloud-edge-0.0.1-SNAPSHOT.jar，将修改好的配置文件放在同一目录下，输入指令执行 +``` +java -Dconfig_file='conf.yaml' -jar .\cloud-edge-0.0.1-SNAPSHOT.jar +``` diff --git a/tools/data-sync-agent/pom.xml b/tools/data-sync-agent/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..c76c5f2da8406cd78cb8ef408ca6e24852a06ad0 --- /dev/null +++ b/tools/data-sync-agent/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.2.0 + + + com.intarkdb + cloud-edge + 0.0.1-SNAPSHOT + cloud-edge + Demo project for Spring Boot + + 17 + 3.2.0 + + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.instardb + InstarDB-jdbc + 1.0 + system + ${project.basedir}/src/main/resources/lib/InstarDB-socket-jdbc-1.0.jar + + + org.opengauss + opengauss-jdbc + 5.0.0 + system + ${project.basedir}/src/main/resources/lib/opengauss-jdbc-5.0.0.jar + + + org.junit.jupiter + junit-jupiter + 5.8.0 + test + + + + org.yaml + snakeyaml + 1.29 + + + + ch.qos.logback + logback-core + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + 11 + UTF-8 + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + com.intarkdb.cloudedge.CloudEdgeApplication + true + + + + repackage + + repackage + + + + + + + + diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/CloudEdgeApplication.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/CloudEdgeApplication.java new file mode 100644 index 0000000000000000000000000000000000000000..16d2caab02363a620829c124f51bbaa44548fdd9 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/CloudEdgeApplication.java @@ -0,0 +1,45 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge; + +import com.intarkdb.cloudedge.config.ConfigProperties; +import com.intarkdb.cloudedge.config.ConfigReader; +import com.intarkdb.cloudedge.mission.SimpleMissionUploadManagerImpl; +import com.intarkdb.cloudedge.mission.UploadMissionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CloudEdgeApplication { + + private static final Logger log = LoggerFactory.getLogger(CloudEdgeApplication.class); + + public static void main(String[] args) { +// SpringApplication.run(CloudEdgeApplication.class, args); + + // read config + ConfigProperties conf = ConfigReader.readConfigFromYaml(); + + UploadMissionManager syncManager = new SimpleMissionUploadManagerImpl(); + // UploadMissionManager syncManager = new AsyncMissionUploadManagerImpl(); + syncManager.init(conf); + log.info("init finished"); + syncManager.start(); + } + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/AsyncCopyManagerHelper.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/AsyncCopyManagerHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..557a472bc0316fea59346f9835123eabdfea30b8 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/AsyncCopyManagerHelper.java @@ -0,0 +1,22 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.center; + +public class AsyncCopyManagerHelper { + + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/CopyManagerHelper.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/CopyManagerHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..3bb6aafab148721f46a0002d4c89db4835b387b3 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/CopyManagerHelper.java @@ -0,0 +1,119 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.center; + +import com.intarkdb.cloudedge.config.DbMsg; +import org.opengauss.copy.CopyManager; +import org.opengauss.core.BaseConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.text.MessageFormat; +import java.util.concurrent.locks.ReentrantLock; + +/** + * 中心数据库相关的方法 + */ + +public class CopyManagerHelper { + + private static final Logger log = LoggerFactory.getLogger(CopyManagerHelper.class); + + private CopyManagerHelper() { + } + + private static CopyManagerHelper singleton = new CopyManagerHelper(); + + public static CopyManagerHelper getCopyManagerHelper() { + return singleton; + } + + private static CopyManager copyManager; + private static Connection connection; + private static ReentrantLock lock = new ReentrantLock(); + + public static void connectToCenterDB(DbMsg dbMsg) { + try { + Class.forName("org.opengauss.Driver"); + String url = MessageFormat.format("jdbc:opengauss://{0}:{1}/postgres", dbMsg.getIp(), dbMsg.getPort()); + connection = DriverManager.getConnection(url, dbMsg.getUser(), dbMsg.getPassword()); + copyManager = new CopyManager((BaseConnection) connection); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + public static void releaseConnection() { + try { + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public boolean uploadData(String tableName, String data) { + if (data == null || "".equals(data)) { + return true; + } + lock.lock(); + try { + copyIn(tableName, data); + return true; + } catch (SQLException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (Throwable t) { + // todo handle duplicate upload error + t.printStackTrace(); + } finally { + lock.unlock(); + } + return false; + } + + private void copyIn(String tableName, String data) throws SQLException, IOException { + long t = System.currentTimeMillis(); + copyManager.copyIn("COPY " + tableName + " FROM STDIN WITH (FORMAT CSV)", new StringReader(data)); + log.debug("copy manager cost {} ms", System.currentTimeMillis() - t); + } + + public boolean uploadDataNoLock(String tableName, String data) { + if (data == null || "".equals(data)) { + return true; + } + try { + copyIn(tableName, data); + return true; + } catch (SQLException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (Throwable t) { + // todo handle duplicate upload error + t.printStackTrace(); + } + return false; + } + + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/MultiCopyManagerHelper.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/MultiCopyManagerHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..f9cf6f82cafb791807eb2162e8a5d3a1349b2088 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/center/MultiCopyManagerHelper.java @@ -0,0 +1,106 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.center; + +import com.intarkdb.cloudedge.config.DbMsg; +import com.intarkdb.cloudedge.config.SyncTableInfo; +import org.opengauss.copy.CopyManager; +import org.opengauss.core.BaseConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class MultiCopyManagerHelper { + private static final Logger log = LoggerFactory.getLogger(MultiCopyManagerHelper.class); + + private static String url; + private static String user; + private static String password; + private static Map connectionMap; + + public static void init(DbMsg dbMsg, List syncTableInfos) { + try { + Class.forName("org.opengauss.Driver"); + + String target = "jdbc:opengauss://center_ip:center_port/postgres" + .replaceFirst("center_ip", dbMsg.getIp()) + .replaceFirst("center_port", dbMsg.getPort()); + url = target; + user = dbMsg.getUser(); + password = dbMsg.getPassword(); + connectionMap = new ConcurrentHashMap<>(); + + for (SyncTableInfo info : syncTableInfos) { + connectionMap.put(info.getTableName(), DriverManager.getConnection(url, user, password)); + } + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + } + + private final static int DUPLICATE_ERR_CODE = 16153; + + public static boolean uploadData(String tableName, String data) { + if (data == null || "".equals(data)) { + return true; + } + long t = System.currentTimeMillis(); + Connection connection = connectionMap.get(tableName); + if (connection == null) { + try { + connection = DriverManager.getConnection(url, user, password); + } catch (SQLException e) { + return false; + } + connectionMap.put(tableName, connection); + } + try { + CopyManager copyManager = new CopyManager((BaseConnection) connection); + copyManager.copyIn("COPY " + tableName + " FROM STDIN WITH (FORMAT CSV)", new StringReader(data)); +// log.debug("copy manager cost {} ms", System.currentTimeMillis() - t); + return true; + } catch (SQLException e) { + if ("Database connection failed when starting copy".equals(e.getMessage())) { + connectionMap.remove(tableName); + try { + connection.close(); + } catch (SQLException e1) { + } + return false; + } else if (e.getErrorCode() == DUPLICATE_ERR_CODE) { + return true; + } + } catch (IOException e) { + e.printStackTrace(); + } catch (Throwable t1) { + // todo handle duplicate upload error + t1.printStackTrace(); + } + return false; + } + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/Constant.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/Constant.java new file mode 100644 index 0000000000000000000000000000000000000000..99c7040c0277defd08d715736d4f2c9f80b1f134 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/Constant.java @@ -0,0 +1,66 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.common; + +public class Constant { + + /** + * 单次读取数量 + */ + public static final int BATCH_SIZE = 500; + + /** + * 扫描窗口宽度 + */ + public static final long SCAN_WINDOW_RANGE = 15 * 1000; + + /** + * 乱序插入等待时间，读取时将只读取到当前时间戳 - 本值 + */ + public static final long SCAN_WAIT_RANGE = 15 * 1000; + + public static final int FAIL_LIMIT = 10; + + public static final String CREATE_SYNC_TABLE_SQL = "CREATE TABLE SYS_CLOUD_SYNC " + + "(id INTEGER, " + + "table_name varchar(100), " + + "sync_column varchar(100), " + + "type int, " + + "begin_timestamp long, " + + "end_timestamp long, " + + "node_id varchar(100), " + + "unique_column varchar(100), " + + "update_time long, " + + "PRIMARY KEY (id), " + + "unique (table_name))"; + + public static final String DESCRIBE_SYNC_TABLE_SQL = "describe SYS_CLOUD_SYNC"; + + public static final String SQL_SELECT_1 = "SELECT * FROM {0} where {1} > ? and {2} <= ? limit {3}"; + public static final String SQL_SELECT_2 = "SELECT count(*) FROM {0} where {1} = ?"; + public static final String SQL_SELECT_3 = "SELECT * FROM {0} where {1} = ? offset ? limit ?"; + + public static final String CENTER_IP = "center_ip"; + public static final String CENTER_PORT = "center_port"; + public static final String CENTER_USER = "center_user"; + public static final String CENTER_PASSWORD = "center_password"; + + public static final String LOCAL_IP = "local_ip"; + public static final String LOCAL_PORT = "local_port"; + public static final String LOCAL_USER = "local_user"; + public static final String LOCAL_PASSWORD = "local_password"; +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/SyncException.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/SyncException.java new file mode 100644 index 0000000000000000000000000000000000000000..00346c29a25e9ddaf263bf7b8ce55c6bd60486e8 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/SyncException.java @@ -0,0 +1,25 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.common; + +public class SyncException extends Exception { + public SyncException(String message) { + super(message); + } + + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/ThreadUncaughtExceptionHandler.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/ThreadUncaughtExceptionHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..571e257a315561543015c9e4a473c1635dc3367e --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/common/ThreadUncaughtExceptionHandler.java @@ -0,0 +1,34 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.common; + +import org.slf4j.Logger; + +public class ThreadUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler { + + private Logger log; + + public ThreadUncaughtExceptionHandler(Logger log) { + this.log = log; + } + + @Override + public void uncaughtException(Thread t, Throwable e) { + log.error("thread run exception: {}", e.getMessage()); + e.printStackTrace(); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigProperties.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..ef9249cfd7d3ed6ea368a30f14720e6e0e660f1c --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigProperties.java @@ -0,0 +1,66 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.config; + +import java.util.List; + +public class ConfigProperties { + + // center database setting + private DbMsg centerDb; + + // local database setting + private List localDb; + + // sync table msg + private List syncTable; + + // loop wait time (ms) + private Integer loopWait; + + public DbMsg getCenterDb() { + return centerDb; + } + + public void setCenterDb(DbMsg centerDb) { + this.centerDb = centerDb; + } + + public List getLocalDb() { + return localDb; + } + + public void setLocalDb(List localDb) { + this.localDb = localDb; + } + + public List getSyncTable() { + return syncTable; + } + + public void setSyncTable(List syncTable) { + this.syncTable = syncTable; + } + + public Integer getLoopWait() { + return loopWait; + } + + public void setLoopWait(Integer loopWait) { + this.loopWait = loopWait; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigReader.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigReader.java new file mode 100644 index 0000000000000000000000000000000000000000..f9f0f84b147720141cf139f1a3e569adbef69ab3 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigReader.java @@ -0,0 +1,178 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.config; + +import com.intarkdb.cloudedge.common.SyncException; +import com.intarkdb.cloudedge.tool.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + + +public class ConfigReader { + + private static final Logger log = LoggerFactory.getLogger(ConfigReader.class); + + /** + * Deprecated + * @param path + * @return + */ + @Deprecated + public static Map readConfiguration(String path) { + if (path == null || "".equals(path)) { + String currentPath = System.getProperty("user.dir"); + path = currentPath + "/conf.properties"; + } + Map result = new HashMap<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(path))) { + String line; + + while ((line = reader.readLine()) != null) { + handle(line.trim(), result); + } + } catch (IOException e) { + e.printStackTrace(); + } + return result; + } + + /** + * + * @param line + * @param result + */ + @Deprecated + private static void handle(String line, Map result) { + int commentIndex = line.indexOf("#"); + if (commentIndex > -1) { + line = line.substring(0, commentIndex); + } + if (line.length() == 0) { + return; + } + String[] kvs = line.split("="); + result.put(kvs[0].trim(), kvs[1].trim()); + } + + /** + * read configurations from yaml file + * @return + */ + public static ConfigProperties readConfigFromYaml() { + ConfigProperties configProperties = new ConfigProperties(); + + String configFile = System.getProperty("config_file"); + if (StringUtil.isBlank(configFile)) { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + URL resourceUrl = classLoader.getResource("conf.yaml"); + if (resourceUrl != null) { + configFile = resourceUrl.getPath(); + } else { + configFile = System.getProperty("user.dir") + "\\src\\main\\resources\\conf.yaml"; + } + } + Yaml yaml = new Yaml(); + try (FileInputStream fileInputStream = new FileInputStream(configFile)) { + ConfigProperties config = yaml.loadAs(fileInputStream, ConfigProperties.class); + + checkAndFixConfig(config); + + return config; + } catch (IOException | SyncException e) { + // config file read failed + log.error("property file read failed!"); + e.printStackTrace(); + } + return null; + } + + /** + * check if there are any required configurations not set + * apply the default value if there are available rules + * @param config + */ + private static void checkAndFixConfig(ConfigProperties config) throws SyncException { + // check local db setting + List msgList = config.getLocalDb(); + for (LocalDbMsg msg: msgList) { + checkDbMsg(msg); + checkNodeId(msg); + } + + // check center db setting + checkDbMsg(config.getCenterDb()); + + // check sync table setting + List syncTableInfos = config.getSyncTable(); + for (SyncTableInfo info : syncTableInfos) { + checkTableInfo(info); + } + } + + private static void checkDbMsg(DbMsg msg) throws SyncException { + String ip = msg.getIp(); + if (StringUtil.isBlank(ip) || !isValidIP(ip)) { + throw new SyncException("invalid ip: " + msg.getIp()); + } + String port = msg.getPort(); + if (StringUtil.isBlank(port) || !isValidPort(port)) { + throw new SyncException("invalid port: " + msg.getPort()); + } + } + + private static void checkNodeId(LocalDbMsg msg) { + String nodeId = msg.getNodeId(); + if (StringUtil.isBlank(nodeId)) { + // set default node id using ip and port + msg.setNodeId(msg.getIp() + "_" + msg.getPort()); + } + } + + private static void checkTableInfo(SyncTableInfo info) throws SyncException { + String tableName = info.getTableName(); + if (StringUtil.isBlank(tableName)) { + throw new SyncException("table name or sync column name missing"); + } + String syncColumn = info.getSyncColumn(); + if (StringUtil.isBlank(syncColumn)) { + throw new SyncException("table name or sync column name missing"); + } + } + + // ip and port pattern + private static final String IP_REGEX = "^((\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])\\.){3}(\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])$"; + private static final String PORT_REGEX = "^\\d{1,5}$"; + + public static boolean isValidIP(String ip) { + return Pattern.matches(IP_REGEX, ip); + } + + public static boolean isValidPort(String port) { + return Pattern.matches(PORT_REGEX, port); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigWatchService.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigWatchService.java new file mode 100644 index 0000000000000000000000000000000000000000..eb2d35909dbf7594a312034e23f7d23f3090356a --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/ConfigWatchService.java @@ -0,0 +1,63 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.config; + +import com.intarkdb.cloudedge.tool.StringUtil; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; + +public class ConfigWatchService { + + public static void configWatch() { + String configFile = System.getProperty("config.file"); + if (StringUtil.isBlank(configFile)) { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + URL resourceUrl = classLoader.getResource("conf.yaml"); + configFile = resourceUrl.getPath(); + } + Path filePath = Paths.get(configFile); + try (WatchService watchService = FileSystems.getDefault().newWatchService()) { + Path directory = filePath.getParent(); + directory.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY); + + WatchKey watchKey; + while ((watchKey = watchService.take()) != null) { + checkWatch(watchKey, filePath); + } + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + } + + private static void checkWatch(WatchKey watchKey, Path filePath) { + for (WatchEvent event : watchKey.pollEvents()) { + if (event.context().equals(filePath.getFileName())) { + // 文件被修改，执行相应的操作 + ConfigProperties newProperties = ConfigReader.readConfigFromYaml(); + } + } + watchKey.reset(); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/DbMsg.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/DbMsg.java new file mode 100644 index 0000000000000000000000000000000000000000..f856d4b9845147413d685a06d4e6814fe22eff74 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/DbMsg.java @@ -0,0 +1,56 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.config; + +public class DbMsg { + private String ip; + private String port; + private String user = ""; + private String password = ""; + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/LocalDbMsg.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/LocalDbMsg.java new file mode 100644 index 0000000000000000000000000000000000000000..b039c6f983dad86d8357059014e56d1a623950d1 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/LocalDbMsg.java @@ -0,0 +1,29 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.config; + +public class LocalDbMsg extends DbMsg { + private String nodeId = ""; + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/SyncTableInfo.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/SyncTableInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..47194cb4604713bc680905279f07a724e406de7b --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/config/SyncTableInfo.java @@ -0,0 +1,58 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.config; + +public class SyncTableInfo { + /** + * 表名 + */ + private String tableName; + + /** + * 同步依据列，必须是该时序表的时间列 + */ + private String syncColumn; + + /** + * 唯一列，非必填 + */ + private String uniqueColumn; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getSyncColumn() { + return syncColumn; + } + + public void setSyncColumn(String syncColumn) { + this.syncColumn = syncColumn; + } + + public String getUniqueColumn() { + return uniqueColumn; + } + + public void setUniqueColumn(String uniqueColumn) { + this.uniqueColumn = uniqueColumn; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/CsvResult.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/CsvResult.java new file mode 100644 index 0000000000000000000000000000000000000000..88fe6825bc9a6fd3657503927e55637264562b7d --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/CsvResult.java @@ -0,0 +1,54 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.local; + +public class CsvResult { + + private int count; + private String date; + private String resultString; + + public CsvResult(int count, String date, String resultString) { + this.count = count; + this.date = date; + this.resultString = resultString; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } + + public String getDate() { + return date; + } + + public void setDate(String date) { + this.date = date; + } + + public String getResultString() { + return resultString; + } + + public void setResultString(String resultString) { + this.resultString = resultString; + } +} \ No newline at end of file diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LastDateInfo.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LastDateInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..a0f5601617c88b186cc9321e6b30df0daadfa809 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LastDateInfo.java @@ -0,0 +1,44 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.local; + +public class LastDateInfo { + + String lastDate; + int lastDateCount; + + public LastDateInfo() { + this.lastDate = null; + this.lastDateCount = 0; + } + + public String getLastDate() { + return lastDate; + } + + public void setLastDate(String lastDate) { + this.lastDate = lastDate; + } + + public int getLastDateCount() { + return lastDateCount; + } + + public void setLastDateCount(int lastDateCount) { + this.lastDateCount = lastDateCount; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LocalDBReader.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LocalDBReader.java new file mode 100644 index 0000000000000000000000000000000000000000..1d8eabd8632f586b2623ecbbf3de794369b0e57a --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LocalDBReader.java @@ -0,0 +1,249 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.local; + +import com.intarkdb.cloudedge.common.Constant; +import com.intarkdb.cloudedge.config.LocalDbMsg; +import com.intarkdb.cloudedge.config.SyncTableInfo; +import com.intarkdb.cloudedge.mission.AsyncUploadMission; +import com.intarkdb.cloudedge.mission.SimpleUploadMission; +import com.intarkdb.cloudedge.mission.SyncTableControlInfo; +import com.intarkdb.cloudedge.mission.UploadMission; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; + +public class LocalDBReader { + + private static final Logger log = LoggerFactory.getLogger(LocalDBReader.class); + + public LocalDBReader(LocalDbMsg localDbMsg) { + this.localDbMsg = localDbMsg; + String projectRootPath = System.getProperty("user.dir"); + try { + Class.forName("org.instardb.jdbc.InstarDriver"); + String url = "jdbc:instardb://local_ip:local_port" + .replaceAll("local_ip", localDbMsg.getIp()) + .replaceAll("local_port", localDbMsg.getPort()); + connection = DriverManager.getConnection(url, localDbMsg.getUser(), localDbMsg.getPassword()); + } catch (SQLException | ClassNotFoundException e) { + e.printStackTrace(); + } + } + + protected Connection connection; + + protected LocalDbMsg localDbMsg; + + + /** + * init the sync manage table + */ + public void initManageTable() { + try (Statement statement = connection.createStatement(); + ResultSet i = statement.executeQuery(Constant.DESCRIBE_SYNC_TABLE_SQL)) { + + return; + } catch (SQLException e) { + log.info("Manage table not existed, build manage table"); + try (Statement statement = connection.createStatement(); + ResultSet j = statement.executeQuery(Constant.CREATE_SYNC_TABLE_SQL)) { + log.debug("create manage table"); + } catch (SQLException e1) { + log.error("create Manage table error"); + e1.printStackTrace(); + } + } + + } + + public void initSimpleMissionQueue(ConcurrentLinkedQueue missionQueue, List syncTableInfo, String nodeId) { + + Map tableMap = syncTableInfo.stream().collect(Collectors.toMap(SyncTableInfo::getTableName, a -> a)); + + try ( + Statement statement = connection.createStatement()) { + + ResultSet resultSet = statement.executeQuery("SELECT table_name, sync_column, begin_timestamp, end_timestamp from SYS_CLOUD_SYNC"); + while (resultSet.next()) { + String tableName = resultSet.getString(1); + String syncColumn = resultSet.getString(2); + Long beginTime = resultSet.getLong(3); + Long endTime = resultSet.getLong(4); + + SyncTableControlInfo controlMsg = new SyncTableControlInfo(tableName, syncColumn, beginTime, endTime, nodeId); + missionQueue.offer(new SimpleUploadMission(controlMsg)); + tableMap.remove(tableName); + } + } catch (SQLException e) { + e.printStackTrace(); + } + + // new table config to upload + int id = missionQueue.size(); + Long endTimeStamp = (Instant.now().toEpochMilli() - Constant.SCAN_WAIT_RANGE) * 1000; + for (Map.Entry entry : tableMap.entrySet()) { + try (PreparedStatement prepareStatement = connection.prepareStatement("insert into SYS_CLOUD_SYNC(id, type, table_name, sync_column, begin_timestamp, end_timestamp, node_id) values (?, ?, ?, ?, ?, ?, ?)")) { + prepareStatement.setInt(1, id++); + prepareStatement.setInt(2, 1); + prepareStatement.setString(3, entry.getKey()); + prepareStatement.setString(4, entry.getValue().getSyncColumn()); + prepareStatement.setLong(5, 0); + prepareStatement.setLong(6, endTimeStamp); + prepareStatement.setString(7, ""); // todo node id + int i = prepareStatement.executeUpdate(); + } catch (SQLException e) { + e.printStackTrace(); + } + // add to mission queue + SyncTableControlInfo controlMsg = new SyncTableControlInfo(entry.getKey(), entry.getValue().getSyncColumn(), 0L, endTimeStamp, nodeId); + missionQueue.offer(new SimpleUploadMission(controlMsg)); + } + + } + + public void initAsyncMissionQueue(ConcurrentLinkedQueue missionQueue, List syncTableInfo, String nodeId) { + + Map tableMap = syncTableInfo.stream().collect(Collectors.toMap(SyncTableInfo::getTableName, a -> a)); + + try ( + Statement statement = connection.createStatement()) { + + ResultSet resultSet = statement.executeQuery("SELECT table_name, sync_column, begin_timestamp, end_timestamp from SYS_CLOUD_SYNC"); + while (resultSet.next()) { + String tableName = resultSet.getString(1); + String syncColumn = resultSet.getString(2); + Long beginTime = resultSet.getLong(3); + Long endTime = resultSet.getLong(4); + + SyncTableControlInfo controlMsg = new SyncTableControlInfo(tableName, syncColumn, beginTime, endTime, nodeId); + missionQueue.offer(new AsyncUploadMission(controlMsg)); + tableMap.remove(tableName); + } + } catch (SQLException e) { + e.printStackTrace(); + } + + // new table config to upload + int id = missionQueue.size(); + Long endTimeStamp = Instant.now().toEpochMilli() - Constant.SCAN_WAIT_RANGE; + for (Map.Entry entry : tableMap.entrySet()) { + try (PreparedStatement prepareStatement = connection.prepareStatement("insert into SYS_CLOUD_SYNC(id, type, table_name, sync_column, begin_timestamp, end_timestamp, node_id) values (?, ?, ?, ?, ?, ?, ?)")) { + prepareStatement.setInt(1, id++); + prepareStatement.setInt(2, 1); + prepareStatement.setString(3, entry.getKey()); + prepareStatement.setString(4, entry.getValue().getSyncColumn()); + prepareStatement.setLong(5, 0); + prepareStatement.setLong(6, endTimeStamp); + prepareStatement.setString(7, ""); // todo node id + int i = prepareStatement.executeUpdate(); + // add to mission queue + SyncTableControlInfo controlMsg = new SyncTableControlInfo(entry.getKey(), entry.getValue().getSyncColumn(), 0L, endTimeStamp, nodeId); + missionQueue.offer(new AsyncUploadMission(controlMsg)); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + } + + public ResultSet readData(String tableName, String columnName, String lastSyncTime, String nowDate, int batchSize) { + String sql = MessageFormat.format(Constant.SQL_SELECT_1, tableName, columnName, columnName, batchSize); + try (PreparedStatement prepareStatement = connection.prepareStatement(sql)) { + + prepareStatement.setString(1, lastSyncTime); + prepareStatement.setString(2, nowDate); + + return prepareStatement.executeQuery(); + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + + public List> readData(String tableName, String columnName, String timeStamp, int beginIndex, int batchSize) { + String countSql = MessageFormat.format(Constant.SQL_SELECT_2, tableName, columnName); + String querySql = MessageFormat.format(Constant.SQL_SELECT_3, tableName, columnName); + try (PreparedStatement countStatement = connection.prepareStatement(countSql); + PreparedStatement queryStatement = connection.prepareStatement(querySql)) { + countStatement.setString(1, timeStamp); + ResultSet resultSet = countStatement.executeQuery(); + int count = ResultParser.countParse(resultSet); + + List> result = new ArrayList<>(); + + // 分页将等于右闭区间的数据全读取 + for (int i = beginIndex; i < count; i += batchSize) { + queryStatement.setString(1, timeStamp); + queryStatement.setInt(2, i); + queryStatement.setInt(3, batchSize); + ResultSet rs = queryStatement.executeQuery(); + result.addAll(ResultParser.simpleParse(tableName, columnName, rs)); + } + return result; + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + + public boolean updateSyncTime(String tableName, long beginTime, long endTime) { + try (PreparedStatement prepareStatement = connection.prepareStatement("UPDATE SYS_CLOUD_SYNC set begin_timestamp = ?, end_timestamp = ? where table_name = ?")) { + prepareStatement.setLong(1, beginTime); + prepareStatement.setLong(2, endTime); + prepareStatement.setString(3, tableName); + int i = prepareStatement.executeUpdate(); + return true; + } catch (SQLException e) { + e.printStackTrace(); + } + return false; + } + + + public static void main(String[] args) throws InterruptedException { + int total = 100; // 总进度 + int progress = 0; // 当前进度 + + while (progress <= total) { + System.out.print("\r进度: " + progress + "%"); // \r 会将光标移动到行首，从而实现覆盖打印的效果 + progress += 10; + Thread.sleep(500); // 暂停 500 毫秒以模拟进度更新 + } + } + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LocalReadResult.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LocalReadResult.java new file mode 100644 index 0000000000000000000000000000000000000000..fe3ce4150335a5604c386480e9f0bb1c909205cd --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/LocalReadResult.java @@ -0,0 +1,73 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.local; + +import java.util.List; + +public class LocalReadResult { + /** + * 结果值 + */ + private List> resultStringList; + + /** + * 本次读取最后一个时间戳 + */ + private String date; + + /** + * 本次读取跟最后一个时间戳一致的数据行数 + */ + private int sameDateCount; + + /** + * 本次读取结果数 + */ + private int count; + + public List> getResultStringList() { + return resultStringList; + } + + public void setResultStringList(List> resultStringList) { + this.resultStringList = resultStringList; + } + + public String getDate() { + return date; + } + + public void setDate(String date) { + this.date = date; + } + + public int getSameDateCount() { + return sameDateCount; + } + + public void setSameDateCount(int sameDateCount) { + this.sameDateCount = sameDateCount; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/ResultParser.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/ResultParser.java new file mode 100644 index 0000000000000000000000000000000000000000..48fdfefe67a0cdddd4bd4a30ce117c8edfa4d09c --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/local/ResultParser.java @@ -0,0 +1,295 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.local; + +import org.opengauss.util.csv.CSVWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; + +public class ResultParser { + + private static final Logger log = LoggerFactory.getLogger(ResultParser.class); + + /** + * cache the trans methods in a map + */ + private static HashMap> parserMap = new HashMap<>(); + private static HashMap dateMap = new HashMap<>(); + + /** + * get the trans method of each column, based on the msg of metadata + * + * @param resultSet + * @return + */ + private static LinkedList initParserChain(String tableName, String columnName, ResultSet resultSet) throws SQLException, NoSuchMethodException { + LinkedList result = new LinkedList<>(); + Class resultSetClass = ResultSet.class; + int columnIndex = -1; + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + if (columnIndex == -1 && columnName.equals(metaData.getColumnName(i))) { + columnIndex = i; + dateMap.put(tableName, columnIndex); + } + switch (metaData.getColumnType(i)) { + case Types.INTEGER: + result.add(resultSetClass.getDeclaredMethod("getInt", int.class)); + break; + case Types.BIGINT: + result.add(resultSetClass.getDeclaredMethod("getLong", int.class)); + break; + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGNVARCHAR: + result.add(resultSetClass.getDeclaredMethod("getString", int.class)); + break; +// case Types.TINYINT: +// case Types.SMALLINT: +// result.add(resultSetClass.getDeclaredMethod("getShort", int.class)); +// break; +// case Types.FLOAT: +// result.add(resultSetClass.getDeclaredMethod("getFloat", int.class)); +// break; +// case Types.DECIMAL: +// result.add(resultSetClass.getDeclaredMethod("getBigDecimal", int.class)); +// break; +// case Types.DATE: +// result.add(resultSetClass.getDeclaredMethod("getTime", int.class)); +// break; +// case Types.TIMESTAMP: +// result.add(resultSetClass.getDeclaredMethod("getTimestamp", int.class)); +// break; +// case Types.BOOLEAN: +// result.add(resultSetClass.getDeclaredMethod("getBoolean", int.class)); +// break; + default: +// System.out.println("unknown type"); + result.add(resultSetClass.getDeclaredMethod("getString", int.class)); + } + } + parserMap.put(tableName, result); + return result; + } + + public static int countParse(ResultSet resultSet) { + try { + if (resultSet.next()) { + return resultSet.getInt(1); + } + } catch (SQLException e) { + log.error("count sql parse error!"); + e.printStackTrace(); + } + return 0; + } + + /** + * 读取时间戳正好等于右闭区间的情况 + * 不需要考虑读取时间戳右闭区间没读完的情况 + * @param tableName + * @param columnName + * @param resultSet + * @return + */ + public static List> simpleParse(String tableName, String columnName, ResultSet resultSet) { + // get the trans method from the cache + LinkedList parserChain = parserMap.get(tableName); + // build it if it doesn't exist + if (parserChain == null) { + try { + parserChain = initParserChain(tableName, columnName, resultSet); + } catch (SQLException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } + } + int colCnt = parserChain.size(); + int rowCnt = 0; + List> listList = new ArrayList<>(); + try { + while (resultSet.next()) { + List line = new ArrayList<>(colCnt); + int column = 1; + for (Method m : parserChain) { + Object o = m.invoke(resultSet, column); + if (o == null) { + line.add(""); + } else { + line.add(o + ""); + } + column++; + } + listList.add(line); + rowCnt++; + } + return listList; + } catch (IllegalAccessException | SQLException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + return null; + } + + /** + * 结果转成列表格式待进一步处理 + * @param tableName + * @param columnName + * @param resultSet + * @return + */ + public static LocalReadResult commonParse(String tableName, String columnName, ResultSet resultSet) + throws SQLException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { + // get the trans method from the cache + LinkedList parserChain = parserMap.get(tableName); + // build it if it doesn't exist + if (parserChain == null) { + parserChain = initParserChain(tableName, columnName, resultSet); + } + int colCnt = parserChain.size(); + int rowCnt = 0; + int dateIndex = dateMap.get(tableName); + LastDateInfo lastDateInfo = new LastDateInfo(); + LocalReadResult result = new LocalReadResult(); + List> listList = new ArrayList<>(); + while (resultSet.next()) { + List line = new ArrayList<>(colCnt); + int column = 1; + for (Method m : parserChain) { + Object o = m.invoke(resultSet, column); + if (o == null) { + line.add(""); + } else { + line.add(o + ""); + } + check(column, dateIndex, line, lastDateInfo); + column++; + } + listList.add(line); + rowCnt++; + } + result.setResultStringList(listList); + result.setDate(lastDateInfo.getLastDate()); + result.setSameDateCount(lastDateInfo.getLastDateCount()); + result.setCount(rowCnt); + return result; + } + + private static void check(int column, int dateIndex, List line, LastDateInfo lastDateInfo) { + if (column == dateIndex) { + String thisDate = line.get(column - 1); + if (thisDate.equals(lastDateInfo.getLastDate())) { + int lastDateCount = lastDateInfo.getLastDateCount(); + lastDateInfo.setLastDateCount(lastDateCount + 1); + } else { + lastDateInfo.setLastDate(thisDate); + lastDateInfo.setLastDateCount(1); + } + } + } + + /** + * 将结果转成csv格式 + * @return + */ + public static String listParseToCsv(List> list, String nodeId) { + StringWriter stringWriter = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(stringWriter); + for (List l : list) { + l.add(0, nodeId); + csvWriter.writeNext(l.toArray(new String[l.size()])); + } + return stringWriter.toString(); + } + + /** + * 结果直接转成csv格式 + * @param tableName + * @param columnName + * @param resultSet + * @return + */ + @Deprecated + public static CsvResult csvParse(String tableName, String columnName, ResultSet resultSet) { + // get the trans method from the cache + LinkedList parserChain = parserMap.get(tableName); + // build it if it doesn't exist + if (parserChain == null) { + try { + parserChain = initParserChain(tableName, columnName, resultSet); + } catch (SQLException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } + } + StringWriter stringWriter = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(stringWriter); + int colCnt = parserChain.size(); + int rowCnt = 0; + int dateIndex = dateMap.get(tableName); + String lastDate = null; + try { + while (resultSet.next()) { + String[] line = new String[colCnt]; + lastDate = runParser(parserChain, line, resultSet, dateIndex, lastDate); + csvWriter.writeNext(line); + rowCnt++; + } + return new CsvResult(rowCnt, lastDate, stringWriter.toString()); + } catch (IllegalAccessException | SQLException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } finally { + try { + csvWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } + + private static String runParser(LinkedList parserChain, String[] line, ResultSet resultSet, int dateIndex, String lastDate) throws InvocationTargetException, IllegalAccessException { + int column = 1; + for (Method m : parserChain) { + line[column - 1] = m.invoke(resultSet, column) + ""; + if (column == dateIndex) { + lastDate = line[column - 1]; + } + column++; + } + return lastDate; + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncExecutor.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncExecutor.java new file mode 100644 index 0000000000000000000000000000000000000000..5573a5e8073e0fd6e4c7417d596cd3b43dadab62 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncExecutor.java @@ -0,0 +1,29 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class AsyncExecutor { + private static ExecutorService executor = Executors.newFixedThreadPool(6); + + public static void run(Runnable runnable) { + executor.execute(runnable); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncMissionUploadManagerImpl.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncMissionUploadManagerImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..7c9df22eabd70f98d5f746895aa940e9ce0e3803 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncMissionUploadManagerImpl.java @@ -0,0 +1,179 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.center.MultiCopyManagerHelper; +import com.intarkdb.cloudedge.common.Constant; +import com.intarkdb.cloudedge.common.ThreadUncaughtExceptionHandler; +import com.intarkdb.cloudedge.config.ConfigProperties; +import com.intarkdb.cloudedge.config.LocalDbMsg; +import com.intarkdb.cloudedge.local.LocalDBReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class AsyncMissionUploadManagerImpl implements UploadMissionManager { + + private static final Logger log = LoggerFactory.getLogger(AsyncMissionUploadManagerImpl.class); + + private ConcurrentLinkedQueue missionQueue; + + private ConcurrentHashMap readerMap; + + public AtomicLong uploadCount = new AtomicLong(0); + + public AtomicLong readCount = new AtomicLong(0); + + private ScheduledExecutorService executorService; + + private boolean closeFlag = false; + + private Integer loopWait; + + + /** + * init the mission + */ + public void init(ConfigProperties conf) { + readerMap = new ConcurrentHashMap<>(); + // init mission + missionQueue = new ConcurrentLinkedQueue<>(); + + int coreSize = 0; + + // init copyManager +// CopyManagerHelper.connectToCenterDB(conf.getCenterDb()); + MultiCopyManagerHelper.init(conf.getCenterDb(), conf.getSyncTable()); + + // get access to local instarDB + for (LocalDbMsg localDbMsg: conf.getLocalDb()) { + LocalDBReader reader = new LocalDBReader(localDbMsg); + // check if the sync manage table exists, if not, build it. + reader.initManageTable(); + readerMap.put(localDbMsg.getNodeId(), reader); + reader.initAsyncMissionQueue(missionQueue, conf.getSyncTable(), localDbMsg.getNodeId()); + coreSize += conf.getSyncTable().size(); + } + + // init thread pool + executorService = Executors.newScheduledThreadPool(coreSize); + + loopWait = conf.getLoopWait(); + } + + @Override + public void refreshConfig(ConfigProperties conf) { + // todo + // shut down old ScheduledExecutorService + + // initialize and start new ScheduledExecutorService + + } + + /** + * add mission to list + * @return + */ + @Override + public boolean addMission(SimpleUploadMission missionMsg) { + return missionQueue.offer(missionMsg); + } + + @Override + public void start() { + speedPrint(); + uploadLoop(); + } + + @Override + public void close() { + closeFlag = true; + } + + public void uploadLoop() { + while (!closeFlag) { + if (missionQueue.isEmpty()) { + try { + Thread.sleep(loopWait); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } else { + UploadMission msg; + while ((msg = missionQueue.poll()) != null) { + handleMission(msg, missionQueue); + } + } + } + executorService.shutdown(); + } + + private void handleMission(UploadMission mission, ConcurrentLinkedQueue missionQueue) { + + Long now = Instant.now().toEpochMilli(); + Long delayTime = Math.max(mission.getControlMsg().getEndTime() / 1000 + Constant.SCAN_WAIT_RANGE - now, 0); + delayTime = Math.max(delayTime, Math.min(mission.getFailCount(), Constant.FAIL_LIMIT) * 5 * 1000); + // put the mission to delay executorService + executorService.schedule(() -> { + long begin = 0; + try { + int handleNum = mission.readAndUpload(readerMap.get(mission.getControlMsg().getNodeId()), mission.getControlMsg(), uploadCount); + readCount.getAndAdd(handleNum); + } catch (Throwable t) { + log.error("handle mission error!" + mission.toString()); + t.printStackTrace(); + mission.addFailCount(); + } finally { + // 将更新完的任务放回列表，下次处理 + long end = Instant.now().toEpochMilli(); + missionQueue.offer(mission); + } + }, delayTime, TimeUnit.MILLISECONDS); + } + + + private void speedPrint() { + Thread thread = new Thread(() -> { + long lastReadCount = 0; + long lastUploadCount = 0; + int secCount = 1; + while (!closeFlag) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + long thisUploadCount = uploadCount.get(); + long thisReadCount = readCount.get(); + log.info("read count: " + thisReadCount + " read speed: " + (thisReadCount - lastReadCount) + " avg speed: " + (thisReadCount / secCount)); + log.info("upload count: " + thisUploadCount + " upload speed: " + (thisUploadCount - lastUploadCount) + " avg speed: " + (thisUploadCount / secCount)); + lastReadCount = thisReadCount; + lastUploadCount = thisUploadCount; + secCount++; + } + }); + thread.setUncaughtExceptionHandler(new ThreadUncaughtExceptionHandler(log)); + thread.start(); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncUploadMission.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncUploadMission.java new file mode 100644 index 0000000000000000000000000000000000000000..3e0fe51d41dc2c2c5a749ce708a7f98eaeaea0e6 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/AsyncUploadMission.java @@ -0,0 +1,117 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.center.MultiCopyManagerHelper; +import com.intarkdb.cloudedge.common.Constant; +import com.intarkdb.cloudedge.local.LocalDBReader; +import com.intarkdb.cloudedge.local.LocalReadResult; +import com.intarkdb.cloudedge.local.ResultParser; +import com.intarkdb.cloudedge.tool.TimeParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +public class AsyncUploadMission extends UploadMission { + private static final Logger log = LoggerFactory.getLogger(AsyncUploadMission.class); + + public AsyncUploadMission(SyncTableControlInfo controlMsg) { + super(controlMsg); + } + + /** + * + * @param reader + * @param controlMsg + * @return 返回本次已读取数据 + */ + @Override + public Integer readAndUpload(LocalDBReader reader, SyncTableControlInfo controlMsg, AtomicLong uploadCounter) { + // default value of next read begin timestamp + long updateBeginTimestamp = controlMsg.getEndTime(); + String nowDate = TimeParser.microsecondToString(updateBeginTimestamp); + +// CopyManagerHelper copyManagerHelper = CopyManagerHelper.getCopyManagerHelper(); + + String lastSyncDate = TimeParser.microsecondToString(controlMsg.getBeginTime()); + + // read data from table + ResultSet dataRaw = reader.readData(controlMsg.getTableName(), controlMsg.getSyncColumn(), lastSyncDate, nowDate, Constant.BATCH_SIZE); + + LocalReadResult localReadResult = null; + try { + localReadResult = ResultParser.commonParse(controlMsg.getTableName(), controlMsg.getSyncColumn(), dataRaw); + } catch (SQLException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + + List> resultStringList = localReadResult.getResultStringList(); + + // 右闭区间是否已经读完？ + if (localReadResult.getCount() == Constant.BATCH_SIZE) { + List> lastDateResult = reader.readData(controlMsg.getTableName(), controlMsg.getSyncColumn(), + localReadResult.getDate(), localReadResult.getSameDateCount(), Constant.BATCH_SIZE); + resultStringList.addAll(lastDateResult); + updateBeginTimestamp = TimeParser.stringToMicrosecond(localReadResult.getDate()); + } + + // upload to center db + asyncUpload(reader, resultStringList, nowDate, uploadCounter); + + // update control message + long scanEndTimestamp = (Instant.now().toEpochMilli() - Constant.SCAN_WAIT_RANGE) * 1000; + if (controlMsg.getEndTime() < scanEndTimestamp) { + controlMsg.setEndTime(scanEndTimestamp); + } + controlMsg.setBeginTime(updateBeginTimestamp); + + + long tt5 = Instant.now().toEpochMilli(); + + if (this.failCount > 0) { + this.failCount = 0; + } + + return resultStringList.size(); + } + + private void asyncUpload(LocalDBReader reader, List> resultStringList, String nowDate, AtomicLong uploadCounter) { + AsyncExecutor.run(() -> { + String csvString = ResultParser.listParseToCsv(resultStringList, controlMsg.getNodeId()); + if (!MultiCopyManagerHelper.uploadData(controlMsg.getTableName(), csvString)) { + log.warn("upload failed! table : " + controlMsg.getTableName() + " time: " + nowDate); + // todo 区分不同的返回异常 + return ; + } else { + uploadCounter.getAndAdd(resultStringList.size()); + reader.updateSyncTime(controlMsg.getTableName(), controlMsg.getBeginTime(), controlMsg.getEndTime()); + } + }); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SimpleMissionUploadManagerImpl.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SimpleMissionUploadManagerImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..8c36444b00c13d497df8c3355715b07c6abe080d --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SimpleMissionUploadManagerImpl.java @@ -0,0 +1,175 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.center.MultiCopyManagerHelper; +import com.intarkdb.cloudedge.common.Constant; +import com.intarkdb.cloudedge.common.ThreadUncaughtExceptionHandler; +import com.intarkdb.cloudedge.config.ConfigProperties; +import com.intarkdb.cloudedge.config.LocalDbMsg; +import com.intarkdb.cloudedge.local.LocalDBReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class SimpleMissionUploadManagerImpl implements UploadMissionManager { + + private static final Logger log = LoggerFactory.getLogger(SimpleMissionUploadManagerImpl.class); + + private ConcurrentLinkedQueue missionQueue; + + private ConcurrentHashMap readerMap; + + public AtomicLong uploadCount = new AtomicLong(0); + + private ScheduledExecutorService executorService; + + private boolean closeFlag = false; + + private Integer loopWait; + + + /** + * init the mission + */ + public void init(ConfigProperties conf) { + readerMap = new ConcurrentHashMap<>(); + // init mission + missionQueue = new ConcurrentLinkedQueue<>(); + + int coreSize = 0; + + // init copyManager +// CopyManagerHelper.connectToCenterDB(conf.getCenterDb()); + MultiCopyManagerHelper.init(conf.getCenterDb(), conf.getSyncTable()); + + // get access to local instarDB + for (LocalDbMsg localDbMsg: conf.getLocalDb()) { + LocalDBReader reader = new LocalDBReader(localDbMsg); + // check if the sync manage table exists, if not, build it. + reader.initManageTable(); + readerMap.put(localDbMsg.getNodeId(), reader); + reader.initSimpleMissionQueue(missionQueue, conf.getSyncTable(), localDbMsg.getNodeId()); + coreSize += conf.getSyncTable().size(); + } + + // init thread pool + executorService = Executors.newScheduledThreadPool(coreSize); + + loopWait = conf.getLoopWait(); + } + + @Override + public void refreshConfig(ConfigProperties conf) { + // todo + // shut down old ScheduledExecutorService + + // initialize and start new ScheduledExecutorService + + } + + /** + * add mission to list + * @return + */ + @Override + public boolean addMission(SimpleUploadMission missionMsg) { + return missionQueue.offer(missionMsg); + } + + @Override + public void start() { + speedPrint(); + uploadLoop(); + } + + @Override + public void close() { + closeFlag = true; + } + + public void uploadLoop() { + while (!closeFlag) { + if (missionQueue.isEmpty()) { + try { + Thread.sleep(loopWait); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } else { + UploadMission msg; + while ((msg = missionQueue.poll()) != null) { + handleMission(msg, missionQueue); + } + } + } + executorService.shutdown(); + } + + private void handleMission(UploadMission mission, ConcurrentLinkedQueue missionQueue) { + + Long now = Instant.now().toEpochMilli(); + Long delayTime = Math.max(mission.getControlMsg().getEndTime() / 1000 + Constant.SCAN_WAIT_RANGE - now, 0); + delayTime = Math.max(delayTime, Math.min(mission.getFailCount(), Constant.FAIL_LIMIT) * 5 * 1000); + // put the mission to delay executorService + executorService.schedule(() -> { + long begin = 0; + try { + begin = Instant.now().toEpochMilli(); + int handleNum = mission.readAndUpload(readerMap.get(mission.getControlMsg().getNodeId()), mission.getControlMsg(), null); + uploadCount.getAndAdd(handleNum); + } catch (Throwable t) { + log.error("handle mission error!" + mission.toString()); + t.printStackTrace(); + mission.addFailCount(); + } finally { + // 将更新完的任务放回列表，下次处理 + long end = Instant.now().toEpochMilli(); +// log.debug("mission update " + mission.getControlMsg().getNodeId() + " " + mission.getControlMsg().getTableName() + " cost " + (end - begin)); + missionQueue.offer(mission); + } + }, delayTime, TimeUnit.MILLISECONDS); + } + + + private void speedPrint() { + Thread thread = new Thread(() -> { + long lastCount = 0; + int secCount = 1; + while (!closeFlag) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + long thisCount = uploadCount.get(); + log.info("upload count: " + thisCount + " upload speed: " + (thisCount - lastCount) + " avg speed: " + (thisCount / secCount)); + lastCount = thisCount; + secCount++; + } + }); + thread.setUncaughtExceptionHandler(new ThreadUncaughtExceptionHandler(log)); + thread.start(); + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SimpleUploadMission.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SimpleUploadMission.java new file mode 100644 index 0000000000000000000000000000000000000000..60208a4d74f815ed87b2df2d0f6817f1c32c7df5 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SimpleUploadMission.java @@ -0,0 +1,108 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.center.MultiCopyManagerHelper; +import com.intarkdb.cloudedge.common.Constant; +import com.intarkdb.cloudedge.local.LocalDBReader; +import com.intarkdb.cloudedge.local.LocalReadResult; +import com.intarkdb.cloudedge.local.ResultParser; +import com.intarkdb.cloudedge.tool.TimeParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +public class SimpleUploadMission extends UploadMission { + + private static final Logger log = LoggerFactory.getLogger(SimpleUploadMission.class); + + public SimpleUploadMission(SyncTableControlInfo controlMsg) { + super(controlMsg); + } + + /** + * + * @param reader + * @param controlMsg + * @return 返回本次已处理数据 + */ + @Override + public Integer readAndUpload(LocalDBReader reader, SyncTableControlInfo controlMsg, AtomicLong uploadCounter) { + // default value of next read begin timestamp + long updateBeginTimestamp = controlMsg.getEndTime(); + String nowDate = TimeParser.microsecondToString(updateBeginTimestamp); + +// CopyManagerHelper copyManagerHelper = CopyManagerHelper.getCopyManagerHelper(); + + String lastSyncDate = TimeParser.microsecondToString(controlMsg.getBeginTime()); + + // read data from table + ResultSet dataRaw = reader.readData(controlMsg.getTableName(), controlMsg.getSyncColumn(), lastSyncDate, nowDate, Constant.BATCH_SIZE); + + LocalReadResult localReadResult = null; + try { + localReadResult = ResultParser.commonParse(controlMsg.getTableName(), controlMsg.getSyncColumn(), dataRaw); + } catch (SQLException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + + List> resultStringList = localReadResult.getResultStringList(); + + // 右闭区间是否已经读完？ + if (localReadResult.getCount() == Constant.BATCH_SIZE) { + List> lastDateResult = reader.readData(controlMsg.getTableName(), controlMsg.getSyncColumn(), + localReadResult.getDate(), localReadResult.getSameDateCount(), Constant.BATCH_SIZE); + resultStringList.addAll(lastDateResult); + updateBeginTimestamp = TimeParser.stringToMicrosecond(localReadResult.getDate()); + } + + String csvString = ResultParser.listParseToCsv(resultStringList, controlMsg.getNodeId()); + // upload to center db + if (!MultiCopyManagerHelper.uploadData(controlMsg.getTableName(), csvString)) { + log.warn("upload failed! table : " + controlMsg.getTableName() + " time: " + nowDate); + // todo 区分不同的返回异常 + return 0; + } + + // update control message + long scanEndTimestamp = Instant.now().toEpochMilli() - Constant.SCAN_WAIT_RANGE; + if (controlMsg.getEndTime() < scanEndTimestamp) { + controlMsg.setEndTime(scanEndTimestamp); + } + controlMsg.setBeginTime(updateBeginTimestamp); + reader.updateSyncTime(controlMsg.getTableName(), controlMsg.getBeginTime(), controlMsg.getEndTime()); + + if (this.failCount > 0) { + this.failCount = 0; + } + + return resultStringList.size(); + } + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SyncTableControlInfo.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SyncTableControlInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..f9cdc28dfecdf2768f8c2b6d9c9b8eb3d2304761 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/SyncTableControlInfo.java @@ -0,0 +1,128 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.common.Constant; + +public class SyncTableControlInfo { + /** + * 表名 + */ + private String tableName; + + /** + * 任务类型 + */ + private int type; + + /** + * 时序表的时间列，由用户指定 + */ + private String syncColumn; + + /** + * 同步任务扫描左区间（左开） + */ + private Long beginTime; + + /** + * 同步任务扫描右区间（右闭） + */ + private Long endTime; + + /** + * 节点id + */ + private String nodeId; + + /** + * 唯一列 + */ + private String uniqueColumn; + + public SyncTableControlInfo(String tableName, String syncColumn, Long beginTime) { + this.tableName = tableName; + this.syncColumn = syncColumn; + this.beginTime = beginTime; + this.endTime = beginTime + Constant.SCAN_WINDOW_RANGE; + } + + public SyncTableControlInfo(String tableName, String syncColumn, Long beginTime, Long endTime, String nodeId) { + this.tableName = tableName; + this.syncColumn = syncColumn; + this.beginTime = beginTime; + this.endTime = endTime; + this.nodeId = nodeId; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public String getSyncColumn() { + return syncColumn; + } + + public void setSyncColumn(String syncColumn) { + this.syncColumn = syncColumn; + } + + public Long getBeginTime() { + return beginTime; + } + + public void setBeginTime(Long beginTime) { + this.beginTime = beginTime; + } + + public Long getEndTime() { + return endTime; + } + + public void setEndTime(Long endTime) { + this.endTime = endTime; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public String getUniqueColumn() { + return uniqueColumn; + } + + public void setUniqueColumn(String uniqueColumn) { + this.uniqueColumn = uniqueColumn; + } + +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/UploadMission.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/UploadMission.java new file mode 100644 index 0000000000000000000000000000000000000000..0d07a0e009bf6989029cb1a96e20c872d660b043 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/UploadMission.java @@ -0,0 +1,55 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.local.LocalDBReader; + +import java.util.concurrent.atomic.AtomicLong; + +public abstract class UploadMission { + protected SyncTableControlInfo controlMsg; + + protected int failCount = 0; + + public UploadMission(SyncTableControlInfo controlMsg) { + this.controlMsg = controlMsg; + } + + public SyncTableControlInfo getControlMsg() { + return controlMsg; + } + + public void setControlMsg(SyncTableControlInfo controlMsg) { + this.controlMsg = controlMsg; + } + + public int getFailCount() { + return failCount; + } + + public void setFailCount(int failCount) { + this.failCount = failCount; + } + + public void addFailCount() { + if (this.failCount < Integer.MAX_VALUE) { + this.failCount++; + } + } + + public abstract Integer readAndUpload(LocalDBReader reader, SyncTableControlInfo controlMsg, AtomicLong uploadCounter); +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/UploadMissionManager.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/UploadMissionManager.java new file mode 100644 index 0000000000000000000000000000000000000000..a991e9e02444f21ede0437524da39ba12af2ea65 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/mission/UploadMissionManager.java @@ -0,0 +1,51 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.mission; + +import com.intarkdb.cloudedge.config.ConfigProperties; + +public interface UploadMissionManager { + + /** + * mission initialize + * @param conf + */ + void init(ConfigProperties conf); + + /** + * refresh config + * @param conf + */ + void refreshConfig(ConfigProperties conf); + + /** + * add mission to the running loop + * @param missionMsg + * @return + */ + boolean addMission(SimpleUploadMission missionMsg); + + /** + * begin to run the mission loop + */ + void start(); + + /** + * stop the mission loop and close the resource + */ + void close(); +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/tool/StringUtil.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/tool/StringUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..a5fba8d519a71f23cc98d8e8803a87bdfb08c2a6 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/tool/StringUtil.java @@ -0,0 +1,27 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.tool; + +public class StringUtil { + public static boolean isBlank(String s) { + if (null == s || "".equals(s)) { + return true; + } else { + return false; + } + } +} diff --git a/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/tool/TimeParser.java b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/tool/TimeParser.java new file mode 100644 index 0000000000000000000000000000000000000000..52bc4a67719194d8a14aaa8a497dd090cadca991 --- /dev/null +++ b/tools/data-sync-agent/src/main/java/com/intarkdb/cloudedge/tool/TimeParser.java @@ -0,0 +1,131 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN + BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +*/ +package com.intarkdb.cloudedge.tool; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; + +public class TimeParser { + + private static final Logger log = LoggerFactory.getLogger(TimeParser.class); + + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static DateTimeFormatter formatter1 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private static DateTimeFormatter formatter2 = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static DateTimeFormatter formatter3 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SS"); + private static DateTimeFormatter formatter4 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S"); + private static DateTimeFormatter formatter5 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS"); + + private static final int FORMATTER_LENGTH = 19; + private static final int FORMATTER_LENGTH1 = 23; + private static final int FORMATTER_LENGTH2 = 10; + private static final int FORMATTER_LENGTH3 = 22; + private static final int FORMATTER_LENGTH4 = 21; + private static final int FORMATTER_LENGTH5 = 26; + + public static LocalDateTime toLocalDatetime(String dateTimeString) { + LocalDateTime localDateTime = null; + + try { + switch (dateTimeString.length()) { + case FORMATTER_LENGTH: + localDateTime = LocalDateTime.parse(dateTimeString, formatter); + break; + case FORMATTER_LENGTH1: + localDateTime = LocalDateTime.parse(dateTimeString, formatter1); + break; + case FORMATTER_LENGTH2: + localDateTime = LocalDateTime.parse(dateTimeString, formatter2); + break; + case FORMATTER_LENGTH3: + localDateTime = LocalDateTime.parse(dateTimeString, formatter3); + break; + case FORMATTER_LENGTH4: + localDateTime = LocalDateTime.parse(dateTimeString, formatter4); + break; + case FORMATTER_LENGTH5: + localDateTime = LocalDateTime.parse(dateTimeString, formatter5); + break; + default: + System.out.println("unsupported datetime string: " + dateTimeString); + return localDateTime; + } + return localDateTime; + } catch (Exception e) { + log.error("date time trans failed, String: " + dateTimeString); + return localDateTime; + } + } + + public static long toTimestamp(String dateTimeString) { + LocalDateTime localDateTime = toLocalDatetime(dateTimeString); + + if (localDateTime != null) { + return localDateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + } else { + return 0; + } + } + + public static String toString(long timestamp) { + LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault()); + + return localDateTime.format(formatter1); + } + + public static String microsecondToString(long microsecondTimestamp) { + long milli = microsecondTimestamp / 1000; + LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(milli), ZoneId.systemDefault()); + return localDateTime.format(formatter1) + String.format("%03d", microsecondTimestamp % 1000); + } + + public static long stringToMicrosecond(String dateTimeString) { + long milli = toTimestamp(dateTimeString); + + return milli * 1000 + Integer.valueOf(dateTimeString.substring(23)); + } + + public static String roundUpTimestamp(String dateTimeString) { + LocalDateTime localDateTime = toLocalDatetime(dateTimeString); + + if (localDateTime != null) { + + if (localDateTime.getNano() != 0) { + LocalDateTime ceilTimestamp = localDateTime.plus(1, ChronoUnit.SECONDS); + return ceilTimestamp.format(formatter5); + } else { + return localDateTime.format(formatter5); + } + } else { + return null; + } + + } + + public static void main(String... args) { +// System.out.println(roundUpTimestamp("2020-01-01 11:11:11.231")); + System.out.println(microsecondToString(1702006505063167L)); + System.out.println(stringToMicrosecond("2023-12-08 11:35:05.063167")); + } +} diff --git a/tools/data-sync-agent/src/main/resources/application.properties b/tools/data-sync-agent/src/main/resources/application.properties new file mode 100644 index 0000000000000000000000000000000000000000..ac6e80bd845043f4e7ccf61c0379b6ea78038bf7 --- /dev/null +++ b/tools/data-sync-agent/src/main/resources/application.properties @@ -0,0 +1 @@ +logging.pattern.console = %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n \ No newline at end of file diff --git a/tools/data-sync-agent/src/main/resources/conf.yaml b/tools/data-sync-agent/src/main/resources/conf.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f484f639fc9896b69a015c53f75da1e95275251b --- /dev/null +++ b/tools/data-sync-agent/src/main/resources/conf.yaml @@ -0,0 +1,58 @@ +# center db setting +centerDb: +# test server +# ip: 192.168.110.41 +# port: 5432 +# user: gaussdb +# password: openGauss@123 + +# ip: 192.168.111.194 +# port: 5432 +# user: gaussdb +# password: openGauss@123 + +#local + ip: 192.168.110.30 + port: 5432 + user: gaussdb + password: Test@1234 + +# local db setting +localDb: + - ip: 10.0.2.15 + port: 3333 + user: + password: + nodeId: +# - ip: 172.21.186.202 +# port: 3336 +# user: +# password: +# nodeId: +# - ip: 192.168.111.194 +# port: 3333 +# user: +# password: +# nodeId: +# - ip: 192.168.110.41 +# port: 3333 +# user: +# password: +# nodeId: + +# sync table [table name]:[sync column] +syncTable: + - tableName: readings + syncColumn: created_time + - tableName: diagnostics + syncColumn: created_time +# - tableName: common_t_table_2 +# syncColumn: t +# - tableName: common_t_table_3 +# syncColumn: t +# - tableName: common_t_table_4 +# syncColumn: t +# - tableName: common_t_table_5 +# syncColumn: t + +loopWait: 30 \ No newline at end of file diff --git a/tools/gradle-7.2-all.zip b/tools/gradle-7.2-all.zip new file mode 100755 index 0000000000000000000000000000000000000000..d43cd93e0f292281d2df8e8e3b3d0f3096a1fa67 Binary files /dev/null and b/tools/gradle-7.2-all.zip differ diff --git a/tools/intarkdb_cli/CMakeLists.txt b/tools/intarkdb_cli/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..7848bc382409243a0c7d4870d1f4b999e15d2e7c --- /dev/null +++ b/tools/intarkdb_cli/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 3.14.1) +# project(intarkdb_cli) + +add_compile_options(-fPIC) +# # add_compile_options(-fPIC -MMD -fno-strict-aliasing -fsigned-char -fms-extensions) + +set(CMAKE_CXX_STANDARD 17) +## create compile_command.json +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +include_directories(${INTARKDB_ZEKERNEL_COMMON_INC_PATH}) + +set(INTARKDB_LINK_LIBS +sqlite3_api_wrapper +intarkdb +cjson +fmt +) + +if(UNIX) + file(GLOB test_main ./*.cpp ./*.c) + add_executable(intarkdb_cli ${test_main}) + # target_link_libraries(intarkdb_cli ${INTARKDB_LINK_LIBS} -Wl,--copy-dt-needed-entries) + if ((${OS_ARCH} STREQUAL "arm32") OR (${OS_ARCH} STREQUAL "aarch64")) + target_link_libraries(intarkdb_cli ${INTARKDB_LINK_LIBS}) + else() + target_link_libraries(intarkdb_cli ${INTARKDB_LINK_LIBS} -Wl,--copy-dt-needed-entries) + endif() + target_link_directories(intarkdb_cli PUBLIC ${INTARKDB_LIB_PATH} ${INTARKDB_THRID_LIB_PATH}) + target_include_directories(intarkdb_cli PUBLIC ${INTARKDB_COMPUTE_SQL_INC_PATH} + ${INTARKDB_PGQUERY_INC_PATH} + ${INTARKDB_SECUREC_INC_PATH} + ${INTARKDB_SRC_PATH} + ${INTARKDB_CJSON_PATH} + ${CMAKE_CURRENT_SOURCE_DIR} + ${INTARKDB_THRID_INC_PATH} + ${INTARKDB_FMT_INC_PATH}) +endif(UNIX) \ No newline at end of file diff --git a/tools/intarkdb_cli/assist.cpp b/tools/intarkdb_cli/assist.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e72ab904ef9a9b3b78c78d204e2f7ec349fe7cf7 --- /dev/null +++ b/tools/intarkdb_cli/assist.cpp @@ -0,0 +1,112 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +* +* assist.cpp +* +* IDENTIFICATION +* openGauss-embedded/tools/intarkdb_cli/assist.cpp +* +* ------------------------------------------------------------------------- +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "assist.h" + + +/* +** Return a pathname which is the user's home directory. A +** 0 return indicates an error of some kind. +*/ +char *find_home_dir(int clearFlag){ + static char *home_dir = nullptr; + if( clearFlag ){ + free(home_dir); + home_dir = nullptr; + return nullptr; + } + if( home_dir ) return home_dir; + +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ + && !defined(__RTP__) && !defined(_WRS_KERNEL) + { + struct passwd *pwent; + uid_t uid = getuid(); + if( (pwent=getpwuid(uid)) != nullptr) { + home_dir = pwent->pw_dir; + } + } +#endif + +#if defined(_WIN32_WCE) + /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() + */ + home_dir = "/"; +#else + +#if defined(_WIN32) || defined(WIN32) + if (!home_dir) { + home_dir = getenv("USERPROFILE"); + } +#endif + + if (!home_dir) { + home_dir = getenv("HOME"); + } + +#if defined(_WIN32) || defined(WIN32) + if (!home_dir) { + char *zDrive, *zPath; + int n; + zDrive = getenv("HOMEDRIVE"); + zPath = getenv("HOMEPATH"); + if( zDrive && zPath ){ + n = strlen30(zDrive) + strlen30(zPath) + 1; + home_dir = malloc( n ); + if( home_dir==0 ) return 0; + sqlite3_snprintf(n, home_dir, "%s%s", zDrive, zPath); + return home_dir; + } + home_dir = "c:\\"; + } +#endif + +#endif /* !_WIN32_WCE */ + + if( home_dir ){ + int n = strlen30(home_dir) + 1; + char *z = (char*)malloc( n ); + if( z ) memcpy(z, home_dir, n); + home_dir = z; + } + + return home_dir; +} + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} \ No newline at end of file diff --git a/tools/intarkdb_cli/assist.h b/tools/intarkdb_cli/assist.h new file mode 100644 index 0000000000000000000000000000000000000000..6642fcf71f89838e72daa08258b2497ef1d0e875 --- /dev/null +++ b/tools/intarkdb_cli/assist.h @@ -0,0 +1,27 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +* +* assist.h +* +* IDENTIFICATION +* openGauss-embedded/tools/intarkdb_cli/assist.h +* +* ------------------------------------------------------------------------- +*/ +#include "interface/sqlite3_api_wrapper/include/sqlite3.h" + +char *find_home_dir(int clearFlag); + +int strlen30(const char *z); \ No newline at end of file diff --git a/tools/intarkdb_cli/cmd.cpp b/tools/intarkdb_cli/cmd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7260dd222462022a992f91c68426149e889713d0 --- /dev/null +++ b/tools/intarkdb_cli/cmd.cpp @@ -0,0 +1,1010 @@ +/* + * Copyright (c) GBA-NCTI-ISDC. 2022-2024. + * + * openGauss embedded is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan + * PSL v2. You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY + * KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * NON-INFRINGEMENT, MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. See the + * Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * cmd.cpp + * + * IDENTIFICATION + * openGauss-embedded/tools/intarkdb_cli/cmd.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "cmd.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cJSON.h" +#include "cm_signal.h" +#include "interface/c/intarkdb_kv.h" +#include "include/intarkdb.h" +#include "linenoise.h" +#include "storage/gstor/zekernel/common/cm_thread.h" +#ifdef SUPPORT_SQLITE +#include +#endif + +#define PLATFORM_INFO_SIZE 1024 +#define CLOSE_TIMES 2 + +ClassCmd::ClassCmd(const std::string &path, Connection *conn, KvOperator *kv_oper) + : sql_conn(conn), kv_operator(kv_oper), db_path(path + "intarkdb"), print_type(box) { + ex_func_map.insert(std::make_pair(".help", &ClassCmd::Help)); + ex_func_map.insert(std::make_pair(".dbinfo", &ClassCmd::Info)); + ex_func_map.insert(std::make_pair(".version", &ClassCmd::Version)); + + ex_func_map.insert(std::make_pair(".dump", &ClassCmd::Dump)); + ex_func_map.insert(std::make_pair(".import", &ClassCmd::Import)); + ex_func_map.insert(std::make_pair(".read", &ClassCmd::Read)); + + ex_func_map.insert(std::make_pair(".table", &ClassCmd::ShowTable)); + ex_func_map.insert(std::make_pair(".index", &ClassCmd::ShowIndex)); + ex_func_map.insert(std::make_pair(".keys", &ClassCmd::SHowKeys)); + ex_func_map.insert(std::make_pair(".schema", &ClassCmd::Schema)); + ex_func_map.insert(std::make_pair(".fullschema", &ClassCmd::FullSchema)); + + ex_func_map.insert(std::make_pair(".mode", &ClassCmd::Mode)); + ex_func_map.insert(std::make_pair(".explain", &ClassCmd::Explain)); + ex_func_map.insert(std::make_pair(".nullvalue", &ClassCmd::SetNullStr)); + ex_func_map.insert(std::make_pair(".width", &ClassCmd::SetWidth)); + ex_func_map.insert(std::make_pair(".echo", &ClassCmd::SetEcho)); + ex_func_map.insert(std::make_pair(".change", &ClassCmd::SetChange)); + ex_func_map.insert(std::make_pair(".timer", &ClassCmd::SetTimer)); + ex_func_map.insert(std::make_pair(".output", &ClassCmd::SetOutFile)); + ex_func_map.insert(std::make_pair(".once", &ClassCmd::SetOutputOnce)); + ex_func_map.insert(std::make_pair(".prompt", &ClassCmd::SetPrompt)); + ex_func_map.insert(std::make_pair(".show", &ClassCmd::ShowOption)); + ex_func_map.insert(std::make_pair(".dbconfig", &ClassCmd::DBConfig)); + + help_msg = "Help: \n"; + std::map help_map; + int length = 20; + help_map.insert(std::make_pair(".echo on|off", "Turn command echo on or off")); + help_map.insert(std::make_pair(".quit", "Exit this program")); + help_map.insert(std::make_pair(".exit", "Exit this program")); + help_map.insert(std::make_pair(".help", "Show help text")); + help_map.insert(std::make_pair(".help kv", "Show help text for kv")); + help_map.insert(std::make_pair(".dbinfo", "Show status information about the database")); + help_map.insert(std::make_pair(".version", "Show version infomation")); + help_map.insert(std::make_pair(".dump", "Render database content as SQL")); + help_map.insert(std::make_pair(".import $FILE $TABLE", "Import data from FILE into TABLE")); + help_map.insert(std::make_pair(".read $FILE", "Read input from FILE")); + + help_map.insert(std::make_pair(".table $TABLE", "List names of tables matching LIKE pattern TABLE")); + help_map.insert(std::make_pair(".index $TABLE", "Show names of indexes")); + help_map.insert(std::make_pair(".schema $PATTERN", "Show the CREATE statements matching PATTERN")); + help_map.insert(std::make_pair(".fullschema", "Show schema and the content of sqlite_stat tables")); + help_map.insert(std::make_pair(".keys", "show all keys")); + + help_map.insert(std::make_pair(".mode", "Set output mode, support: " + GetModeType(","))); + help_map.insert(std::make_pair(".explain on|off", "Turn command echo on or off")); + help_map.insert(std::make_pair(".nullvalue", "Use STRING in place of NULL values")); + help_map.insert(std::make_pair(".width $NUM", "Set minimum column widths")); + help_map.insert(std::make_pair(".echo on|off", "Turn command echo on or off")); + help_map.insert(std::make_pair(".change on|off", "Show number of rows changed by SQL")); + help_map.insert(std::make_pair(".timer on|off", "Turn SQL timer on or off")); + help_map.insert(std::make_pair(".output $FILE", "Send output to FILE or stdout if FILE is omitted")); + help_map.insert(std::make_pair(".once $FILE", "Output for the next SQL command only to FILE")); + help_map.insert(std::make_pair(".prompt", "Replace the standard prompts")); + help_map.insert(std::make_pair(".show", "Show the current values for various settings")); + help_map.insert(std::make_pair(".dbconfig $OP $VAL", " List or change onfig optionss")); + for (auto item : help_map) { + fmt::format_to(std::back_inserter(help_msg), "{:<{}}: {}\n", item.first, length, item.second); + } +} + +std::function shutdown_handler; + +static void signal_handler(int sig_no) { shutdown_handler(sig_no); } + +void ClassCmd::main() { + int HistorySetMaxLen = 30; + uint close_num = 0; + auto prompt = GetBeginPrompt(); + linenoiseSetMultiLine(1); + linenoiseHistorySetMaxLen(HistorySetMaxLen); + shutdown_handler = [&close_num](int sig_no) { + close_num++; + if (close_num >= CLOSE_TIMES) { + std::cout << "Closing..." << std::endl; + exit(1); + } + }; + (void)cm_regist_signal(SIGINT, signal_handler); + + print_delegate_ = GetPrintDelegate("stdout"); + std::cout << "WelCome intarkdb shell." << std::endl; + while (true) { + std::string query; + while (true) { + char *input = linenoise(prompt); + if (input == nullptr) { + close_num++; + if (close_num < CLOSE_TIMES) { + std::cout << "If you want to close, press ctrl+c again" << std::endl; + continue; + } else { + std::cout << "Closing..." << std::endl; + return; + } + } else { + close_num = 0; + } + + linenoiseHistoryAdd(input); + query += std::string(input); + if (IsFinishQuery(query)) { + prompt = GetBeginPrompt(); + break; + } + prompt = GetContinuePrompt(); + query += " \n "; + linenoiseFree(input); + } + if (execute(query) == false) { + std::cout << "Closing..." << std::endl; + break; + } + } + return; +} + +std::string ClassCmd::GetTableName(const std::string &str) { + if (str[0] == '\"' && str[str.size() - 1] == '\"') return str.substr(1, str.size() - 2); + + std::string tmp = str; + transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower); + return tmp; +} + +void ClassCmd::GetCmd(std::string str, const char split, std::vector &res) { + std::istringstream iss(str); + std::string buf; + while (getline(iss, buf, split)) { + if (buf.length() > 0 && buf != "\n") res.push_back(buf); + } + // 去掉";" + if (res.size() > 0) { + int last = res.size() - 1; + if (res[last][res[last].size() - 1] == ';') res[last] = res[last].substr(0, res[last].size() - 1); + if (res[last].size() == 0) res.pop_back(); + } +} + +bool ClassCmd::execute(std::string &query) { + std::vector strList; + GetCmd(query, ' ', strList); + std::string result; + try { + if (IsQuitCmd(strList[0])) { + return false; + } + + func build_func = nullptr; + if (IsBuildInCmd(strList[0], build_func)) { + PrintBuildInCmdResult(query, build_func, strList); + return true; + } + + KvOperator::kv_func kv_func = nullptr; + if (IsKVCmd(strList[0], kv_func) && !IsSqlCmd(query, strList)) { + PrintKVCmdResult(query, kv_func, strList); + return true; + } + + PrintSQLCmdResult(query); + } catch (const std::exception &ex) { + std::cout << "Error:" << std::string(ex.what()) << std::endl; + } + return true; +} + +bool ClassCmd::IsSqlCmd(std::string query, std::vector &strList) { + std::string cmd = strList[0]; + std::transform(cmd.begin(), cmd.end(), cmd.begin(), ::tolower); + + std::string sql = query; + std::transform(sql.begin(), sql.end(), sql.begin(), ::tolower); + if (cmd == "set") { + if (sql.find("auto_commit") != std::string::npos && sql.find("=") != std::string::npos) { + return true; + } + } + + return false; +} + +std::string ClassCmd::Help(const std::vector &strList) { + if (strList.size() == 1) + return help_msg; + else if (strList.size() == 2) { + if (strList[1] == "kv") return KvOperator::show_kv_help(); + } + + return help_msg; +} + +std::string ClassCmd::GetModeType(const std::string &separator) { + std::string s_mode_type_msg; + for (const auto &item : m_print_type) s_mode_type_msg += item.first + separator; + s_mode_type_msg = s_mode_type_msg.substr(0, s_mode_type_msg.size() - 1); + return s_mode_type_msg; +} + +std::string ClassCmd::Mode(const std::vector &strList) { + std::string prompt_msg = "Useage: .mode " + GetModeType("|") + "\n"; + if (strList.size() != 2) return prompt_msg; + auto item = m_print_type.find(strList[1]); + if (item == m_print_type.end()) return prompt_msg; + print_type = item->second; + return "change mode = " + strList[1] + "\n"; +} + +std::string ClassCmd::Info(const std::vector &strList) { + std::stringstream msg; + msg << "Info: " << std::endl; + msg << "version: " << get_version() << std::endl; + msg << "git commit id: " << get_commit_id() << std::endl; + msg << "DB_PATH: " << db_path << std::endl; + return msg.str(); +} + +std::string ClassCmd::Version(const std::vector &strList) { + std::stringstream msg; + msg << "Info: " << std::endl; + msg << "version: " << get_version() << std::endl; + msg << "git commit id: " << get_commit_id() << std::endl; + return msg.str(); +} + +std::string ClassCmd::SetNullStr(const std::vector &strList) { + std::string prompt_msg = "Useage: .nullvalue $null\n"; + if (strList.size() != 2) return prompt_msg; + null_str = strList[1]; + return "change nullvalue = " + strList[1] + "\n"; +} + +std::string ClassCmd::SetWidth(const std::vector &strList) { + std::string prompt_msg = "Useage: .width $num\n"; + if (strList.size() != 2) return prompt_msg; + min_col_len = atoi(strList[1].c_str()); + return "change min width = " + strList[1]; +} + +void ClassCmd::DrowTableLine(const std::vector &vlength, std::string_view symbols, size_t symbols_size, + std::string_view line) { + const uint32_t SYMBOLS_AND_INTERVAL_SIZE = 3; + std::stringstream s_format; + s_format << "{:" << line << ">{}}"; + print_delegate_->Print(symbols.substr(0, symbols_size)); + for (std::size_t i = 0; i < vlength.size() - 1; i++) { + print_delegate_->Print(fmt::format(s_format.str(), symbols.substr(symbols_size, symbols_size), + vlength[i] + SYMBOLS_AND_INTERVAL_SIZE)); + } + s_format << "\n"; + print_delegate_->Print(fmt::format(s_format.str(), symbols.substr(symbols_size * 2, symbols_size), + vlength[vlength.size() - 1] + SYMBOLS_AND_INTERVAL_SIZE)); +} + +static auto UpdateWidth(const std::vector &record, std::vector &vlength) -> void { + for (std::size_t i = 0; i < record.size(); i++) { + if (record[i].size() > vlength[i]) { + vlength[i] = record[i].size(); + } + } +} + +static auto UpdateWidth(const std::vector> &record_batch, std::vector &vlength) + -> void { + for (const auto &row : record_batch) { + UpdateWidth(row, vlength); + } +} + +auto ClassCmd::PrintHeader(const std::vector &headers, const std::vector &widths, + const std::string &symbols, int symbols_size) -> void { + std::string_view symbols_view = symbols; + std::string_view first_line_symbol(symbols_view.substr(symbols_size * 2, symbols_size * 3)); + std::string_view line_symbol(symbols_view.substr(symbols_size, symbols_size)); + std::string_view separator(symbols_view.substr(0, symbols_size)); + std::string_view second_line_symbol(symbols_view.substr(symbols_size * 5, symbols_size * 3)); + + DrowTableLine(widths, first_line_symbol, symbols_size, line_symbol); + for (std::size_t i = 0; i < headers.size(); i++) { + print_delegate_->Print(fmt::format(fmt::emphasis::bold, "{} {:<{}} ", separator, headers[i], widths[i])); + } + print_delegate_->Print(separator); + print_delegate_->Print("\n"); + DrowTableLine(widths, second_line_symbol, symbols_size, line_symbol); +} + +auto ClassCmd::PrintFooter(const std::vector &widths, const std::string &symbols, int symbols_size) -> void { + std::string_view symbols_view = symbols; + std::string_view end_line_symbol(symbols_view.substr(symbols_size * 8, symbols_size * 3)); + std::string_view line_symbol(symbols_view.substr(symbols_size, symbols_size)); + DrowTableLine(widths, end_line_symbol, symbols_size, line_symbol); +} + +auto ClassCmd::PrintRows(const std::vector &headers, const std::vector> &rows, + std::vector &widths, const std::string &symbols, int symbols_size, bool first_page) + -> void { + std::string_view symbols_view = symbols; + std::string_view separator(symbols_view.substr(0, symbols_size)); + if (first_page) { + PrintHeader(headers, widths, symbols, symbols_size); + } + for (size_t i = 0; i < rows.size(); ++i) { + for (size_t j = 0; j < rows[i].size(); ++j) { + print_delegate_->Print(fmt::format("{} {:<{}} ", separator, rows[i][j], widths[j])); + } + print_delegate_->Print(separator); + print_delegate_->Print("\n"); + } +} + +auto ClassCmd::PrintHeaderAndBody(RecordIterator &records, const std::vector &headers, + std::vector &widths, const std::string &symbols, int symbols_size) -> void { + constexpr int PAGE_SIZE = 100; // 每次打印的行数 + int row_count = 0; + std::vector> record_batch; + record_batch.reserve(PAGE_SIZE); + bool first_page = true; + UpdateWidth(headers, widths); // 根据header更新列宽 + while (true) { + const auto &[record, eof] = records.Next(); + if (eof) { + break; + } + row_count++; + record_batch.push_back(RecordToString(record, null_str)); + if (row_count == PAGE_SIZE) { + UpdateWidth(record_batch, widths); // 根据新批次的数据更新列宽 + PrintRows(headers, record_batch, widths, symbols, symbols_size, first_page); + first_page = false; + record_batch.clear(); + row_count = 0; + } + } + + if (row_count > 0) { + UpdateWidth(record_batch, widths); // 根据新批次的数据更新列宽 + PrintRows(headers, record_batch, widths, symbols, symbols_size, first_page); + first_page = false; + } + if (first_page) { + // empty table , 上面都没有打印 + PrintRows(headers, record_batch, widths, symbols, symbols_size, first_page); + } +} + +// 打印整个结果集 +auto ClassCmd::PrintTable(RecordIterator &records, const std::string &symbols, size_t symbols_size) -> void { + auto headers = records.GetHeader(); + std::vector widths(headers.size(), min_col_len); + PrintHeaderAndBody(records, headers, widths, symbols, symbols_size); + PrintFooter(widths, symbols, symbols_size); +} + +auto ClassCmd::PrintSelectRecords(RecordIterator &record_iterator) -> void { + switch (b_explain ? box : print_type) { + case box: { + PrintTable(record_iterator, "│─┌┬┐├┼┤└┴┘", 3); // 3 bytes + break; + } + case table: { + PrintTable(record_iterator, "|-+++++++++", 1); + break; + } + case csv: { + PrintCSVFormat(record_iterator); + break; + } + case json: { + PrintJSONFormat(record_iterator); + break; + } + case insert: { + PrintInsert(record_iterator, "\"table\""); + break; + } + default: + break; + } +} + +static auto PrintInsertValue(std::shared_ptr &print_delegate, const Value &v) { + if (v.IsNull()) { + print_delegate->Print("NULL"); + } else if (v.IsNumeric()) { + print_delegate->Print(fmt::format("{}", v.GetCastAs())); + } else if (v.GetLogicalType().TypeId() == GS_TYPE_BOOLEAN) { + print_delegate->Print(fmt::format("{}", v.GetCastAs())); + } else { + print_delegate->Print(fmt::format("{}", v.ToString())); + } +} + +auto ClassCmd::PrintInsert(RecordIterator &record_iterator, const std::string &table_name) -> void { + while (true) { + const auto &[record, eof] = record_iterator.Next(); + if (eof) { + break; + } + print_delegate_->Print("INSERT INTO " + table_name + " VALUES"); + auto col_num = record.ColumnCount(); + for (uint32_t i = 0; i < col_num; ++i) { + const auto &v = record.FieldRef(i); + if (i == 0) { + print_delegate_->Print("("); + } else { + print_delegate_->Print(","); + } + PrintInsertValue(print_delegate_, v); + } + print_delegate_->Print(");\n"); + } +} + +auto ClassCmd::PrintRecords(RecordIterator &record_iterator) -> void { + if (record_iterator.GetRetCode() != 0) { + print_delegate_->Print( + fmt::format("Error Code:{} Msg:{}\n", record_iterator.GetRetCode(), record_iterator.GetRetMsg())); + return; + } + + switch (record_iterator.GetStmtType()) { + case StatementType::INVALID_STATEMENT: { + return; + } + case StatementType::SHOW_STATEMENT: + case StatementType::SELECT_STATEMENT: { + PrintSelectRecords(record_iterator); + break; + } + case StatementType::CALL_STATEMENT: { + if (record_iterator.GetBatchType() == RecordBatchType::Select) { + PrintSelectRecords(record_iterator); + } + else { + print_delegate_->Print("Query OK\n"); + } + break; + } + default: { + print_delegate_->Print("Query OK\n"); + break; + } + } + UpdateTotalChanges(record_iterator.GetEffectRow()); + PrintEffectRow(record_iterator.GetEffectRow()); +} + +void ClassCmd::PrintTimer(const std::chrono::steady_clock::time_point &start, + const std::chrono::steady_clock::time_point &end) { + if (b_timer) { + print_delegate_->Print( + fmt::format("Run Time: {:.4f} ms \n", std::chrono::duration(end - start).count())); + } +} + +void ClassCmd::UpdatePrintDelegate(uint16_t &output_times, std::string &tmp_out_file) { + if (output_times > 0) { + if (tmp_out_file.size() > 0) { + print_delegate_ = GetPrintDelegate(tmp_out_file); + tmp_out_file.clear(); + } else { + output_times--; + if (output_times == 0) { + print_delegate_ = GetPrintDelegate("stdout"); + } + } + } +} + +void ClassCmd::PrintResult(const std::string &str) { + print_delegate_->Print(str); + // 更新print_delegate_ + UpdatePrintDelegate(output_times, tmp_output_file); +} + +std::string ClassCmd::SetEcho(const std::vector &strList) { return SetSwitch(".echo", b_echo, strList); } + +std::string ClassCmd::SetChange(const std::vector &strList) { + return SetSwitch(".change", b_show_change, strList); +} + +std::string ClassCmd::SetTimer(const std::vector &strList) { + return SetSwitch(".timer", b_timer, strList); +} + +std::string ClassCmd::Explain(const std::vector &strList) { + return SetSwitch(".explain", b_explain, strList); +} + +std::string ClassCmd::SetSwitch(const std::string &cmd, bool &bswitch, const std::vector &strList) { + static std::string prompt_msg = "Useage: " + cmd + " on|off \n"; // 只创建一次 + if (strList.size() != 2) { + return prompt_msg; + } + + if (strList[1] == "on") { + bswitch = true; + } else if (strList[1] == "off") { + bswitch = false; + } else { + return prompt_msg; + } + return ""; +} + +std::string ClassCmd::SetOutFile(const std::vector &strList) { + std::string prompt_msg = "Useage: .output FILE|off\n"; + if (strList.size() != 2) return prompt_msg; + if (strList[1] == "off") { + print_delegate_ = GetPrintDelegate("stdout"); + } else { + print_delegate_ = GetPrintDelegate(strList[1]); + } + output_times = 0; + return ""; +} + +std::string ClassCmd::SetOutputOnce(const std::vector &strList) { + std::string prompt_msg = "Useage: .once FILE\n"; + if (strList.size() != 2) return prompt_msg; + tmp_output_file = strList[1]; + output_times = 1; + return ""; +} + +std::string ClassCmd::SetPrompt(const std::vector &strList) { + std::string prompt_msg = "Useage: .prompt your_prompt\n"; + if (strList.size() != 2) return prompt_msg; + begin_prompt.clear(); + continue_prompt.clear(); + fmt::format_to(std::back_inserter(begin_prompt), "{:<{}}> ", strList[1], strList[1].size()); + fmt::format_to(std::back_inserter(continue_prompt), "{:>{}} ", ">>>", strList[1].size() + 1); + return ""; +} + +std::string ClassCmd::ShowOption(const std::vector &strList) { + std::string msg; + int length = 10; + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "path", length, db_path); + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "output", length, print_delegate_->GetFileName()); + + for (auto item : m_print_type) + if (item.second == print_type) { + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "mode", length, item.first); + break; + } + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "nullvalue", length, null_str); + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "echo", length, b_echo ? "on" : "off"); + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "change", length, b_show_change ? "on" : "off"); + fmt::format_to(std::back_inserter(msg), "{:>{}}: {}\n", "timer", length, b_timer ? "on" : "off"); + return msg; +} + +std::string ClassCmd::ShowTable(const std::vector &strList) { + std::string result; + std::stringstream sql_query; + sql_query << "select NAME from \"SYS_TABLES\" where \"SPACE#\"=" << SQL_SPACE_TYPE_USERS; + if (strList.size() > 1) { + std::string sName = strList[1]; + transform(sName.begin(), sName.end(), sName.begin(), ::tolower); + sql_query << " and (NAME like '" << sName << "' or NAME like '" << strList[1] << "')"; + } + sql_query << ";"; + auto records = sql_conn->Query(sql_query.str().c_str())->GetRecords(); + for (std::size_t i = 1; i < records.size(); i++) { + for (auto item : records[i]) { + result += item + "\n"; + } + } + return result; +} + +std::string ClassCmd::ShowIndex(const std::vector &strList) { + std::string result; + std::stringstream sql_query; + sql_query << "select ti.NAME from \"SYS_INDEXES\" ti join (select \"ID\", " + "\"NAME\" from \"SYS_TABLES\" " + << "where \"SPACE#\"=" << SQL_SPACE_TYPE_USERS; + if (strList.size() > 1) { + std::string sName = strList[1]; + transform(sName.begin(), sName.end(), sName.begin(), ::tolower); + sql_query << " and (NAME like '" << sName << "' or NAME like '" << strList[1] << "')"; + } + sql_query << ") tt on ti.\"TABLE#\" = tt.\"ID\";"; + auto records = sql_conn->Query(sql_query.str().c_str())->GetRecords(); + for (std::size_t i = 1; i < records.size(); i++) { + for (auto item : records[i]) { + result += item + "\n"; + } + } + return result; +} + +std::string ClassCmd::SHowKeys(const std::vector &strList) { + std::string result; + std::stringstream sql_query; + sql_query << "SELECT KEY FROM \"" << kv_operator->getKVTable() << "\";"; + auto records = sql_conn->Query(sql_query.str().c_str())->GetRecords(); + for (std::size_t i = 1; i < records.size(); i++) { + for (auto item : records[i]) { + result += item + "\n"; + } + } + return result; +} + +std::string ClassCmd::Schema(const std::vector &strList) { + std::string prompt_msg = "Useage: .schema $TABLE\n"; + if (strList.size() != 2) return prompt_msg; + std::vector v_index_sqls; + std::string table = GetTableName(strList[1]); + std::string result = sql_conn->ShowCreateTable(table, v_index_sqls) + "\n"; + for (auto item : v_index_sqls) { + result += item + "\n"; + } + return result; +} + +std::string ClassCmd::FullSchema(const std::vector &strList) { + std::string result; + std::stringstream sql_query; + sql_query << "show tables;"; + auto records = sql_conn->Query(sql_query.str().c_str())->GetRecords(); + for (std::size_t i = 1; i < records.size(); i++) { + for (auto item : records[i]) { + std::vector v_index_sqls; + result += sql_conn->ShowCreateTable(item, v_index_sqls) + "\n"; + for (auto item : v_index_sqls) { + result += item + "\n"; + } + result += "\n"; + } + } + return result; +} + +// TODO: 内置函数的输出也不一定是短小的，需要考虑机制让内置函数也可以选择 +// 构造字符串，或直接使用 print_delegate 输出 +std::string ClassCmd::Dump(const std::vector &strList) { + // NOTE： 先特殊处理让结果直接输出，并返回一个空字符串 + std::string prompt_msg = "Useage: .dump $TABLE\n"; + if (strList.size() != 2) { + return prompt_msg; + } + print_delegate_->Print("BEGIN;\n"); + std::vector v_index_sqls; + auto create_table_sql = sql_conn->ShowCreateTable(strList[1], v_index_sqls); + print_delegate_->Print(create_table_sql); + print_delegate_->Print("\n"); + std::string sql_query = "SELECT * FROM " + strList[1]; + auto record_iterator = sql_conn->QueryIterator(sql_query.c_str()); + PrintInsert(*record_iterator, strList[1]); + print_delegate_->Print("COMMIT;\n"); + return ""; +} + +void ClassCmd::StringSplit(const std::string &str, const char split, std::vector &res) { + std::istringstream iss(str + split); + std::string buf; + while (getline(iss, buf, split)) { + res.push_back(buf); + } +} + +std::string ClassCmd::ImportCSV(const std::string &file, const std::string &table, uint16 skip) { + std::ifstream fin; + fin.open(file.c_str(), std::ios::in); + if (!fin.is_open()) { + return "open file: " + file + " error\n"; + } + + std::string str_line; + // skip n + for (uint16 i = 0; i < skip; i++) getline(fin, str_line); + + sql_conn->Query("BEGIN"); + + // 判断数据表是否存在，不存在则创建 + if (sql_conn->GetTableInfo(table) == nullptr) { + getline(fin, str_line); + std::vector col_name; + StringSplit(str_line, ',', col_name); + std::stringstream sql; + sql << "CREATE TABLE " << table << " ("; + for (size_t i = 0; i < col_name.size(); i++) { + if (i != 0) sql << ", "; + sql << "\"" << col_name[i] << "\" VARCHAR"; + } + sql << ");"; + auto r = sql_conn->Query(sql.str().c_str()); + if (r->GetRetCode() != 0) { + std::stringstream sErrMsg; + sErrMsg << "Error Code:" << r->GetRetCode() << " Msg:" << r->GetRetMsg() << std::endl; + return sErrMsg.str(); + } + } + std::vector> rows; + bool bResult = true; + std::string result = "Import OK\n"; + while (getline(fin, str_line) && str_line.size() > 0) { + std::vector row; + StringSplit(str_line, ',', row); + std::stringstream insert_sql; + insert_sql << "INSERT INTO " << table << " VALUES("; + for (size_t i = 0; i < row.size(); i++) { + if (i != 0) insert_sql << ", "; + if (row[i].size() > 0) + insert_sql << "'" << row[i] << "'"; + else + insert_sql << "NULL"; + } + insert_sql << ");"; + auto r = sql_conn->Query(insert_sql.str().c_str()); + if (r->GetRetCode() != 0) { + std::stringstream sErrMsg; + sErrMsg << "Error Code:" << r->GetRetCode() << " Msg:" << r->GetRetMsg() << std::endl; + result = sErrMsg.str(); + sql_conn->Query("ROLLBACK"); + bResult = false; + break; + } + } + if (bResult == true) sql_conn->Query("COMMIT"); + return result; +} + +std::string ClassCmd::Import(const std::vector &strList) { + std::string prompt_msg = + "Useage: .import $FILE $TABLE [--OPTION]" + " Options: \n" + " --csv Use , and \\n as column and row separators\n" +#ifdef SUPPORT_SQLITE + " --sqlite TABLE Use SQLite table\n" +#endif + " --skip N Skip the first N rows of input\n"; + if (strList.size() < 4) return prompt_msg; + + EImportType format_type = EImportType::invalid; + uint16 skip = 0; + std::vector strcmd; + for (size_t i = 1; i < strList.size(); i++) { + if (strList[i].substr(0, 2) == "--") { + std::string option = strList[i].substr(2); + if (option == "csv") + format_type = EImportType::csv; + else if (option == "skip" && i < strList.size() - 1) + skip = atoi(strList[++i].c_str()); + } else { + strcmd.push_back(strList[i]); + } + } + + switch (format_type) { + case EImportType::csv: { + return ImportCSV(strcmd[0], GetTableName(strcmd[1]), skip); + } + default: { + return "FORMAT OPTION ERROR! \n" + prompt_msg; + } + } +} + +std::string ClassCmd::Read(const std::vector &strList) { + std::string prompt_msg = "Useage: .read $FILE\n"; + if (strList.size() < 2) return prompt_msg; + std::ifstream fin; + fin.open(strList[1].c_str(), std::ios::in); + if (!fin.is_open()) { + return "open file: " + strList[1] + " error\n"; + } + std::string str_line; + std::string cmd; + while (getline(fin, str_line)) { + cmd += str_line; + if (IsFinishQuery(cmd)) { + execute(cmd); + cmd.clear(); + } + } + return ""; +} + +std::string ClassCmd::DBConfig(const std::vector &strList) { + std::string result; + int length = 10; + if (strList.size() == 1) { // show all config + fmt::format_to(std::back_inserter(result), "{:>{}}: {}\n", "log_level", length, + cm_log_param_instance()->log_level); + } + if (strList.size() >= 3) { + if (strList[1] == "log_level") { + cm_log_param_instance()->log_level = atoi(strList[2].c_str()); + fmt::format_to(std::back_inserter(result), "{:>{}}: {}\n", "log_level", length, + cm_log_param_instance()->log_level); + } + } + return result; +} + +std::shared_ptr ClassCmd::GetPrintDelegate(const std::string &filename) { + if (filename == "stdout") { + return std::make_shared(); + } else { + return std::make_shared(filename.c_str()); + } +} + +auto ClassCmd::PrintCmd(const std::string &cmd) -> void { + if (b_echo) { + print_delegate_->Print(cmd); + print_delegate_->Print("\n"); + } +} + +auto ClassCmd::UpdateTotalChanges(uint64_t current_changes) -> void { total_changes += current_changes; } + +auto ClassCmd::PrintEffectRow(uint64_t current_changes) -> void { + if (b_show_change) { + print_delegate_->Print(fmt::format("changes: {}\ttotal_changes: {}\n", current_changes, total_changes)); + } +} + +auto ClassCmd::IsBuildInCmd(const std::string &cmd, func &build_func) -> bool { + auto iter = ex_func_map.find(cmd); + if (iter != ex_func_map.end()) { + build_func = iter->second; + } + return iter != ex_func_map.end(); +} + +auto ClassCmd::IsKVCmd(const std::string &cmd, KvOperator::kv_func &kv_func) -> bool { + auto iter = kv_operator->kv_func_map.find(cmd); + if (iter != kv_operator->kv_func_map.end()) { + kv_func = iter->second; + } + return iter != kv_operator->kv_func_map.end(); +} + +auto ClassCmd::PrintBuildInCmdResult(const std::string &query, const func &build_func, + const std::vector &args) -> void { + auto result = (this->*build_func)(args); + // 短结果直接打印 + PrintCmd(query); + PrintResult(result); +} + +auto ClassCmd::PrintKVCmdResult(const std::string &query, const KvOperator::kv_func &kv_func, + const std::vector &args) -> void { + auto start = GetTimer(); + auto result = (kv_operator->*kv_func)(args); + // 短结果直接打印 + PrintCmd(query); + PrintResult(result); + auto end = GetTimer(); + PrintTimer(start, end); +} + +auto ClassCmd::PrintSQLCmdResult(const std::string &query) -> void { + // handle sql + auto start = GetTimer(); + auto r = sql_conn->QueryIterator(query.c_str()); + PrintCmd(query); + PrintRecords(*r); + auto end = GetTimer(); // RecordBatch改为RecordIterator后，需要Print之后，才执行完成。 + PrintTimer(start, end); +} + +static auto PrintCSVRow(std::shared_ptr &print_delegate, + const std::vector &row_content) { + for (size_t i = 0; i < row_content.size(); i++) { + if (i > 0) { + print_delegate->Print(","); + } + print_delegate->Print(row_content[i]); + } + print_delegate->Print("\n"); +} + +auto ClassCmd::PrintCSVFormat(RecordIterator &record_iterator) -> void { + auto headers = record_iterator.GetHeader(); + PrintCSVRow(print_delegate_, headers); + + while (true) { + const auto &[record, eof] = record_iterator.Next(); + if (eof) { + break; + } + auto row_content = RecordToString(record, null_str); + PrintCSVRow(print_delegate_, row_content); + } +} + +auto ClassCmd::RecordToString(const Record &record, const std::string &null_format) -> std::vector { + std::vector row_content; + for (size_t i = 0; i < record.ColumnCount(); i++) { + const auto &val = record.FieldRef(i); + row_content.emplace_back(val.IsNull() ? null_format : val.ToString()); + } + return row_content; +} + +// TODO: 和 PrintInsertValue 逻辑重复了，考虑抽象合并 +static auto AddJsonValue(cJSON *jrow, const char *field_name, const Value &val) -> void { + if (val.IsNull()) { + cJSON_AddItemToObject(jrow, field_name, cJSON_CreateNull()); + } else if (val.IsNumeric()) { + cJSON_AddItemToObject(jrow, field_name, cJSON_CreateNumber(val.GetCastAs())); + } else if (val.GetLogicalType().TypeId() == GS_TYPE_BOOLEAN) { + cJSON_AddItemToObject(jrow, field_name, cJSON_CreateBool(val.GetCastAs())); + } else { + cJSON_AddItemToObject(jrow, field_name, cJSON_CreateString(val.ToString().c_str())); + } +} + +static auto RecordToJSONRow(const std::vector &headers, const Record &record) -> std::string { + cJSON *jrow = cJSON_CreateObject(); + for (size_t j = 0; j < headers.size(); ++j) { + const auto &header = headers[j]; + const auto &v = record.FieldRef(j); + AddJsonValue(jrow, header.c_str(), v); + } + char *str = cJSON_PrintUnformatted(jrow); + std::string json_row = str; + // 注意释放内存，使用方法参考cJSON测试用例, + // 需要保证前面不会抛出异常，否则会导致free语句执行不到 + // 最好使用RAII的方法保证内存能够被正确释放 + free(str); + cJSON_Delete(jrow); + return json_row; +} + +auto ClassCmd::PrintJSONFormat(RecordIterator &record_iterator) -> void { + auto headers = record_iterator.GetHeader(); + print_delegate_->Print("[\n"); + bool first = true; + while (true) { + const auto &[record, eof] = record_iterator.Next(); + if (!eof && !first) { + print_delegate_->Print(",\n"); + } + if (eof) { + print_delegate_->Print("\n"); + break; + } + first = false; + print_delegate_->Print(RecordToJSONRow(headers, record)); + } + print_delegate_->Print("]\n"); +} + +auto ClassCmd::IsFinishQuery(const std::string &query) -> bool { + return (query.length() > 0 && (query[query.length() - 1] == ';' || query[0] == '.')); +} diff --git a/tools/intarkdb_cli/cmd.h b/tools/intarkdb_cli/cmd.h new file mode 100644 index 0000000000000000000000000000000000000000..5f2568f2775dd49ad135469843c5e50f37a22d0d --- /dev/null +++ b/tools/intarkdb_cli/cmd.h @@ -0,0 +1,186 @@ +/* + * Copyright (c) GBA-NCTI-ISDC. 2022-2024. + * + * openGauss embedded is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * cmd.h + * + * IDENTIFICATION + * openGauss-embedded/tools/intarkdb_cli/cmd.h + * + * ------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include + +#include "common/record_batch.h" +#include "kv_operator.h" +#include "main/connection.h" + +enum EPrintType { invalid = 0, box, table, csv, json, insert }; +const std::map m_print_type = { + {"box", box}, {"table", table}, {"csv", csv}, {"json", json}, {"insert", insert}}; + +enum class EImportType { invalid = 0, csv, sqlite }; + +class ClassCmd { + public: + ClassCmd(const std::string& path, Connection* conn, KvOperator* kv_oper); + ~ClassCmd(){}; + void main(); + bool execute(std::string& query); + // 进一步判断是否SQL + bool IsSqlCmd(std::string query, std::vector& strList); + + typedef std::string (ClassCmd::*func)(const std::vector& strList); + std::string Help(const std::vector& strList); + std::string Info(const std::vector& strList); + std::string Version(const std::vector& strList); + std::string Dump(const std::vector& strList); + std::string Import(const std::vector& strList); + std::string Read(const std::vector& strList); + std::string ShowTable(const std::vector& strList); + std::string ShowIndex(const std::vector& strList); + std::string SHowKeys(const std::vector& strList); + std::string Schema(const std::vector& strList); + std::string FullSchema(const std::vector& strList); + + std::string Mode(const std::vector& strList); + std::string Explain(const std::vector& strList); + std::string SetNullStr(const std::vector& strList); + std::string SetWidth(const std::vector& strList); + std::string SetEcho(const std::vector& strList); + std::string SetChange(const std::vector& strList); + std::string SetTimer(const std::vector& strList); + std::string SetOutFile(const std::vector& strList); + std::string SetOutputOnce(const std::vector& strList); + std::string SetPrompt(const std::vector& strList); + std::string ShowOption(const std::vector& strList); + std::string DBConfig(const std::vector& strList); + + std::map ex_func_map; // + + class PrintDelegate { + public: + PrintDelegate() : fp(nullptr), filename_("stdout"){}; + PrintDelegate(const char* file) : fp(fopen(file, "a")), filename_(file){}; + ~PrintDelegate() { + if (fp) { + fflush(fp); + fclose(fp); + } + }; + + template + typename std::enable_if::value, void>::type Print(const T& str) const { + if (fp == nullptr) { + fmt::print(stdout, "{}", str); + } else { + fmt::print(fp, "{}", str); + } + } + + const char* GetFileName() const { return filename_.c_str(); } + + private: + FILE* fp; + std::string filename_; + }; + + void PrintResult(const std::string& str); + + const char* GetBeginPrompt() { return begin_prompt.c_str(); }; + const char* GetContinuePrompt() { return continue_prompt.c_str(); }; + + private: + std::string GetModeType(const std::string& separator); + void DrowTableLine(const std::vector& vlength, std::string_view symbols, size_t symbols_size, + std::string_view line); + void PrintTable(RecordIterator& record_batch, const std::string& symbols, size_t symbols_size); + std::function PrintTable(const RecordBatch& record_batch, const std::string& symbols, + size_t symbols_size); + auto PrintInsert(RecordIterator& record_iterator, const std::string& table_name) -> void; + auto PrintSelectRecords(RecordIterator& record_batch) -> void; + std::string SetSwitch(const std::string& cmd, bool& bswitch, const std::vector& strList); + double timeDiff(const timeval& pStart, const timeval& pEnd); + std::string ImportCSV(const std::string& file, const std::string& table, uint16 skip); + + void StringSplit(const std::string& str, const char split, std::vector& res); + void GetCmd(std::string str, const char split, std::vector& res); + std::string GetTableName(const std::string& str); + + auto IsQuitCmd(const std::string& cmd) -> bool { return cmd == ".quit" || cmd == ".exit"; } + // 是否内置命令 + auto IsBuildInCmd(const std::string& cmd, func& build_func) -> bool; + auto IsKVCmd(const std::string& cmd, KvOperator::kv_func& kv_func) -> bool; + auto GetPrintDelegate(const std::string& output_file) -> std::shared_ptr; + auto UpdatePrintDelegate(uint16_t& output_times, std::string& tmp_out_file) -> void; + auto PrintCmd(const std::string& cmd) -> void; + auto UpdateTotalChanges(uint64_t current_changes) -> void; + auto PrintEffectRow(uint64_t current_changes) -> void; + + auto PrintBuildInCmdResult(const std::string& query, const func& build_func, const std::vector& args) + -> void; + auto PrintKVCmdResult(const std::string& query, const KvOperator::kv_func& kv_func, + const std::vector& args) -> void; + auto PrintSQLCmdResult(const std::string& query) -> void; + + auto PrintRecords(RecordIterator& record_batch) -> void; + auto PrintRows(const std::vector& headers, const std::vector>& rows, + std::vector& widths, const std::string& symbols, int symbols_size, bool first_page) -> void; + + auto GetTimer() const { return std::chrono::steady_clock::now(); } + + auto PrintTimer(const std::chrono::steady_clock::time_point& start, + const std::chrono::steady_clock::time_point& end) -> void; + + auto PrintCSVFormat(RecordIterator& record_iterator) -> void; + auto PrintJSONFormat(RecordIterator& record_iterator) -> void; + + auto PrintHeader(const std::vector& headers, const std::vector& widths, + const std::string& symbols, int symbols_size) -> void; + auto PrintHeaderAndBody(RecordIterator& iterator, const std::vector& headers, + std::vector& widths, const std::string& symbols, int symbols_size) -> void; + auto PrintFooter(const std::vector& widths, const std::string& symbols, int symbols_size) -> void; + + auto RecordToString(const Record& record, const std::string& null_format) -> std::vector; + + auto IsFinishQuery(const std::string&) -> bool; + + Connection* sql_conn; + KvOperator* kv_operator; + + std::string db_path; + std::shared_ptr print_delegate_; + uint16_t output_times{0}; + std::string tmp_output_file{""}; + EPrintType print_type; + std::string null_str{""}; + size_t min_col_len{8}; + uint64_t total_changes{0}; + std::string begin_prompt{"intarkdb> "}; + std::string continue_prompt{" ...> "}; + std::string help_msg; + + // switch + bool b_echo{0}; + bool b_show_change{0}; + bool b_timer{0}; + bool b_explain{0}; +}; diff --git a/tools/intarkdb_cli/kv_operator.cpp b/tools/intarkdb_cli/kv_operator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9e5ce16fcdae73e7348ed0c79ca24c169ac858a4 --- /dev/null +++ b/tools/intarkdb_cli/kv_operator.cpp @@ -0,0 +1,164 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +* +* kv_operator.cpp +* +* IDENTIFICATION +* openGauss-embedded/tools/intarkdb_cli/kv_operator.cpp +* +* ------------------------------------------------------------------------- +*/ + +#include "kv_operator.h" +#include + + +static std::string KV_USEAGE = + "KV Usage: \n" + " set key value : set key's value \n" + " get key : get key's value \n" + " del key : delete key \n" + " kvtb tablename : change table, create if not exist \n" + " multi : start a transaction \n" + " exec : commit current transaction \n" + " discard : discard current transaction \n" + " help kv: show kv usage \n"; + +// kv_operator::kv_operator(std::shared_ptr instance) +KvOperator::KvOperator(std::unique_ptr kvconn) : kv_connection_(std::move(kvconn)), auto_commit(true) +{ + kv_func_map.insert(std::make_pair("set", &KvOperator::kv_set)); + kv_func_map.insert(std::make_pair("get", &KvOperator::kv_get)); + kv_func_map.insert(std::make_pair("del", &KvOperator::kv_del)); + kv_func_map.insert(std::make_pair("kvtb", &KvOperator::kv_change_table)); + //事务 仿照redis命令 + kv_func_map.insert(std::make_pair("multi", &KvOperator::kv_begin)); + kv_func_map.insert(std::make_pair("exec", &KvOperator::kv_commit)); + kv_func_map.insert(std::make_pair("discard", &KvOperator::kv_rollback)); +} + +KvOperator::~KvOperator() +{ + // +} + +std::string KvOperator::kv_set(const std::vector& strList) +{ + if(strList.size() != 3) { + return "cmd err usage: set key value;\n"; + } + + std::stringstream res; + auto reply = kv_connection_->Set(strList[1].c_str(), strList[2].c_str()); + if (reply->type == GS_SUCCESS) { + if (auto_commit) { + kv_commit({}); + res << "Success"; + } else { + res << "Success in this Transaction"; + } + } else { + res << "kv set failed key:" << strList[1] << " value:" << strList[2]; + } + res << std::endl; + return res.str(); +} + +std::string KvOperator::kv_get(const std::vector& strList) +{ + if(strList.size() != 2) { + return "cmd err usage: get key;\n"; + } + + std::stringstream res; + auto reply = kv_connection_->Get(strList[1].c_str()); + if (reply->type == GS_SUCCESS) { + if (reply->len > 0) { + res << reply->str; + } else { + res << "key: " << strList[1] << " not exist"; + } + } else { + printf("get failed...\r\n"); + } + res << std::endl; + return res.str(); +} + +std::string KvOperator::kv_del(const std::vector& strList) +{ + if(strList.size() != 2) { + return "cmd err usage: del key;\n"; + } + + std::stringstream res; + auto reply = kv_connection_->Del(strList[1].c_str()); + if (reply->type == GS_SUCCESS) { + if (auto_commit) { + kv_commit({}); + res << "Success"; + } else { + res << "Success in this Transaction"; + } + } else { + res << "kv del failed key:" << strList[1]; + } + res << std::endl; + return res.str(); +} + +std::string KvOperator::kv_change_table(const std::vector& strList) +{ + if(strList.size() != 2) { + return "cmd err usage: kvtb tablename;\n"; + } + + std::stringstream res; + if (kv_connection_->OpenTable(strList[1].c_str()) != GS_SUCCESS) { + res << "kv change table: "<< strList[1] << " fail"; + } else { + res << "kv change table: "<< strList[1] << " success"; + kv_table = strList[1]; + } + + res << std::endl; + return res.str(); +} + +std::string KvOperator::kv_begin(const std::vector& strList) +{ + auto_commit = false; + kv_connection_->Begin(); + return "Begin Success\n"; +} + +std::string KvOperator::kv_commit(const std::vector& strList) +{ + auto_commit = true; + kv_connection_->Commit(); + return "Commit Success\n"; +} + +std::string KvOperator::kv_rollback(const std::vector& strList) +{ + auto_commit = true; + kv_connection_->Rollback(); + return "Rollback Success\n"; +} + +std::string KvOperator::show_kv_help() +{ + return KV_USEAGE; +} diff --git a/tools/intarkdb_cli/kv_operator.h b/tools/intarkdb_cli/kv_operator.h new file mode 100644 index 0000000000000000000000000000000000000000..dd4f12e731578f86625f3ae1227abb882688d163 --- /dev/null +++ b/tools/intarkdb_cli/kv_operator.h @@ -0,0 +1,60 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +* +* kv_operator.h +* +* IDENTIFICATION +* openGauss-embedded/tools/intarkdb_cli/kv_operator.h +* +* ------------------------------------------------------------------------- +*/ + +#pragma once + +#include +#include +#include + +#include "compute/kv/kv_connection.h" + +class KvOperator +{ +public: + KvOperator(std::unique_ptr kvconn); + ~KvOperator(); + + typedef std::string(KvOperator::*kv_func)(const std::vector& strList); + //kv_func(); + std::string kv_set(const std::vector& strList); + std::string kv_get(const std::vector& strList); + std::string kv_del(const std::vector& strList); + std::string kv_change_table(const std::vector& strList); //change table, create if not exist + + std::string kv_begin(const std::vector& strList); + std::string kv_commit(const std::vector& strList); + std::string kv_rollback(const std::vector& strList); + + static std::string show_kv_help(); + + std::map kv_func_map; // + std::string getKVTable() {return kv_table;} + + + +private: + std::unique_ptr kv_connection_; + bool auto_commit; + std::string kv_table{"SYS_KV"}; +}; diff --git a/tools/intarkdb_cli/linenoise.c b/tools/intarkdb_cli/linenoise.c new file mode 100644 index 0000000000000000000000000000000000000000..28e6d3bdcb8a52f1e9a1c0122df82c1b41b49993 --- /dev/null +++ b/tools/intarkdb_cli/linenoise.c @@ -0,0 +1,1229 @@ +/* linenoise.c -- guerrilla line editing library against the idea that a + * line editing lib needs to be 20,000 lines of C code. + * + * You can find the latest source code at: + * + * http://github.com/antirez/linenoise + * + * Does a number of crazy assumptions that happen to be true in 99.9999% of + * the 2010 UNIX computers around. + * + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2016, Salvatore Sanfilippo + * Copyright (c) 2010-2013, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ------------------------------------------------------------------------ + * + * References: + * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html + * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html + * + * Todo list: + * - Filter bogus Ctrl+ combinations. + * - Win32 support + * + * Bloat: + * - History search like Ctrl+r in readline? + * + * List of escape sequences used by this program, we do everything just + * with three sequences. In order to be so cheap we may have some + * flickering effect with some slow terminal, but the lesser sequences + * the more compatible. + * + * EL (Erase Line) + * Sequence: ESC [ n K + * Effect: if n is 0 or missing, clear from cursor to end of line + * Effect: if n is 1, clear from beginning of line to cursor + * Effect: if n is 2, clear entire line + * + * CUF (CUrsor Forward) + * Sequence: ESC [ n C + * Effect: moves cursor forward n chars + * + * CUB (CUrsor Backward) + * Sequence: ESC [ n D + * Effect: moves cursor backward n chars + * + * The following is used to get the terminal width if getting + * the width with the TIOCGWINSZ ioctl fails + * + * DSR (Device Status Report) + * Sequence: ESC [ 6 n + * Effect: reports the current cusor position as ESC [ n ; m R + * where n is the row and m is the column + * + * When multi line mode is enabled, we also use an additional escape + * sequence. However multi line editing is disabled by default. + * + * CUU (Cursor Up) + * Sequence: ESC [ n A + * Effect: moves cursor up of n chars. + * + * CUD (Cursor Down) + * Sequence: ESC [ n B + * Effect: moves cursor down of n chars. + * + * When linenoiseClearScreen() is called, two additional escape sequences + * are used in order to clear the screen and position the cursor at home + * position. + * + * CUP (Cursor position) + * Sequence: ESC [ H + * Effect: moves the cursor to upper left corner + * + * ED (Erase display) + * Sequence: ESC [ 2 J + * Effect: clear the whole screen + * + */ + +#define _DEFAULT_SOURCE /* For fchmod() */ +#define _BSD_SOURCE /* For fchmod() */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "linenoise.h" + +#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 +#define LINENOISE_MAX_LINE 8192 +static char *unsupported_term[] = {"dumb","cons25","emacs",NULL}; +static linenoiseCompletionCallback *completionCallback = NULL; +static linenoiseHintsCallback *hintsCallback = NULL; +static linenoiseFreeHintsCallback *freeHintsCallback = NULL; + +static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ +static int rawmode = 0; /* For atexit() function to check if restore is needed*/ +static int mlmode = 0; /* Multi line mode. Default is single line. */ +static int atexit_registered = 0; /* Register atexit just 1 time. */ +static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN; +static int history_len = 0; +static char **history = NULL; + +/* The linenoiseState structure represents the state during line editing. + * We pass this state to functions implementing specific editing + * functionalities. */ +struct linenoiseState { + int ifd; /* Terminal stdin file descriptor. */ + int ofd; /* Terminal stdout file descriptor. */ + char *buf; /* Edited line buffer. */ + size_t buflen; /* Edited line buffer size. */ + const char *prompt; /* Prompt to display. */ + size_t plen; /* Prompt length. */ + size_t pos; /* Current cursor position. */ + size_t oldpos; /* Previous refresh cursor position. */ + size_t len; /* Current edited line length. */ + size_t cols; /* Number of columns in terminal. */ + size_t maxrows; /* Maximum num of rows used so far (multiline mode) */ + int history_index; /* The history index we are currently editing. */ +}; + +enum KEY_ACTION{ + KEY_NULL = 0, /* NULL */ + CTRL_A = 1, /* Ctrl+a */ + CTRL_B = 2, /* Ctrl-b */ + CTRL_C = 3, /* Ctrl-c */ + CTRL_D = 4, /* Ctrl-d */ + CTRL_E = 5, /* Ctrl-e */ + CTRL_F = 6, /* Ctrl-f */ + CTRL_H = 8, /* Ctrl-h */ + TAB = 9, /* Tab */ + NL = 10, /* Enter typed before raw mode was enabled */ + CTRL_K = 11, /* Ctrl+k */ + CTRL_L = 12, /* Ctrl+l */ + ENTER = 13, /* Enter */ + CTRL_N = 14, /* Ctrl-n */ + CTRL_P = 16, /* Ctrl-p */ + CTRL_T = 20, /* Ctrl-t */ + CTRL_U = 21, /* Ctrl+u */ + CTRL_W = 23, /* Ctrl+w */ + ESC = 27, /* Escape */ + BACKSPACE = 127 /* Backspace */ +}; + +static void linenoiseAtExit(void); +int linenoiseHistoryAdd(const char *line); +static void refreshLine(struct linenoiseState *l); + +/* Debugging macro. */ +#if 0 +FILE *lndebug_fp = NULL; +#define lndebug(...) \ + do { \ + if (lndebug_fp == NULL) { \ + lndebug_fp = fopen("/tmp/lndebug.txt","a"); \ + fprintf(lndebug_fp, \ + "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \ + (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \ + (int)l->maxrows,old_rows); \ + } \ + fprintf(lndebug_fp, ", " __VA_ARGS__); \ + fflush(lndebug_fp); \ + } while (0) +#else +#define lndebug(fmt, ...) +#endif + +/* ======================= Low level terminal handling ====================== */ + +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + +/* Set if to use or not the multi line mode. */ +void linenoiseSetMultiLine(int ml) { + mlmode = ml; +} + +/* Return true if the terminal name is in the list of terminals we know are + * not able to understand basic escape sequences. */ +static int isUnsupportedTerm(void) { + char *term = getenv("TERM"); + int j; + + if (term == NULL) return 0; + for (j = 0; unsupported_term[j]; j++) + if (!strcasecmp(term,unsupported_term[j])) return 1; + return 0; +} + +/* Raw mode: 1960 magic shit. */ +static int enableRawMode(int fd) { + struct termios raw; + + if (!isatty(STDIN_FILENO)) goto fatal; + if (!atexit_registered) { + atexit(linenoiseAtExit); + atexit_registered = 1; + } + if (tcgetattr(fd,&orig_termios) == -1) goto fatal; + + raw = orig_termios; /* modify the original mode */ + /* input modes: no break, no CR to NL, no parity check, no strip char, + * no start/stop output control. */ + raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON); + /* output modes - disable post processing */ + raw.c_oflag &= ~(OPOST); + /* control modes - set 8 bit chars */ + raw.c_cflag |= (CS8); + /* local modes - choing off, canonical off, no extended functions, + * no signal chars (^Z,^C) */ + raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); + /* control chars - set return condition: min number of bytes and timer. + * We want read to return every single byte, without timeout. */ + raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ + + /* put terminal in raw mode */ + if (tcsetattr(fd,TCSANOW,&raw) < 0) goto fatal; + rawmode = 1; + return 0; + +fatal: + errno = ENOTTY; + return -1; +} + +static void disableRawMode(int fd) { + /* Don't even check the return value as it's too late. */ + if (rawmode && tcsetattr(fd,TCSANOW,&orig_termios) != -1) + rawmode = 0; +} + +/* Use the ESC [6n escape sequence to query the horizontal cursor position + * and return it. On error -1 is returned, on success the position of the + * cursor. */ +static int getCursorPosition(int ifd, int ofd) { + char buf[32]; + int cols, rows; + unsigned int i = 0; + + /* Report cursor location */ + if (write(ofd, "\x1b[6n", 4) != 4) return -1; + + /* Read the response: ESC [ rows ; cols R */ + while (i < sizeof(buf)-1) { + if (read(ifd,buf+i,1) != 1) break; + if (buf[i] == 'R') break; + i++; + } + buf[i] = '\0'; + + /* Parse it. */ + if (buf[0] != ESC || buf[1] != '[') return -1; + if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1; + return cols; +} + +/* Try to get the number of columns in the current terminal, or assume 80 + * if it fails. */ +static int getColumns(int ifd, int ofd) { + struct winsize ws; + + if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) { + /* ioctl() failed. Try to query the terminal itself. */ + int start, cols; + + /* Get the initial position so we can restore it later. */ + start = getCursorPosition(ifd,ofd); + if (start == -1) goto failed; + + /* Go to right margin and get position. */ + if (write(ofd,"\x1b[999C",6) != 6) goto failed; + cols = getCursorPosition(ifd,ofd); + if (cols == -1) goto failed; + + /* Restore position. */ + if (cols > start) { + char seq[32]; + snprintf(seq,32,"\x1b[%dD",cols-start); + if (write(ofd,seq,strlen(seq)) == -1) { + /* Can't recover... */ + } + } + return cols; + } else { + return ws.ws_col; + } + +failed: + return 80; +} + +/* Clear the screen. Used to handle ctrl+l */ +void linenoiseClearScreen(void) { + if (write(STDOUT_FILENO,"\x1b[H\x1b[2J",7) <= 0) { + /* nothing to do, just to avoid warning. */ + } +} + +/* Beep, used for completion when there is nothing to complete or when all + * the choices were already shown. */ +static void linenoiseBeep(void) { + fprintf(stderr, "\x7"); + fflush(stderr); +} + +/* ============================== Completion ================================ */ + +/* Free a list of completion option populated by linenoiseAddCompletion(). */ +static void freeCompletions(linenoiseCompletions *lc) { + size_t i; + for (i = 0; i < lc->len; i++) + free(lc->cvec[i]); + if (lc->cvec != NULL) + free(lc->cvec); +} + +/* This is an helper function for linenoiseEdit() and is called when the + * user types the key in order to complete the string currently in the + * input. + * + * The state of the editing is encapsulated into the pointed linenoiseState + * structure as described in the structure definition. */ +static int completeLine(struct linenoiseState *ls) { + linenoiseCompletions lc = { 0, NULL }; + int nread, nwritten; + char c = 0; + + completionCallback(ls->buf,&lc); + if (lc.len == 0) { + linenoiseBeep(); + } else { + size_t stop = 0, i = 0; + + while(!stop) { + /* Show completion or original buffer */ + if (i < lc.len) { + struct linenoiseState saved = *ls; + + ls->len = ls->pos = strlen(lc.cvec[i]); + ls->buf = lc.cvec[i]; + refreshLine(ls); + ls->len = saved.len; + ls->pos = saved.pos; + ls->buf = saved.buf; + } else { + refreshLine(ls); + } + + nread = read(ls->ifd,&c,1); + if (nread <= 0) { + freeCompletions(&lc); + return -1; + } + + switch(c) { + case 9: /* tab */ + i = (i+1) % (lc.len+1); + if (i == lc.len) linenoiseBeep(); + break; + case 27: /* escape */ + /* Re-show original buffer */ + if (i < lc.len) refreshLine(ls); + stop = 1; + break; + default: + /* Update buffer and return */ + if (i < lc.len) { + nwritten = snprintf(ls->buf,ls->buflen,"%s",lc.cvec[i]); + ls->len = ls->pos = nwritten; + } + stop = 1; + break; + } + } + } + + freeCompletions(&lc); + return c; /* Return last read character */ +} + +/* Register a callback function to be called for tab-completion. */ +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { + completionCallback = fn; +} + +/* Register a hits function to be called to show hits to the user at the + * right of the prompt. */ +void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) { + hintsCallback = fn; +} + +/* Register a function to free the hints returned by the hints callback + * registered with linenoiseSetHintsCallback(). */ +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) { + freeHintsCallback = fn; +} + +/* This function is used by the callback function registered by the user + * in order to add completion options given the input string when the + * user typed . See the example.c source code for a very easy to + * understand example. */ +void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { + size_t len = strlen(str); + char *copy, **cvec; + + copy = malloc(len+1); + if (copy == NULL) return; + memcpy(copy,str,len+1); + cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); + if (cvec == NULL) { + free(copy); + return; + } + lc->cvec = cvec; + lc->cvec[lc->len++] = copy; +} + +/* =========================== Line editing ================================= */ + +/* We define a very simple "append buffer" structure, that is an heap + * allocated string where we can append to. This is useful in order to + * write all the escape sequences in a buffer and flush them to the standard + * output in a single call, to avoid flickering effects. */ +struct abuf { + char *b; + int len; +}; + +static void abInit(struct abuf *ab) { + ab->b = NULL; + ab->len = 0; +} + +static void abAppend(struct abuf *ab, const char *s, int len) { + char *new = realloc(ab->b,ab->len+len); + + if (new == NULL) return; + memcpy(new+ab->len,s,len); + ab->b = new; + ab->len += len; +} + +static void abFree(struct abuf *ab) { + free(ab->b); +} + +/* Helper of refreshSingleLine() and refreshMultiLine() to show hints + * to the right of the prompt. */ +void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { + char seq[64]; + if (hintsCallback && plen+l->len < l->cols) { + int color = -1, bold = 0; + char *hint = hintsCallback(l->buf,&color,&bold); + if (hint) { + int hintlen = strlen(hint); + int hintmaxlen = l->cols-(plen+l->len); + if (hintlen > hintmaxlen) hintlen = hintmaxlen; + if (bold == 1 && color == -1) color = 37; + if (color != -1 || bold != 0) + snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; + abAppend(ab,seq,strlen(seq)); + abAppend(ab,hint,hintlen); + if (color != -1 || bold != 0) + abAppend(ab,"\033[0m",4); + /* Call the function to free the hint returned. */ + if (freeHintsCallback) freeHintsCallback(hint); + } + } +} + +/* Single line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshSingleLine(struct linenoiseState *l) { + char seq[64]; + size_t plen = strlen(l->prompt); + int fd = l->ofd; + char *buf = l->buf; + size_t len = l->len; + size_t pos = l->pos; + struct abuf ab; + + while((plen+pos) >= l->cols) { + buf++; + len--; + pos--; + } + while (plen+len > l->cols) { + len--; + } + + abInit(&ab); + /* Cursor to left edge */ + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + /* Erase to right */ + snprintf(seq,64,"\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + /* Move cursor to original position. */ + snprintf(seq,64,"\r\x1b[%dC", (int)(pos+plen)); + abAppend(&ab,seq,strlen(seq)); + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Multi line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshMultiLine(struct linenoiseState *l) { + char seq[64]; + int plen = strlen(l->prompt); + int rows = (plen+l->len+l->cols-1)/l->cols; /* rows used by current buf. */ + int rpos = (plen+l->oldpos+l->cols)/l->cols; /* cursor relative row. */ + int rpos2; /* rpos after refresh. */ + int col; /* colum position, zero-based. */ + int old_rows = l->maxrows; + int fd = l->ofd, j; + struct abuf ab; + + /* Update maxrows if needed. */ + if (rows > (int)l->maxrows) l->maxrows = rows; + + /* First step: clear all the lines used before. To do so start by + * going to the last row. */ + abInit(&ab); + if (old_rows-rpos > 0) { + lndebug("go down %d", old_rows-rpos); + snprintf(seq,64,"\x1b[%dB", old_rows-rpos); + abAppend(&ab,seq,strlen(seq)); + } + + /* Now for every row clear it, go up. */ + for (j = 0; j < old_rows-1; j++) { + lndebug("clear+up"); + snprintf(seq,64,"\r\x1b[0K\x1b[1A"); + abAppend(&ab,seq,strlen(seq)); + } + + /* Clean the top line. */ + lndebug("clear"); + snprintf(seq,64,"\r\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + if (maskmode == 1) { + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } + + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + + /* If we are at the very end of the screen with our prompt, we need to + * emit a newline and move the prompt to the first column. */ + if (l->pos && + l->pos == l->len && + (l->pos+plen) % l->cols == 0) + { + lndebug(""); + abAppend(&ab,"\n",1); + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + rows++; + if (rows > (int)l->maxrows) l->maxrows = rows; + } + + /* Move cursor to right position. */ + rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */ + lndebug("rpos2 %d", rpos2); + + /* Go up till we reach the expected position. */ + if (rows-rpos2 > 0) { + lndebug("go-up %d", rows-rpos2); + snprintf(seq,64,"\x1b[%dA", rows-rpos2); + abAppend(&ab,seq,strlen(seq)); + } + + /* Set column. */ + col = (plen+(int)l->pos) % (int)l->cols; + lndebug("set col %d", 1+col); + if (col) + snprintf(seq,64,"\r\x1b[%dC", col); + else + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + + lndebug("\n"); + l->oldpos = l->pos; + + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Calls the two low level functions refreshSingleLine() or + * refreshMultiLine() according to the selected mode. */ +static void refreshLine(struct linenoiseState *l) { + if (mlmode) + refreshMultiLine(l); + else + refreshSingleLine(l); +} + +/* Insert the character 'c' at cursor current position. + * + * On error writing to the terminal -1 is returned, otherwise 0. */ +int linenoiseEditInsert(struct linenoiseState *l, char c) { + if (l->len < l->buflen) { + if (l->len == l->pos) { + l->buf[l->pos] = c; + l->pos++; + l->len++; + l->buf[l->len] = '\0'; + if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { + /* Avoid a full update of the line in the + * trivial case. */ + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; + } else { + refreshLine(l); + } + } else { + memmove(l->buf+l->pos+1,l->buf+l->pos,l->len-l->pos); + l->buf[l->pos] = c; + l->len++; + l->pos++; + l->buf[l->len] = '\0'; + refreshLine(l); + } + } + return 0; +} + +/* Move cursor on the left. */ +void linenoiseEditMoveLeft(struct linenoiseState *l) { + if (l->pos > 0) { + l->pos--; + refreshLine(l); + } +} + +/* Move cursor on the right. */ +void linenoiseEditMoveRight(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos++; + refreshLine(l); + } +} + +/* Move cursor to the start of the line. */ +void linenoiseEditMoveHome(struct linenoiseState *l) { + if (l->pos != 0) { + l->pos = 0; + refreshLine(l); + } +} + +/* Move cursor to the end of the line. */ +void linenoiseEditMoveEnd(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos = l->len; + refreshLine(l); + } +} + +/* Substitute the currently edited line with the next or previous history + * entry as specified by 'dir'. */ +#define LINENOISE_HISTORY_NEXT 0 +#define LINENOISE_HISTORY_PREV 1 +void linenoiseEditHistoryNext(struct linenoiseState *l, int dir) { + if (history_len > 1) { + /* Update the current history entry before to + * overwrite it with the next one. */ + free(history[history_len - 1 - l->history_index]); + history[history_len - 1 - l->history_index] = strdup(l->buf); + /* Show the new entry */ + l->history_index += (dir == LINENOISE_HISTORY_PREV) ? 1 : -1; + if (l->history_index < 0) { + l->history_index = 0; + return; + } else if (l->history_index >= history_len) { + l->history_index = history_len-1; + return; + } + strncpy(l->buf,history[history_len - 1 - l->history_index],l->buflen); + l->buf[l->buflen-1] = '\0'; + l->len = l->pos = strlen(l->buf); + refreshLine(l); + } +} + +/* Delete the character at the right of the cursor without altering the cursor + * position. Basically this is what happens with the "Delete" keyboard key. */ +void linenoiseEditDelete(struct linenoiseState *l) { + if (l->len > 0 && l->pos < l->len) { + memmove(l->buf+l->pos,l->buf+l->pos+1,l->len-l->pos-1); + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Backspace implementation. */ +void linenoiseEditBackspace(struct linenoiseState *l) { + if (l->pos > 0 && l->len > 0) { + memmove(l->buf+l->pos-1,l->buf+l->pos,l->len-l->pos); + l->pos--; + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Delete the previous word, maintaining the cursor at the start of the + * current word. */ +void linenoiseEditDeletePrevWord(struct linenoiseState *l) { + size_t old_pos = l->pos; + size_t diff; + + while (l->pos > 0 && l->buf[l->pos-1] == ' ') + l->pos--; + while (l->pos > 0 && l->buf[l->pos-1] != ' ') + l->pos--; + diff = old_pos - l->pos; + memmove(l->buf+l->pos,l->buf+old_pos,l->len-old_pos+1); + l->len -= diff; + refreshLine(l); +} + +/* This function is the core of the line editing capability of linenoise. + * It expects 'fd' to be already in "raw mode" so that every key pressed + * will be returned ASAP to read(). + * + * The resulting string is put into 'buf' when the user type enter, or + * when ctrl+d is typed. + * + * The function returns the length of the current buffer. */ +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +{ + struct linenoiseState l; + + /* Populate the linenoise state that we pass to functions implementing + * specific editing functionalities. */ + l.ifd = stdin_fd; + l.ofd = stdout_fd; + l.buf = buf; + l.buflen = buflen; + l.prompt = prompt; + l.plen = strlen(prompt); + l.oldpos = l.pos = 0; + l.len = 0; + l.cols = getColumns(stdin_fd, stdout_fd); + l.maxrows = 0; + l.history_index = 0; + + /* Buffer starts empty. */ + l.buf[0] = '\0'; + l.buflen--; /* Make sure there is always space for the nulterm */ + + /* The latest history entry is always our current buffer, that + * initially is just an empty string. */ + linenoiseHistoryAdd(""); + + if (write(l.ofd,prompt,l.plen) == -1) return -1; + while(1) { + char c; + int nread; + char seq[3]; + + nread = read(l.ifd,&c,1); + if (nread <= 0) return l.len; + + /* Only autocomplete when the callback is set. It returns < 0 when + * there was an error reading from fd. Otherwise it will return the + * character that should be handled next. */ + if (c == 9 && completionCallback != NULL) { + c = completeLine(&l); + /* Return on errors */ + if (c < 0) return l.len; + /* Read next character when 0 */ + if (c == 0) continue; + } + + switch(c) { + case NL: /* enter, typed before raw mode was enabled */ + case ENTER: /* enter */ + history_len--; + free(history[history_len]); + if (mlmode) linenoiseEditMoveEnd(&l); + if (hintsCallback) { + /* Force a refresh without hints to leave the previous + * line as the user typed it after a newline. */ + linenoiseHintsCallback *hc = hintsCallback; + hintsCallback = NULL; + refreshLine(&l); + hintsCallback = hc; + } + return (int)l.len; + case CTRL_C: /* ctrl-c */ + errno = EAGAIN; + return -1; + case BACKSPACE: /* backspace */ + case 8: /* ctrl-h */ + linenoiseEditBackspace(&l); + break; + case CTRL_D: /* ctrl-d, remove char at right of cursor, or if the + line is empty, act as end-of-file. */ + if (l.len > 0) { + linenoiseEditDelete(&l); + } else { + history_len--; + free(history[history_len]); + return -1; + } + break; + case CTRL_T: /* ctrl-t, swaps current character with previous. */ + if (l.pos > 0 && l.pos < l.len) { + int aux = buf[l.pos-1]; + buf[l.pos-1] = buf[l.pos]; + buf[l.pos] = aux; + if (l.pos != l.len-1) l.pos++; + refreshLine(&l); + } + break; + case CTRL_B: /* ctrl-b */ + linenoiseEditMoveLeft(&l); + break; + case CTRL_F: /* ctrl-f */ + linenoiseEditMoveRight(&l); + break; + case CTRL_P: /* ctrl-p */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case CTRL_N: /* ctrl-n */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case ESC: /* escape sequence */ + /* Read the next two bytes representing the escape sequence. + * Use two calls to handle slow terminals returning the two + * chars at different times. */ + if (read(l.ifd,seq,1) == -1) break; + if (read(l.ifd,seq+1,1) == -1) break; + + /* ESC [ sequences. */ + if (seq[0] == '[') { + if (seq[1] >= '0' && seq[1] <= '9') { + /* Extended escape, read additional byte. */ + if (read(l.ifd,seq+2,1) == -1) break; + if (seq[2] == '~') { + switch(seq[1]) { + case '3': /* Delete key. */ + linenoiseEditDelete(&l); + break; + } + } + } else { + switch(seq[1]) { + case 'A': /* Up */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case 'B': /* Down */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case 'C': /* Right */ + linenoiseEditMoveRight(&l); + break; + case 'D': /* Left */ + linenoiseEditMoveLeft(&l); + break; + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; + } + } + } + + /* ESC O sequences. */ + else if (seq[0] == 'O') { + switch(seq[1]) { + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; + } + } + break; + default: + if (linenoiseEditInsert(&l,c)) return -1; + break; + case CTRL_U: /* Ctrl+u, delete the whole line. */ + buf[0] = '\0'; + l.pos = l.len = 0; + refreshLine(&l); + break; + case CTRL_K: /* Ctrl+k, delete from current to end of line. */ + buf[l.pos] = '\0'; + l.len = l.pos; + refreshLine(&l); + break; + case CTRL_A: /* Ctrl+a, go to the start of the line */ + linenoiseEditMoveHome(&l); + break; + case CTRL_E: /* ctrl+e, go to the end of the line */ + linenoiseEditMoveEnd(&l); + break; + case CTRL_L: /* ctrl+l, clear screen */ + linenoiseClearScreen(); + refreshLine(&l); + break; + case CTRL_W: /* ctrl+w, delete previous word */ + linenoiseEditDeletePrevWord(&l); + break; + } + } + return l.len; +} + +/* This special mode is used by linenoise in order to print scan codes + * on screen for debugging / development purposes. It is implemented + * by the linenoise_example program using the --keycodes option. */ +void linenoisePrintKeyCodes(void) { + char quit[4]; + + printf("Linenoise key codes debugging mode.\n" + "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); + if (enableRawMode(STDIN_FILENO) == -1) return; + memset(quit,' ',4); + while(1) { + char c; + int nread; + + nread = read(STDIN_FILENO,&c,1); + if (nread <= 0) continue; + memmove(quit,quit+1,sizeof(quit)-1); /* shift string to left. */ + quit[sizeof(quit)-1] = c; /* Insert current char on the right. */ + if (memcmp(quit,"quit",sizeof(quit)) == 0) break; + + printf("'%c' %02x (%d) (type quit to exit)\n", + isprint(c) ? c : '?', (int)c, (int)c); + printf("\r"); /* Go left edge manually, we are in raw mode. */ + fflush(stdout); + } + disableRawMode(STDIN_FILENO); +} + +/* This function calls the line editing function linenoiseEdit() using + * the STDIN file descriptor set in raw mode. */ +static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { + int count; + + if (buflen == 0) { + errno = EINVAL; + return -1; + } + + if (enableRawMode(STDIN_FILENO) == -1) return -1; + count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); + disableRawMode(STDIN_FILENO); + printf("\n"); + return count; +} + +/* This function is called when linenoise() is called with the standard + * input file descriptor not attached to a TTY. So for example when the + * program using linenoise is called in pipe or with a file redirected + * to its standard input. In this case, we want to be able to return the + * line regardless of its length (by default we are limited to 4k). */ +static char *linenoiseNoTTY(void) { + char *line = NULL; + size_t len = 0, maxlen = 0; + + while(1) { + if (len == maxlen) { + if (maxlen == 0) maxlen = 16; + maxlen *= 2; + char *oldval = line; + line = realloc(line,maxlen); + if (line == NULL) { + if (oldval) free(oldval); + return NULL; + } + } + int c = fgetc(stdin); + if (c == EOF || c == '\n') { + if (c == EOF && len == 0) { + free(line); + return NULL; + } else { + line[len] = '\0'; + return line; + } + } else { + line[len] = c; + len++; + } + } +} + +/* The high level function that is the main API of the linenoise library. + * This function checks if the terminal has basic capabilities, just checking + * for a blacklist of stupid terminals, and later either calls the line + * editing function or uses dummy fgets() so that you will be able to type + * something even in the most desperate of the conditions. */ +char *linenoise(const char *prompt) { + char buf[LINENOISE_MAX_LINE]; + int count; + + if (!isatty(STDIN_FILENO)) { + /* Not a tty: read from file / pipe. In this mode we don't want any + * limit to the line size, so we call a function to handle that. */ + return linenoiseNoTTY(); + } else if (isUnsupportedTerm()) { + size_t len; + + printf("%s",prompt); + fflush(stdout); + if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL; + len = strlen(buf); + while(len && (buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + buf[len] = '\0'; + } + return strdup(buf); + } else { + count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt); + if (count == -1) return NULL; + return strdup(buf); + } +} + +/* This is just a wrapper the user may want to call in order to make sure + * the linenoise returned buffer is freed with the same allocator it was + * created with. Useful when the main program is using an alternative + * allocator. */ +void linenoiseFree(void *ptr) { + free(ptr); +} + +/* ================================ History ================================= */ + +/* Free the history, but does not reset it. Only used when we have to + * exit() to avoid memory leaks are reported by valgrind & co. */ +static void freeHistory(void) { + if (history) { + int j; + + for (j = 0; j < history_len; j++) + free(history[j]); + free(history); + } +} + +/* At exit we'll try to fix the terminal to the initial conditions. */ +static void linenoiseAtExit(void) { + disableRawMode(STDIN_FILENO); + freeHistory(); +} + +/* This is the API call to add a new entry in the linenoise history. + * It uses a fixed array of char pointers that are shifted (memmoved) + * when the history max length is reached in order to remove the older + * entry and make room for the new one, so it is not exactly suitable for huge + * histories, but will work well for a few hundred of entries. + * + * Using a circular buffer is smarter, but a bit more complex to handle. */ +int linenoiseHistoryAdd(const char *line) { + char *linecopy; + + if (history_max_len == 0) return 0; + + /* Initialization on first call. */ + if (history == NULL) { + history = malloc(sizeof(char*)*history_max_len); + if (history == NULL) return 0; + memset(history,0,(sizeof(char*)*history_max_len)); + } + + /* Don't add duplicated lines. */ + if (history_len && !strcmp(history[history_len-1], line)) return 0; + + /* Add an heap allocated copy of the line in the history. + * If we reached the max length, remove the older line. */ + linecopy = strdup(line); + if (!linecopy) return 0; + if (history_len == history_max_len) { + free(history[0]); + memmove(history,history+1,sizeof(char*)*(history_max_len-1)); + history_len--; + } + history[history_len] = linecopy; + history_len++; + return 1; +} + +/* Set the maximum length for the history. This function can be called even + * if there is already some history, the function will make sure to retain + * just the latest 'len' elements if the new history length value is smaller + * than the amount of items already inside the history. */ +int linenoiseHistorySetMaxLen(int len) { + char **new; + + if (len < 1) return 0; + if (history) { + int tocopy = history_len; + + new = malloc(sizeof(char*)*len); + if (new == NULL) return 0; + + /* If we can't copy everything, free the elements we'll not use. */ + if (len < tocopy) { + int j; + + for (j = 0; j < tocopy-len; j++) free(history[j]); + tocopy = len; + } + memset(new,0,sizeof(char*)*len); + memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy); + free(history); + history = new; + } + history_max_len = len; + if (history_len > history_max_len) + history_len = history_max_len; + return 1; +} + +/* Save the history in the specified file. On success 0 is returned + * otherwise -1 is returned. */ +int linenoiseHistorySave(const char *filename) { + mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO); + FILE *fp; + int j; + + fp = fopen(filename,"w"); + umask(old_umask); + if (fp == NULL) return -1; + fchmod(fileno(fp),S_IRUSR|S_IWUSR); + for (j = 0; j < history_len; j++) + fprintf(fp,"%s\n",history[j]); + fclose(fp); + return 0; +} + +/* Load the history from the specified file. If the file does not exist + * zero is returned and no operation is performed. + * + * If the file exists and the operation succeeded 0 is returned, otherwise + * on error -1 is returned. */ +int linenoiseHistoryLoad(const char *filename) { + FILE *fp = fopen(filename,"r"); + char buf[LINENOISE_MAX_LINE]; + + if (fp == NULL) return -1; + + while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) { + char *p; + + p = strchr(buf,'\r'); + if (!p) p = strchr(buf,'\n'); + if (p) *p = '\0'; + linenoiseHistoryAdd(buf); + } + fclose(fp); + return 0; +} diff --git a/tools/intarkdb_cli/linenoise.h b/tools/intarkdb_cli/linenoise.h new file mode 100644 index 0000000000000000000000000000000000000000..6dfee73bcd41fb9efca5e0d68b48d891622e363a --- /dev/null +++ b/tools/intarkdb_cli/linenoise.h @@ -0,0 +1,75 @@ +/* linenoise.h -- VERSION 1.0 + * + * Guerrilla line editing library against the idea that a line editing lib + * needs to be 20,000 lines of C code. + * + * See linenoise.c for more information. + * + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2014, Salvatore Sanfilippo + * Copyright (c) 2010-2013, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __LINENOISE_H +#define __LINENOISE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct linenoiseCompletions { + size_t len; + char **cvec; +} linenoiseCompletions; + +typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); +typedef void(linenoiseFreeHintsCallback)(void *); +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); +void linenoiseSetHintsCallback(linenoiseHintsCallback *); +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); +void linenoiseAddCompletion(linenoiseCompletions *, const char *); + +char *linenoise(const char *prompt); +void linenoiseFree(void *ptr); +int linenoiseHistoryAdd(const char *line); +int linenoiseHistorySetMaxLen(int len); +int linenoiseHistorySave(const char *filename); +int linenoiseHistoryLoad(const char *filename); +void linenoiseClearScreen(void); +void linenoiseSetMultiLine(int ml); +void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __LINENOISE_H */ diff --git a/tools/intarkdb_cli/main.cpp b/tools/intarkdb_cli/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..557f98ea5b3aaf49005132c1b44da4324e5bb8c3 --- /dev/null +++ b/tools/intarkdb_cli/main.cpp @@ -0,0 +1,93 @@ +/* +* Copyright (c) GBA-NCTI-ISDC. 2022-2024. +* +* openGauss embedded is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +* MERCHANTABILITY OR FITFOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +* ------------------------------------------------------------------------- +* +* main.cpp +* +* IDENTIFICATION +* openGauss-embedded/tools/intarkdb_cli/main.cpp +* +* ------------------------------------------------------------------------- +*/ +#include +#include + +#include +#include +#include + +#include "linenoise.h" + +#include "binder/binder.h" +#include "catalog/catalog.h" +#include "main/connection.h" +#include "main/database.h" +#include "kv_operator.h" +#include "cmd.h" +#include "assist.h" +#include "function/version/pragma_version.h" + +#include "compute/kv/kv_connection.h" + +auto main(int argc, char** argv) -> int { + std::string path = "./"; + if (argc >= 2) { + if (strcmp(argv[1], "--version")==0){ + std::cout << intarkdb::LibraryVersion() << " " << intarkdb::SourceID() << std::endl; + return 0; + } + else{ + path = argv[1]; + path += "/"; + } + } + try { + auto db_instance = std::shared_ptr(IntarkDB::GetInstance(path.c_str())); + // 启动db + db_instance->Init(); + + Connection conn(db_instance); + conn.Init(); + + auto kv_conn = std::make_unique(db_instance); + kv_conn->Init(); + KvOperator kv_operator(std::move(kv_conn)); + ClassCmd cmd(path, &conn, &kv_operator); + + char *zHome; + int nHistory; + char *zHistory = getenv("INTARKDB_HISTORY"); + if( zHistory ){ + zHistory = strdup(zHistory); + }else if( (zHome = find_home_dir(0))!=nullptr ){ + nHistory = strlen30(zHome) + 20; + if( (zHistory = (char*)malloc(nHistory))!=nullptr ){ + sqlite3_snprintf(nHistory, zHistory,"%s/.intarkdb_history", zHome); + } + } + if( zHistory ){ + linenoiseHistoryLoad(zHistory); + } + cmd.main(); + if( zHistory ){ + linenoiseHistorySetMaxLen(2000); + linenoiseHistorySave(zHistory); + free(zHistory); + } + } catch (const std::exception& e) { + std::cout<<" ERROR MSG:"< +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char* DEFAULT_TABLE = "SYS_KV"; +#define TIMEOUT 600 +#define thread_num 8 +static int time_count = 0; // 10分钟超时自动结束 +static int is_run = 0; +static int is_start = 0; +static int dbindex = 0; +static int old_index = 0; +static int qps = 0; +static char db_type[16] = "\0"; +const static int ReadTotalNum = 100000; +void *instar_handles[32]; +void *sqlite_handles[32]; +void *sqlitedb; + +pthread_t tid[thread_num]; + +void stop_presstest() +{ + is_start = 0; + dbindex = 0; + old_index = 0; + qps = 0; + strcpy(db_type, "\0"); +} + +void get_format_time_st(char *str_time) +{ + struct tm *tm_t; + struct timeval time; + + gettimeofday(&time, NULL); + tm_t = localtime(&time.tv_sec); + if (NULL != tm_t) + { + sprintf(str_time, "%04d-%02d-%02d %02d:%02d:%02d.%03ld", + tm_t->tm_year + 1900, + tm_t->tm_mon + 1, + tm_t->tm_mday, + tm_t->tm_hour, + tm_t->tm_min, + tm_t->tm_sec, + time.tv_usec / 1000); + } + return; +} + +void timer_thread() +{ + pthread_t tid; + char str_time[64]; + while (is_start) + { + get_format_time_st(str_time); + qps = dbindex - old_index; + GS_LOG_RUN_INF("%s qps %d total %d status running", db_type, dbindex - old_index, dbindex); + old_index = dbindex; + + time_count++; + if (time_count > TIMEOUT) + stop_presstest(); + + sleep(1); + } + return; +} + +void instar_read_test_thread(void *handle) +{ + prctl(PR_SET_NAME, "instar_read_thread"); + gstorReply *reply_get; + char str[20]; + while (is_start) + { + int tmp_index = (dbindex++) % ReadTotalNum; + sprintf(str, "%d", tmp_index); + reply_get = (gstorReply *)gstor_command_get(handle, str); // 测读 + if (reply_get->type != GS_SUCCESS) + { + GS_LOG_RUN_INF("InstarDB error: %d", reply_get->type); + stop_presstest(); + } + gstor_freeReplyObject(reply_get); + } +} + +void instar_prepare_data() +{ + gstorReply *reply_set; + char str[20]; + while (is_run && dbindex < ReadTotalNum) + { + int tmp_index = ++dbindex; + sprintf(str, "%d", tmp_index); + reply_set = (gstorReply *)gstor_command_set(instar_handles[0], str, str); // 测写入 + gstor_freeReplyObject(reply_set); + if (tmp_index % 1000 == 0) + { + gstor_kv_commit(instar_handles[0]); + } + } + dbindex = 0; + GS_LOG_RUN_INF("instarDB finish prepare data num:%d", ReadTotalNum); +} + +void sqlite_read_test_thread(void *handle) +{ + prctl(PR_SET_NAME, "sqlite_read_thread"); + int result; + char str[128]; + char *err_msg = NULL; + int data = 0; + + while (is_start) + { + int tmp_index = (dbindex++) % ReadTotalNum; + sprintf(str, "SELECT * from COMPANY where id =%d", tmp_index); + result = sqlite3_exec(handle, str, NULL, &data, &err_msg); + if (result != SQLITE_OK) + { + GS_LOG_RUN_INF("SQLite error: %s", err_msg); + stop_presstest(); + } + sqlite3_free(err_msg); + } +} +void sqlite_prepare_data() +{ + gstorReply *reply_set; + char str[128]; + sqlite3_exec(sqlitedb, "begin", NULL, NULL, NULL); + while (is_run && dbindex <= ReadTotalNum) + { + int tmp_index = ++dbindex; + sprintf(str, "INSERT INTO COMPANY (name1) VALUES (%d);", tmp_index); + sqlite3_exec(sqlitedb, str, NULL, NULL, NULL); + + if (tmp_index % 1000 == 0) + { + sqlite3_exec(sqlitedb, "commit", NULL, NULL, NULL); + sqlite3_exec(sqlitedb, "begin", NULL, NULL, NULL); + } + } + dbindex = 0; + GS_LOG_RUN_INF("sqlite finish prepare data num:%d", ReadTotalNum); +} + +status_t instarDB_func(const char* buf_cmd, char* res_msg) +{ + if(strcmp(buf_cmd, "prepare") == 0) + { + instar_prepare_data(); + } + else if (strcmp(buf_cmd, "start") == 0) + { + gstorReply *reply_get; + char str[20]; + sprintf(str, "%d", ReadTotalNum); + reply_get = (gstorReply *)gstor_command_get(instar_handles[0], str); + if (reply_get->type != GS_SUCCESS) + { + GS_LOG_RUN_INF("InstarDB error: %d", reply_get->type); + return RES_SERVER_ERR; + } + else if(reply_get->str == NULL) + { + instar_prepare_data(); + } + gstor_freeReplyObject(reply_get); + is_start = 1; + pthread_t timer_tid; + pthread_create(&timer_tid, NULL, timer_thread, NULL); + for (int i = 0; i < thread_num; i++) + { + pthread_create(&tid[i], NULL, instar_read_test_thread, instar_handles[i]); + } + } + else + { + GS_LOG_DEBUG_WAR("err cmd:%s", buf_cmd); + sprintf(res_msg, "err cmd:%s", buf_cmd); + return RES_CMD_ERR; + } + return RES_SUCCESS; +} + +status_t sqlite_func(const char* buf_cmd, char* res_msg) +{ + if(strcmp(buf_cmd, "prepare") == 0) + { + sqlite_prepare_data(); + } + else if (strcmp(buf_cmd, "start") ==0 ) + { + //检查是否需要prepare + char str[128]; + int data = 0; + int result; + char *err_msg = NULL; + int nRow; + int nCol; + char** pResult; + sprintf(str, "SELECT * from COMPANY where id = %d", ReadTotalNum); + result = sqlite3_get_table(sqlitedb, str, &pResult, &nRow, &nCol, &err_msg); + //result = sqlite3_exec(sqlitedb, str, NULL, &data, &err_msg); + if (result != SQLITE_OK) + { + GS_LOG_RUN_INF("SQLite error: %s", err_msg); + } + if(nRow == 0) + { + sqlite_prepare_data(); + } + sqlite3_free(err_msg); + + is_start = 1; + pthread_t timer_tid; + pthread_create(&timer_tid, NULL, timer_thread, NULL); + for (int i = 0; i < thread_num; i++) + { + pthread_create(&tid[i], NULL, sqlite_read_test_thread, sqlite_handles[i]); + } + } + else + { + GS_LOG_DEBUG_WAR("err cmd:%s", buf_cmd); + sprintf(res_msg, "err cmd:%s", buf_cmd); + return RES_CMD_ERR; + } + return RES_SUCCESS; +} + + +status_t pressure_proto_handle(cs_pipe_t *pipe, const cJSON * proto) +{ + GS_LOG_DEBUG_INF("%s proto: %s", __func__, cJSON_PrintUnformatted(proto)); + char buf_cmd[KV_CMD_SIZE]; + cJSON *res_proto = cJSON_CreateObject(); + double seqId = 0; + uint32 rescode = RES_SUCCESS; + char res_msg[RES_MSG_SIZE] = ""; + do + { + rescode = RES_FIELD_ERR; + GS_BREAK_IF_ERROR(json_get_number(proto, "seqId", &seqId)); + GS_BREAK_IF_ERROR(json_get_string(proto, "cmd", buf_cmd)); + if(strcmp(buf_cmd, "status") != 0) + { + GS_BREAK_IF_ERROR(json_get_string(proto, "value", db_type)); + } + rescode = RES_SUCCESS; + } while(0); + cJSON_AddItemToObjectCS(res_proto, "seqId", cJSON_CreateNumber((double)seqId)); + + if (strcmp(buf_cmd, "status") == 0) + { + cJSON *json_rows = cJSON_CreateObject(); + cJSON_AddItemToObjectCS(json_rows, "isRunning", cJSON_CreateString(db_type)); + cJSON_AddItemToObjectCS(json_rows, "qps", cJSON_CreateNumber(qps)); + cJSON_AddItemToObjectCS(res_proto, "value", json_rows); + } + else if (strcmp(buf_cmd, "stop") == 0) + { + stop_presstest(); + } + else + { + if (strcmp(db_type, "instarDB") == 0) + { + rescode = instarDB_func(buf_cmd, res_msg); + } + else if (strcmp(db_type, "sqlite") == 0) + { + rescode = sqlite_func(buf_cmd, res_msg); + } + else + { + strcpy(res_msg, "err db type"); + rescode = RES_CMD_ERR; + } + } + cJSON_AddItemToObjectCS(res_proto, "rescode", cJSON_CreateNumber((double)rescode)); + cJSON_AddItemToObjectCS(res_proto, "msg", cJSON_CreateString(res_msg)); + status_t status = tcp_send_set_end(pipe, cJSON_PrintUnformatted(res_proto)); +} + + + +status_t sqlite_init(const char* path) +{ + sqlite3 *db; + char *zErrMsg = 0; + int rc; + float task_tps = 0; + struct timeval start; + struct timeval end; + char *sql_create_table; + + char db_file[64]; + printf("%s/sqlites.db", path); + sprintf(db_file, "%s/sqlites.db", path); + rc = sqlite3_open(db_file, &db); + if (rc) + { + GS_LOG_RUN_INF("Can't open database: %s", sqlite3_errmsg(db)); + exit(0); + } + else + { + GS_LOG_RUN_INF("Opened database successfully"); + } + + sql_create_table = "CREATE TABLE IF NOT EXISTS COMPANY(id INTEGER PRIMARY KEY AUTOINCREMENT,name1 CHAR(20) NOT NULL);"; + rc = sqlite3_exec(db, sql_create_table, NULL, 0, &zErrMsg); + if (rc != SQLITE_OK) + { + GS_LOG_RUN_INF("SQL error: %s", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlitedb = db; + + for (int i = 0; i < thread_num; i++) + { + sqlite3 *tmp_db; + rc = sqlite3_open_v2(db_file, &tmp_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, NULL); + if (rc) + { + GS_LOG_RUN_INF("Can't open database: %s", sqlite3_errmsg(db)); + exit(0); + } + sqlite_handles[i] = tmp_db; + } + + return GS_SUCCESS; +} + +status_t instar_init() +{ + for (int i = 0; i < thread_num; i++) + { + + if (alloc_kv_handle(&instar_handles[i]) != GS_SUCCESS) + { + GS_LOG_RUN_INF("get instar_handles[%d] failed", i); + } + if (create_or_open_kv_table(instar_handles[i], "tb_pressure_test") != GS_SUCCESS) + { + GS_LOG_RUN_INF("open table failed i:%d", i); + return GS_ERROR; + } + gstor_kv_begin(instar_handles[i]); + } + return GS_SUCCESS; +} + +void sig_handler(int signo) +{ + is_run = 0; +} + +int main(int argc, char *argv[]) +{ + intarkdb_database db; + char path[1024] = "."; + if (argc >= 2) + { + memcpy(path, argv[1], strlen(argv[1])); + } + GS_LOG_RUN_INF("db path:%s", path); + if (intarkdb_open(path, &db) != SQL_SUCCESS) + { + GS_LOG_RUN_INF("intarkdb_open failed"); + return GS_ERROR; + } + + if (srv_start_om_agent_by_config(db, "./om_agent_cfg.json", path) != GS_SUCCESS) { + GS_LOG_RUN_INF("[API] om_agent failed"); + return GS_ERROR; + } + + GS_RETURN_IFERR(sqlite_init(path)); + GS_RETURN_IFERR(instar_init()); + + register_proto_handles(8, pressure_proto_handle); + + GS_LOG_RUN_INF("path=%s thread_num=%d", path, thread_num); + + if (signal(SIGUSR1, sig_handler) == SIG_ERR) + { + printf("signal error\n"); + } + is_run = 1; + while(is_run) + { + sleep(1); + } + + srv_close_om_agent(); + intarkdb_close(&db); + sqlite3_close(db); +} \ No newline at end of file diff --git a/tools/pressure_test/embedded_test_prepare.sh b/tools/pressure_test/embedded_test_prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b2e27d2c8a6da7db17e9f2871fe486715415355 --- /dev/null +++ b/tools/pressure_test/embedded_test_prepare.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ ! -d "test" ]; then + mkdir test +fi +./pressure_test ReadTest 1 P +echo "prepare success" + diff --git a/tools/pressure_test/embedded_test_start.sh b/tools/pressure_test/embedded_test_start.sh new file mode 100644 index 0000000000000000000000000000000000000000..b5eb11052a345cbf4f344dbaf6f3937917afcf06 --- /dev/null +++ b/tools/pressure_test/embedded_test_start.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ ! -d "test" ]; then + mkdir test +fi +nohup ./pressure_test ReadTest 8 R > nohup.out 2>&1 & + diff --git a/tools/pressure_test/embedded_test_stop.sh b/tools/pressure_test/embedded_test_stop.sh new file mode 100644 index 0000000000000000000000000000000000000000..93fd36392eea3a40c1cbb0fed27d345de2b422f7 --- /dev/null +++ b/tools/pressure_test/embedded_test_stop.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +pid=`pidof -s pressure_test` +echo $pid +if [ ! -z $pid ]; then + kill -10 $pid +else + echo "no pressure_test start" +fi + +##等进程停止后删除目录 +#pid=`pidof -s pressure_test` +#while [ ! -z $pid ]; do +# sleep 0.1 +# pid=`pidof -s pressure_test` +#done +#rm -rf test/gstor +##创建库文件，防止启动过慢 +#./pressure_test test +#echo "create db file" diff --git a/tools/pressure_test/pressure_test.c b/tools/pressure_test/pressure_test.c new file mode 100644 index 0000000000000000000000000000000000000000..41741ef393a1159880d3d5ee8159dee6cf394bb5 --- /dev/null +++ b/tools/pressure_test/pressure_test.c @@ -0,0 +1,250 @@ +#include "stdio.h" +#include +#include +#include +#include +#include +#include "interface/c/gstor.h" + + +#ifdef __cplusplus +extern "C" +{ +#endif + #define TIMEOUT 600 + static int time_count = 0; //10分钟超时自动结束 + static int is_run = 0; + static int kv_index = 0; + static int old_kv_index = 0; + static char *log_file = "embedded_test.log"; + void *handles[32]; + static uint32 thread_num = 1; + const static int ReadTotalNum = 100000; + + typedef enum + { + ReadTest = 0, + WriteTest = 1 + } test_type; + + + + __fortify_function void mylog(const char *__restrict __fmt, ...) + { + FILE *fp = fopen(log_file, "a+"); + fprintf(fp, __fmt, __va_arg_pack()); + fclose(fp); + } + + void get_format_time_st(char * str_time) + { + struct tm *tm_t; + struct timeval time; + + gettimeofday(&time, NULL); + tm_t = localtime(&time.tv_sec); + if (NULL != tm_t) + { + sprintf(str_time, "%04d-%02d-%02d %02d:%02d:%02d.%03ld", + tm_t->tm_year + 1900, + tm_t->tm_mon + 1, + tm_t->tm_mday, + tm_t->tm_hour, + tm_t->tm_min, + tm_t->tm_sec, + time.tv_usec / 1000); + } + + return; + } + + void log_stat() + { + if (is_run == 0) + return; + + char str_time[64]; + get_format_time_st(str_time); + int diff = kv_index - old_kv_index; + mylog("%s qps %d total %d status running\n", str_time, kv_index - old_kv_index, kv_index); + old_kv_index = kv_index; + time_count++; + if(time_count > TIMEOUT) + is_run = 0; + return; + } + + void timer_thread() + { + //printf("timer thread start \n"); + prctl(PR_SET_NAME, "my_timer_thread"); + pthread_t tid; + while (is_run) + { + sleep(1); + pthread_create(&tid, NULL, log_stat, NULL); + } + + return; + } + + void pressure_read_test_thread(void * handle) + { + prctl(PR_SET_NAME, "my_read_thread"); + gstorReply *reply_set; + gstorReply *reply_get; + char str[20]; + while (is_run) + { + int index = (kv_index++) % ReadTotalNum; + sprintf(str, "%d", index ); + //mylog("str:%s \n", str); + reply_get = (gstorReply *)gstor_command_get(handle, str); // 测读 + gstor_freeReplyObject(reply_get); + } + } + void pressure_write_test_thread(void *handle) + { + prctl(PR_SET_NAME, "my_write_thread"); + gstorReply *reply_set; + char str[20]; + while (is_run) + { + int index = kv_index++; + sprintf(str, "%d", index); + //mylog("str:%s \n", str); + reply_set = (gstorReply *)gstor_command_set(handle, str, str); // 测写入 + gstor_freeReplyObject(reply_set); + //gstor_kv_commit(handle); + if (kv_index % 1000 == 0) + { + gstor_kv_commit(handle); + } + } + } + + void gstor_client(uint32 thread_num, test_type type) + { + pthread_t tid[32]; + + struct timeval start; + struct timeval end; + + kv_index = 0; + + pthread_t timer_tid; + pthread_create(&timer_tid, NULL, timer_thread, NULL); + gettimeofday(&start, NULL); + + for (int i = 0; i < thread_num; i++) + { + if(type == ReadTest) + { + pthread_create(&tid[i], NULL, pressure_read_test_thread, handles[i]); + } + else + { + pthread_create(&tid[i], NULL, pressure_write_test_thread, handles[i]); + } + //kv_index++; + } + for (int i = 0; i < thread_num; i++) + { + pthread_join(tid[i], NULL); + } + gettimeofday(&end, NULL); + float time_use = (end.tv_sec - start.tv_sec) + (1.0 * (end.tv_usec - start.tv_usec)) / 1000000; + char str_time[64]; + get_format_time_st(str_time); + mylog("%s avg_qps %lf total %d status stop\n", str_time, kv_index/time_use, kv_index); + return; + } + + void sig_handler(int signo) + { + //printf("catch the signal SIGUSR1 %d run num:%d\n", signo, kv_index); + is_run = 0; + } + + int main(int argc, char *argv[]) + { + char path[1024] = "\0"; + char cmd[16] = "\0"; + if (argc >= 2) + { + memcpy(path, argv[1], strlen(argv[1]) + 1); + } + else + { + printf("command err!!! ./pressure_test ${path} ${thread_num} eg:./pressure_test test 8"); + return 0; + } + + int db_type = 0; + if (gstor_startup_db(db_type, path) != GS_SUCCESS) + { + printf("[API] gstor_startup_db failed\n"); + return GS_ERROR; + } + if (argc >= 3) + { + thread_num = atoi(argv[2]); + } + + printf("path=%s thread_num=%d\n", path, thread_num); + for (int i = 0; i < thread_num; i++) + { + + if (alloc_kv_handle(&handles[i]) != GS_SUCCESS) + { + printf("get handles[%d] failed\n", i); + } + if (create_or_open_kv_table(handles[i], EXC_DCC_KV_TABLE) != GS_SUCCESS) + { + printf("open table failed i:%d\n", i); + return GS_ERROR; + } + gstor_kv_begin(handles[i]); + } + + if (signal(SIGUSR1, sig_handler) == SIG_ERR) + { + printf("signal error\n"); + } + if (argc >= 4 && strcmp(argv[3], "R") == 0) //不设置的话只创建库文件 + { + is_run = 1; + gstor_client(thread_num, ReadTest); + } + else if (argc >= 4 && strcmp(argv[3], "W") == 0) //测写 + { + is_run = 1; + gstor_client(thread_num, WriteTest); + } + else if (argc >= 4 && strcmp(argv[3], "P") == 0) // 测读前准备数据 + { + gstorReply *reply_set; + char str[20]; + pthread_t timer_tid; + pthread_create(&timer_tid, NULL, timer_thread, NULL); + while (is_run && kv_index < ReadTotalNum) + { + int index = kv_index++; + sprintf(str, "%d", index); + // mylog("str:%s \n", str); + reply_set = (gstorReply *)gstor_command_set(handles[0], str, str); // 测写入 + gstor_freeReplyObject(reply_set); + if (kv_index % 1000 == 0) + { + gstor_kv_commit(handles[0]); + } + } + } + + gstor_shutdown_db(); + return 0; // 指定程序的退出状态码为 0，表示正常退出 + } + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/tools/pressure_test/sqlite_test.c b/tools/pressure_test/sqlite_test.c new file mode 100644 index 0000000000000000000000000000000000000000..2880b1162fbb6a67d9631e307c739843ad99f5a4 --- /dev/null +++ b/tools/pressure_test/sqlite_test.c @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define TIMEOUT 600 +static int time_count = 0; // 10分钟超时自动结束 +static int is_run = 0; +static int sqlite_index = 0; +static int old_sqlite_index = 0; +char *log_file = "sqlite_test.log"; + +__fortify_function void mylog(const char *__restrict __fmt, ...) +{ + FILE *fp = fopen(log_file, "a+"); + fprintf(fp, __fmt, __va_arg_pack()); + fclose(fp); +} + +void get_format_time_st(char *str_time) +{ + struct tm *tm_t; + struct timeval time; + + gettimeofday(&time, NULL); + tm_t = localtime(&time.tv_sec); + if (NULL != tm_t) + { + sprintf(str_time, "%04d-%02d-%02d %02d:%02d:%02d.%03ld", + tm_t->tm_year + 1900, + tm_t->tm_mon + 1, + tm_t->tm_mday, + tm_t->tm_hour, + tm_t->tm_min, + tm_t->tm_sec, + time.tv_usec / 1000); + } + return; +} + +void log_stat() +{ + if (is_run == 0) + return; + + char str_time[64]; + get_format_time_st(str_time); + mylog("%s qps %d total %d status running\n", str_time, sqlite_index - old_sqlite_index, sqlite_index); + old_sqlite_index = sqlite_index; + time_count++; + if (time_count > TIMEOUT) + is_run = 0; + return; +} + +void timer_thread() +{ + // printf("timer thread start \n"); + pthread_t tid; + while (is_run) + { + sleep(1); + pthread_create(&tid, NULL, log_stat, NULL); + } + + return; +} + +void sig_handler(int signo) +{ + //mylog("catch the signal SIGUSR1 %d run num:%d\n", signo, sqlite_index); + is_run = 0; +} + +int exec_handle(void *data, int argc, char **argv, char **colname) +{ + /* 计数器*/ + int i = *(int *)(data); + *(int *)(data) = i + 1; + + /* 取出结果*/ + printf("NO.%d message: [%s] is [%s], [%s] is [%s]...", *(int *)(data), colname[0], colname[1], argv[0], argv[1]); + return 0; +} + +int sqlite_test() +{ + sqlite3 *db; + char *zErrMsg = 0; + int rc; + int rc2; + char *sql; + float time_use = 0; + float task_tps = 0; + struct timeval start; + struct timeval end; + char *sql_create_table; + char *sql_create_index; + char *sql_create_index2; + char buf[120]; + int count = 0; + char str[100]; + int i, j, k, h; + int index; + int result, ret; + int data = 0; + char *err_msg = NULL; + char **dbResult; // 是 char ** 类型，两个*号 + int nRow, nColumn; + + rc = sqlite3_open("test/sqlites.db", &db); + if (rc) + { + printf("Can't open database: %s\n", sqlite3_errmsg(db)); + exit(0); + } + else + { + printf("Opened database successfully\n"); + } + + sql_create_table = "CREATE TABLE IF NOT EXISTS COMPANY(id INTEGER PRIMARY KEY AUTOINCREMENT,name1 CHAR(20) NOT NULL);"; + // sql_create_index = "create index index_name1 on COMPANY (name1);"; + rc = sqlite3_exec(db, sql_create_table, NULL, 0, &zErrMsg); + if (rc != SQLITE_OK) + { + printf("SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + + // 使用事务提升插入效率，否则每个独立的事务都要打开和关闭数据库文件 + pthread_t tid; + pthread_create(&tid, NULL, timer_thread, NULL); + gettimeofday(&start, NULL); + sqlite3_exec(db, "begin", NULL, NULL, NULL); + + while (is_run) + { + sprintf(buf, "INSERT INTO COMPANY (name1) VALUES (%d);", sqlite_index); + sqlite3_exec(db, buf, NULL, NULL, NULL); + sqlite3_exec(db, "commit", NULL, NULL, NULL); + sqlite3_exec(db, "begin", NULL, NULL, NULL); + sprintf(buf, "SELECT * from COMPANY where id =%d", sqlite_index); + ret = sqlite3_exec(db, str, NULL, &data, &err_msg); + sqlite_index++; + /*if (sqlite_index % 3000 == 0) + { + sqlite3_exec(db, "commit", NULL, NULL, NULL); + sqlite3_exec(db, "begin", NULL, NULL, NULL); + }*/ + } + gettimeofday(&end, NULL); + time_use = (end.tv_sec - start.tv_sec) + (1.0 * (end.tv_usec - start.tv_usec)) / 1000000; + char str_time[64]; + get_format_time_st(str_time); + mylog("%s avg_qps %lf total %d status stop\n", str_time, sqlite_index / time_use, sqlite_index); + + sqlite3_close(db); + return 0; +} +int main(int argc, char *argv[]) +{ + printf("run \n"); + if (signal(SIGUSR1, sig_handler) == SIG_ERR) + { + printf("signal error\n"); + } + is_run = 1; + sqlite_test(); + return 0; +} \ No newline at end of file diff --git a/tools/pressure_test/sqlite_test_start.sh b/tools/pressure_test/sqlite_test_start.sh new file mode 100644 index 0000000000000000000000000000000000000000..916854fb3285543e54dff6f7e04a8a89c99c12ac --- /dev/null +++ b/tools/pressure_test/sqlite_test_start.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ ! -d "test" ]; then + mkdir test +fi +nohup ./sqlite_test > nohup.out 2>&1 & diff --git a/tools/pressure_test/sqlite_test_stop.sh b/tools/pressure_test/sqlite_test_stop.sh new file mode 100644 index 0000000000000000000000000000000000000000..723046ae14ff8049c6411baebc387e8752256298 --- /dev/null +++ b/tools/pressure_test/sqlite_test_stop.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +pid=`pidof -s sqlite_test` +echo $pid +if [ ! -z $pid ]; then + kill -10 $pid +else + echo "no sqlite_test start" +fi + +#等进程停止后删除目录 +#pid=`pidof -s sqlite_test` +#while [ ! -z $pid ]; do +# sleep 0.1 +# pid=`pidof -s sqlite_test` +#done +#rm -rf test/sqlites.db* \ No newline at end of file diff --git a/tools/rust-gstor-multithreading/.gitignore b/tools/rust-gstor-multithreading/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f161057e19c490ac2820acf166160c2e59aa5a2c --- /dev/null +++ b/tools/rust-gstor-multithreading/.gitignore @@ -0,0 +1,7 @@ +# Generated files +/target/ + +# The library shouldn't decide about the exact versions of +# its dependencies, but let the downstream crate decide. +Cargo.lock + diff --git a/tools/rust-gstor-multithreading/Cargo.toml b/tools/rust-gstor-multithreading/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..772aed1eb0994e90d833ff8c91cbcc841f42fba6 --- /dev/null +++ b/tools/rust-gstor-multithreading/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rust-gstor" +version = "0.1.0" +edition = "2021" +build = "build.rs" +links = "gstor" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libc = "0.2" +rand = "0.8.5" \ No newline at end of file diff --git a/tools/rust-gstor-multithreading/build.rs b/tools/rust-gstor-multithreading/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..deec37b99f3d69d484b0be26c67903cfbeb4fe4f --- /dev/null +++ b/tools/rust-gstor-multithreading/build.rs @@ -0,0 +1,13 @@ +use std::process::Command; +use std::env; + +fn main() { + println!("pre build ..."); + let pwd = Command::new("pwd").output().expect("get pwd error"); + println!("[INFO]: {}", String::from_utf8_lossy(&pwd.stdout)); + let src_dir = env::var("GSTOR_SRC").unwrap(); + let binarylibs = env::var("BINARYLIBS").unwrap(); + Command::new(&format!("{}/build.sh", src_dir)).arg("-3rd").arg(binarylibs).arg("-m").arg("Debug").spawn().expect("build error"); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rustc-link-search=native={}", &format!("{}/output/lib", src_dir)); +} \ No newline at end of file diff --git a/tools/rust-gstor-multithreading/src/main.rs b/tools/rust-gstor-multithreading/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..4c6ae469698a80859910b78fa57f59e4b6b01cfd --- /dev/null +++ b/tools/rust-gstor-multithreading/src/main.rs @@ -0,0 +1,250 @@ +extern crate libc; +use std::ffi::CString; +use std::ffi::CStr; +use libc::c_char; +use rand::distributions::{Alphanumeric,DistString}; +use std::{thread, time, env, process}; +use std::time::{Duration, Instant}; + + +#[link(name = "gstor")] + +extern "C" { + fn gstor_startup(data_path: CString) -> libc::c_int; + fn gstor_set_param(name: *const c_char, value: *const c_char, data_path: *const c_char) -> libc::c_int; + fn gstor_alloc(handle: *const u64) -> libc::c_int; + fn gstor_open_table(handle: u64, table_name: CString) -> libc::c_int; + fn gstor_put(handle: u64, key: *const c_char, key_len: libc::c_uint, val: *const c_char, val_len: libc::c_uint) -> libc::c_int; + fn rust_gstor_get(handle: u64, key: *const c_char) -> *const c_char; + fn gstor_commit(handle: u64) -> libc::c_int; + fn gstor_shutdown(); +} + +fn main() { + + let args: Vec = env::args().collect(); + // println!("{:?}", args); + + if args.len() < 3 { + println!("Usage: cmd -D data_path"); + process::exit(-1); + } + + let op = args.get(1); + let param = args.get(2); + + if op.unwrap() != "-D" { + println!("Usage: cmd -D data_path"); + process::exit(-1); + } + + if param.unwrap().len() == 0 { + println!("param error"); + process::exit(-1); + } + + let thread_num : u16; + let total : u16; + if args.len()==5{ + thread_num = args[3].parse().expect("Wanted a number"); + total = args[4].parse().expect("Wanted a number"); + if thread_num==0||total==0||thread_num>total{ + println!("Invalid parameter args:{:?}", args); + process::exit(-1); + } + }else{ + thread_num = 8; + total = 200; + println!("Invalid parameter args:{:?}", args); + } + + let data_path = CString::new(String::from(param.unwrap())).unwrap(); + let handle = Box::::new(0); + let mut handles = vec![Box::::new(0);thread_num.into()]; + + /* 初始化参数 */ + let value = CString::new("134217728").unwrap(); + let name = CString::new("DATA_BUFFER_SIZE").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("BUF_POOL_NUM").unwrap(); + let value = CString::new("1").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("LOG_BUFFER_SIZE").unwrap(); + let value = CString::new("16777216").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("LOG_BUFFER_COUNT").unwrap(); + let value = CString::new("4").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("SPACE_SIZE").unwrap(); + let value = CString::new("134217728").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("LOG_LEVEL").unwrap(); + let value = CString::new("55").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + /* 初始化参数 */ + + /* 数据库启动 */ + let ret = unsafe { + gstor_startup(CString::new(String::from(param.unwrap())).unwrap()) + }; + /* 数据库启动 */ + + /* 申请handle */ + // let ret = unsafe { + // gstor_alloc(handle.as_ref()) + // }; + for i in &handles{ + let ret = unsafe { + gstor_alloc(i.as_ref()) + }; + } + /* 申请handle */ + + /* 打开表 */ + // let ret = unsafe { + // gstor_open_table(*handle, CString::new("SYS_KV").unwrap()) + // }; + for i in &handles{ + let ret = unsafe { + gstor_open_table(**i, CString::new("SYS_KV").unwrap()) + }; + } + /* 打开表 */ + + +// /* 插入数据 */ +// let ret = unsafe { +// let key = CString::new("key1").unwrap(); +// let val = CString::new("val1").unwrap(); +// gstor_put(*handle, key.as_ptr(), 4, val.as_ptr(), 4) +// }; +// /* 插入数据 */ +// let key = CString::new("key1").unwrap(); +// /* 读取数据 */ +// let ret = unsafe { +// rust_gstor_get(*handle, key.as_ptr()) +// }; +// let c_str: &CStr = unsafe { CStr::from_ptr(ret) }; +// let str_slice: &str = c_str.to_str().unwrap(); +// let str_buf: String = str_slice.to_owned(); +// println!("[INFO]: key = {:?} value = {}", key, str_buf); +// /* 读取数据 */ + + // while true { + // let mut rng = rand::thread_rng(); + // let random_code = Alphanumeric.sample_string(&mut rng, 32); + // let key: String = "key-".to_string() + random_code.as_ref(); + // let val: String = "val-".to_string() + random_code.as_ref(); + // // println!("{} {}", key, val); + // /* 插入数据 */ + // let ret = unsafe { + // let key = CString::new(key.clone()).unwrap(); + // let val = CString::new(val).unwrap(); + // gstor_put(*handles[0], key.as_ptr(), key.as_bytes().len().try_into().unwrap(), val.as_ptr(), val.as_bytes().len().try_into().unwrap()) + // }; + // /* 插入数据 */ + + // /* 提交 */ + // let ret = unsafe { + // gstor_commit(*handles[0]) + // }; + // /* 提交 */ + + // /* 读取数据 */ + // let key = CString::new(key).unwrap(); + // let ret = unsafe { + // rust_gstor_get(*handles[0], key.as_ptr()) + // }; + // let c_str: &CStr = unsafe { CStr::from_ptr(ret) }; + // let str_slice: &str = c_str.to_str().unwrap(); + // let str_buf: String = str_slice.to_owned(); + // println!("[INFO]: key = {:?} value = {}", key, str_buf); + // /* 读取数据 */ + + // let ten_millis = time::Duration::from_millis(1000); + // // let now = time::Instant::now(); + // thread::sleep(ten_millis); + // } + + println!("thread_num:{} total:{}",thread_num,total); + let now = Instant::now(); + + let avg_num = total/thread_num; + let mut thread_handles = vec![]; + for num in 0..thread_num{ + //let index: _ = num.into(); + let index: _ = >::into(num); + let h = *handles[index]; + let thread_handle = thread::spawn(move || do_something(h,num,avg_num)); + thread_handles.push(thread_handle); + } + + for thread_handle in thread_handles { + thread_handle.join().unwrap(); + } + +/* 关闭数据库 */ + unsafe { gstor_shutdown() }; +/* 关闭数据库 */ + println!("end..."); + println!("process done! thread_num:{} total:{} Time spent:{}",thread_num,total,now.elapsed().as_secs()); +} + +fn do_something(handle: u64, thread_num:u16, total:u16) { + let now = Instant::now(); + for i in 0..total{ + let mut rng = rand::thread_rng(); + let random_code = Alphanumeric.sample_string(&mut rng, 32); + let key: String = "key-".to_string() + random_code.as_ref(); + let val: String = "val-".to_string() + random_code.as_ref(); + // println!("{} {}", key, val); + /* 插入数据 */ + let ret = unsafe { + let key = CString::new(key.clone()).unwrap(); + let val = CString::new(val).unwrap(); + gstor_put(handle, key.as_ptr(), key.as_bytes().len().try_into().unwrap(), val.as_ptr(), val.as_bytes().len().try_into().unwrap()) + }; + /* 插入数据 */ + + /* 提交 */ + let ret = unsafe { + gstor_commit(handle) + }; + /* 提交 */ + + /* 读取数据 */ + let key = CString::new(key).unwrap(); + let ret = unsafe { + rust_gstor_get(handle, key.as_ptr()) + }; + let c_str: &CStr = unsafe { CStr::from_ptr(ret) }; + let str_slice: &str = c_str.to_str().unwrap(); + let str_buf: String = str_slice.to_owned(); + if i % 10 == 0{ + println!("[INFO]: thread_num:{}, data number:{} key = {:?} value = {}", thread_num, i, key, str_buf); + } + /* 读取数据 */ + + //let ten_millis = time::Duration::from_millis(1000); + // let now = time::Instant::now(); + //thread::sleep(ten_millis); + } + println!("thread_num:{} done, deal with number:{}, Time spent:{}",thread_num,total,now.elapsed().as_secs()); +} \ No newline at end of file diff --git a/tools/rust-gstor/.gitignore b/tools/rust-gstor/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2c96eb1b6517f2617f9ddeae9f07f5fd7bd7ddef --- /dev/null +++ b/tools/rust-gstor/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock diff --git a/tools/rust-gstor/Cargo.toml b/tools/rust-gstor/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..772aed1eb0994e90d833ff8c91cbcc841f42fba6 --- /dev/null +++ b/tools/rust-gstor/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rust-gstor" +version = "0.1.0" +edition = "2021" +build = "build.rs" +links = "gstor" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libc = "0.2" +rand = "0.8.5" \ No newline at end of file diff --git a/tools/rust-gstor/build.rs b/tools/rust-gstor/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..deec37b99f3d69d484b0be26c67903cfbeb4fe4f --- /dev/null +++ b/tools/rust-gstor/build.rs @@ -0,0 +1,13 @@ +use std::process::Command; +use std::env; + +fn main() { + println!("pre build ..."); + let pwd = Command::new("pwd").output().expect("get pwd error"); + println!("[INFO]: {}", String::from_utf8_lossy(&pwd.stdout)); + let src_dir = env::var("GSTOR_SRC").unwrap(); + let binarylibs = env::var("BINARYLIBS").unwrap(); + Command::new(&format!("{}/build.sh", src_dir)).arg("-3rd").arg(binarylibs).arg("-m").arg("Debug").spawn().expect("build error"); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rustc-link-search=native={}", &format!("{}/output/lib", src_dir)); +} \ No newline at end of file diff --git a/tools/rust-gstor/src/main.rs b/tools/rust-gstor/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..c26f4764b2bcbe17005f0a672fb6d5cc14c40eec --- /dev/null +++ b/tools/rust-gstor/src/main.rs @@ -0,0 +1,161 @@ +extern crate libc; +use std::ffi::CString; +use std::ffi::CStr; +use libc::c_char; +use rand::distributions::{Alphanumeric,DistString}; +use std::{thread, time, env, process}; + +#[link(name = "gstor")] + +extern "C" { + fn gstor_startup(data_path: CString) -> libc::c_int; + fn gstor_set_param(name: *const c_char, value: *const c_char, data_path: *const c_char) -> libc::c_int; + fn gstor_alloc(handle: *const u64) -> libc::c_int; + fn gstor_open_table(handle: u64, table_name: CString) -> libc::c_int; + fn gstor_put(handle: u64, key: *const c_char, key_len: libc::c_uint, val: *const c_char, val_len: libc::c_uint) -> libc::c_int; + fn rust_gstor_get(handle: u64, key: *const c_char) -> *const c_char; + fn gstor_commit(handle: u64) -> libc::c_int; + fn gstor_shutdown(); +} + +fn main() { + + let args: Vec = env::args().collect(); + // println!("{:?}", args); + + if args.len() != 3 { + println!("Usage: cmd -D data_path"); + process::exit(-1); + } + + let op = args.get(1); + let param = args.get(2); + + if op.unwrap() != "-D" { + println!("Usage: cmd -D data_path"); + process::exit(-1); + } + + if param.unwrap().len() == 0 { + println!("param error"); + process::exit(-1); + } + let data_path = CString::new(String::from(param.unwrap())).unwrap(); + let handle = Box::::new(0); + + /* 初始化参数 */ + let value = CString::new("134217728").unwrap(); + let name = CString::new("DATA_BUFFER_SIZE").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("BUF_POOL_NUM").unwrap(); + let value = CString::new("1").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("LOG_BUFFER_SIZE").unwrap(); + let value = CString::new("16777216").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("LOG_BUFFER_COUNT").unwrap(); + let value = CString::new("4").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("SPACE_SIZE").unwrap(); + let value = CString::new("134217728").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + + let name = CString::new("LOG_LEVEL").unwrap(); + let value = CString::new("55").unwrap(); + let ret = unsafe { + gstor_set_param(name.as_ptr(), value.as_ptr(), data_path.as_ptr()) + }; + /* 初始化参数 */ + + /* 数据库启动 */ + let ret = unsafe { + gstor_startup(CString::new(String::from(param.unwrap())).unwrap()) + }; + /* 数据库启动 */ + + /* 申请handle */ + let ret = unsafe { + gstor_alloc(handle.as_ref()) + }; + /* 申请handle */ + + /* 打开表 */ + let ret = unsafe { + gstor_open_table(*handle, CString::new("SYS_KV").unwrap()) + }; + /* 打开表 */ + + +// /* 插入数据 */ +// let ret = unsafe { +// let key = CString::new("key1").unwrap(); +// let val = CString::new("val1").unwrap(); +// gstor_put(*handle, key.as_ptr(), 4, val.as_ptr(), 4) +// }; +// /* 插入数据 */ +// let key = CString::new("key1").unwrap(); +// /* 读取数据 */ +// let ret = unsafe { +// rust_gstor_get(*handle, key.as_ptr()) +// }; +// let c_str: &CStr = unsafe { CStr::from_ptr(ret) }; +// let str_slice: &str = c_str.to_str().unwrap(); +// let str_buf: String = str_slice.to_owned(); +// println!("[INFO]: key = {:?} value = {}", key, str_buf); +// /* 读取数据 */ + + while true { + let mut rng = rand::thread_rng(); + let random_code = Alphanumeric.sample_string(&mut rng, 32); + let key: String = "key-".to_string() + random_code.as_ref(); + let val: String = "val-".to_string() + random_code.as_ref(); + // println!("{} {}", key, val); + /* 插入数据 */ + let ret = unsafe { + let key = CString::new(key.clone()).unwrap(); + let val = CString::new(val).unwrap(); + gstor_put(*handle, key.as_ptr(), key.as_bytes().len().try_into().unwrap(), val.as_ptr(), val.as_bytes().len().try_into().unwrap()) + }; + /* 插入数据 */ + + /* 提交 */ + let ret = unsafe { + gstor_commit(*handle) + }; + /* 提交 */ + + /* 读取数据 */ + let key = CString::new(key).unwrap(); + let ret = unsafe { + rust_gstor_get(*handle, key.as_ptr()) + }; + let c_str: &CStr = unsafe { CStr::from_ptr(ret) }; + let str_slice: &str = c_str.to_str().unwrap(); + let str_buf: String = str_slice.to_owned(); + println!("[INFO]: key = {:?} value = {}", key, str_buf); + /* 读取数据 */ + + let ten_millis = time::Duration::from_millis(1000); + // let now = time::Instant::now(); + thread::sleep(ten_millis); + } + +/* 关闭数据库 */ + unsafe { gstor_shutdown() }; +/* 关闭数据库 */ + println!("end..."); +} diff --git a/tools/rust-memory-allocator/.gitignore b/tools/rust-memory-allocator/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..43e2c77ec1da2a24069234f6ccd34c0a0971eedf --- /dev/null +++ b/tools/rust-memory-allocator/.gitignore @@ -0,0 +1,4 @@ +.vscode/ +target/ +Cargo.lock +src/build/ diff --git a/tools/rust-memory-allocator/Cargo.toml b/tools/rust-memory-allocator/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..7ee3a6def4f86ebdbc20da27b36cc7a5c07e09a0 --- /dev/null +++ b/tools/rust-memory-allocator/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "c_call_rust" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib"] + +[dependencies] +lazy_static = "1.4.0" diff --git a/tools/rust-memory-allocator/README.md b/tools/rust-memory-allocator/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2b4da9d22fbbaa49f3a6320fb79eb1c927fe2419 --- /dev/null +++ b/tools/rust-memory-allocator/README.md @@ -0,0 +1,16 @@ +# 使用说明 + +## 编译 + +在本项目顶级目录下直接运行 + +```bash +make +``` + +即可。 + +## 生成文件说明 + +- `./target/debug/libc_call_rust.so`：Rust 编译出来的供 C 调用的动态库； +- `./src/build/main`：C 编译出的可执行程序。 diff --git a/tools/rust-memory-allocator/src/CMakeLists.txt b/tools/rust-memory-allocator/src/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e7dc743ae9d811230711fcd51357c3a3b771fd9 --- /dev/null +++ b/tools/rust-memory-allocator/src/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.0) + +project(main VERSION 1.0) + +link_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/../target/debug +) + +message(STATUS "[INFO]: ${CMAKE_CURRENT_SOURCE_DIR}") + +add_executable(main main.c) +target_link_libraries(main c_call_rust) diff --git a/tools/rust-memory-allocator/src/lib.rs b/tools/rust-memory-allocator/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..0d38e42c05b644a240d04fefc00cee66ee876e38 --- /dev/null +++ b/tools/rust-memory-allocator/src/lib.rs @@ -0,0 +1,179 @@ +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +// #[repr(C)] +// pub struct User { +// age: u8, +// name: String +// } + +// #[no_mangle] +// pub extern fn struct_to_c() -> User { +// // let user = User { name: CString::new("catalina").unwrap(), age: 25 }; +// let user = User { name: String::from("struct in rust"), age: 255 }; +// user +// } + +// #[no_mangle] +// pub extern fn say_hello(name: *const std::os::raw::c_char) { +// let c_str: &CStr = unsafe { +// CStr::from_ptr(name) +// }; +// println!("{}", "hello, ".to_string() + c_str.to_str().unwrap()); +// } + +#[no_mangle] +pub extern fn rust_alloc_init(block_initial_size: usize, page_cnt_per_block: usize, page_size: usize) -> i32 { + let mut alloc = ALLOC_8K.lock().unwrap(); + + if alloc.blocks.len() > 0 { + return 1; + } + // alloc.page_cnt_per_block = 200; + alloc.init(block_initial_size, page_cnt_per_block, page_size); + 0 +} + +#[no_mangle] +pub extern fn rust_get_page(index: usize) -> *const Page { + let mut alloc = ALLOC_8K.lock().unwrap(); + + let mut page = match alloc.get_page(index) { + Ok(page) => page, + Err(msg) => { + println!("{}", msg); + null_mut() + } + }; + println!("0: {:?}", page); + + page +} + + +use std::ptr::null_mut; +use std::vec::Vec; +use lazy_static::lazy_static; +use std::{sync::Mutex}; + +pub struct Page { + buf: Vec, +} + +impl Page { + fn new() -> Self { + Page { + buf: Vec::new(), + } + } + + fn init(&mut self, length: usize) { + self.buf.resize(length, 0u8); + } +} +struct Block { + elements: Vec, + used_count: u64, +} + +impl Block { + fn new() -> Self { + Block { elements: Vec::new(), used_count: 0 } + } + + fn init(&mut self, page_cnt: usize, page_size: usize) { + self.elements.reserve(page_cnt); + for _ in 0..page_cnt { + let mut p = Page::new(); + p.init(page_size); + self.elements.push(p); + } + } +} + + +pub struct MyAllocator { + blocks: Vec, + page_cnt_per_block: usize, +} + +impl MyAllocator { + pub fn new() -> Self { + MyAllocator{ + blocks: Vec::new(), + page_cnt_per_block: 0, + } + } + pub fn init(&mut self, block_initial_size: usize, page_cnt_per_block: usize, page_size: usize) { + self.blocks.reserve(block_initial_size); + println!("self.blocks = {}", self.blocks.len()); + + self.page_cnt_per_block = page_cnt_per_block; + for i in 0.. block_initial_size { + let mut b = Block::new(); + b.init(page_cnt_per_block, page_size); + self.blocks.push(b); + self.blocks[i].used_count = 0; + } + println!("self.blocks = {}", self.blocks.len()); + } +} + +// impl FromStr for Endian { +// type Err = String; + +// fn from_str(s: &str) -> Result { +// match s { +// "little" => Ok(Self::Little), +// "big" => Ok(Self::Big), +// _ => Err(format!(r#"unknown endian: "{}""#, s)), +// } +// } +// } + +type AllocatorErr = String; + +pub trait MemoryAllocator { + // add code here + // fn malloc() -> (AllocatorErr, *const Page, usize){ + + // } + + fn free(_idx: usize) -> Option { + None + } + + fn get_page(&self, idx: usize) -> Result<*const Page, AllocatorErr>; + + fn reszie_buffer(_new_size: usize) -> Option; +} + +impl MemoryAllocator for MyAllocator { + fn get_page(&self, idx: usize) -> Result<*const Page, AllocatorErr> { + let block_cap = self.blocks.capacity(); + if idx >= self.page_cnt_per_block * block_cap{ + Err(AllocatorErr::from("The index is not in range!")) + } else { + let mut block_index = idx / self.page_cnt_per_block; + let mut page_index = idx % self.page_cnt_per_block; + if page_index == 0 { + page_index = self.page_cnt_per_block - 1; + block_index = block_index - 1; + } else { + page_index = page_index - 1; + } + Ok(&self.blocks[block_index].elements[page_index]) + } + } + + fn reszie_buffer(_new_size: usize) -> Option { + None + } +} + +lazy_static! { + static ref ALLOC_8K: Mutex = { + let a = MyAllocator::new(); + Mutex::new(a) + }; +} \ No newline at end of file diff --git a/tools/rust-memory-allocator/src/main.c b/tools/rust-memory-allocator/src/main.c new file mode 100644 index 0000000000000000000000000000000000000000..055d35e0d2128078ba9c029ec9bd5d3ea6c3d2d9 --- /dev/null +++ b/tools/rust-memory-allocator/src/main.c @@ -0,0 +1,45 @@ +#include "main.h" +#include "string.h" + +#define P(...) printf("\033[1;32m[C]\033[0m " __VA_ARGS__) +#define E(...) printf("\033[1;32m[C]\033[1;31m ERROR.\033[0m " __VA_ARGS__) + +void PrintUser(const User *u) +{ + if (u == NULL) + { + E("user return null\n"); + return; + } + P("user[%p] .name = %s, .age = %d\n", u, u->name, u->age); + // P("user[%p]\n", u); +} + +int main() +{ + int result; + result = rust_alloc_init(10, 100, 8 * 1024); + P("result = %d\n", result); + User *user = NULL; + + user = (User *)rust_get_page(100); + PrintUser(user); + + strncpy(user->name, "ssss", 64); + user->age = 10; + P("User -----------\n"); + PrintUser(user); + + P("User2 -----------\n"); + User *user2 = (User *)rust_get_page(100); + PrintUser(user2); + + user = (User *)rust_get_page(100000); + PrintUser(user); + if (NULL == user) + { + return 1; + } + + return 0; +} diff --git a/tools/rust-memory-allocator/src/main.h b/tools/rust-memory-allocator/src/main.h new file mode 100644 index 0000000000000000000000000000000000000000..038bd1d98fa75e4f841f927e2eeb1b97351b3737 --- /dev/null +++ b/tools/rust-memory-allocator/src/main.h @@ -0,0 +1,10 @@ +#include + +typedef struct User +{ + unsigned int age; + char name[64]; +} User; + +int rust_alloc_init(int block_initial_size, int page_cnt_per_block, int page_size); +User *rust_get_page(int index); diff --git a/tools/shell_tools/git_add_tag.sh b/tools/shell_tools/git_add_tag.sh new file mode 100644 index 0000000000000000000000000000000000000000..5e66ecf027daebb348df2536325614010188061e --- /dev/null +++ b/tools/shell_tools/git_add_tag.sh @@ -0,0 +1,16 @@ + +version=$1 + +mkdir -p build/release +git tag -a $version -m "version" +cd ../../build/release && cmake -DCMAKE_BUILD_TYPE=Release -DUT=OFF -DENABLE_GCOV=OFF -DENABLE_DCC_LITE=ON \ + -DENABLE_MEMCHECK=OFF -DENABLE_EXPORT_API=ON -DSTATISTICS=OFF -DENABLE_BACKUP=OFF \ + -DENABLE_LIBAIO=OFF -DSSL=OFF -DCRYPTO=OFF -DBUILD_TESTS=OFF ../.. +cd ../.. +git add VERSION.h +git commit -m $version +git tag -f $version -m "version" +echo "add tag $version success" +#提交commit和tag +#git push +#git push origin $version #origin替换成对应远程仓库别名 \ No newline at end of file diff --git a/tools/sqlite3-api-test/CMakeLists.txt b/tools/sqlite3-api-test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b3142fccae74bc987c9c1f8d3d5c6d107bbbb7b --- /dev/null +++ b/tools/sqlite3-api-test/CMakeLists.txt @@ -0,0 +1,24 @@ +## patch +set(GSTOR_SRC_HOME ${INTARKDB_SRC_PATH}) +set(GSTOR_LIB_PATH ${INTARKDB_THRID_LIB_PATH}) +set(SQL_LIB_PATH ${INTARKDB_LIB_PATH}) +## include +include_directories(${GSTOR_SRC_HOME}) +include_directories(${INTARKDB_COMPUTE_SQL_INC_PATH}) + +## include 3rd +include_directories(${INTARKDB_SECUREC_INC_PATH}) +MESSAGE(STATUS "INTARKDB_SECUREC_INC_PATH: ${INTARKDB_SECUREC_INC_PATH}") + +link_directories(${GSTOR_LIB_PATH}) +link_directories(${SQL_LIB_PATH}) + +set(SQLITE3_API_TEST + ${CMAKE_CURRENT_SOURCE_DIR}/sqlite3_api_test.c +) + +add_executable(sqlite3_api_test ${SQLITE3_API_TEST}) +target_link_libraries(sqlite3_api_test sqlite3_api_wrapper) +target_include_directories(sqlite3_api_test PUBLIC ${GSTOR_LIB_PATH} ${SQL_LIB_PATH} ) +set_target_properties(sqlite3_api_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) +set_target_properties(sqlite3_api_test PROPERTIES LINKER_LANGUAGE "C") diff --git a/tools/sqlite3-api-test/sqlite3_api_test.c b/tools/sqlite3-api-test/sqlite3_api_test.c new file mode 100644 index 0000000000000000000000000000000000000000..c25aa953229a1ee3974b637b55eaaa968c9f97b2 --- /dev/null +++ b/tools/sqlite3-api-test/sqlite3_api_test.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include + +#include "interface/c/intarkdb_sql.h" +#include "interface/sqlite3_api_wrapper/include/sqlite3.h" + + +int NUM_Threads[20] = {5, 10, 15, 20, 25, 30}; +char result[100] = {'\0'}; // 初始化为全零 +int NUM_Insert = 10; + +int *thread_index[100]; +void *handle_multi[100]; + +int exec_handle(void *data, int argc, char **argv, char **colname) +{ + /* 计数器*/ + int i = *(int *)(data); + *(int *)(data) = i + 1; + + /* 取出结果*/ + printf("[%s] is [%s], [%s] is [%s]...\n", colname[0], argv[0], colname[1], argv[1]); + return 0; +} + +int main(int argc, char *argv[]) +{ + sqlite3 *db; + int h = 0; + int rc; + + float time_use = 0; + float task_tps = 0; + struct timeval start; + struct timeval end; + const char *data = "Callback function called"; + + char str[100]; + char *err_msg = NULL; + + rc = sqlite3_open("sqlites.db", &db); + if (rc) + { + printf("Can't open database: %s\n", "sqlites.db"); + exit(0); + } + printf("open database success\n"); + /********************第二步，创建数据库表*************************/ + char *sql_create_table; + char buf[120]; + + sql_create_table = "CREATE TABLE PLC_DATA(ID INTEGER, ACCTID INTEGER, CHGAMT VARCHAR(20), PRIMARY KEY (ID,ACCTID));"; + + rc = sqlite3_exec(db, sql_create_table, NULL, 0, &err_msg); + if (rc != SQLITE_OK) + { + fprintf(stdout, "SQL error: %s\n", err_msg); + } else { + fprintf(stdout, "Table created successfully\n"); + } + + sql_create_table = "INSERT INTO PLC_DATA(ID,ACCTID,CHGAMT) VALUES (1,11,'AAA'), (2,22,'BBB');"; + rc = sqlite3_exec(db, sql_create_table, NULL, 0, &err_msg); + if (rc != SQLITE_OK){ + printf("Can't insert data \n"); + } else { + printf("insert data success\n"); + } + + sql_create_table = "SELECT ID,ACCTID,CHGAMT FROM PLC_DATA;"; + rc = sqlite3_exec(db, sql_create_table, exec_handle, &data, &err_msg); + if (rc != SQLITE_OK){ + printf("Can't select data \n"); + } else { + printf("select data success\n"); + } + + + sqlite3_stmt *pstmt; + const char *sql = "INSERT INTO PLC_DATA(ID,ACCTID,CHGAMT) VALUES(?,?,?);"; + int nRet = sqlite3_prepare_v2(db, sql, strlen(sql), &pstmt, NULL); + rc = sqlite3_bind_int(pstmt, 1, 3); + printf("sqlite3_bind_int rc = %d \n", rc); + rc = sqlite3_bind_int(pstmt, 2, 33); + printf("sqlite3_bind_int rc = %d \n", rc); + rc = sqlite3_bind_text(pstmt, 3, "Jonn", strlen("Jonn"), NULL); + printf("sqlite3_bind_text rc = %d \n", rc); + + sqlite3_step(pstmt); + sqlite3_reset(pstmt); + + sql_create_table = "SELECT ID,ACCTID,CHGAMT FROM PLC_DATA;"; + nRet = sqlite3_prepare_v2(db, sql_create_table, strlen(sql_create_table), &pstmt, NULL); + + while (SQLITE_ROW == sqlite3_step(pstmt)) { + printf("ID = %s\n", (char *)sqlite3_column_text(pstmt, 0)); + printf("ACCTID = %s\n", (char *)sqlite3_column_text(pstmt, 1)); + printf("CHGAMT = %s\n", (char *)sqlite3_column_text(pstmt, 2)); + } + sqlite3_reset(pstmt); + + + sql_create_table = "SELECT ID,ACCTID,CHGAMT FROM PLC_DATA;"; + rc = sqlite3_exec(db, sql_create_table, exec_handle, &data, &err_msg); + if (rc != SQLITE_OK){ + printf("Can't select data \n"); + } else { + printf("select data success\n"); + } + sqlite3_close(db); + printf("sqlite3_close success\n"); + + return 0; +} \ No newline at end of file diff --git a/tools/wget-log b/tools/wget-log new file mode 100644 index 0000000000000000000000000000000000000000..8e8d3b8854639faf75aecd59a0bb47fd520cee53 --- /dev/null +++ b/tools/wget-log @@ -0,0 +1,6 @@ +--2024-02-22 11:24:42-- https://objects.githubusercontent.com/github-production-release-asset-2e65be/696192900/87a5c612-6762-416d-8e6d-5e990139f1f4?X-Amz-Algorithm=AWS4-HMAC-SHA256 +Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.110.133 +Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.110.133|:443... connected. +HTTP request sent, awaiting response... 401 Unauthorized + +Username/Password Authentication Failed.