diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml index c2ae1ffca2c36354498da88df08ae4f4539416f1..f6f45dccae023bacf8406144c159ee099b8897ca 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml @@ -52,6 +52,7 @@ along with this program. If not, see .--> + @@ -104,6 +105,7 @@ along with this program. If not, see .--> a.module_id as moduleId, a.data_count as dataCount, a.db_type as dbType, + a.database_id as databaseId, b.id as fieldId, b.name as fieldName, b.label as fieldLabel, @@ -203,6 +205,7 @@ along with this program. If not, see .--> expire_unit = #{expireUnit}, is_active = #{isActive}, db_type = #{dbType}, + database_id = #{databaseId}, xml = #{xml,typeHandler=CompressHandler} where id = #{id} @@ -238,6 +241,7 @@ along with this program. If not, see .--> expire_unit, xml, db_type, + database_id, is_active) values (#{id}, #{name}, @@ -250,6 +254,7 @@ along with this program. If not, see .--> #{expireUnit}, #{xml,typeHandler=CompressHandler}, #{dbType}, + #{databaseId}, #{isActive}) diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.java b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..5ec0cd2d26562f3a8a3d039dc6426ac21425f686 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.dao.mapper; + +import neatlogic.framework.datawarehouse.dto.DatabaseVo; + +import java.util.List; + +public interface DatabaseMapper { + + DatabaseVo getDataBaseById(Long id); + + int checkDatabaseNameIsRepeat(DatabaseVo dataBaseVo); + + int getDataBaseCount(DatabaseVo DataBaseVo); + + List getDataBaseList(DatabaseVo DataBaseVo); + + int insertDataBase(DatabaseVo DataBaseVo); + + int deleteDataBaseById(Long id); +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.xml b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.xml new file mode 100644 index 0000000000000000000000000000000000000000..1c7052e57038e548b42257c993db16de1b8dbb7b --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.xml @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + INSERT INTO `database` (`id`, `name`, `type`, `config`, `file_id_list`) + VALUES (#{id}, #{name}, #{type}, #{configStr, typeHandler=CompressHandler}, #{fileIdListStr}) + ON DUPLICATE KEY UPDATE + `name` = #{name}, + `type` = #{type}, + `config` = #{configStr, typeHandler=CompressHandler}, + `file_id_list` = #{fileIdListStr} + + + + DELETE FROM `database` WHERE `id` = #{value} + + \ No newline at end of file diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java index 085b83b26a3299260deb2ff4cc304e19216e20a1..d7f9c9b3176acd78b7be1ffa7dceffbc1b2e2a50 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java +++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java @@ -76,6 +76,8 @@ public class DataSourceVo extends BasePageVo { private List fieldList = new ArrayList<>();//需要默认值为空数组，避免空指针异常 @EntityField(name = "nfdd.datasourcevo.entityfield.name.connectionid", type = ApiParamType.LONG) private Long connectionId; + @EntityField(name = "nfdd.datasourcevo.entityfield.name.databaseid", type = ApiParamType.LONG) + private Long databaseId; @JSONField(serialize = false)//数据列表 private List dataList; @EntityField(name = "nfdd.datasourcevo.entityfield.name.paramlist", type = ApiParamType.JSONARRAY) @@ -249,6 +251,14 @@ public class DataSourceVo extends BasePageVo { this.connectionId = connectionId; } + public Long getDatabaseId() { + return databaseId; + } + + public void setDatabaseId(Long databaseId) { + this.databaseId = databaseId; + } + public void setQueryTimeout(Integer queryTimeout) { this.queryTimeout = queryTimeout; } diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java new file mode 100644 index 0000000000000000000000000000000000000000..71a736110e785dbab932acc7d75e016eae55a213 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.dto; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.annotation.JSONField; +import neatlogic.framework.common.dto.BasePageVo; +import neatlogic.framework.file.dto.FileVo; + +import java.util.List; + +public class DatabaseVo extends BasePageVo { + private Long id; + private String name; + private String type; + private JSONObject config; + private List fileIdList; + private List fileList; + @JSONField(serialize = false) + private String configStr; + @JSONField(serialize = false) + private String fileIdListStr; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public JSONObject getConfig() { + if (config == null && configStr != null) { + try { + config = JSONObject.parseObject(configStr); + } catch (Exception ignored) { + + } + } + return this.config; + } + + public void setConfig(JSONObject config) { + this.config = config; + } + + public String getConfigStr() { + if (config != null) { + configStr = config.toJSONString(); + } + return configStr; + } + + public void setConfigStr(String configStr) { + this.configStr = configStr; + } + + public List getFileIdList() { + if (fileIdList == null && fileIdListStr != null) { + try { + fileIdList = JSONArray.parseArray(fileIdListStr, Long.class); + } catch (Exception ignored) { + + } + } + return fileIdList; + } + + public void setFileIdList(List fileIdList) { + this.fileIdList = fileIdList; + } + + public List getFileList() { + return fileList; + } + + public void setFileList(List fileList) { + this.fileList = fileList; + } + + public String getFileIdListStr() { + if (fileIdListStr == null && fileIdList != null) { + fileIdListStr = JSONArray.toJSONString(fileIdList); + } + return fileIdListStr; + } + + public void setFileIdListStr(String fileIdListStr) { + this.fileIdListStr = fileIdListStr; + } +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java new file mode 100644 index 0000000000000000000000000000000000000000..eaca60a5851483a9a15c43ece7660e3975e162e8 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.exceptions; + +import neatlogic.framework.exception.core.ApiRuntimeException; +import neatlogic.framework.util.$; + +public class DatabaseConnectionFailedException extends ApiRuntimeException { + + public enum Type { + CONFIG_IS_EMPTY, FILE_ID_LIST_IS_EMPTY + } + public DatabaseConnectionFailedException(Type type, String name) { + super(getMessage(type, name)); + } + + private static String getMessage(Type type, String name) { + if (type == Type.FILE_ID_LIST_IS_EMPTY) { + return $.t("nfde.databaseconnectionfailedexception.fileidlistisempty", name); + } else { + return $.t("{0}nfde.databaseconnectionfailedexception.configisempty", name); + } + } +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNameRepeatException.java b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNameRepeatException.java new file mode 100644 index 0000000000000000000000000000000000000000..d82f5afe28bc9e6c172a67fb35e53e477b905339 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNameRepeatException.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.exceptions; + +import neatlogic.framework.exception.core.ApiRuntimeException; + +public class DatabaseNameRepeatException extends ApiRuntimeException { + public DatabaseNameRepeatException(String name) { + super("nfde.databasenamerepeatexception.databasenamerepeatexception", name); + } +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java new file mode 100644 index 0000000000000000000000000000000000000000..d5c412c4c0e936ca7a38d960a24c64d2694f5453 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.exceptions; + +import neatlogic.framework.exception.core.ApiRuntimeException; + +public class DatabaseNotFoundException extends ApiRuntimeException { + + public DatabaseNotFoundException(Long id) { + super("nfde.databasenotfoundexception.databasenotfoundexception", id); + } +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java new file mode 100644 index 0000000000000000000000000000000000000000..0a91a6d66d2f94aa41edd01c83af0163830d8d3a --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.service; + +import java.sql.Connection; +import java.sql.SQLException; + +public interface DatabaseService { + + Connection getConnectionByDatabaseId(Long databaseId) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException; +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..d171a6c6c8e4c76aade599e76acd75af2d0fff0f --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.service; + +import com.alibaba.fastjson.JSONObject; +import neatlogic.framework.common.util.FileUtil; +import neatlogic.framework.datawarehouse.dao.mapper.DatabaseMapper; +import neatlogic.framework.datawarehouse.dto.DatabaseVo; +import neatlogic.framework.datawarehouse.exceptions.DatabaseConnectionFailedException; +import neatlogic.framework.datawarehouse.exceptions.DatabaseNotFoundException; +import neatlogic.framework.exception.file.FileNotFoundException; +import neatlogic.framework.file.dao.mapper.FileMapper; +import neatlogic.framework.file.dto.FileVo; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; + +@Service +public class DatabaseServiceImpl implements DatabaseService { + + static Logger logger = LoggerFactory.getLogger(DatabaseServiceImpl.class); + + @Resource + private DatabaseMapper databaseMapper; + + @Resource + private FileMapper fileMapper; + + + @Override + public Connection getConnectionByDatabaseId(Long databaseId) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException { + DatabaseVo dataBaseVo = databaseMapper.getDataBaseById(databaseId); + if (dataBaseVo == null) { + throw new DatabaseNotFoundException(databaseId); + } + List fileIdList = dataBaseVo.getFileIdList(); + if (CollectionUtils.isNotEmpty(fileIdList)) { + URL[] urls = new URL[fileIdList.size()]; + for (int i = 0; i < fileIdList.size(); i++) { + Long fileId = fileIdList.get(i); + FileVo fileVo = fileMapper.getFileById(fileId); + if (fileVo == null) { + throw new FileNotFoundException(fileId); + } + try (InputStream is = FileUtil.getData(fileVo.getPath())) { + String fileName = fileVo.getName(); + String suffix = ".jar"; + String prefix = fileName.substring(0, fileName.length() - suffix.length()); + File file = new File(prefix + "-" + fileVo.getId() + suffix); + if (!file.exists()) { + Path path = Paths.get(file.toURI()); + Files.copy(is, path); + } + urls[i] = file.toURI().toURL(); + }catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + JSONObject config = dataBaseVo.getConfig(); + if (MapUtils.isNotEmpty(config)) { + String user = config.getString("user"); + String password = config.getString("password"); + String url = config.getString("url"); + String driverClassName = config.getString("driverClassName"); + Properties props = new Properties(); + if (StringUtils.isNoneBlank(user)) { + props.put("user", user); + } + if (StringUtils.isNotBlank(password)) { + props.put("password", password); + } + URLClassLoader loader = new URLClassLoader(urls, null); + Class clazz = loader.loadClass(driverClassName); + Driver driver = ((Driver) clazz.newInstance()); + return driver.connect(url, props); + } else { + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.CONFIG_IS_EMPTY, dataBaseVo.getName()); + } + } else { + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, dataBaseVo.getName()); + } + } +} diff --git a/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..72baee90da03a815214df12a8a68a07d73d9e631 --- /dev/null +++ b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.module.framework.datawarehouse.handler; + +import neatlogic.framework.datawarehouse.core.DataSourceServiceHandlerBase; +import neatlogic.framework.datawarehouse.dao.mapper.DataWarehouseDataSourceMapper; +import neatlogic.framework.datawarehouse.dto.*; +import neatlogic.framework.datawarehouse.exceptions.ReportDataSourceSyncException; +import neatlogic.framework.datawarehouse.service.DatabaseService; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.DocumentException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.net.URLClassLoader; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { + static Logger logger = LoggerFactory.getLogger(JDBCDataSourceHandler.class); + int FETCH_SIZE = 1000; + @Resource + private DataWarehouseDataSourceMapper dataSourceMapper; + @Resource + private DatabaseService databaseService; + + @Override + public String getHandler() { + return "jdbc"; + } + + @Override + public void mySyncData(DataSourceVo dataSourceVo, DataSourceAuditVo reportDataSourceAuditVo) { + Connection conn = null; + PreparedStatement queryStatement = null; + ResultSet resultSet = null; + + try { + List selectList = getSqlFromDataSource(dataSourceVo); + conn = getConnection(dataSourceVo); + for (SelectVo select : selectList) { + String sqlText = select.getSql(); + queryStatement = conn.prepareStatement(sqlText, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + queryStatement.setFetchSize(FETCH_SIZE); + queryStatement.setFetchDirection(ResultSet.FETCH_FORWARD); + if (dataSourceVo.getQueryTimeout() != null && dataSourceVo.getQueryTimeout() > 0) { + queryStatement.setQueryTimeout(dataSourceVo.getQueryTimeout()); + } + + if (CollectionUtils.isNotEmpty(select.getParamList())) { + for (int p = 0; p < select.getParamList().size(); p++) { + if (select.getParamList().get(p) instanceof String || select.getParamList().get(p) instanceof Number) { + queryStatement.setObject(p + 1, select.getParamList().get(p)); + } else if (select.getParamList().get(p) instanceof String[]) { + // 数组参数有待处理 + queryStatement.setObject(p + 1, ((String[]) select.getParamList().get(p))[0]); + } else if (select.getParamList().get(p) instanceof Number[]) { + // 数组参数有待处理 + queryStatement.setObject(p + 1, ((Number[]) select.getParamList().get(p))[0]); + } + } + } + /* + 新增日志记录 + */ + if (logger.isInfoEnabled()) { + logger.info("REPORT RUN SQL::" + sqlText); + } + + resultSet = queryStatement.executeQuery(); + + ResultSetMetaData metaData = resultSet.getMetaData(); + Map fieldMap = new HashMap<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + fieldMap.put(metaData.getColumnLabel(i).toLowerCase(), i); + } + + while (resultSet.next()) { + DataSourceDataVo reportDataSourceDataVo = new DataSourceDataVo(dataSourceVo.getId()); + reportDataSourceDataVo.setExpireMinute(dataSourceVo.getExpireMinute()); + List aggregateFieldList = new ArrayList<>(); + List keyFieldList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(dataSourceVo.getParamList())) { + for (DataSourceParamVo paramVo : dataSourceVo.getParamList()) { + if (fieldMap.containsKey(paramVo.getName().toLowerCase())) { + Object v = resultSet.getObject(fieldMap.get(paramVo.getName().toLowerCase())); + Long lv = null; + try { + lv = (Long) v; + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + if (lv != null) { + if (paramVo.getCurrentValue() == null) { + paramVo.setCurrentValue(lv); + } else if (lv > paramVo.getCurrentValue()) { + paramVo.setCurrentValue(lv); + } + } + } + } + } + for (DataSourceFieldVo fieldVo : dataSourceVo.getFieldList()) { + if (fieldMap.containsKey(fieldVo.getName().toLowerCase())) { + Object v = resultSet.getObject(fieldMap.get(fieldVo.getName().toLowerCase())); + fieldVo.setValue(v != null ? v : "");//把所有的null值都转成空字符串 + } + reportDataSourceDataVo.addField(fieldVo); + if (StringUtils.isNotBlank(fieldVo.getAggregate())) { + aggregateFieldList.add(fieldVo); + } + if (fieldVo.getIsKey().equals(1)) { + keyFieldList.add(fieldVo); + } + } + aggregateAndInsertData(aggregateFieldList, keyFieldList, reportDataSourceDataVo, reportDataSourceAuditVo); + } + if (CollectionUtils.isNotEmpty(dataSourceVo.getParamList())) { + for (DataSourceParamVo param : dataSourceVo.getParamList()) { + dataSourceMapper.updateDataSourceParamCurrentValue(param); + } + } + } + } catch (SQLException | DocumentException | InstantiationException | IllegalAccessException | + ClassNotFoundException e) { + logger.error(e.getMessage(), e); + reportDataSourceAuditVo.setError(e.getMessage()); + throw new ReportDataSourceSyncException(dataSourceVo, e); + } finally { + try { + if (resultSet != null) { + resultSet.close(); + } + if (queryStatement != null) { + queryStatement.close(); + } + if (conn != null) { + ClassLoader classLoader = conn.getClass().getClassLoader(); + conn.close(); + if (classLoader instanceof URLClassLoader) { + ((URLClassLoader) classLoader).close(); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + + private Connection getConnection(DataSourceVo dataSourceVo) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException { + return databaseService.getConnectionByDatabaseId(dataSourceVo.getDatabaseId()); + } +} diff --git a/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql new file mode 100644 index 0000000000000000000000000000000000000000..f3424461b53e7c09dd53d95d280ffc2d2bfedfef --- /dev/null +++ b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql @@ -0,0 +1,10 @@ +CREATE TABLE `database` ( + `id` bigint NOT NULL COMMENT '主键ID', + `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL COMMENT '名称', + `type` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '类型', + `config` text COLLATE utf8mb4_general_ci NOT NULL COMMENT '配置信息', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +ALTER TABLE `datawarehouse_datasource` + ADD COLUMN `database_id` BIGINT NULL COMMENT '数据源' AFTER `db_type`; \ No newline at end of file diff --git a/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json new file mode 100644 index 0000000000000000000000000000000000000000..6ac9cae5001cf39c2f456b519236ffb36ffa9c12 --- /dev/null +++ b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json @@ -0,0 +1,12 @@ +{ + "content": [ + { + "type": "功能", + "detail": [ + { + "msg": "1.数据仓库支持配置第三方数据库获取数据" + } + ] + } + ] +} diff --git a/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql b/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql index 8987dc4f70e658a2e938c36893bf1746636e87b3..2881a7a9ac89c38babc56f203c7f4859f59d49a2 100644 --- a/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql +++ b/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql @@ -112,6 +112,17 @@ CREATE TABLE IF NOT EXISTS `database_view_info` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '视图信息表'; +-- ---------------------------- +-- Table structure for database +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `database` ( + `id` bigint NOT NULL COMMENT '主键ID', + `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL COMMENT '名称', + `type` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '类型', + `config` text COLLATE utf8mb4_general_ci NOT NULL COMMENT '配置信息', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + -- ---------------------------- -- Table structure for datawarehouse_datasource -- ---------------------------- @@ -131,6 +142,7 @@ CREATE TABLE IF NOT EXISTS `datawarehouse_datasource` `data_count` int NULL DEFAULT NULL COMMENT '数据量', `expire_unit` enum ('minute','hour','day','month','year') CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '过期单位', `db_type` enum ('mysql','mongodb') CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT 'mysql' COMMENT '数据库类型', + `database_id` bigint NULL DEFAULT NULL COMMENT '数据库ID', `last_fire_time` timestamp(3) NULL DEFAULT NULL COMMENT '最后一次激活时间', `last_finish_time` timestamp(3) NULL DEFAULT NULL COMMENT '最后一次完成时间', `next_fire_time` timestamp(3) NULL DEFAULT NULL COMMENT '下一次激活时间',