From cd7f32a49f4448aaf108d3f4fcb2734094fd3ade Mon Sep 17 00:00:00 2001
From: "1437892690@qq.com" <1437892690@qq.com>
Date: Mon, 29 Dec 2025 18:28:53 +0800
Subject: [PATCH] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20=E6=95=B0=E6=8D=AE?=
=?UTF-8?q?=E4=BB=93=E5=BA=93=E6=95=B0=E6=8D=AE=E6=BA=90=E5=A2=9E=E5=8A=A0?=
=?UTF-8?q?=E6=94=AF=E6=8C=81=E9=9B=86=E6=88=90=E9=85=8D=E7=BD=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
关联 #[1583948054888451]数据仓库数据源增加支持集成配置 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1583948054888451
---
.../mapper/DataWarehouseDataSourceMapper.xml | 5 +
.../datawarehouse/dto/DataSourceVo.java | 10 ++
.../handler/IntegrationDataSourceHandler.java | 139 ++++++++++++++++++
.../DataWareHouseIntegrationHandler.java | 122 +++++++++++++++
.../handler/FrameworkRequestFrom.java | 2 +-
.../changelog/2025-12-29/neatlogic_tenant.sql | 12 ++
6 files changed, 289 insertions(+), 1 deletion(-)
create mode 100644 src/main/java/neatlogic/module/framework/datawarehouse/handler/IntegrationDataSourceHandler.java
create mode 100644 src/main/java/neatlogic/module/framework/datawarehouse/integration/handler/DataWareHouseIntegrationHandler.java
create mode 100644 src/main/resources/neatlogic/resources/framework/changelog/2025-12-29/neatlogic_tenant.sql
diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml
index 8d661b2db..ac804c755 100644
--- a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml
+++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml
@@ -40,6 +40,7 @@
+
@@ -92,6 +93,7 @@
a.module_id as moduleId,
a.data_count as dataCount,
a.db_type as dbType,
+ a.integration_uuid as integrationUuid,
a.database_id as databaseId,
b.id as fieldId,
b.name as fieldName,
@@ -193,6 +195,7 @@
is_active = #{isActive},
db_type = #{dbType},
database_id = #{databaseId},
+ integration_uuid = #{integrationUuid},
xml = #{xml,typeHandler=CompressHandler}
where id = #{id}
@@ -229,6 +232,7 @@
xml,
db_type,
database_id,
+ integration_uuid,
is_active)
values (#{id},
#{name},
@@ -242,6 +246,7 @@
#{xml,typeHandler=CompressHandler},
#{dbType},
#{databaseId},
+ #{integrationUuid},
#{isActive})
diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java
index b97836384..57e8843c2 100644
--- a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java
+++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java
@@ -75,6 +75,8 @@ public class DataSourceVo extends BasePageVo {
private Long connectionId;
@EntityField(name = "nfdd.datasourcevo.entityfield.name.databaseid", type = ApiParamType.LONG)
private Long databaseId;
+ @EntityField(name = "term.framework.integrationuuid", type = ApiParamType.STRING)
+ private String integrationUuid;
@JSONField(serialize = false)//数据列表
private List dataList;
@EntityField(name = "nfdd.datasourcevo.entityfield.name.paramlist", type = ApiParamType.JSONARRAY)
@@ -256,6 +258,14 @@ public class DataSourceVo extends BasePageVo {
this.databaseId = databaseId;
}
+ public String getIntegrationUuid() {
+ return integrationUuid;
+ }
+
+ public void setIntegrationUuid(String integrationUuid) {
+ this.integrationUuid = integrationUuid;
+ }
+
public void setQueryTimeout(Integer queryTimeout) {
this.queryTimeout = queryTimeout;
}
diff --git a/src/main/java/neatlogic/module/framework/datawarehouse/handler/IntegrationDataSourceHandler.java b/src/main/java/neatlogic/module/framework/datawarehouse/handler/IntegrationDataSourceHandler.java
new file mode 100644
index 000000000..ed3415c04
--- /dev/null
+++ b/src/main/java/neatlogic/module/framework/datawarehouse/handler/IntegrationDataSourceHandler.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2025 TechSure Co., Ltd. All Rights Reserved.
+ * This file is part of the NeatLogic software.
+ * Licensed under the NeatLogic Sustainable Use License (NSUL), Version 4.x – 2025.
+ * You may use this file only in compliance with the License.
+ * See the LICENSE file distributed with this work for the full license text.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+package neatlogic.module.framework.datawarehouse.handler;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import neatlogic.framework.common.dto.BasePageVo;
+import neatlogic.framework.datawarehouse.core.DataSourceServiceHandlerBase;
+import neatlogic.framework.datawarehouse.dto.*;
+import neatlogic.framework.datawarehouse.exceptions.ReportDataSourceSyncException;
+import neatlogic.framework.exception.integration.IntegrationHandlerNotFoundException;
+import neatlogic.framework.exception.integration.IntegrationNotFoundException;
+import neatlogic.framework.integration.core.IIntegrationHandler;
+import neatlogic.framework.integration.core.IntegrationHandlerFactory;
+import neatlogic.framework.integration.dao.mapper.IntegrationMapper;
+import neatlogic.framework.integration.dto.IntegrationResultVo;
+import neatlogic.framework.integration.dto.IntegrationVo;
+import neatlogic.framework.matrix.exception.MatrixExternalAccessException;
+import neatlogic.module.framework.integration.handler.FrameworkRequestFrom;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.*;
+
+@Component
+public class IntegrationDataSourceHandler extends DataSourceServiceHandlerBase {
+
+ static Logger logger = LoggerFactory.getLogger(JDBCDataSourceHandler.class);
+ int PAGE_SIZE = 10;
+ @Resource
+ private IntegrationMapper integrationMapper;
+
+ @Override
+ public String getHandler() {
+ return "integration";
+ }
+
+ @Override
+ public void mySyncData(DataSourceVo dataSourceVo, DataSourceAuditVo reportDataSourceAuditVo) {
+ try {
+ JSONObject inputParamObj = new JSONObject();
+ List selectList = getSqlFromDataSource(dataSourceVo);
+ if (CollectionUtils.isNotEmpty(selectList)) {
+ String inputParamStr = selectList.get(0).getSql();
+ if (StringUtils.isNotBlank(inputParamStr)) {
+ try {
+ inputParamObj = JSON.parseObject(inputParamStr);
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+ }
+ String integrationUuid = dataSourceVo.getIntegrationUuid();
+ IntegrationVo integrationVo = integrationMapper.getIntegrationByUuid(integrationUuid);
+ if (integrationVo == null) {
+ throw new IntegrationNotFoundException(integrationUuid);
+ }
+ IIntegrationHandler handler = IntegrationHandlerFactory.getHandler(integrationVo.getHandler());
+ if (handler == null) {
+ throw new IntegrationHandlerNotFoundException(integrationVo.getHandler());
+ }
+ JSONObject paramObj = integrationVo.getParamObj();
+ paramObj.putAll(inputParamObj);
+ paramObj.put("currentPage", 1);
+ paramObj.put("pageSize", PAGE_SIZE);
+ IntegrationResultVo resultVo = handler.sendRequest(integrationVo, FrameworkRequestFrom.DATAWAREHOUSE);
+ if (StringUtils.isNotBlank(resultVo.getError())) {
+ logger.error(resultVo.getError());
+ throw new MatrixExternalAccessException(integrationVo.getName());
+ }
+ handler.validate(resultVo);
+ JSONObject transformedResult = JSONObject.parseObject(resultVo.getTransformedResult());
+ Integer rowNum = transformedResult.getInteger("rowNum");
+ JSONArray tbodyList = transformedResult.getJSONArray("tbodyList");
+ if (CollectionUtils.isNotEmpty(tbodyList)) {
+ saveTbodyList(tbodyList, dataSourceVo, reportDataSourceAuditVo);
+ }
+ if (rowNum != null && rowNum > 0) {
+ BasePageVo basePageVo = new BasePageVo();
+ basePageVo.setPageSize(PAGE_SIZE);
+ basePageVo.setRowNum(rowNum);
+ Integer pageCount = basePageVo.getPageCount();
+ for (int currentPage = 2; currentPage <= pageCount; currentPage++) {
+ paramObj.put("currentPage", currentPage);
+ resultVo = handler.sendRequest(integrationVo, FrameworkRequestFrom.DATAWAREHOUSE);
+ if (StringUtils.isNotBlank(resultVo.getError())) {
+ logger.error(resultVo.getError());
+ throw new MatrixExternalAccessException(integrationVo.getName());
+ }
+ handler.validate(resultVo);
+ transformedResult = JSONObject.parseObject(resultVo.getTransformedResult());
+ tbodyList = transformedResult.getJSONArray("tbodyList");
+ if (CollectionUtils.isNotEmpty(tbodyList)) {
+ saveTbodyList(tbodyList, dataSourceVo, reportDataSourceAuditVo);
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ reportDataSourceAuditVo.setError(e.getMessage());
+ throw new ReportDataSourceSyncException(dataSourceVo, e);
+ }
+ }
+
+ private void saveTbodyList(JSONArray tbodyList, DataSourceVo dataSourceVo, DataSourceAuditVo reportDataSourceAuditVo) {
+ for (int i = 0; i < tbodyList.size(); i++) {
+ JSONObject tbodyObj = tbodyList.getJSONObject(i);
+ DataSourceDataVo reportDataSourceDataVo = new DataSourceDataVo(dataSourceVo.getId());
+ reportDataSourceDataVo.setExpireMinute(dataSourceVo.getExpireMinute());
+ List aggregateFieldList = new ArrayList<>();
+ List keyFieldList = new ArrayList<>();
+ for (DataSourceFieldVo fieldVo : dataSourceVo.getFieldList()) {
+ Object v = tbodyObj.get(fieldVo.getName());
+ fieldVo.setValue(v != null ? v : "");//把所有的null值都转成空字符串
+ reportDataSourceDataVo.addField(fieldVo);
+ if (StringUtils.isNotBlank(fieldVo.getAggregate())) {
+ aggregateFieldList.add(fieldVo);
+ }
+ if (fieldVo.getIsKey().equals(1)) {
+ keyFieldList.add(fieldVo);
+ }
+ }
+ aggregateAndInsertData(aggregateFieldList, keyFieldList, reportDataSourceDataVo, reportDataSourceAuditVo);
+ }
+ }
+}
diff --git a/src/main/java/neatlogic/module/framework/datawarehouse/integration/handler/DataWareHouseIntegrationHandler.java b/src/main/java/neatlogic/module/framework/datawarehouse/integration/handler/DataWareHouseIntegrationHandler.java
new file mode 100644
index 000000000..633c2dd0e
--- /dev/null
+++ b/src/main/java/neatlogic/module/framework/datawarehouse/integration/handler/DataWareHouseIntegrationHandler.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2025 TechSure Co., Ltd. All Rights Reserved.
+ * This file is part of the NeatLogic software.
+ * Licensed under the NeatLogic Sustainable Use License (NSUL), Version 4.x – 2025.
+ * You may use this file only in compliance with the License.
+ * See the LICENSE file distributed with this work for the full license text.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+package neatlogic.module.framework.datawarehouse.integration.handler;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import neatlogic.framework.common.constvalue.ApiParamType;
+import neatlogic.framework.exception.core.ApiRuntimeException;
+import neatlogic.framework.integration.core.IntegrationHandlerBase;
+import neatlogic.framework.integration.dto.IntegrationResultVo;
+import neatlogic.framework.integration.dto.IntegrationVo;
+import neatlogic.framework.integration.dto.PatternVo;
+import neatlogic.framework.matrix.exception.MatrixExternalDataIsNotJsonException;
+import neatlogic.framework.matrix.exception.MatrixExternalDataLostKeyOrTitleInTheadListException;
+import neatlogic.framework.matrix.exception.MatrixExternalDataNotFormattedException;
+import neatlogic.framework.matrix.exception.MatrixExternalNoReturnException;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+@Component
+public class DataWareHouseIntegrationHandler extends IntegrationHandlerBase {
+
+ @Override
+ public String getName() {
+ return "数据仓库数据规范";
+ }
+
+ @Override
+ public Integer hasPattern() {
+ return 1;
+ }
+
+ @Override
+ public List getInputPattern() {
+ List jsonList = new ArrayList<>();
+ jsonList.add(new PatternVo("keyword", "input", ApiParamType.STRING, 0, "关键字"));
+ jsonList.add(new PatternVo("currentPage", "input", ApiParamType.INTEGER, 0, "当前页"));
+ jsonList.add(new PatternVo("pageSize", "input", ApiParamType.INTEGER, 0, "每页大小"));
+ jsonList.add(new PatternVo("needPage", "input", ApiParamType.BOOLEAN, 0, "是否分页"));
+ return jsonList;
+ }
+
+ @Override
+ public List getOutputPattern() {
+ List jsonList = new ArrayList<>();
+ PatternVo theadList = new PatternVo("theadList", "output", ApiParamType.JSONARRAY, 1, "表头列表");
+ theadList.addChild(new PatternVo("key", "output", ApiParamType.STRING, 1, "表头键值"));
+ theadList.addChild(new PatternVo("title", "output", ApiParamType.STRING, 1, "表头名称"));
+ theadList.addChild(new PatternVo("isSearchable", "output", ApiParamType.INTEGER, 1, "该字段是否可以搜索过滤"));
+ theadList.addChild(new PatternVo("primaryKey", "output", ApiParamType.INTEGER, 1, "该字段是否是唯一主键"));
+ jsonList.add(theadList);
+ PatternVo tbodyList = new PatternVo("tbodyList", "output", ApiParamType.JSONARRAY, 1, "数据列表");
+ jsonList.add(tbodyList);
+ jsonList.add(new PatternVo("currentPage", "output", ApiParamType.INTEGER, 0, "当前页"));
+ jsonList.add(new PatternVo("rowNum", "output", ApiParamType.INTEGER, 0, "条目数量"));
+ jsonList.add(new PatternVo("pageSize", "output", ApiParamType.INTEGER, 0, "每页大小"));
+ jsonList.add(new PatternVo("pageCount", "output", ApiParamType.INTEGER, 0, "页数"));
+ return jsonList;
+ }
+
+ @Override
+ public void validate(IntegrationResultVo resultVo) throws ApiRuntimeException {
+ if (StringUtils.isBlank(resultVo.getError())) {
+ if (StringUtils.isNotBlank(resultVo.getTransformedResult())) {
+ JSONObject transformedResult;
+ try {
+ transformedResult = JSONObject.parseObject(resultVo.getTransformedResult());
+ } catch (Exception ex) {
+ throw new MatrixExternalDataIsNotJsonException();
+ }
+ if (MapUtils.isNotEmpty(transformedResult)) {
+ Set keys = transformedResult.keySet();
+ Set keySet = new HashSet<>();
+ getOutputPattern().forEach(o -> keySet.add(o.getName()));
+ if (!CollectionUtils.containsAll(keys, keySet)) {
+ throw new MatrixExternalDataNotFormattedException(JSON.toJSONString(CollectionUtils.removeAll(keySet, keys)));
+ }
+ JSONArray theadList = transformedResult.getJSONArray("theadList");
+ if (CollectionUtils.isNotEmpty(theadList)) {
+ for (int i = 0; i < theadList.size(); i++) {
+ if (!theadList.getJSONObject(i).containsKey("key") || !theadList.getJSONObject(i).containsKey("title")) {
+ throw new MatrixExternalDataLostKeyOrTitleInTheadListException();
+ }
+ }
+ } else {
+ throw new MatrixExternalDataNotFormattedException("theadList");
+ }
+ } else {
+ throw new MatrixExternalNoReturnException();
+ }
+ } else {
+ throw new MatrixExternalNoReturnException();
+ }
+ }
+ }
+
+ @Override
+ protected void beforeSend(IntegrationVo integrationVo) {
+
+ }
+
+ @Override
+ protected void afterReturn(IntegrationVo integrationVo) {
+
+ }
+}
diff --git a/src/main/java/neatlogic/module/framework/integration/handler/FrameworkRequestFrom.java b/src/main/java/neatlogic/module/framework/integration/handler/FrameworkRequestFrom.java
index 3a8061d85..bcef88b86 100755
--- a/src/main/java/neatlogic/module/framework/integration/handler/FrameworkRequestFrom.java
+++ b/src/main/java/neatlogic/module/framework/integration/handler/FrameworkRequestFrom.java
@@ -18,5 +18,5 @@ package neatlogic.module.framework.integration.handler;
import neatlogic.framework.integration.core.IRequestFrom;
public enum FrameworkRequestFrom implements IRequestFrom {
- API, TEST, FORM, SCHEDULE;
+ API, TEST, FORM, SCHEDULE, DATAWAREHOUSE;
}
diff --git a/src/main/resources/neatlogic/resources/framework/changelog/2025-12-29/neatlogic_tenant.sql b/src/main/resources/neatlogic/resources/framework/changelog/2025-12-29/neatlogic_tenant.sql
new file mode 100644
index 000000000..27b143ded
--- /dev/null
+++ b/src/main/resources/neatlogic/resources/framework/changelog/2025-12-29/neatlogic_tenant.sql
@@ -0,0 +1,12 @@
+/*
+ * Copyright (C) 2025 TechSure Co., Ltd. All Rights Reserved.
+ * This file is part of the NeatLogic software.
+ * Licensed under the NeatLogic Sustainable Use License (NSUL), Version 4.x – 2025.
+ * You may use this file only in compliance with the License.
+ * See the LICENSE file distributed with this work for the full license text.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+ALTER TABLE `datawarehouse_datasource`
+ ADD COLUMN `integration_uuid` CHAR(32) NULL COMMENT '集成UUID' AFTER `database_id`;
--
Gitee