From 9c819219ab9110f0e22d11a7ed2bfee444acad32 Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Thu, 12 Jun 2025 18:00:22 +0800 Subject: [PATCH 1/5] =?UTF-8?q?[=E4=BF=AE=E5=A4=8D]=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=AF=BC=E8=87=B4java.lang.OutOfMemoryError:=20Metasp?= =?UTF-8?q?ace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1442121129623552]数据仓库第三方数据源导致java.lang.OutOfMemoryError: Metaspace http://192.168.0.96:8090/demo/rdm.html#/bug-detail/939050947543040/939050947543057/1442121129623552 --- .../service/DatabaseService.java | 26 ----- .../DriverHolder.java} | 98 ++++++++----------- .../handler/JDBCDataSourceHandler.java | 53 +++++++--- 3 files changed, 80 insertions(+), 97 deletions(-) delete mode 100644 src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java rename src/main/java/neatlogic/framework/datawarehouse/{service/DatabaseServiceImpl.java => utils/DriverHolder.java} (43%) diff --git a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java deleted file mode 100644 index 0a91a6d66..000000000 --- a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package neatlogic.framework.datawarehouse.service; - -import java.sql.Connection; -import java.sql.SQLException; - -public interface DatabaseService { - - Connection getConnectionByDatabaseId(Long databaseId) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException; -} diff --git a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java similarity index 43% rename from src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java rename to src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java index d171a6c6c..779a57ac5 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java +++ b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java @@ -15,65 +15,47 @@ * along with this program. If not, see . */ -package neatlogic.framework.datawarehouse.service; +package neatlogic.framework.datawarehouse.utils; -import com.alibaba.fastjson.JSONObject; import neatlogic.framework.common.util.FileUtil; -import neatlogic.framework.datawarehouse.dao.mapper.DatabaseMapper; import neatlogic.framework.datawarehouse.dto.DatabaseVo; import neatlogic.framework.datawarehouse.exceptions.DatabaseConnectionFailedException; -import neatlogic.framework.datawarehouse.exceptions.DatabaseNotFoundException; -import neatlogic.framework.exception.file.FileNotFoundException; -import neatlogic.framework.file.dao.mapper.FileMapper; import neatlogic.framework.file.dto.FileVo; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import javax.annotation.Resource; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.sql.Connection; import java.sql.Driver; +import java.sql.DriverManager; import java.sql.SQLException; import java.util.List; -import java.util.Properties; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; -@Service -public class DatabaseServiceImpl implements DatabaseService { +public class DriverHolder { + private static final Logger logger = LoggerFactory.getLogger(DriverHolder.class); + private static final Map LOADER_MAP = new ConcurrentHashMap<>(); + private static final Map DRIVER_MAP = new ConcurrentHashMap<>(); - static Logger logger = LoggerFactory.getLogger(DatabaseServiceImpl.class); + public static synchronized Driver borrowDriver(DatabaseVo databaseVo) throws ClassNotFoundException, InstantiationException, IllegalAccessException { + Long id = databaseVo.getId(); + Driver driver = DRIVER_MAP.get(id); + if (driver != null) return driver; - @Resource - private DatabaseMapper databaseMapper; - - @Resource - private FileMapper fileMapper; - - - @Override - public Connection getConnectionByDatabaseId(Long databaseId) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException { - DatabaseVo dataBaseVo = databaseMapper.getDataBaseById(databaseId); - if (dataBaseVo == null) { - throw new DatabaseNotFoundException(databaseId); - } - List fileIdList = dataBaseVo.getFileIdList(); - if (CollectionUtils.isNotEmpty(fileIdList)) { - URL[] urls = new URL[fileIdList.size()]; - for (int i = 0; i < fileIdList.size(); i++) { - Long fileId = fileIdList.get(i); - FileVo fileVo = fileMapper.getFileById(fileId); - if (fileVo == null) { - throw new FileNotFoundException(fileId); - } + // 只在第一次创建 + List fileList = databaseVo.getFileList(); + if (CollectionUtils.isNotEmpty(fileList)) { + URL[] urls = new URL[fileList.size()]; + for (int i = 0; i < fileList.size(); i++) { + FileVo fileVo = fileList.get(i); try (InputStream is = FileUtil.getData(fileVo.getPath())) { String fileName = fileVo.getName(); String suffix = ".jar"; @@ -84,33 +66,31 @@ public class DatabaseServiceImpl implements DatabaseService { Files.copy(is, path); } urls[i] = file.toURI().toURL(); - }catch (Exception e) { + } catch (Exception e) { logger.error(e.getMessage(), e); throw new RuntimeException(e); } } - JSONObject config = dataBaseVo.getConfig(); - if (MapUtils.isNotEmpty(config)) { - String user = config.getString("user"); - String password = config.getString("password"); - String url = config.getString("url"); - String driverClassName = config.getString("driverClassName"); - Properties props = new Properties(); - if (StringUtils.isNoneBlank(user)) { - props.put("user", user); - } - if (StringUtils.isNotBlank(password)) { - props.put("password", password); - } - URLClassLoader loader = new URLClassLoader(urls, null); - Class clazz = loader.loadClass(driverClassName); - Driver driver = ((Driver) clazz.newInstance()); - return driver.connect(url, props); - } else { - throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.CONFIG_IS_EMPTY, dataBaseVo.getName()); - } + URLClassLoader loader = new URLClassLoader(urls, DriverHolder.class.getClassLoader()); + Class clazz = loader.loadClass(databaseVo.getConfig().getString("driverClassName")); + driver = (Driver) clazz.newInstance(); + // 驱动已经自动注册到 DriverManager，这里再记一次 + LOADER_MAP.put(id, loader); + DRIVER_MAP.put(id, driver); + return driver; } else { - throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, dataBaseVo.getName()); + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, databaseVo.getName()); + } + } + + public static synchronized void destroyDriver(Long id) throws SQLException, IOException { + Driver driver = DRIVER_MAP.remove(id); + if (driver != null) { + DriverManager.deregisterDriver(driver); + } + URLClassLoader classLoader = LOADER_MAP.remove(id); + if (classLoader != null) { + classLoader.close(); } } } diff --git a/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java index 72baee90d..bf018d826 100644 --- a/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java +++ b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java @@ -17,12 +17,19 @@ package neatlogic.module.framework.datawarehouse.handler; +import com.alibaba.fastjson.JSONObject; import neatlogic.framework.datawarehouse.core.DataSourceServiceHandlerBase; import neatlogic.framework.datawarehouse.dao.mapper.DataWarehouseDataSourceMapper; +import neatlogic.framework.datawarehouse.dao.mapper.DatabaseMapper; import neatlogic.framework.datawarehouse.dto.*; +import neatlogic.framework.datawarehouse.exceptions.DatabaseConnectionFailedException; +import neatlogic.framework.datawarehouse.exceptions.DatabaseNotFoundException; import neatlogic.framework.datawarehouse.exceptions.ReportDataSourceSyncException; -import neatlogic.framework.datawarehouse.service.DatabaseService; +import neatlogic.framework.datawarehouse.utils.DriverHolder; +import neatlogic.framework.file.dao.mapper.FileMapper; +import neatlogic.framework.file.dto.FileVo; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.dom4j.DocumentException; import org.slf4j.Logger; @@ -30,12 +37,8 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.net.URLClassLoader; import java.sql.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Component public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { @@ -43,8 +46,12 @@ public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { int FETCH_SIZE = 1000; @Resource private DataWarehouseDataSourceMapper dataSourceMapper; + + @Resource + private DatabaseMapper databaseMapper; + @Resource - private DatabaseService databaseService; + private FileMapper fileMapper; @Override public String getHandler() { @@ -157,11 +164,7 @@ public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { queryStatement.close(); } if (conn != null) { - ClassLoader classLoader = conn.getClass().getClassLoader(); conn.close(); - if (classLoader instanceof URLClassLoader) { - ((URLClassLoader) classLoader).close(); - } } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -170,6 +173,32 @@ public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { } private Connection getConnection(DataSourceVo dataSourceVo) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException { - return databaseService.getConnectionByDatabaseId(dataSourceVo.getDatabaseId()); + DatabaseVo databaseVo = databaseMapper.getDataBaseById(dataSourceVo.getDatabaseId()); + if (databaseVo == null) { + throw new DatabaseNotFoundException(dataSourceVo.getDatabaseId()); + } + List fileIdList = databaseVo.getFileIdList(); + if (CollectionUtils.isNotEmpty(fileIdList)) { + List fileList = fileMapper.getFileListByIdList(fileIdList); + databaseVo.setFileList(fileList); + } else { + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, databaseVo.getName()); + } + Driver driver = DriverHolder.borrowDriver(databaseVo); + JSONObject config = databaseVo.getConfig(); + if (MapUtils.isNotEmpty(config)) { + String user = config.getString("user"); + String password = config.getString("password"); + String url = config.getString("url"); + Properties props = new Properties(); + if (StringUtils.isNoneBlank(user)) { + props.put("user", user); + } + if (StringUtils.isNotBlank(password)) { + props.put("password", password); + } + return driver.connect(url, props); + } + return null; } } -- Gitee From 3b8dc5f73d0db7df4891717e53531eea7f12c6aa Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Thu, 12 Jun 2025 18:02:32 +0800 Subject: [PATCH 2/5] =?UTF-8?q?[=E4=BF=AE=E5=A4=8D]=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=AF=BC=E8=87=B4java.lang.OutOfMemoryError:=20Metasp?= =?UTF-8?q?ace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1442121129623552]数据仓库第三方数据源导致java.lang.OutOfMemoryError: Metaspace http://192.168.0.96:8090/demo/rdm.html#/bug-detail/939050947543040/939050947543057/1442121129623552 --- .../framework/datawarehouse/handler/JDBCDataSourceHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java index bf018d826..bccd76af6 100644 --- a/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java +++ b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java @@ -198,7 +198,8 @@ public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { props.put("password", password); } return driver.connect(url, props); + } else { + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.CONFIG_IS_EMPTY, databaseVo.getName()); } - return null; } } -- Gitee From e9c74817ca812781bcac75ee9676d419c1a44918 Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Thu, 12 Jun 2025 18:20:40 +0800 Subject: [PATCH 3/5] =?UTF-8?q?[=E4=BF=AE=E5=A4=8D]=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=AF=BC=E8=87=B4java.lang.OutOfMemoryError:=20Metasp?= =?UTF-8?q?ace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1442121129623552]数据仓库第三方数据源导致java.lang.OutOfMemoryError: Metaspace http://192.168.0.96:8090/demo/rdm.html#/bug-detail/939050947543040/939050947543057/1442121129623552 --- .../neatlogic/framework/datawarehouse/utils/DriverHolder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java index 779a57ac5..3fa7cd787 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java +++ b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java @@ -74,7 +74,6 @@ public class DriverHolder { URLClassLoader loader = new URLClassLoader(urls, DriverHolder.class.getClassLoader()); Class clazz = loader.loadClass(databaseVo.getConfig().getString("driverClassName")); driver = (Driver) clazz.newInstance(); - // 驱动已经自动注册到 DriverManager，这里再记一次 LOADER_MAP.put(id, loader); DRIVER_MAP.put(id, driver); return driver; -- Gitee From 0e805c8263c1920a03927ad467a4982d19424d1c Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Thu, 12 Jun 2025 18:28:27 +0800 Subject: [PATCH 4/5] =?UTF-8?q?[=E4=BF=AE=E5=A4=8D]=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=AF=BC=E8=87=B4java.lang.OutOfMemoryError:=20Metasp?= =?UTF-8?q?ace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1442121129623552]数据仓库第三方数据源导致java.lang.OutOfMemoryError: Metaspace http://192.168.0.96:8090/demo/rdm.html#/bug-detail/939050947543040/939050947543057/1442121129623552 --- .../framework/datawarehouse/utils/DriverHolder.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java index 3fa7cd787..ee2c04e0a 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java +++ b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java @@ -85,11 +85,19 @@ public class DriverHolder { public static synchronized void destroyDriver(Long id) throws SQLException, IOException { Driver driver = DRIVER_MAP.remove(id); if (driver != null) { - DriverManager.deregisterDriver(driver); + try { + DriverManager.deregisterDriver(driver); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } } URLClassLoader classLoader = LOADER_MAP.remove(id); if (classLoader != null) { - classLoader.close(); + try { + classLoader.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } } } } -- Gitee From a2b19690521730feba73eb7c91c3a58e7abf81db Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Thu, 12 Jun 2025 18:30:36 +0800 Subject: [PATCH 5/5] =?UTF-8?q?[=E4=BF=AE=E5=A4=8D]=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=AF=BC=E8=87=B4java.lang.OutOfMemoryError:=20Metasp?= =?UTF-8?q?ace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1442121129623552]数据仓库第三方数据源导致java.lang.OutOfMemoryError: Metaspace http://192.168.0.96:8090/demo/rdm.html#/bug-detail/939050947543040/939050947543057/1442121129623552 --- .../datawarehouse/utils/DriverHolder.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java index ee2c04e0a..70cf4534e 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java +++ b/src/main/java/neatlogic/framework/datawarehouse/utils/DriverHolder.java @@ -56,19 +56,19 @@ public class DriverHolder { URL[] urls = new URL[fileList.size()]; for (int i = 0; i < fileList.size(); i++) { FileVo fileVo = fileList.get(i); - try (InputStream is = FileUtil.getData(fileVo.getPath())) { - String fileName = fileVo.getName(); - String suffix = ".jar"; - String prefix = fileName.substring(0, fileName.length() - suffix.length()); - File file = new File(prefix + "-" + fileVo.getId() + suffix); - if (!file.exists()) { + String fileName = fileVo.getName(); + String suffix = ".jar"; + String prefix = fileName.substring(0, fileName.length() - suffix.length()); + File file = new File(prefix + "-" + fileVo.getId() + suffix); + if (!file.exists()) { + try (InputStream is = FileUtil.getData(fileVo.getPath())) { Path path = Paths.get(file.toURI()); Files.copy(is, path); + urls[i] = file.toURI().toURL(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); } - urls[i] = file.toURI().toURL(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException(e); } } URLClassLoader loader = new URLClassLoader(urls, DriverHolder.class.getClassLoader()); -- Gitee