From 499a9959b9307edb63baebec8e7898b532151363 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 31 Jul 2020 14:08:29 +0800 Subject: [PATCH 01/42] =?UTF-8?q?=E6=94=B9=E6=88=90=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=8F=91=E9=80=81=E3=80=82=E5=B7=B2=E9=81=BF=E5=85=8D=E7=B2=98?= =?UTF-8?q?=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/client/core/key/NettyKeyPusher.java | 4 ++-- .../hotkey/worker/netty/flush/FlushUtil.java | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index d5afa72..f985489 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -44,7 +44,7 @@ public class NettyKeyPusher implements IKeyPusher { for (Channel channel : map.keySet()) { try { List batch = map.get(channel); - channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_NEW_KEY, FastJsonUtils.convertObjectToJSON(batch)))); + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_NEW_KEY, FastJsonUtils.convertObjectToJSON(batch)))).sync(); } catch (Exception e) { try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); @@ -78,7 +78,7 @@ public class NettyKeyPusher implements IKeyPusher { try { List batch = map.get(channel); channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(Context.APP_NAME, - MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))); + MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))).sync(); } catch (Exception e) { try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java index cdc1299..04f1354 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java @@ -16,19 +16,19 @@ public class FlushUtil { * 往channel里输出消息 */ public static void flush(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) { - if (channelHandlerContext.channel().isWritable()) { - channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { - if (!future.isSuccess()) { - logger.warn("flush error " + future.cause().getMessage()); - } - }); - } else { +// if (channelHandlerContext.channel().isWritable()) { +// channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { +// if (!future.isSuccess()) { +// logger.warn("flush error " + future.cause().getMessage()); +// } +// }); +// } else { try { //同步发送 channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); } catch (InterruptedException e) { logger.error("flush error " + e.getMessage()); } - } +// } } } -- Gitee From c409f4af72fd2a743921ba07c8d530cd59384235 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Sat, 1 Aug 2020 17:00:55 +0800 Subject: [PATCH 02/42] =?UTF-8?q?=E6=8B=89=E5=8F=96=E8=A7=84=E5=88=99?= =?UTF-8?q?=E5=AE=8C=E6=AF=95=E5=90=8E=EF=BC=8C=E5=86=8D=E6=8B=89=E5=8F=96?= =?UTF-8?q?=E5=B7=B2=E5=AD=98=E5=9C=A8=E7=9A=84=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java index c129fb1..216160a 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java @@ -38,8 +38,6 @@ public class EtcdStarter { fetchRule(); - fetchExistHotKey(); - // startWatchWorker(); startWatchRule(); @@ -243,6 +241,9 @@ public class EtcdStarter { JdLogger.info(getClass(), "trying to connect to etcd and fetch rule info"); boolean success = fetchRuleFromEtcd(); if (success) { + //拉取已存在的热key + fetchExistHotKey(); + scheduledExecutorService.shutdown(); } -- Gitee From 460af1a6fac7d85e4d103290c1d8a112b07ae962 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Mon, 3 Aug 2020 23:37:44 +0800 Subject: [PATCH 03/42] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E6=B8=85=E7=90=86?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=92=8C=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 8 + .../dashboard/DashboardApplication.java | 13 +- .../dashboard/common/domain/Constant.java | 6 + .../common/domain/vo/ClearCfgVo.java | 47 +++++ .../dashboard/common/monitor/DataHandler.java | 37 ++++ .../dashboard/controller/ClearController.java | 62 +++++++ .../dashboard/mapper/KeyRecordMapper.java | 3 + .../dashboard/mapper/StatisticsMapper.java | 4 +- .../dashboard/mapper/SummaryMapper.java | 3 + .../dashboard/service/ClearService.java | 23 +++ .../service/impl/ClearServiceImpl.java | 85 +++++++++ .../service/impl/KeyServiceImpl.java | 167 ------------------ .../service/impl/UserServiceImpl.java | 2 +- .../dashboard/mapper/KeyRecordMapper.xml | 6 + .../dashboard/mapper/StatisticsMapper.xml | 7 +- .../hotkey/dashboard/mapper/SummaryMapper.xml | 7 + .../static/admin/common/js/clearcfg-edit.js | 25 +++ .../resources/templates/admin/clear/edit.html | 69 ++++++++ .../resources/templates/admin/clear/list.html | 157 ++++++++++++++++ .../templates/admin/common/html/leftMenu.html | 7 +- 20 files changed, 553 insertions(+), 185 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java create mode 100644 dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js create mode 100644 dashboard/src/main/resources/templates/admin/clear/edit.html create mode 100644 dashboard/src/main/resources/templates/admin/clear/list.html diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index e88389d..7382d8d 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -1,5 +1,7 @@ package com.jd.platform.hotkey.common.configcenter; +import java.util.concurrent.TimeUnit; + /** * @author wuweifeng wrote on 2019-12-06 * @version 1.0 @@ -54,4 +56,10 @@ public interface ConfigConstant { * 存放客户端hotKey访问次数和总访问次数的path */ String keyHitCountPath = "/jd/keyHitCount/"; + + /** + * 清理历史数据的配置的path + * time unit : day + */ + String clearCfgPath = "/jd/clearCfg/"; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 2b361c6..074e45d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,12 +1,7 @@ package com.jd.platform.hotkey.dashboard; -import com.alibaba.fastjson.JSON; -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; -import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; @@ -14,10 +9,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; - import javax.annotation.Resource; -import java.util.HashMap; -import java.util.Map; @EnableAsync @@ -25,7 +17,6 @@ import java.util.Map; @SpringBootApplication public class DashboardApplication implements CommandLineRunner { - private Logger logger = LoggerFactory.getLogger(getClass()); @Resource @@ -37,11 +28,11 @@ public class DashboardApplication implements CommandLineRunner { }catch (Exception e){ e.printStackTrace(); } - } + @Override - public void run(String... args) { + public void run(String... args) { int row = timelyMapper.clear(); logger.info("clear db timely hotKey, effect row : {}",row); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index b11fa73..952aa2a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -35,6 +35,9 @@ public class Constant { public static final String WORKER_VIEW = "节点信息"; + public static final String CLEAR_VIEW = "数据清理"; + + public static final int MAX_DAY_RANGE = 3; public static final int VERSION = 1; @@ -55,6 +58,9 @@ public class Constant { public static final String INFO = "info"; + public static final String THIRTY_DAY = "30"; + + public static final List HEAD = new ArrayList<>(); static { HEAD.add("热点key"); HEAD.add("次数"); HEAD.add("所属APP");} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java new file mode 100644 index 0000000..14b872b --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java @@ -0,0 +1,47 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearCfgVo + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:54 + */ +public class ClearCfgVo { + + private String app; + + private String ttl; + + private Long version; + + public ClearCfgVo(String app, String ttl, Long version) { + this.app = app; + this.ttl = ttl; + this.version = version; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public String getTtl() { + return ttl; + } + + public void setTtl(String ttl) { + this.ttl = ttl; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index b5b1a1e..8a5dac9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -3,12 +3,16 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import com.alibaba.fastjson.JSON; import com.ibm.etcd.api.Event; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; @@ -39,6 +43,13 @@ public class DataHandler { @Resource private StatisticsMapper statisticsMapper; + @Resource + private SummaryMapper summaryMapper; + + @Resource + private IConfigCenter configCenter; + + /** * 队列 */ @@ -210,4 +221,30 @@ public class DataHandler { } + + + /** + * 每天根据app的配置清理过期数据 + */ + @Scheduled(cron = "0 0 1 * * ?") + public void clearExpireData() { + try { + LocalDateTime now = LocalDateTime.now(); + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + for (KeyValue kv : keyValues) { + String key = kv.getKey().toStringUtf8(); + String ttl = kv.getValue().toStringUtf8(); + String app = key.replace(ConfigConstant.clearCfgPath,""); + Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); + summaryMapper.clearExpireData(app, expireDate); + keyRecordMapper.clearExpireData(app, expireDate); + statisticsMapper.clearExpireData(app, expireDate); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java new file mode 100644 index 0000000..0ce81f0 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java @@ -0,0 +1,62 @@ +package com.jd.platform.hotkey.dashboard.controller; + +import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; +import com.jd.platform.hotkey.dashboard.service.ClearService; +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + +/** + * @ProjectName: hotkey + * @ClassName: ClearController + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:48 + */ +@Controller +@RequestMapping("/clear") +public class ClearController { + + + private String prefix = "admin/clear"; + + + @Resource + private ClearService clearService; + + @GetMapping("/view") + public String view(ModelMap modelMap){ + modelMap.put("title", Constant.CLEAR_VIEW); + return prefix + "/list"; + } + + + @PostMapping("/list") + @ResponseBody + public Page list(PageReq page, String app){ + PageInfo info = clearService.pageClearCfg(page, app); + return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + } + + + @GetMapping("/edit/{app}") + public String edit(@PathVariable("app") String app, ModelMap modelMap){ + modelMap.put("clearCfg", clearService.selectClearCfg(app)); + return prefix + "/edit"; + } + + @PostMapping("/save") + @ResponseBody + public Result save(ClearCfgVo cfg){ + int b = clearService.saveClearCfg(cfg); + return b == 0 ? Result.fail():Result.success(); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java index 3ef46b3..56c05ab 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java @@ -6,6 +6,7 @@ import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -25,4 +26,6 @@ public interface KeyRecordMapper { List maxHotKey(SearchReq req); List statisticsByRule(SearchReq req); + + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java index dabdfc5..daf848e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java @@ -4,6 +4,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -42,7 +43,6 @@ public interface StatisticsMapper { /** * 清理 - * @param type type */ - void clear(int type); + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java index 4685e76..b12ff0d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java @@ -6,6 +6,7 @@ import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -22,4 +23,6 @@ public interface SummaryMapper { int saveOrUpdate(Summary records); List listRuleHitCount(SearchReq req); + + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java new file mode 100644 index 0000000..5266ef3 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java @@ -0,0 +1,23 @@ +package com.jd.platform.hotkey.dashboard.service; + +import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearService + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:51 + */ +public interface ClearService { + + + PageInfo pageClearCfg(PageReq page, String app); + + ClearCfgVo selectClearCfg(String app); + + int saveClearCfg(ClearCfgVo cfg); + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java new file mode 100644 index 0000000..8227a95 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java @@ -0,0 +1,85 @@ +package com.jd.platform.hotkey.dashboard.service.impl; + +import com.github.pagehelper.PageInfo; +import com.github.pagehelper.util.StringUtil; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; +import com.jd.platform.hotkey.dashboard.service.ClearService; +import com.jd.platform.hotkey.dashboard.service.UserService; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; + +/** + * @ProjectName: hotkey + * @ClassName: ClearServiceImpl + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:57 + */ +@Service +public class ClearServiceImpl implements ClearService { + + + @Resource + private IConfigCenter configCenter; + + @Resource + private UserService userService; + + + @Override + public PageInfo pageClearCfg(PageReq page, String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + if(CollectionUtils.isEmpty(keyValues)){ + List apps = userService.listApp(); + for (String ap : apps) { + configCenter.put(ConfigConstant.clearCfgPath + ap, Constant.THIRTY_DAY); + } + keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + } + List cfgVos = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + String key = kv.getKey().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + configCenter.put(key, Constant.THIRTY_DAY); + continue; + } + long version = kv.getModRevision(); + String k = key.replace(ConfigConstant.clearCfgPath,""); + if(StringUtils.isEmpty(app)){ + cfgVos.add(new ClearCfgVo(k, v, version)); + }else{ + if(k.equals(app)){ + cfgVos.add(new ClearCfgVo(k, v, version)); + } + } + } + return new PageInfo<>(cfgVos); + } + + @Override + public ClearCfgVo selectClearCfg(String app) { + KeyValue kv = configCenter.getKv(ConfigConstant.clearCfgPath + app); + if(kv == null || kv.getValue() == null){ + configCenter.put(ConfigConstant.clearCfgPath + app, Constant.THIRTY_DAY); + return new ClearCfgVo(app, Constant.THIRTY_DAY, 0L); + } + String v = kv.getValue().toStringUtf8(); + return new ClearCfgVo(app,v,kv.getModRevision()); + } + + @Override + public int saveClearCfg(ClearCfgVo cfg) { + configCenter.put(ConfigConstant.clearCfgPath + cfg.getApp(), cfg.getTtl()); + return 1; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 88a4263..04f80c6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -255,173 +255,6 @@ public class KeyServiceImpl implements KeyService { }*/ } - - private static List statisticsList(){ - Random rd = new Random(); - List records = new ArrayList<>(); - // statisticsMapper.clear(1); - for (int i = 0; i < 35; i++) { - if(i== 2||i== 3||i== 4||i== 25||i== 12||i== 13||i== 14||i== 30||i==32||i==33||i== 34){ - continue; - } - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusMinutes(35-i); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - int minus = DateUtil.nowMinus(now); - - Statistics s1 = new Statistics(); - s1.setKeyName("k21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(800)); - s1.setBizType(1); - s1.setMinutes(minus); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("k22"); - s11.setApp("key22APP"); - s11.setBizType(1); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setMinutes(minus); - s11.setCount(rd.nextInt(600)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - } - - if(1==1){ - return records; - } - // statisticsMapper.batchInsert(records); - List list = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 12 || i ==13){ - }else{ - Statistics st = new Statistics(); - st.setApp("k21"); - st.setKeyName("k21"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); - // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); - int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); - // System.out.println(time); - st.setMinutes(time); - // list.add(st); - } - } - List list2 = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ - - }else{ - Statistics st2 = new Statistics(); - st2.setApp("k22"); - st2.setKeyName("k22"); - st2.setCount(rd.nextInt(100)); - st2.setBizType(1); - st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); - list2.add(st2); - } - } - // list.addAll(list2); - return list; - - } - - private List statisticsList1(int type){ - Random rd = new Random(); - List records = new ArrayList<>(); - statisticsMapper.clear(6); - - if(type == 3){ - int temp = 1; - for (int j = 0; j < 9; j++) { - for (int i = 0; i < 24; i++) { - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusHours(9 * 24 - temp); - - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - Statistics s1 = new Statistics(); - s1.setKeyName("k21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(200)); - s1.setBizType(6); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("k22"); - s11.setApp("key22APP"); - s11.setBizType(6); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setCount(rd.nextInt(500)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - - Statistics s33 = new Statistics(); - s33.setKeyName("k33"); - s33.setApp("key33APP"); - s33.setBizType(6); - s33.setCreateTime(nowTime); - s33.setDays(day); - s33.setCount(rd.nextInt(300)); - s33.setHours(hour); - s33.setUuid(3 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s33); - temp++; - } - } - - - }else{ - for (int i = 0; i < 30; i++) { - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusHours(30-i); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - Statistics s1 = new Statistics(); - s1.setKeyName("key21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(800)); - s1.setBizType(2); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("key22"); - s11.setApp("key22APP"); - s11.setBizType(2); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setCount(rd.nextInt(600)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - } - } - statisticsMapper.batchInsert(records); - return records; - } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index 5cb83e5..c03ea2e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -34,7 +34,7 @@ import java.util.Set; @Service public class UserServiceImpl implements UserService { - @Value("${erp.defaultPwd}") + @Value("${erp.defaultPwd:123}") private String defaultPwd; @Resource private UserMapper userMapper; diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml index f869987..94037b7 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml @@ -199,4 +199,10 @@ + + delete from hk_key_record + where app_name = #{app} + + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml index 8829d12..ac7f464 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml @@ -74,8 +74,9 @@ - + where app = #{app} + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml index 60f41a2..bd04545 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml @@ -51,4 +51,11 @@ + + delete from hk_summary + where app = #{app} + + + + \ No newline at end of file diff --git a/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js b/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js new file mode 100644 index 0000000..0d8de56 --- /dev/null +++ b/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js @@ -0,0 +1,25 @@ +$("#form-edit").validate({ + submitHandler:function(form){ + edit(); + } +}); + +function edit() { + let dataFormJson=$("#form-edit").serialize(); + $.ajax({ + cache : true, + type : "POST", + url : "/clear/save", + data : dataFormJson, + headers: { + "Authorization":getCookie("token") + }, + async : false, + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); + }, + success : function(data) { + $.operate.saveSuccess(data); + } + }); +} diff --git a/dashboard/src/main/resources/templates/admin/clear/edit.html b/dashboard/src/main/resources/templates/admin/clear/edit.html new file mode 100644 index 0000000..7432300 --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/clear/edit.html @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+
+ + +
+
+
+
+
+
+
+
+
+ + +
+ +
+ + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/clear/list.html b/dashboard/src/main/resources/templates/admin/clear/list.html new file mode 100644 index 0000000..7087f79 --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/clear/list.html @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + +
+
+ +
+ +
+
+ +
+
+
搜索条件
+
+ +
+
+
+
+ + +
+ +
+
+
+
+ + +
+
+
表单
+
+ + + +
+
+ + + +
+
+ +
+ +
+
+
+
+ +
+ +
+
+
+ + + +
+ +
+ + + + + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 29baffe..668341d 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -52,7 +52,7 @@ - + 热点记录 变更记录 + + + 数据清理配置 + -- Gitee From efbc179aea7bc66fa1e5b2f738ed928a08bda607 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 4 Aug 2020 00:10:31 +0800 Subject: [PATCH 04/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E5=92=8CHothit=E7=BB=9F=E8=AE=A1=E5=A4=A7=E8=A1=A8?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=20=E9=BB=98=E8=AE=A4=E7=BC=A9?= =?UTF-8?q?=E7=9F=AD=E6=97=B6=E9=97=B4=E4=B8=BA=E6=9C=80=E8=BF=915?= =?UTF-8?q?=E5=B0=8F=E6=97=B6=EF=BC=8C=E4=BF=AE=E6=94=B9=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/db.sql | 2 + .../admin/assets/js/bootstrap/js/base_list.js | 17 ++++---- .../resources/templates/admin/key/list.html | 36 +++++++++------- .../templates/admin/key/listmaxhot.html | 4 +- .../templates/admin/rule/listhitcount.html | 43 +++++++++++-------- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 06e6b0c..c959191 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -45,6 +45,7 @@ CREATE TABLE `hk_key_record` ( `rule` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '''' COMMENT '规则', PRIMARY KEY (`id`) USING BTREE, UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引' + INDEX `idx_app_ct`(`app_name`, `create_time`) USING BTREE COMMENT '查询索引' ) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; @@ -76,6 +77,7 @@ CREATE TABLE `hk_statistics` ( `create_time` datetime(0) NOT NULL COMMENT '创建时间', PRIMARY KEY (`id`) USING BTREE, UNIQUE INDEX `uniq_uuid`(`uuid`) USING BTREE COMMENT '防重唯一索引' + INDEX `idx_app_ct`(`app`, `create_time`) USING BTREE COMMENT '查询索引' ) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Compact; diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 05eb56e..01763f2 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -40,13 +40,6 @@ ajaxOptions:{ headers: {"Authorization": getCookie("token")} }, - exportOptions:{ - ignoreColumn: [0,1], //忽略某一列的索引 - fileName: '报表导出', //文件名称设置 - worksheetName: 'sheet1', //表格工作区名称 - tableName: '报表导出', - excelstyles: ['background-color', 'color', 'font-size', 'font-weight'] - }, onLoadSuccess: function(){ //加载成功时执行 var datas = $('#dataTable').bootstrapTable('getData'); console.info("加载成功"); @@ -70,7 +63,15 @@ if(app!==undefined && app != null && app!==""){ search.app = app; } - return search; + let time1 = $("#startTime").val(); + if(time1!==undefined && time1 != null && time1!==""){ + search.startTime = time1; + } + let time2 = $("#endTime").val(); + if(time2!==undefined && time2 != null && time2!==""){ + search.endTime = time2; + } + return search; }, search:function(my){//查询条件 diff --git a/dashboard/src/main/resources/templates/admin/key/list.html b/dashboard/src/main/resources/templates/admin/key/list.html index 38c596f..27ff05d 100644 --- a/dashboard/src/main/resources/templates/admin/key/list.html +++ b/dashboard/src/main/resources/templates/admin/key/list.html @@ -174,8 +174,27 @@ var options = { { field: 'createTime', title: '创建时间' }] }; -$(function(){ + + $('.datetimepicker').datetimepicker({ + language: 'zh-CN', + CustomFormat: 'yyyy-mm-dd HH:ii:ss', + weekStart: 1, + todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 + autoclose: 1, //选完时间自动关闭 + todayHighlight: 1, //当天时间高亮 + startView: 2, //从月视图开始，选天 + minView: 0, //提供选择分钟的视图 + forceParse: 0, + minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 + }); + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); + + +$(function(){ $.ajax({ cache : true, type : "POST", @@ -206,24 +225,9 @@ $(function(){ } }); - var oTab=$.table.oTableInit(options); oTab.Init(); - $('.datetimepicker').datetimepicker({ - language: 'zh-CN', - CustomFormat: 'yyyy-mm-dd HH:ii:ss', - weekStart: 1, - todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 - autoclose: 1, //选完时间自动关闭 - todayHighlight: 1, //当天时间高亮 - startView: 2, //从月视图开始，选天 - minView: 0, //提供选择分钟的视图 - forceParse: 0, - minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 -}); - - }) function exportHot() { diff --git a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html index 366b156..5d7fc22 100644 --- a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html +++ b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html @@ -35,14 +35,14 @@
- + - + diff --git a/dashboard/src/main/resources/templates/admin/rule/listhitcount.html b/dashboard/src/main/resources/templates/admin/rule/listhitcount.html index 7e501fa..f1be6e3 100644 --- a/dashboard/src/main/resources/templates/admin/rule/listhitcount.html +++ b/dashboard/src/main/resources/templates/admin/rule/listhitcount.html @@ -46,7 +46,7 @@
- + @@ -55,7 +55,7 @@
- + @@ -134,6 +134,27 @@ } }] }; + + + + $('.datetimepicker').datetimepicker({ + language: 'zh-CN', + CustomFormat: 'yyyy-mm-dd HH:ii:ss', + weekStart: 1, + todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 + autoclose: 1, //选完时间自动关闭 + todayHighlight: 1, //当天时间高亮 + startView: 2, //从月视图开始，选天 + minView: 0, //提供选择分钟的视图 + forceParse: 0, + minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 + }); + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); + + $(function(){ $.ajax({ cache : true, @@ -164,26 +185,12 @@ } } }); - var oTab=$.table.oTableInit(options); + + let oTab=$.table.oTableInit(options); oTab.Init(); }) - - $('.datetimepicker').datetimepicker({ - language: 'zh-CN', - CustomFormat: 'yyyy-mm-dd HH:ii:ss', - weekStart: 1, - todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 - autoclose: 1, //选完时间自动关闭 - todayHighlight: 1, //当天时间高亮 - startView: 2, //从月视图开始，选天 - minView: 0, //提供选择分钟的视图 - forceParse: 0, - minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 - }); - - -- Gitee From ce0c4f6835dfe66b094c349b675d6baaef8de23f Mon Sep 17 00:00:00 2001 From: wangyu106 Date: Thu, 6 Aug 2020 14:16:40 +0800 Subject: [PATCH 05/42] =?UTF-8?q?io=E7=BA=BF=E7=A8=8B=E4=B8=8E=E4=B8=9A?= =?UTF-8?q?=E5=8A=A1=E7=BA=BF=E7=A8=8B=E4=B8=80=E4=B8=80=E7=BB=91=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- worker/pom.xml | 19 ++-------- .../keydispatcher/DispatcherConfig.java | 28 +++++++++++--- .../worker/keydispatcher/KeyConsumer.java | 38 +++++++++++++------ .../worker/keydispatcher/KeyProducer.java | 38 +++++++++++++++++-- .../worker/netty/filter/HotKeyFilter.java | 4 +- 5 files changed, 89 insertions(+), 38 deletions(-) diff --git a/worker/pom.xml b/worker/pom.xml index 5f234d9..aec68d8 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -19,18 +19,6 @@ - - io.protostuff - protostuff-core - 1.7.2 - - - - io.protostuff - protostuff-runtime - 1.7.2 - - com.jd.platform.hotkey common @@ -38,10 +26,11 @@ compile + - com.jd.ump - jannotation - 4.0.5 + org.jctools + jctools-core + 2.1.2 diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java index 3d620b1..5866cf6 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java @@ -3,6 +3,8 @@ package com.jd.platform.hotkey.worker.keydispatcher; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.keylistener.IKeyListener; import com.jd.platform.hotkey.worker.tool.CpuNum; +import com.lmax.disruptor.dsl.Disruptor; +import org.jctools.queues.SpscArrayQueue; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -10,10 +12,9 @@ import org.springframework.context.annotation.Configuration; import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.*; /** * @author wuweifeng @@ -34,12 +35,24 @@ public class DispatcherConfig { * 队列 */ public static BlockingQueue QUEUE = new LinkedBlockingQueue<>(2000000); + public static Map> MAPQUEUE = new ConcurrentHashMap<>(); + + static { + // + // https://www.jianshu.com/p/fde38db97318 + int nowCount = CpuNum.workerCount(); + int num = 2000000 / nowCount; + for (int i = 0; i < nowCount; i++) { + Queue spscArrayQueue = new SpscArrayQueue<>(num); + MAPQUEUE.put(i + "", spscArrayQueue); + } + } @Bean public Consumer consumer() { int nowCount = CpuNum.workerCount(); //将实际值赋给static变量 - if(threadCount != 0) { + if (threadCount != 0) { nowCount = threadCount; } @@ -47,10 +60,15 @@ public class DispatcherConfig { for (int i = 0; i < nowCount; i++) { KeyConsumer keyConsumer = new KeyConsumer(); keyConsumer.setKeyListener(iKeyListener); + keyConsumer.setQueue(MAPQUEUE.get(i+"")); consumerList.add(keyConsumer); threadPoolExecutor.submit(keyConsumer::beginConsume); } return new Consumer(consumerList); } + + public static void main(String[] args) { + Queue spscArrayQueue = new SpscArrayQueue<>(16); + } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java index a85d972..408189b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java @@ -5,6 +5,9 @@ import com.jd.platform.hotkey.worker.keylistener.IKeyListener; import com.jd.platform.hotkey.worker.keylistener.KeyEventOriginal; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; + import static com.jd.platform.hotkey.worker.keydispatcher.DispatcherConfig.QUEUE; import static com.jd.platform.hotkey.worker.tool.InitConstant.totalDealCount; @@ -18,26 +21,37 @@ public class KeyConsumer { private IKeyListener iKeyListener; + private Queue queue; + + public void setKeyListener(IKeyListener iKeyListener) { this.iKeyListener = iKeyListener; } + public Queue getQueue() { + return queue; + } + + public void setQueue(Queue queue) { + this.queue = queue; + } + public void beginConsume() { while (true) { - try { - HotKeyModel model = QUEUE.take(); - if (model.isRemove()) { - iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); - } else { - iKeyListener.newKey(model, KeyEventOriginal.CLIENT); - } - - //处理完毕，将数量加1 - totalDealCount.increment(); - } catch (InterruptedException e) { - e.printStackTrace(); + HotKeyModel model = queue.poll(); + if (model == null) { + continue; + } + if (model.isRemove()) { + iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); + } else { + iKeyListener.newKey(model, KeyEventOriginal.CLIENT); } + //处理完毕，将数量加1 + totalDealCount.increment(); + + } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java index fc86ecc..88df4fd 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java @@ -3,8 +3,15 @@ package com.jd.platform.hotkey.worker.keydispatcher; import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.tool.InitConstant; +import io.netty.channel.ChannelHandlerContext; import org.springframework.stereotype.Component; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +import static com.jd.platform.hotkey.worker.keydispatcher.DispatcherConfig.MAPQUEUE; import static com.jd.platform.hotkey.worker.keydispatcher.DispatcherConfig.QUEUE; import static com.jd.platform.hotkey.worker.tool.InitConstant.expireTotalCount; import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @@ -17,7 +24,12 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @Component public class KeyProducer { - public void push(HotKeyModel model) { + // io线程与业务线程一一绑定 + Map map = new ConcurrentHashMap<>(); + AtomicInteger atomicInteger = new AtomicInteger(0); + ReentrantLock lock = new ReentrantLock(); + + public void push(HotKeyModel model, ChannelHandlerContext ctx) { if (model == null || model.getKey() == null) { return; } @@ -27,12 +39,30 @@ public class KeyProducer { return; } + + String threadId = ctx.channel().eventLoop().toString(); + lock.lock(); try { - QUEUE.put(model); - totalOfferCount.increment(); - } catch (InterruptedException e) { + if (map.containsKey(threadId)) { + String index = map.get(threadId); + MAPQUEUE.get(index).add(model); + + } else { + int index = atomicInteger.getAndIncrement(); + map.put(threadId, index + ""); + MAPQUEUE.get(index).add(model); + } + + + } catch (Exception e) { e.printStackTrace(); + + } finally { + lock.unlock(); } + + + totalOfferCount.increment(); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 080a9d9..e6ceb84 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -60,7 +60,7 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { if (WhiteListHolder.contains(model.getKey())) { return; } - keyProducer.push(model); + keyProducer.push(model, ctx); return; } //老版的用的单个HotKeyModel，新版用的数组 @@ -74,7 +74,7 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { if (timeOut > 1000) { logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); } - keyProducer.push(model); + keyProducer.push(model,ctx); } } -- Gitee From 7f406a6ddc2434a75e6ac8a2e860dcdd50db2305 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 6 Aug 2020 16:51:56 +0800 Subject: [PATCH 06/42] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sample/SampleApplicationTests.java | 9 ++++- worker/pom.xml | 7 ++-- .../keydispatcher/DispatcherConfig.java | 26 ++----------- .../worker/keydispatcher/KeyConsumer.java | 38 ++++++------------- .../worker/keydispatcher/KeyProducer.java | 38 ++----------------- 5 files changed, 31 insertions(+), 87 deletions(-) diff --git a/sample/src/test/java/com/jd/platform/sample/SampleApplicationTests.java b/sample/src/test/java/com/jd/platform/sample/SampleApplicationTests.java index 741fdca..1f7e12b 100644 --- a/sample/src/test/java/com/jd/platform/sample/SampleApplicationTests.java +++ b/sample/src/test/java/com/jd/platform/sample/SampleApplicationTests.java @@ -1,13 +1,20 @@ package com.jd.platform.sample; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest class SampleApplicationTests { - + private Logger logger = LoggerFactory.getLogger(getClass()); @Test void contextLoads() { +// if (JdHotKeyStore.isHotKey("VENDER_ID_2111")) { +// logger.error("isHot"); +// } else { +// logger.error("noHot"); +// } } } diff --git a/worker/pom.xml b/worker/pom.xml index aec68d8..0d64df6 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -26,11 +26,10 @@ compile - - org.jctools - jctools-core - 2.1.2 + com.jd.ump + jannotation + 4.0.5 diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java index 5866cf6..defcf67 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java @@ -3,8 +3,6 @@ package com.jd.platform.hotkey.worker.keydispatcher; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.keylistener.IKeyListener; import com.jd.platform.hotkey.worker.tool.CpuNum; -import com.lmax.disruptor.dsl.Disruptor; -import org.jctools.queues.SpscArrayQueue; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -12,9 +10,10 @@ import org.springframework.context.annotation.Configuration; import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; /** * @author wuweifeng @@ -35,18 +34,6 @@ public class DispatcherConfig { * 队列 */ public static BlockingQueue QUEUE = new LinkedBlockingQueue<>(2000000); - public static Map> MAPQUEUE = new ConcurrentHashMap<>(); - - static { - // - // https://www.jianshu.com/p/fde38db97318 - int nowCount = CpuNum.workerCount(); - int num = 2000000 / nowCount; - for (int i = 0; i < nowCount; i++) { - Queue spscArrayQueue = new SpscArrayQueue<>(num); - MAPQUEUE.put(i + "", spscArrayQueue); - } - } @Bean public Consumer consumer() { @@ -60,15 +47,10 @@ public class DispatcherConfig { for (int i = 0; i < nowCount; i++) { KeyConsumer keyConsumer = new KeyConsumer(); keyConsumer.setKeyListener(iKeyListener); - keyConsumer.setQueue(MAPQUEUE.get(i+"")); consumerList.add(keyConsumer); threadPoolExecutor.submit(keyConsumer::beginConsume); } return new Consumer(consumerList); } - - public static void main(String[] args) { - Queue spscArrayQueue = new SpscArrayQueue<>(16); - } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java index 408189b..a85d972 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyConsumer.java @@ -5,9 +5,6 @@ import com.jd.platform.hotkey.worker.keylistener.IKeyListener; import com.jd.platform.hotkey.worker.keylistener.KeyEventOriginal; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; - import static com.jd.platform.hotkey.worker.keydispatcher.DispatcherConfig.QUEUE; import static com.jd.platform.hotkey.worker.tool.InitConstant.totalDealCount; @@ -21,37 +18,26 @@ public class KeyConsumer { private IKeyListener iKeyListener; - private Queue queue; - - public void setKeyListener(IKeyListener iKeyListener) { this.iKeyListener = iKeyListener; } - public Queue getQueue() { - return queue; - } - - public void setQueue(Queue queue) { - this.queue = queue; - } - public void beginConsume() { while (true) { - HotKeyModel model = queue.poll(); - if (model == null) { - continue; - } - if (model.isRemove()) { - iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); - } else { - iKeyListener.newKey(model, KeyEventOriginal.CLIENT); + try { + HotKeyModel model = QUEUE.take(); + if (model.isRemove()) { + iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); + } else { + iKeyListener.newKey(model, KeyEventOriginal.CLIENT); + } + + //处理完毕，将数量加1 + totalDealCount.increment(); + } catch (InterruptedException e) { + e.printStackTrace(); } - //处理完毕，将数量加1 - totalDealCount.increment(); - - } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java index 88df4fd..fc86ecc 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java @@ -3,15 +3,8 @@ package com.jd.platform.hotkey.worker.keydispatcher; import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.tool.InitConstant; -import io.netty.channel.ChannelHandlerContext; import org.springframework.stereotype.Component; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -import static com.jd.platform.hotkey.worker.keydispatcher.DispatcherConfig.MAPQUEUE; import static com.jd.platform.hotkey.worker.keydispatcher.DispatcherConfig.QUEUE; import static com.jd.platform.hotkey.worker.tool.InitConstant.expireTotalCount; import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @@ -24,12 +17,7 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @Component public class KeyProducer { - // io线程与业务线程一一绑定 - Map map = new ConcurrentHashMap<>(); - AtomicInteger atomicInteger = new AtomicInteger(0); - ReentrantLock lock = new ReentrantLock(); - - public void push(HotKeyModel model, ChannelHandlerContext ctx) { + public void push(HotKeyModel model) { if (model == null || model.getKey() == null) { return; } @@ -39,30 +27,12 @@ public class KeyProducer { return; } - - String threadId = ctx.channel().eventLoop().toString(); - lock.lock(); try { - if (map.containsKey(threadId)) { - String index = map.get(threadId); - MAPQUEUE.get(index).add(model); - - } else { - int index = atomicInteger.getAndIncrement(); - map.put(threadId, index + ""); - MAPQUEUE.get(index).add(model); - } - - - } catch (Exception e) { + QUEUE.put(model); + totalOfferCount.increment(); + } catch (InterruptedException e) { e.printStackTrace(); - - } finally { - lock.unlock(); } - - - totalOfferCount.increment(); } } -- Gitee From aec79d0be86d6168e51b9940e4d23f03e6e7e06f Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 6 Aug 2020 18:44:51 +0800 Subject: [PATCH 07/42] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../worker/cache/CaffeineCacheHolder.java | 6 +- .../worker/netty/filter/HotKeyFilter.java | 4 +- .../hotkey/worker/tool/ProtostuffUtils.java | 77 ------------------- worker/src/test/java/Test.java | 49 ++++++------ 4 files changed, 29 insertions(+), 107 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java index ec8a185..aa09758 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java @@ -42,7 +42,6 @@ public class CaffeineCacheHolder { public static void clearCacheByAppName(String appName) { if(CACHE_MAP.get(appName) != null) { CACHE_MAP.get(appName).invalidateAll(); - CACHE_MAP.put(appName, null); } } @@ -60,4 +59,9 @@ public class CaffeineCacheHolder { return map; } + + public static void main(String[] args) { + Cache cache = getCache("a"); + clearCacheByAppName("a"); + } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index e6ceb84..080a9d9 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -60,7 +60,7 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { if (WhiteListHolder.contains(model.getKey())) { return; } - keyProducer.push(model, ctx); + keyProducer.push(model); return; } //老版的用的单个HotKeyModel，新版用的数组 @@ -74,7 +74,7 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { if (timeOut > 1000) { logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); } - keyProducer.push(model,ctx); + keyProducer.push(model); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java deleted file mode 100644 index 0cb88c5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.jd.platform.hotkey.worker.tool; - -import io.protostuff.LinkedBuffer; -import io.protostuff.ProtostuffIOUtil; -import io.protostuff.Schema; -import io.protostuff.runtime.RuntimeSchema; - -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -/** - * - * @author 周志刚 - * @date 2019/6/18 - **/ -public class ProtostuffUtils { - /** - * 避免每次序列化都重新申请Buffer空间 - */ - private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); - /** - * 缓存Schema - */ - private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); - - /** - * 序列化方法，把指定对象序列化成字节数组 - * - * @param obj - * @param - * @return - */ - @SuppressWarnings("unchecked") - public static byte[] serialize(T obj) { - Class clazz = (Class) obj.getClass(); - Schema schema = getSchema(clazz); - byte[] data; - try { - data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); - } finally { - buffer.clear(); - } - - return data; - } - - /** - * 反序列化方法，将字节数组反序列化成指定Class类型 - * - * @param data - * @param clazz - * @param - * @return - */ - public static T deserialize(byte[] data, Class clazz) { - Schema schema = getSchema(clazz); - T obj = schema.newMessage(); - ProtostuffIOUtil.mergeFrom(data, obj, schema); - return obj; - } - - @SuppressWarnings("unchecked") - private static Schema getSchema(Class clazz) { - Schema schema = (Schema) schemaCache.get(clazz); - if (Objects.isNull(schema)) { - //这个schema通过RuntimeSchema进行懒创建并缓存 - //所以可以一直调用RuntimeSchema.getSchema(),这个方法是线程安全的 - schema = RuntimeSchema.getSchema(clazz); - if (Objects.nonNull(schema)) { - schemaCache.put(clazz, schema); - } - } - - return schema; - } -} \ No newline at end of file diff --git a/worker/src/test/java/Test.java b/worker/src/test/java/Test.java index 92a993c..f041c9c 100644 --- a/worker/src/test/java/Test.java +++ b/worker/src/test/java/Test.java @@ -1,8 +1,3 @@ -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.worker.tool.ProtostuffUtils; - /** * @author wuweifeng * @version 1.0 @@ -10,28 +5,28 @@ import com.jd.platform.hotkey.worker.tool.ProtostuffUtils; */ public class Test { public static void main(String[] args) { - HotKeyMsg hotKeyMsg = new HotKeyMsg(); - hotKeyMsg.setAppName("cartsoa"); - HotKeyModel hotKeyModel = new HotKeyModel(); - hotKeyModel.setCount(1); - hotKeyModel.setKey("pin_xx"); - hotKeyModel.setAppName("cartsoa"); - hotKeyMsg.setBody(FastJsonUtils.convertObjectToJSON(hotKeyModel)); - - byte[] serialize = ProtostuffUtils.serialize(hotKeyMsg); - String msg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); - - long time1 = System.currentTimeMillis(); - for (int i = 0; i < 300000; i++) { - HotKeyMsg hhh = ProtostuffUtils.deserialize(serialize, HotKeyMsg.class); - } - System.out.println(System.currentTimeMillis() - time1); - - long time = System.currentTimeMillis(); - for (int i = 0; i < 300000; i++) { - HotKeyMsg hhh = FastJsonUtils.toBean(msg, HotKeyMsg.class); - } - System.out.println(System.currentTimeMillis() - time); +// HotKeyMsg hotKeyMsg = new HotKeyMsg(); +// hotKeyMsg.setAppName("cartsoa"); +// HotKeyModel hotKeyModel = new HotKeyModel(); +// hotKeyModel.setCount(1); +// hotKeyModel.setKey("pin_xx"); +// hotKeyModel.setAppName("cartsoa"); +// hotKeyMsg.setBody(FastJsonUtils.convertObjectToJSON(hotKeyModel)); +// +// byte[] serialize = ProtostuffUtils.serialize(hotKeyMsg); +// String msg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); +// +// long time1 = System.currentTimeMillis(); +// for (int i = 0; i < 300000; i++) { +// HotKeyMsg hhh = ProtostuffUtils.deserialize(serialize, HotKeyMsg.class); +// } +// System.out.println(System.currentTimeMillis() - time1); +// +// long time = System.currentTimeMillis(); +// for (int i = 0; i < 300000; i++) { +// HotKeyMsg hhh = FastJsonUtils.toBean(msg, HotKeyMsg.class); +// } +// System.out.println(System.currentTimeMillis() - time); } -- Gitee From 6cbd781fe551a58b048c69cc3993487600667ed6 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 6 Aug 2020 21:17:18 +0800 Subject: [PATCH 08/42] =?UTF-8?q?=E9=87=87=E7=94=A8ChannelGroup=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E4=B9=8B=E5=89=8D=E7=9A=84hashmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/worker/model/AppInfo.java | 42 ++++++++++--------- .../netty/client/ClientChangeListener.java | 22 ++++------ .../netty/client/IClientChangeListener.java | 2 +- .../worker/netty/pusher/AppServerPusher.java | 17 ++------ .../netty/server/NodesServerHandler.java | 3 +- .../hotkey/worker/starters/EtcdStarter.java | 5 +-- 6 files changed, 36 insertions(+), 55 deletions(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java b/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java index 4e106a3..599edde 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java @@ -1,9 +1,9 @@ package com.jd.platform.hotkey.worker.model; import io.netty.channel.ChannelHandlerContext; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import io.netty.channel.group.ChannelGroup; +import io.netty.channel.group.DefaultChannelGroup; +import io.netty.util.concurrent.GlobalEventExecutor; /** * @author wuweifeng wrote on 2019-12-05 @@ -15,31 +15,33 @@ public class AppInfo { */ private String appName; /** - * 客户端ip 和 channel的映射关系 + * 某app的全部channel */ - private Map map = new ConcurrentHashMap<>(); - - @Override - public String toString() { - return "AppInfo{" + - "appName='" + appName + '\'' + - ", map=" + map + - '}'; + private ChannelGroup channelGroup; + + public AppInfo(String appName) { + this.appName = appName; + channelGroup = new DefaultChannelGroup(appName, GlobalEventExecutor.INSTANCE); } - public String getAppName() { - return appName; + public void groupPush(Object object) { + channelGroup.writeAndFlush(object); } - public void setAppName(String appName) { - this.appName = appName; + public void add(ChannelHandlerContext ctx) { + channelGroup.add(ctx.channel()); } - public Map getMap() { - return map; + public void remove(ChannelHandlerContext ctx) { + channelGroup.remove(ctx.channel()); } - public void setMap(Map map) { - this.map = map; + public String getAppName() { + return appName; } + + public int size() { + return channelGroup.size(); + } + } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java index daca0c9..f2362b7 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java @@ -1,13 +1,12 @@ package com.jd.platform.hotkey.worker.netty.client; +import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; - /** * 对客户端的管理，新来、断线的管理 * @@ -20,7 +19,7 @@ public class ClientChangeListener implements IClientChangeListener { private static final String NEW_CLIENT = "监听到事件"; private static final String NEW_CLIENT_JOIN = "new client join"; - private static final String CLIENT_LOSE = "client removed"; + private static final String CLIENT_LOSE = "client removed "; /** * 客户端新增 @@ -33,29 +32,24 @@ public class ClientChangeListener implements IClientChangeListener { for (AppInfo appInfo : ClientInfoHolder.apps) { if (appName.equals(appInfo.getAppName())) { appExist = true; - appInfo.getMap().put(ip, ctx); + appInfo.add(ctx); break; } } if (!appExist) { - AppInfo appInfo = new AppInfo(); - appInfo.setAppName(appName); + AppInfo appInfo = new AppInfo(appName); ClientInfoHolder.apps.add(appInfo); - appInfo.getMap().put(ip, ctx); + appInfo.add(ctx); } logger.info(NEW_CLIENT_JOIN); } @Override - public synchronized void loseClient(String ip) { + public synchronized void loseClient(ChannelHandlerContext ctx) { for (AppInfo appInfo : ClientInfoHolder.apps) { - Map map = appInfo.getMap(); - if (map.containsKey(ip)) { - map.remove(ip); - break; - } + appInfo.remove(ctx); } - logger.info(CLIENT_LOSE); + logger.info(CLIENT_LOSE + NettyIpUtil.clientIp(ctx)); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java index a6656b8..b97825b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java @@ -15,5 +15,5 @@ public interface IClientChangeListener { /** * 客户端掉线 */ - void loseClient(String channelId); + void loseClient(ChannelHandlerContext ctx); } \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java index db89b85..99058cf 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java @@ -6,14 +6,10 @@ import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.worker.model.AppInfo; -import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; import org.springframework.stereotype.Component; -import java.util.Map; - /** * 推送到各客户端服务器 * @author wuweifeng wrote on 2020-02-24 @@ -29,21 +25,14 @@ public class AppServerPusher implements IPusher { public void push(HotKeyModel model) { for (AppInfo appInfo : ClientInfoHolder.apps) { if (model.getAppName().equals(appInfo.getAppName())) { - Map map = appInfo.getMap(); HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.RESPONSE_NEW_KEY, FastJsonUtils.convertObjectToJSON(model)); String hotMsg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); -// for (ChannelHandlerContext channel : map.values()) { -// ByteBuf byteBuf = MsgBuilder.buildByteBuf(hotMsg); -// FlushUtil.flush(channel, byteBuf); -// } + ByteBuf byteBuf = MsgBuilder.buildByteBuf(hotMsg); - //并行发送 - map.values().parallelStream().forEach(channel -> { - ByteBuf byteBuf = MsgBuilder.buildByteBuf(hotMsg); - FlushUtil.flush(channel, byteBuf); - }); + //整个app全部发送 + appInfo.groupPush(byteBuf); return; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java index 64f2a2b..d6b5fcc 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java @@ -2,7 +2,6 @@ package com.jd.platform.hotkey.worker.netty.server; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import io.netty.channel.ChannelHandlerContext; @@ -59,7 +58,7 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { if (clientEventListener != null) { - clientEventListener.loseClient(NettyIpUtil.clientIp(ctx)); + clientEventListener.loseClient(ctx); } ctx.close(); super.channelInactive(ctx); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 9ce9cc9..9fe62a4 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -20,7 +20,6 @@ import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; import com.jd.platform.hotkey.worker.tool.AsyncPool; import com.jd.platform.hotkey.worker.tool.InitConstant; import io.grpc.StatusRuntimeException; -import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -31,7 +30,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.annotation.Resource; import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -196,8 +194,7 @@ public class EtcdStarter { String ip = IpUtils.getIp(); for (AppInfo appInfo : ClientInfoHolder.apps) { String appName = appInfo.getAppName(); - Map map = appInfo.getMap(); - int count = map.values().size(); + int count = appInfo.size(); //即便是full gc也不能超过3秒 configCenter.putAndGrant(ConfigConstant.clientCountPath + appName + "/" + ip, count + "", 13); } -- Gitee From f0947b8b87d8f8a004a6865eba55c4031d28e5d7 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 17 Aug 2020 09:52:48 +0800 Subject: [PATCH 09/42] =?UTF-8?q?=E9=87=87=E7=94=A8ChannelGroup=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E4=B9=8B=E5=89=8D=E7=9A=84hashmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/sample/Starter.java | 19 +++++++++++++++++++ .../sample/controller/TestController.java | 4 ++-- sample/src/main/resources/application.yml | 4 ++-- .../worker/netty/server/NodesServer.java | 5 +++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index d63e3a0..244c54b 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -2,7 +2,10 @@ package com.jd.platform.sample; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.client.ClientStarter; +import com.jd.platform.hotkey.client.callback.JdHotKeyStore; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -23,6 +26,7 @@ public class Starter { @Resource private IConfigCenter iConfigCenter; + private Logger logger = LoggerFactory.getLogger(getClass()); @@ -36,6 +40,21 @@ public class Starter { for (KeyValue keyValue : list) { System.out.println(keyValue.getKey() + keyValue.getValue().toStringUtf8()); } + +// a(); + } + + private void a() { + System.out.println("begin"); + while (true) { + if (JdHotKeyStore.getValue("a") != null) { + + } else { + JdHotKeyStore.smartSet("a", 1); + logger.error("smartSet"); + } + } + } } diff --git a/sample/src/main/java/com/jd/platform/sample/controller/TestController.java b/sample/src/main/java/com/jd/platform/sample/controller/TestController.java index bf643fb..8063822 100644 --- a/sample/src/main/java/com/jd/platform/sample/controller/TestController.java +++ b/sample/src/main/java/com/jd/platform/sample/controller/TestController.java @@ -76,8 +76,8 @@ public class TestController { // } @RequestMapping("") - public Object a() { - if (JdHotKeyStore.isHotKey("pin_tianyalei")) { + public Object a(String s) { + if (JdHotKeyStore.isHotKey("ecitemcenter_itemV2_" + s)) { logger.error("isHot"); } else { logger.error("noHot"); diff --git a/sample/src/main/resources/application.yml b/sample/src/main/resources/application.yml index 41fc686..6424c32 100644 --- a/sample/src/main/resources/application.yml +++ b/sample/src/main/resources/application.yml @@ -1,8 +1,8 @@ #etcd的地址，如有多个用逗号分隔 etcd: -# server: ${etcdServer:https://127.0.0.1:2379} + server: ${etcdServer:https://127.0.0.1:2379} # server: http://10.170.161.91:2379 - server: http://open-etcd.jd.com:2000 + # server: http://open-etcd.jd.com:2000 spring: application: name: sample diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java index 6ea9b4f..b0eacd0 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java @@ -17,6 +17,7 @@ import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import java.util.List; +import java.util.concurrent.TimeUnit; /** * 该server用于给各个微服务实例连接用。 @@ -44,6 +45,10 @@ public class NodesServer { .childHandler(new ChildChannelHandler()); //绑定端口，同步等待成功 ChannelFuture future = bootstrap.bind(port).sync(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bossGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + workerGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + })); //等待服务器监听端口关闭 future.channel().closeFuture().sync(); } catch (Exception e) { -- Gitee From 927dcde57693638b660e3a63cde6fdcd37c18890 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Tue, 18 Aug 2020 20:55:06 +0800 Subject: [PATCH 10/42] update LICENSE. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 261eeb9..48d4f57 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright [2020] [JD.com, Inc.] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -- Gitee From 0190cc71c76aab2bb05136db992be9a8e3678d59 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 21 Aug 2020 11:19:44 +0800 Subject: [PATCH 11/42] =?UTF-8?q?fix=20bd.sql=20=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E7=83=AD=E7=82=B9=E5=88=A0=E9=99=A4=E4=BF=AE=E6=94=B9=E5=86=85?= =?UTF-8?q?=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/service/impl/KeyServiceImpl.java | 2 +- dashboard/src/main/resources/db.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 04f80c6..d41ebb6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -198,7 +198,7 @@ public class KeyServiceImpl implements KeyService { KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); - return logMapper.insertSelective(new ChangeLog(arr[0], Constant.HOTKEY_CHANGE, arr[1],"", keyTimely.getUpdater(), SystemClock.now() + "")); + return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(),"", keyTimely.getUpdater(), SystemClock.now() + "")); } @Override diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index c959191..8cbb0fa 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -44,7 +44,7 @@ CREATE TABLE `hk_key_record` ( `uuid` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '防重ID', `rule` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '''' COMMENT '规则', PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引' + UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引', INDEX `idx_app_ct`(`app_name`, `create_time`) USING BTREE COMMENT '查询索引' ) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; -- Gitee From 93c421e3f6b2efc8cc0c6efb4e401db56dede134 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 24 Aug 2020 16:30:40 +0800 Subject: [PATCH 12/42] =?UTF-8?q?=E4=BF=AE=E6=94=B9IO=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/worker/cache/CaffeineCacheHolder.java | 5 ----- .../worker/keydispatcher/DispatcherConfig.java | 4 ++++ .../hotkey/worker/netty/filter/HotKeyFilter.java | 9 --------- .../hotkey/worker/netty/server/NodesServer.java | 8 +------- .../worker/starters/NodesServerStarter.java | 4 ---- .../jd/platform/hotkey/worker/tool/CpuNum.java | 16 ++++++---------- 6 files changed, 11 insertions(+), 35 deletions(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java index aa09758..d3801e8 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java @@ -59,9 +59,4 @@ public class CaffeineCacheHolder { return map; } - - public static void main(String[] args) { - Cache cache = getCache("a"); - clearCacheByAppName("a"); - } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java index defcf67..7e3094a 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java @@ -41,6 +41,10 @@ public class DispatcherConfig { //将实际值赋给static变量 if (threadCount != 0) { nowCount = threadCount; + } else { + if (nowCount >= 8) { + nowCount = nowCount / 2; + } } List consumerList = new ArrayList<>(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 080a9d9..c893c79 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -54,15 +54,6 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { } private void publishMsg(String message, ChannelHandlerContext ctx) { - //这个是给测试用的，实际走的是下面那个 - if (message.startsWith("{")) { - HotKeyModel model = FastJsonUtils.toBean(message, HotKeyModel.class); - if (WhiteListHolder.contains(model.getKey())) { - return; - } - keyProducer.push(model); - return; - } //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, HotKeyModel.class); for (HotKeyModel model : models) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java index b0eacd0..0762742 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.netty.server; -import com.jd.platform.hotkey.common.coder.Codec; import com.jd.platform.hotkey.common.tool.Constant; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; @@ -27,7 +26,6 @@ import java.util.concurrent.TimeUnit; public class NodesServer { private IClientChangeListener clientChangeListener; private List messageFilters; - private Codec codec; public void startNettyServer(int port) throws Exception { //boss单线程 @@ -52,6 +50,7 @@ public class NodesServer { //等待服务器监听端口关闭 future.channel().closeFuture().sync(); } catch (Exception e) { + e.printStackTrace(); //do nothing System.out.println("netty stop"); } finally { @@ -75,8 +74,6 @@ public class NodesServer { ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); ch.pipeline() .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) -// .addLast(codec.newEncoder()) -// .addLast(codec.newDecoder()) .addLast(new StringDecoder()) .addLast(serverHandler); } @@ -90,7 +87,4 @@ public class NodesServer { this.messageFilters = messageFilters; } - public void setCodec(Codec codec) { - this.codec = codec; - } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java index 0b9c522..5759f5e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.starters; -import com.jd.platform.hotkey.common.coder.Codec; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import com.jd.platform.hotkey.worker.netty.server.NodesServer; @@ -29,8 +28,6 @@ public class NodesServerStarter { private IClientChangeListener iClientChangeListener; @Resource private List messageFilters; - @Resource - private Codec codec; @PostConstruct public void start() { @@ -40,7 +37,6 @@ public class NodesServerStarter { NodesServer nodesServer = new NodesServer(); nodesServer.setClientChangeListener(iClientChangeListener); nodesServer.setMessageFilters(messageFilters); - nodesServer.setCodec(codec); try { nodesServer.startNettyServer(port); } catch (Exception e) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java index acaa70e..55bd842 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java @@ -13,17 +13,13 @@ public class CpuNum { public static int workerCount() { //取cpu核数，新版jdk在docker里取的就是真实分配的，老版jdk取的是宿主机的，可能特别大，如32核 int count = Runtime.getRuntime().availableProcessors(); -// if (isNewerVersion()) { -// if (count >= 4) { -// count = count / 2; -// } -// } else { -// if (count >= 8) { -// count = 4; -// } -// } - if (count >= 4) { + if (isNewerVersion()) { + return count; + } else { count = count / 2; + if (count == 0) { + count = 1; + } } return count; } -- Gitee From a7b9666cd224c75a29f222a08fd18d49be1df3f2 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 25 Aug 2020 11:50:01 +0800 Subject: [PATCH 13/42] =?UTF-8?q?=E9=87=87=E7=94=A8ChannelGroup=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E4=B9=8B=E5=89=8D=E7=9A=84hashmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pom.xml | 2 +- .../common/configcenter/etcd/JdEtcdClient.java | 1 - .../java/com/jd/platform/sample/Starter.java | 17 +++++++++++++++++ .../hotkey/worker/netty/pusher/EtcdPusher.java | 17 ++++++++++++++--- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 59a7d76..a548184 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -18,7 +18,7 @@ 3.4.2 1.1.7.3 1.2.70 - 0.0.14 + 0.0.16 2.8.0 5.1.0 diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java index 06e85e9..fb64d9e 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java @@ -3,7 +3,6 @@ package com.jd.platform.hotkey.common.configcenter.etcd; import cn.hutool.core.collection.CollectionUtil; import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.api.Kv; import com.ibm.etcd.api.LeaseGrantResponse; import com.ibm.etcd.api.RangeResponse; import com.ibm.etcd.client.KvStoreClient; diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index 244c54b..2386483 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -12,6 +12,9 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; /** * @author wuweifeng wrote on 2020-01-14 @@ -28,6 +31,7 @@ public class Starter { private IConfigCenter iConfigCenter; private Logger logger = LoggerFactory.getLogger(getClass()); + private AtomicLong counter = new AtomicLong(); @PostConstruct @@ -41,6 +45,19 @@ public class Starter { System.out.println(keyValue.getKey() + keyValue.getValue().toStringUtf8()); } + + ExecutorService executorService = Executors.newFixedThreadPool(128); + +// for (int i = 0; i < 200000; i++) { +// final int j = i; +// executorService.submit(() -> { +// iConfigCenter.putAndGrant( j + "", UUID.randomUUID().toString(), 10000); +// counter.incrementAndGet(); +// if (counter.get() % 5000 == 0) { +// logger.info("counter:" + counter.get()); +// } +// }); +// } // a(); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java index 627d3eb..aa1971b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java @@ -8,9 +8,12 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** + * 将热key推送到etcd，dashboard监听后留做备份 * @author wuweifeng wrote on 2020-02-24 * @version 1.0 */ @@ -19,12 +22,20 @@ public class EtcdPusher implements IPusher { @Resource private IConfigCenter iConfigCenter; + /** + * 将推送key到etcd的任务都丢到这个线程池里 + */ + private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(128); @Override public void push(HotKeyModel model) { - //推送到etcd，供dashboard监听入库 - iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), - KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); + //2020-8-25日，新增线程池写入etcd的逻辑。本地测试发现8线程写入etcd，每秒最多500个key-value，128线程能写2500个以上 + EXECUTOR_SERVICE.submit(() -> { + //推送到etcd，供dashboard监听入库。 + iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), + KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); + }); + } @Override -- Gitee From 89928fb6e44350a83bfe620d01f8636355e74fe9 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 25 Aug 2020 16:06:02 +0800 Subject: [PATCH 14/42] =?UTF-8?q?=E8=A7=A3=E5=86=B3dashboard=E7=9B=91?= =?UTF-8?q?=E5=90=AC=E6=96=B0key=E4=BA=A7=E7=94=9F=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=97=B6=E9=97=B4=E5=B0=8F=E4=BA=8E0?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/monitor/EtcdMonitor.java | 16 ++++++++++------ .../java/com/jd/platform/sample/Starter.java | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 92fa472..c69312c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -19,7 +19,6 @@ import io.grpc.StatusRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; import javax.annotation.PostConstruct; import javax.annotation.Resource; @@ -55,7 +54,7 @@ public class EtcdMonitor { private DataHandler dataHandler; - public static final ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); + public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(256); /** * 监听新来的热key，该key的产生是来自于手工在控制台添加 @@ -82,12 +81,17 @@ public class EtcdMonitor { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyRecordPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); - EventWrapper eventWrapper = build(event); - String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyRecordPath, ""); - eventWrapper.setKey(appKey); + //提交获取该key的任务到线程池 + threadPoolExecutor.submit(() -> { + EventWrapper eventWrapper = build(event); + + String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyRecordPath, ""); + eventWrapper.setKey(appKey); + + dataHandler.offer(eventWrapper); + }); - dataHandler.offer(eventWrapper); } }); } diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index 2386483..a754862 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -51,7 +51,7 @@ public class Starter { // for (int i = 0; i < 200000; i++) { // final int j = i; // executorService.submit(() -> { -// iConfigCenter.putAndGrant( j + "", UUID.randomUUID().toString(), 10000); +// iConfigCenter.putAndGrant( ConfigConstant.hotKeyRecordPath + "abc/" + j, UUID.randomUUID().toString(), 10000); // counter.incrementAndGet(); // if (counter.get() % 5000 == 0) { // logger.info("counter:" + counter.get()); -- Gitee From c6e97413c1efe102b1a303020b9b86e42ffd5db7 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 27 Aug 2020 14:59:41 +0800 Subject: [PATCH 15/42] =?UTF-8?q?worker=E5=8F=98=E6=9B=B4=E4=B8=8D?= =?UTF-8?q?=E5=85=A5=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/EtcdMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index c69312c..d70bb87 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -128,7 +128,7 @@ public class EtcdMonitor { //监听rule变化 watchRule(); - watchWorkers(); +// watchWorkers(); //观察热key访问次数和总访问次数，并做统计 watchHitCount(); -- Gitee From 7702b8090a312914f7c11d5c28494e930d624251 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 27 Aug 2020 16:07:27 +0800 Subject: [PATCH 16/42] =?UTF-8?q?=E5=8F=98=E6=9B=B4=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E7=9A=84=E5=88=9B=E5=BB=BA=E6=97=B6=E9=97=B4=E5=B1=95=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/templates/admin/changeLog/list.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dashboard/src/main/resources/templates/admin/changeLog/list.html b/dashboard/src/main/resources/templates/admin/changeLog/list.html index a6fbf8d..d6ed97a 100644 --- a/dashboard/src/main/resources/templates/admin/changeLog/list.html +++ b/dashboard/src/main/resources/templates/admin/changeLog/list.html @@ -142,6 +142,11 @@ var options = { } } }, + { field: 'createTime', title: '创建时间', + formatter:function (val,row,index) { + return changeDateFormat(val); + } + }, { field: 'from', title: '初始值', -- Gitee From 20a5546f639804fcdfa1478c015ff03009a0cdbc Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 28 Aug 2020 18:37:43 +0800 Subject: [PATCH 17/42] =?UTF-8?q?worker=E6=94=B9=E4=B8=BA=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=8E=A8=E9=80=81hotkey=E5=88=B0batch=E5=9C=B0?= =?UTF-8?q?=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 6 +- .../dashboard/common/monitor/DataHandler.java | 48 ++++++++------ .../dashboard/common/monitor/EtcdMonitor.java | 41 ++++++++++-- .../java/com/jd/platform/sample/Starter.java | 38 ++++++++++- .../worker/netty/pusher/EtcdPusher.java | 65 +++++++++++++++---- worker/src/test/java/Test.java | 11 +++- worker/src/test/java/TestBatch.java | 36 ++++++++++ 7 files changed, 202 insertions(+), 43 deletions(-) create mode 100644 worker/src/test/java/TestBatch.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 7382d8d..9a8d62c 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -1,7 +1,5 @@ package com.jd.platform.hotkey.common.configcenter; -import java.util.concurrent.TimeUnit; - /** * @author wuweifeng wrote on 2019-12-06 * @version 1.0 @@ -39,6 +37,10 @@ public interface ConfigConstant { * 每个app的热key记录放这里，供控制台监听入库用。格式如：jd/records/app1/userA */ String hotKeyRecordPath = "/jd/keyRecords/"; + /** + * 每个app的热key记录放这里，供控制台监听入库用。格式如：jd/batchRecords/ jd/records/app1/userA,jd/records/app1/userB 一个逗号分隔的巨大的key集合 + */ + String hotKeyBatchRecordPath = "/jd/keyBatchRecords/"; /** * caffeine的size */ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 8a5dac9..1b58d45 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -28,6 +28,7 @@ import javax.annotation.Resource; import java.time.LocalDateTime; import java.util.Date; import java.util.List; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -68,34 +69,41 @@ public class DataHandler { public void insertRecords() { while (true) { - TwoTuple twoTuple; + EventWrapper eventWrapper; try { - twoTuple = handHotKey(queue.take()); + eventWrapper = queue.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + continue; + } + try { + TwoTuple twoTuple = handHotKey(eventWrapper); if (twoTuple == null) { continue; } + KeyRecord keyRecord = twoTuple.getSecond(); + KeyTimely keyTimely = twoTuple.getFirst(); + + if (keyTimely.getUuid() == null) { + keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); + } else { + try { + keyTimelyMapper.saveOrUpdate(keyTimely); + } catch (Exception e) { + log.info("insert timely error",e); + } + } + + if (keyRecord != null) { + //插入记录 + keyRecordMapper.insertSelective(keyRecord); + } } catch (Exception e) { + log.error("eventWrapper:" + eventWrapper); e.printStackTrace(); log.error("handHotKey error ," + e.getCause()); - continue; - } - KeyRecord keyRecord = twoTuple.getSecond(); - KeyTimely keyTimely = twoTuple.getFirst(); - - if (keyTimely.getUuid() == null) { - keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); - } else { - try { - keyTimelyMapper.saveOrUpdate(keyTimely); - } catch (Exception e) { - log.info("insert timely error",e); - } } - if (keyRecord != null) { - //插入记录 - keyRecordMapper.insertSelective(keyRecord); - } } @@ -115,7 +123,7 @@ public class DataHandler { String[] arr = appKey.split("/"); String appName = arr[0]; String key = arr[1]; - String uuid = eventWrapper.getUuid(); + String uuid = UUID.randomUUID().toString(); int type = eventType.getNumber(); //组建成对象，供累计后批量插入、删除 diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index d70bb87..cdb362a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -22,10 +22,7 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -96,6 +93,37 @@ public class EtcdMonitor { }); } + /** + * 监听新来的批量热key，该key的产生是来自于worker集群推送过来的 + */ + public void watchBatchHotKeyRecord() { + threadPoolExecutor.submit(() -> { + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyBatchRecordPath); + while (watchIterator.hasNext()) { + Event event = event(watchIterator); + + //提交获取该key的任务到线程池 + threadPoolExecutor.submit(() -> { + EventWrapper eventWrapper = build(event); + + String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyBatchRecordPath, ""); + String[] keyArray = appKey.split(","); + for (int i = 0; i < keyArray.length; i++) { + String key = keyArray[i]; + if (StrUtil.isEmpty(key)) { + continue; + } + eventWrapper.setKey(key.replace(ConfigConstant.hotKeyRecordPath, "")); + + dataHandler.offer(eventWrapper); + } + + }); + + } + }); + } + private EventWrapper build(Event event) { KeyValue kv = event.getKv(); long ttl = configCenter.timeToLive(kv.getLease()); @@ -107,7 +135,6 @@ public class EtcdMonitor { eventWrapper.setTtl(ttl); eventWrapper.setVersion(kv.getVersion()); eventWrapper.setEventType(eventType); - eventWrapper.setUuid(v); return eventWrapper; } @@ -123,6 +150,10 @@ public class EtcdMonitor { //开始监听热key产生 watchHotKeyRecord(); + //监听批量发来的热key + watchBatchHotKeyRecord(); + + //监听手工创建的key watchHandOperationHotKey(); //监听rule变化 diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index a754862..a94e187 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -1,5 +1,7 @@ package com.jd.platform.sample; +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.base.Joiner; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.client.ClientStarter; import com.jd.platform.hotkey.client.callback.JdHotKeyStore; @@ -12,6 +14,7 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicLong; @@ -44,10 +47,18 @@ public class Starter { for (KeyValue keyValue : list) { System.out.println(keyValue.getKey() + keyValue.getValue().toStringUtf8()); } - + //监听巨大的key +// watchBigKey(); ExecutorService executorService = Executors.newFixedThreadPool(128); + + StringBuilder builder1 = new StringBuilder(); + for (int i = 0; i < 10000; i++) { + builder1.append(UUID.randomUUID()); + } + logger.info("begin : " + System.currentTimeMillis()); + iConfigCenter.putAndGrant( "/abc/" + builder1.toString(), UUID.randomUUID().toString(), 60); // for (int i = 0; i < 200000; i++) { // final int j = i; // executorService.submit(() -> { @@ -61,6 +72,20 @@ public class Starter { // a(); } +// private void watchBigKey() { +// CompletableFuture.runAsync(() -> { +// KvClient.WatchIterator watchIterator = iConfigCenter.watchPrefix("/abc"); +// //如果有新事件，即rule的变更，就重新拉取所有的信息 +// while (watchIterator.hasNext()) { +// //这句必须写，next会让他卡住，除非真的有新rule变更 +// Event event = watchIterator.next().getEvents().get(0); +// double m = ObjectSizeCalculator.getObjectSize(event.getKv().getKey().toStringUtf8()); +// logger.error("length:" + m); +// } +// +// }); +// } + private void a() { System.out.println("begin"); while (true) { @@ -74,4 +99,15 @@ public class Starter { } + + public static void main(String[] args) { + Joiner joiner = Joiner.on(","); + String s = "1"; + List list = CollectionUtil.list(true, s); + String j = joiner.join(list); + System.out.println(j); + + String[] ss = j.split(","); + System.out.println(ss); + } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java index aa1971b..e588aa2 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java @@ -1,19 +1,26 @@ package com.jd.platform.hotkey.worker.netty.pusher; +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.base.Joiner; +import com.google.common.collect.Queues; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.tool.HotKeyPathTool; import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; +import com.jd.platform.hotkey.worker.tool.AsyncPool; import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.*; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; /** * 将热key推送到etcd，dashboard监听后留做备份 + * * @author wuweifeng wrote on 2020-02-24 * @version 1.0 */ @@ -22,20 +29,52 @@ public class EtcdPusher implements IPusher { @Resource private IConfigCenter iConfigCenter; - /** - * 将推送key到etcd的任务都丢到这个线程池里 - */ - private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(128); + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); @Override public void push(HotKeyModel model) { - //2020-8-25日，新增线程池写入etcd的逻辑。本地测试发现8线程写入etcd，每秒最多500个key-value，128线程能写2500个以上 - EXECUTOR_SERVICE.submit(() -> { - //推送到etcd，供dashboard监听入库。 - iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), - KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); - }); + hotKeyStoreQueue.offer(model); + } + + @PostConstruct + public void uploadToEtcd() { + Map> map = new HashMap<>(); + AsyncPool.asyncDo(() -> { + while (true) { + try { + //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 + List tempModels = new ArrayList<>(); + Queues.drain(hotKeyStoreQueue, tempModels, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + //区分出来不同的duration，不同的duration往etcd放时方法不一样 + for (int i = 0; i < tempModels.size(); i++) { + HotKeyModel hotKeyModel = tempModels.get(i); + int duration = KeyRuleHolder.getRuleByAppAndKey(hotKeyModel).getDuration(); + //取到value + List keys = map.computeIfAbsent(duration, s -> new ArrayList<>()); + //将该model的key存进去 + keys.add(HotKeyPathTool.keyRecordPath(hotKeyModel)); + } + for (Integer duration : map.keySet()) { + List list = map.get(duration); + if (CollectionUtil.isEmpty(list)) { + continue; + } + Joiner joiner = Joiner.on(","); + String s = joiner.join(list); + iConfigCenter.putAndGrant(ConfigConstant.hotKeyBatchRecordPath + s, UUID.randomUUID().toString(), duration); + map.get(duration).clear(); + } + + + } catch (Exception e) { + e.printStackTrace(); + } + } + }); } @Override diff --git a/worker/src/test/java/Test.java b/worker/src/test/java/Test.java index f041c9c..c7e6940 100644 --- a/worker/src/test/java/Test.java +++ b/worker/src/test/java/Test.java @@ -6,13 +6,20 @@ public class Test { public static void main(String[] args) { // HotKeyMsg hotKeyMsg = new HotKeyMsg(); -// hotKeyMsg.setAppName("cartsoa"); +// hotKeyMsg.setAppName("cartsoafewwfwfwfw"); // HotKeyModel hotKeyModel = new HotKeyModel(); // hotKeyModel.setCount(1); -// hotKeyModel.setKey("pin_xx"); +// hotKeyModel.setKey("pin_xwefwfaregegax"); // hotKeyModel.setAppName("cartsoa"); // hotKeyMsg.setBody(FastJsonUtils.convertObjectToJSON(hotKeyModel)); // + +// for (int i = 0; i < 10000; i++) { +// +// } +// String s = ConfigConstant.hotKeyRecordPath + hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); +// double m = ObjectSizeCalculator.getObjectSize(s); +// System.out.println(m * 10000 / 1024); // byte[] serialize = ProtostuffUtils.serialize(hotKeyMsg); // String msg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); // diff --git a/worker/src/test/java/TestBatch.java b/worker/src/test/java/TestBatch.java new file mode 100644 index 0000000..524dcaa --- /dev/null +++ b/worker/src/test/java/TestBatch.java @@ -0,0 +1,36 @@ +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-08-27 + */ +//public class TestBatch { +// public static BlockingQueue QUEUE = new LinkedBlockingQueue<>(2000000); +// private static Logger logger = LoggerFactory.getLogger("tttt"); +// +// public static void main(String[] args) { +// new Thread(new Runnable() { +// @Override +// public void run() { +// while (true) { +// try { +// Thread.sleep(100); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// QUEUE.offer("1"); +// } +// } +// }).start(); +// +// while (true) { +// List list = new ArrayList<>(); +// try { +// Queues.drain(QUEUE, list, 1000, 1, TimeUnit.SECONDS); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// logger.info(list.toString()); +// } +// +// } +//} -- Gitee From e2877467cfa65ab2e94d96ceeff37518e87ef5c8 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 1 Sep 2020 13:34:07 +0800 Subject: [PATCH 18/42] =?UTF-8?q?worker=E6=8E=A2=E6=B5=8B=E5=90=8E?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5=E6=8E=A8=E5=88=B0dashboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 8 + .../hotkey/common/model/HotKeyModel.java | 1 + .../common/model/typeenum/MessageType.java | 3 +- .../hotkey/common/tool}/flush/FlushUtil.java | 2 +- .../dashboard/cache/CaffeineBuilder.java | 30 ++++ .../hotkey/dashboard/common/domain/Page.java | 18 +- .../dashboard/common/monitor/DataHandler.java | 19 ++ .../dashboard/controller/KeyController.java | 3 +- .../{common/monitor => etcd}/EtcdMonitor.java | 107 +++++++---- .../hotkey/dashboard/etcd/EtcdRegister.java | 63 +++++++ .../hotkey/dashboard/model/KeyTimely.java | 15 ++ .../dashboard/netty/HotKeyReceiver.java | 166 ++++++++++++++++++ .../hotkey/dashboard/netty/NodesServer.java | 78 ++++++++ .../dashboard/netty/NodesServerHandler.java | 66 +++++++ .../hotkey/dashboard/service/KeyService.java | 3 +- .../service/impl/KeyServiceImpl.java | 73 ++++---- .../hotkey/dashboard/util/PageUtil.java | 35 ++++ .../hotkey/dashboard/util/RuleUtil.java | 4 +- .../java/com/jd/platform/sample/Starter.java | 15 +- .../worker/keydispatcher/KeyProducer.java | 5 +- .../netty/dashboard/DashboardHolder.java | 26 +++ .../worker/netty/dashboard/NettyClient.java | 86 +++++++++ .../netty/dashboard/NettyClientHandler.java | 58 ++++++ .../worker/netty/filter/HeartBeatFilter.java | 2 +- .../worker/netty/filter/HotKeyFilter.java | 5 +- .../netty/pusher/BatchToEtcdScheduler.java | 45 ----- .../worker/netty/pusher/DashboardPusher.java | 24 +++ .../worker/netty/pusher/EtcdPusher.java | 88 ---------- .../worker/netty/pusher/KeyCollector.java | 53 ------ .../worker/netty/pusher/KeyUploader.java | 46 +++++ .../netty/pusher/store/HotkeyTempStore.java | 27 +++ .../hotkey/worker/starters/EtcdStarter.java | 25 +++ worker/src/test/java/TestBatch.java | 48 ++++- 33 files changed, 967 insertions(+), 280 deletions(-) rename {worker/src/main/java/com/jd/platform/hotkey/worker/netty => common/src/main/java/com/jd/platform/hotkey/common/tool}/flush/FlushUtil.java (95%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{common/monitor => etcd}/EtcdMonitor.java (77%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java create mode 100755 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java create mode 100755 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java create mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java create mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 9a8d62c..e115056 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -13,6 +13,10 @@ public interface ConfigConstant { * 所有的workers，存这里 */ String workersPath = "/jd/workers/"; + /** + * dashboard的ip存这里 + */ + String dashboardPath = "/jd/dashboard/"; /** * 该app所有的workers地址的path。需要手工分配，默认每个app都用所有的worker */ @@ -64,4 +68,8 @@ public interface ConfigConstant { * time unit : day */ String clearCfgPath = "/jd/clearCfg/"; + /** + * 控制台启动的netty端口 + */ + int dashboardPort = 11112; } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java index 0451fce..c82323a 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java @@ -53,4 +53,5 @@ public class HotKeyModel extends BaseModel { public void setKeyType(KeyType keyType) { this.keyType = keyType; } + } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java index cd3f983..5b07624 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java @@ -8,7 +8,8 @@ public enum MessageType { APP_NAME((byte) 1), REQUEST_NEW_KEY((byte) 2), RESPONSE_NEW_KEY((byte) 3), - REQUEST_HIT_COUNT((byte) 7), + REQUEST_HIT_COUNT((byte) 7), //命中率 + REQUEST_HOT_KEY((byte) 8), //热key，worker->dashboard PING((byte) 4), PONG((byte) 5), EMPTY((byte) 6); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java similarity index 95% rename from worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java rename to common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java index 04f1354..009eb47 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.worker.netty.flush; +package com.jd.platform.hotkey.common.tool.flush; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java new file mode 100644 index 0000000..63612e8 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java @@ -0,0 +1,30 @@ +package com.jd.platform.hotkey.dashboard.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import java.util.concurrent.TimeUnit; + +/** + * @author wuweifeng wrote on 2019-12-12 + * @version 1.0 + */ +public class CaffeineBuilder { + + public static Cache cache(int duration) { + return cache(128, 1000000, duration); + } + + + /** + * 构建所有来的要缓存的key getCache + */ + public static Cache cache(int minSize, int maxSize, int expireSeconds) { + return Caffeine.newBuilder() + .initialCapacity(minSize)//初始大小 + .maximumSize(maxSize)//最大数量 + .expireAfterWrite(expireSeconds, TimeUnit.SECONDS)//过期时间 + .build(); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java index 5f64c75..d10c3a5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java @@ -7,9 +7,13 @@ import java.util.List; public class Page implements Serializable { private static final long serialVersionUID = 1L; - + /** + * 当前第几页 + */ private Integer page; - + /** + * 总共多少条 + */ private int total; private List rows; @@ -20,6 +24,7 @@ public class Page implements Serializable { this.rows = rows; } + public Integer getPage() { return page; } @@ -43,4 +48,13 @@ public class Page implements Serializable { public void setRows(List rows) { this.rows = rows; } + + @Override + public String toString() { + return "Page{" + + "page=" + page + + ", total=" + total + + ", rows=" + rows + + '}'; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 1b58d45..0b8a50c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -6,6 +6,7 @@ import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; @@ -109,6 +110,24 @@ public class DataHandler { } + /** + * 插入记录表 + * @param type 0是新增，1是删除 + */ + public void insertRecord(String appKey, int type) { + String source = Constant.SYSTEM; + String rule = RuleUtil.rule(appKey); + KeyRule keyRule = RuleUtil.findByKey(appKey); + //appName+"/"+"key" + String[] arr = appKey.split("/"); + String appName = arr[0]; + String key = arr[1]; + String uuid = UUID.randomUUID().toString(); + KeyRecord keyRecord = new KeyRecord(key, rule, appName, (long)keyRule.getDuration(), source, type, uuid, new Date()); + keyRecord.setRule(rule); + keyRecordMapper.insertSelective(keyRecord); + } + /** * 处理热点key和记录 diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index 0b287a8..330e3b4 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -79,8 +79,7 @@ public class KeyController extends BaseController { @PostMapping("/listTimely") @ResponseBody public Page listTimely(PageReq page, SearchReq searchReq){ - PageInfo info = keyService.pageKeyTimely(page, searchReq); - return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + return keyService.pageKeyTimely(page, searchReq); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java similarity index 77% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index cdb362a..16f7e48 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.common.monitor; +package com.jd.platform.hotkey.dashboard.etcd; import cn.hutool.core.util.StrUtil; import com.ibm.etcd.api.Event; @@ -6,23 +6,30 @@ import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.client.kv.KvClient; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.Worker; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.service.WorkerService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import io.grpc.StatusRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -96,33 +103,33 @@ public class EtcdMonitor { /** * 监听新来的批量热key，该key的产生是来自于worker集群推送过来的 */ - public void watchBatchHotKeyRecord() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyBatchRecordPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - - //提交获取该key的任务到线程池 - threadPoolExecutor.submit(() -> { - EventWrapper eventWrapper = build(event); - - String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyBatchRecordPath, ""); - String[] keyArray = appKey.split(","); - for (int i = 0; i < keyArray.length; i++) { - String key = keyArray[i]; - if (StrUtil.isEmpty(key)) { - continue; - } - eventWrapper.setKey(key.replace(ConfigConstant.hotKeyRecordPath, "")); - - dataHandler.offer(eventWrapper); - } - - }); - - } - }); - } +// public void watchBatchHotKeyRecord() { +// threadPoolExecutor.submit(() -> { +// KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyBatchRecordPath); +// while (watchIterator.hasNext()) { +// Event event = event(watchIterator); +// +// //提交获取该key的任务到线程池 +// threadPoolExecutor.submit(() -> { +// EventWrapper eventWrapper = build(event); +// +// String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyBatchRecordPath, ""); +// String[] keyArray = appKey.split(","); +// for (int i = 0; i < keyArray.length; i++) { +// String key = keyArray[i]; +// if (StrUtil.isEmpty(key)) { +// continue; +// } +// eventWrapper.setKey(key.replace(ConfigConstant.hotKeyRecordPath, "")); +// +// dataHandler.offer(eventWrapper); +// } +// +// }); +// +// } +// }); +// } private EventWrapper build(Event event) { KeyValue kv = event.getKv(); @@ -147,11 +154,14 @@ public class EtcdMonitor { //规则拉取完毕后才能去开始入库 insertRecords(); + //开始入库 + dealHotkey(); + //开始监听热key产生 watchHotKeyRecord(); //监听批量发来的热key - watchBatchHotKeyRecord(); +// watchBatchHotKeyRecord(); //监听手工创建的key watchHandOperationHotKey(); @@ -165,12 +175,48 @@ public class EtcdMonitor { watchHitCount(); } + /** + * 每秒去清理一次caffeine + */ + @Scheduled(fixedRate = 1000) + public void fetchDashboardIp() { + try { + HotKeyReceiver.cleanUpCaffeine(); + } catch (Exception e) { + e.printStackTrace(); + } + } + private void insertRecords() { threadPoolExecutor.submit(() -> { dataHandler.insertRecords(); }); } + + /** + * 开始消费各worker发来的热key + */ + private void dealHotkey() { + threadPoolExecutor.submit(() -> { + try { + while (true) { + //获取发来的这个热key，存入本地caffeine，设置过期时间 + HotKeyModel model = HotKeyReceiver.getQueue().take(); + + //将该key放入实时热key本地缓存中 + HotKeyReceiver.put(model); + + //入库record表，TODO 需要改成batch入库 + dataHandler.insertRecord(model.getAppName() + "/" + model.getKey(), 0); + } + + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + } + /** * 异步监听rule规则变化 */ @@ -252,7 +298,6 @@ public class EtcdMonitor { } - /** * 监听热key访问次数和总访问次数 */ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java new file mode 100644 index 0000000..d0a44c5 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java @@ -0,0 +1,63 @@ +package com.jd.platform.hotkey.dashboard.etcd; + +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.tool.IpUtils; +import com.jd.platform.hotkey.dashboard.netty.NodesServer; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * dashboard也注册到etcd，供各个worker连接 + * + * @author wuweifeng + * @version 1.0 + * @date 2020-08-28 + */ +@Component +public class EtcdRegister { + @Resource + private IConfigCenter configCenter; + @Resource + private NodesServer nodesServer; + + /** + * 每隔一会去check一下，自己还在不在etcd里 + */ + @PostConstruct + public void makeSureSelfOn() { + new Thread(() -> nodesServer.startNettyServer(ConfigConstant.dashboardPort)).start(); + + //开启上传worker信息 + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.scheduleAtFixedRate(() -> { + + try { + uploadSelfInfo(); + } catch (Exception e) { + //do nothing + } + + }, 3, 30, TimeUnit.SECONDS); + } + + private void uploadSelfInfo() { + configCenter.putAndGrant(buildKey(), buildValue(), 32); + } + + private String buildKey() { + String hostName = IpUtils.getHostName(); + return ConfigConstant.dashboardPath + hostName; + } + + private String buildValue() { + String ip = IpUtils.getIp(); + return ip + ":" + ConfigConstant.dashboardPort; + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java index 6efd7c5..0dd7907 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java @@ -50,6 +50,21 @@ public class KeyTimely implements Serializable { this.createTime = createTime; } + @Override + public String toString() { + return "KeyTimely{" + + "id=" + id + + ", key='" + key + '\'' + + ", appName='" + appName + '\'' + + ", val='" + val + '\'' + + ", duration=" + duration + + ", createTime=" + createTime + + ", uuid='" + uuid + '\'' + + ", ruleDesc='" + ruleDesc + '\'' + + ", updater='" + updater + '\'' + + '}'; + } + public String getRuleDesc() { return ruleDesc; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java new file mode 100644 index 0000000..ff2f558 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -0,0 +1,166 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import cn.hutool.core.util.StrUtil; +import com.github.benmanes.caffeine.cache.Cache; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.rule.KeyRule; +import com.jd.platform.hotkey.dashboard.cache.CaffeineBuilder; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.model.KeyTimely; +import com.jd.platform.hotkey.dashboard.util.PageUtil; +import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; + +/** + * 此处存储所有发来的热key，统一处理入库 + * + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class HotKeyReceiver { + + /** + * 发来的热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + /** + * 存储实时hotkey，供界面查询实时热key + */ + private static Map> aliveKeyStore = new ConcurrentHashMap<>(); + + private static Logger logger = LoggerFactory.getLogger("HotKeyReceiver"); + + public static void push(HotKeyModel model) { + hotKeyStoreQueue.offer(model); + } + + public static LinkedBlockingQueue getQueue() { + return hotKeyStoreQueue; + } + + /** + * 将热key存入本地缓存，设置过期时间 + */ + public static void put(HotKeyModel hotKeyModel) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + logger.error("rule is null, hotkeyModel " + hotKeyModel.getAppName() + "-" + hotKeyModel.getKey()); + return; + } + Cache cache = aliveKeyStore.computeIfAbsent(keyRule.getDuration(), s -> CaffeineBuilder.cache(keyRule.getDuration())); + cache.put(hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(), hotKeyModel); + } + + /** + * 展示当前所有的实时热key + */ + public static List list(SearchReq searchReq) { + List timelyList = new ArrayList<>(); + + for (Integer duration : aliveKeyStore.keySet()) { + Cache cache = aliveKeyStore.get(duration); + ConcurrentMap concurrentHashMap = cache.asMap(); + for (Map.Entry entry : concurrentHashMap.entrySet()) { + KeyTimely keyTimely = parse((HotKeyModel) entry.getValue()); + if (keyTimely == null) { + continue; + } + timelyList.add(keyTimely); + } + } + + if (searchReq != null) { + if (StrUtil.isNotEmpty(searchReq.getApp())) { + timelyList = timelyList.stream().filter(keyTimely -> searchReq.getApp().equals(keyTimely.getAppName())).collect(Collectors.toList()); + } + if (StrUtil.isNotEmpty(searchReq.getKey())) { + timelyList = timelyList.stream().filter(keyTimely -> searchReq.getKey().equals(keyTimely.getKey())).collect(Collectors.toList()); + } + + } + + timelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); + + return timelyList; + + } + + public static void main(String[] args) { + List keyTimelyList = new ArrayList<>(); + for (int i = 0; i < 24; i++) { + KeyTimely keyTimely = new KeyTimely(); + keyTimely.setKey(i + ""); + keyTimely.setCreateTime(new Date()); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + keyTimelyList.add(keyTimely); + } + keyTimelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); + + Page page =PageUtil.pagination(keyTimelyList, 10, 3); + System.out.println(page.getTotal()); + System.out.println(page.getPage()); + System.out.println(page.getRows()); + } + + + + /** + * 将hotkeyModel变成前端需要的对象 + */ + private static KeyTimely parse(HotKeyModel hotKeyModel) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return null; + } + long remainTime = keyRule.getDuration() * 1000 - (System.currentTimeMillis() - hotKeyModel.getCreateTime()); + return KeyTimely.aKeyTimely() + .key(hotKeyModel.getKey()) + .val(UUID.randomUUID().toString()) + .appName(hotKeyModel.getAppName()) + .duration(remainTime / 1000) + .ruleDesc(RuleUtil.ruleDesc(appNameKey)) + .createTime(new Date(hotKeyModel.getCreateTime())).build(); + } + + /** + * 删除实时热key + */ + public static boolean delete(String appNameKey) { + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return false; + } + Cache cache = aliveKeyStore.get(keyRule.getDuration()); + if (cache == null) { + return false; + } + cache.invalidate(appNameKey); + return true; + } + + /** + * 定时清理caffeine + */ + public static void cleanUpCaffeine() { + for (Integer duration : aliveKeyStore.keySet()) { + Cache cache = aliveKeyStore.get(duration); + cache.cleanUp(); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java new file mode 100755 index 0000000..a3dc7ee --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java @@ -0,0 +1,78 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import com.jd.platform.hotkey.common.tool.Constant; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.DelimiterBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.concurrent.TimeUnit; + +/** + * 该server用于给各个worker实例连接用。 + * + * @author wuweifeng wrote on 2019-11-05. + */ +@Component +public class NodesServer { + private Logger logger = LoggerFactory.getLogger(getClass()); + + public void startNettyServer(int port) { + //boss单线程 + EventLoopGroup bossGroup = new NioEventLoopGroup(1); + EventLoopGroup workerGroup = new NioEventLoopGroup(4); + try { + ServerBootstrap bootstrap = new ServerBootstrap(); + bootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .handler(new LoggingHandler(LogLevel.INFO)) + .option(ChannelOption.SO_BACKLOG, 1024) + //保持长连接 + .childOption(ChannelOption.SO_KEEPALIVE, true) + //出来网络io事件，如记录日志、对消息编解码等 + .childHandler(new ChildChannelHandler()); + //绑定端口，同步等待成功 + ChannelFuture future = bootstrap.bind(port).sync(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bossGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + workerGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + })); + //等待服务器监听端口关闭 + future.channel().closeFuture().sync(); + } catch (Exception e) { + e.printStackTrace(); + logger.error("dashboard netty server start failure"); + } finally { + //优雅退出，释放线程池资源 + bossGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + } + } + + /** + * handler类 + */ + private class ChildChannelHandler extends ChannelInitializer { + + @Override + protected void initChannel(Channel ch) { + NodesServerHandler serverHandler = new NodesServerHandler(); + + ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); + ch.pipeline() + .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) + .addLast(new StringDecoder()) + .addLast(serverHandler); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java new file mode 100755 index 0000000..2a2a435 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java @@ -0,0 +1,66 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.common.tool.flush.FlushUtil; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import java.util.List; + +import static com.jd.platform.hotkey.common.tool.Constant.PONG; + +/** + * 这里处理所有netty事件。 + * + * @author wuweifeng wrote on 2019-11-05. + */ +public class NodesServerHandler extends SimpleChannelInboundHandler { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, String message) { + if (StringUtils.isEmpty(message)) { + return; + } + try { + HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + if (MessageType.PING == msg.getMessageType()) { + String hotMsg = FastJsonUtils.convertObjectToJSON(new HotKeyMsg(MessageType.PONG, PONG)); + FlushUtil.flush(ctx, MsgBuilder.buildByteBuf(hotMsg)); + } else if (MessageType.REQUEST_HOT_KEY == msg.getMessageType()) { + List list = FastJsonUtils.toList(msg.getBody(), HotKeyModel.class); + for (int i = 0; i < list.size(); i++) { + HotKeyReceiver.push(list.get(0)); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("some thing is error , " + cause.getMessage()); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + ctx.close(); + super.channelInactive(ctx); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java index 3dfcf18..785793a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.service; import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; @@ -33,7 +34,7 @@ public interface KeyService { KeyTimely selectByPk(Long key); - PageInfo pageKeyTimely(PageReq page, SearchReq param); + Page pageKeyTimely(PageReq page, SearchReq param); PageInfo pageMaxHot(PageReq page, SearchReq param); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index d41ebb6..6c54995 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -9,6 +9,7 @@ import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; @@ -21,11 +22,15 @@ import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.service.KeyService; import com.jd.platform.hotkey.dashboard.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.PageUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -58,8 +63,11 @@ public class KeyServiceImpl implements KeyService { @Resource private ChangeLogMapper logMapper; + private Logger logger = LoggerFactory.getLogger(getClass()); + /** * 折线图 + * * @param req req * @return vo */ @@ -68,52 +76,48 @@ public class KeyServiceImpl implements KeyService { int type = req.getType(); String appReq = req.getApp(); // admin 全查 - if(StrUtil.isNotEmpty(appReq)){ + if (StrUtil.isNotEmpty(appReq)) { app = appReq; } req.setApp(null); LocalDateTime now = LocalDateTime.now(); req.setEndTime(req.getEndTime() == null ? DateUtil.ldtToDate(now) : req.getEndTime()); List rules = ruleService.listRules(null); - if(type == 4){ - LocalDateTime st = req.getStartTime() == null? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); + if (type == 4) { + LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); req.setStartTime(DateUtil.ldtToDate(st)); LocalDateTime et = DateUtil.dateToLdt(req.getEndTime()); - boolean longTime = Duration.between(st,et).toHours()>2; + boolean longTime = Duration.between(st, et).toHours() > 2; req.setType(longTime ? 6 : 5); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(st,et,list,!longTime,rules,app); + return CommonUtil.processData(st, et, list, !longTime, rules, app); } - if(type == 5){ + if (type == 5) { LocalDateTime startTime = now.minusMinutes(31); req.setStartTime(DateUtil.ldtToDate(startTime)); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime,now,list,true,rules,app); - }else if(type == 6){ + return CommonUtil.processData(startTime, now, list, true, rules, app); + } else if (type == 6) { LocalDateTime startTime2 = now.minusHours(25); req.setStartTime(DateUtil.ldtToDate(startTime2)); List list2 = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime2,now,list2,false,rules,app); - }else{ + return CommonUtil.processData(startTime2, now, list2, false, rules, app); + } else { LocalDateTime startTime3 = now.minusDays(7).minusHours(1); req.setStartTime(DateUtil.ldtToDate(startTime3)); req.setType(6); List list3 = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime3,now,list3,false,rules,app); + return CommonUtil.processData(startTime3, now, list3, false, rules, app); } } @Override - public PageInfo pageKeyTimely(PageReq page, SearchReq param) { - PageHelper.startPage(page.getPageNum(), page.getPageSize()); - List listKey = keyTimelyMapper.listKeyTimely(param); - for (KeyTimely timely : listKey) { - timely.setKey(CommonUtil.keyName(timely.getKey())); - timely.setRuleDesc(RuleUtil.ruleDesc(timely.getAppName() + "/" + timely.getKey())); - } - return new PageInfo<>(listKey); + public Page pageKeyTimely(PageReq page, SearchReq param) { + List keyTimelies = HotKeyReceiver.list(param); + return PageUtil.pagination(keyTimelies, page.getPageSize(), page.getPageNum()); + } @Override @@ -143,12 +147,12 @@ public class KeyServiceImpl implements KeyService { Map keyDateMap = keyDateMap(statistics, hours); // 获取时间x轴 List list = new ArrayList<>(); - for (int i = hours; i >0 ; i--) { - LocalDateTime time = LocalDateTime.now().minusHours(i-1); + for (int i = hours; i > 0; i--) { + LocalDateTime time = LocalDateTime.now().minusHours(i - 1); int hour = time.getHour(); - list.add(hour+"时"); + list.add(hour + "时"); } - return new HotKeyLineChartVo(list,keyDateMap); + return new HotKeyLineChartVo(list, keyDateMap); } @@ -198,7 +202,7 @@ public class KeyServiceImpl implements KeyService { KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); - return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(),"", keyTimely.getUpdater(), SystemClock.now() + "")); + return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(), "", keyTimely.getUpdater(), SystemClock.now() + "")); } @Override @@ -212,29 +216,29 @@ public class KeyServiceImpl implements KeyService { } - - - private Map keyDateMap(List statistics, int hours){ + private Map keyDateMap(List statistics, int hours) { Map map = new HashMap<>(10); Map> listMap = statistics.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); for (Map.Entry> m : listMap.entrySet()) { int start = DateUtil.preHoursInt(5); - map.put(m.getKey(),new int[hours]); + map.put(m.getKey(), new int[hours]); int[] data = map.get(m.getKey()); int tmp = 0; for (int i = 0; i < hours; i++) { Statistics st; try { st = m.getValue().get(tmp); - if(String.valueOf(start).endsWith("24")){ start = start + 77; } - if(start != st.getHours()){ + if (String.valueOf(start).endsWith("24")) { + start = start + 77; + } + if (start != st.getHours()) { data[i] = 0; - }else{ - tmp ++; + } else { + tmp++; data[i] = st.getCount(); } start++; - }catch (Exception e){ + } catch (Exception e) { data[i] = 0; } } @@ -243,9 +247,8 @@ public class KeyServiceImpl implements KeyService { } - private void checkParam(SearchReq req) { - if(req.getStartTime() == null || req.getEndTime() == null){ + if (req.getStartTime() == null || req.getEndTime() == null) { req.setStartTime(DateUtil.preTime(5)); req.setEndTime(new Date()); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java new file mode 100644 index 0000000..e809f3d --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java @@ -0,0 +1,35 @@ +package com.jd.platform.hotkey.dashboard.util; + +import com.google.common.collect.Lists; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-09-01 + */ +public class PageUtil { + /** + * 通用分页工具类 + * + * @param data + * @param pageSize + * @param pageNum + * @param + * @return + */ + public static Page pagination(final List data, final int pageSize, final int pageNum) { + if (CollectionUtils.isEmpty(data)) { + return new Page<>(0, 0, null); + } + List> lists = Lists.partition(data, pageSize); + int localPageNum = pageNum; + if (localPageNum >= lists.size()) { + localPageNum = lists.size() - 1; + } + return new Page<>(localPageNum, data.size(), lists.get(localPageNum)); + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java index 40d82ff..d9352dc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java @@ -33,7 +33,7 @@ public class RuleUtil { } /** - * 根据APP的key，获取该key对应的rule + * 根据APP的key，获取该key对应的rule.如 cartpc-pu__ */ public static String rule(String key) { try { @@ -62,7 +62,7 @@ public class RuleUtil { return ""; } - private static KeyRule findByKey(String appNameKey) { + public static KeyRule findByKey(String appNameKey) { synchronized (RULE_MAP) { if (StrUtil.isEmpty(appNameKey)) { return null; diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index a94e187..23bd5e6 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -57,8 +57,10 @@ public class Starter { for (int i = 0; i < 10000; i++) { builder1.append(UUID.randomUUID()); } - logger.info("begin : " + System.currentTimeMillis()); - iConfigCenter.putAndGrant( "/abc/" + builder1.toString(), UUID.randomUUID().toString(), 60); + + a(); +// logger.info("begin : " + System.currentTimeMillis()); +// iConfigCenter.putAndGrant( "/abc/" + builder1.toString(), UUID.randomUUID().toString(), 60); // for (int i = 0; i < 200000; i++) { // final int j = i; // executorService.submit(() -> { @@ -89,11 +91,16 @@ public class Starter { private void a() { System.out.println("begin"); while (true) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } if (JdHotKeyStore.getValue("a") != null) { - + System.out.println("ishot"); } else { JdHotKeyStore.smartSet("a", 1); - logger.error("smartSet"); +// logger.error("smartSet"); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java index fc86ecc..2f3320b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.keydispatcher; -import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.tool.InitConstant; import org.springframework.stereotype.Component; @@ -17,12 +16,12 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @Component public class KeyProducer { - public void push(HotKeyModel model) { + public void push(HotKeyModel model, long now) { if (model == null || model.getKey() == null) { return; } //5秒前的过时消息就不处理了 - if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { + if (now - model.getCreateTime() > InitConstant.timeOut) { expireTotalCount.increment(); return; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java new file mode 100644 index 0000000..6e0d57e --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java @@ -0,0 +1,26 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import io.netty.channel.Channel; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class DashboardHolder { + /** + * 是否连上了dashboard + */ + public static boolean hasConnected = false; + /** + * channel + */ + public static Channel channel = null; + + public static void flushToDashboard(String message) { + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HOT_KEY, message))); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java new file mode 100755 index 0000000..47d1ef2 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java @@ -0,0 +1,86 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.tool.Constant; +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.DelimiterBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.timeout.IdleStateHandler; + +/** + * netty连接器 + * + * @author wuweifeng wrote on 2019-11-05. + */ +public class NettyClient { + private static final NettyClient nettyClient = new NettyClient(); + + private Bootstrap bootstrap; + + + public static NettyClient getInstance() { + return nettyClient; + } + + private NettyClient() { + if (bootstrap == null) { + bootstrap = initBootstrap(); + } + } + + private Bootstrap initBootstrap() { + //少线程 + EventLoopGroup group = new NioEventLoopGroup(2); + + Bootstrap bootstrap = new Bootstrap(); + NettyClientHandler nettyClientHandler = new NettyClientHandler(); + bootstrap.group(group).channel(NioSocketChannel.class) + .option(ChannelOption.SO_KEEPALIVE, true) + .option(ChannelOption.TCP_NODELAY, true) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); + ch.pipeline() + .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) + .addLast(new StringDecoder()) + //10秒没消息时，就发心跳包过去 + .addLast(new IdleStateHandler(0, 0, 30)) + .addLast(nettyClientHandler); + } + }); + return bootstrap; + } + + public synchronized void connect(String address) { + if (DashboardHolder.hasConnected) { + return; + } + String[] ss = address.split(":"); + try { + ChannelFuture channelFuture = bootstrap.connect(ss[0], Integer.parseInt(ss[1])).sync(); + DashboardHolder.channel = channelFuture.channel(); + DashboardHolder.hasConnected = true; + } catch (Exception e) { + DashboardHolder.hasConnected = false; + DashboardHolder.channel = null; + e.printStackTrace(); + } + + //这一步就阻塞了 +// channelFuture.channel().closeFuture().sync(); + //当server断开后才会走下面的 +// System.out.println("server is down"); + } + + public synchronized void disConnect() { + DashboardHolder.channel = null; + DashboardHolder.hasConnected = false; + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java new file mode 100755 index 0000000..5fdbe3a --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java @@ -0,0 +1,58 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.Constant; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author wuweifeng wrote on 2019-11-05. + */ +@ChannelHandler.Sharable +public class NettyClientHandler extends SimpleChannelInboundHandler { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent idleStateEvent = (IdleStateEvent) evt; + + if (idleStateEvent.state() == IdleState.ALL_IDLE) { + //向服务端发送消息 + ctx.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.PING, Constant.PING))); + } + } + + super.userEventTriggered(ctx, evt); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + logger.info("channelActive:" + ctx.name()); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + NettyClient.getInstance().disConnect(); + } + + @Override + protected void channelRead0(ChannelHandlerContext channelHandlerContext, String message) { + HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + if (MessageType.PONG == msg.getMessageType()) { + logger.info("heart beat"); + } + + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java index a013d53..7ef2cb8 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java @@ -4,7 +4,7 @@ import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; +import com.jd.platform.hotkey.common.tool.flush.FlushUtil; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index c893c79..0a1521a 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -56,16 +56,17 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { private void publishMsg(String message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, HotKeyModel.class); + long now = SystemClock.now(); for (HotKeyModel model : models) { //白名单key不处理 if (WhiteListHolder.contains(model.getKey())) { continue; } - long timeOut = SystemClock.now() - model.getCreateTime(); + long timeOut = now - model.getCreateTime(); if (timeOut > 1000) { logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); } - keyProducer.push(model); + keyProducer.push(model, now); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java deleted file mode 100644 index fe958d5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.Set; -import java.util.UUID; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-26 - */ -@Component -@Deprecated -public class BatchToEtcdScheduler { - @Resource - private KeyCollector keyCollector; - @Resource - private IConfigCenter iConfigCenter; - - /** - * 每隔0.5秒上传一下已探测出的热key发往etcd供入库 - */ -// @Scheduled(fixedRate = 500) - public void uploadClientCount() { - Set set = keyCollector.lockAndGetResult(); - if (set.size() == 0) { - return; - } - - //worker将热key推送到该地址，供dashboard监听入库做记录 - String hotKeyRecordPath = ConfigConstant.hotKeyRecordPath + UUID.randomUUID().toString(); - try { - //推送到etcd，供dashboard监听入库 - iConfigCenter.putAndGrant(hotKeyRecordPath, FastJsonUtils.convertObjectToJSON(set), 10); - } catch (Exception e) { - e.printStackTrace(); - } - - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java new file mode 100644 index 0000000..aa420cd --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java @@ -0,0 +1,24 @@ +package com.jd.platform.hotkey.worker.netty.pusher; + +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; +import org.springframework.stereotype.Component; + +/** + * 将热key推送到dashboard供入库 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +@Component +public class DashboardPusher implements IPusher { + @Override + public void push(HotKeyModel model) { + HotkeyTempStore.push(model); + } + + @Override + public void remove(HotKeyModel model) { + + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java deleted file mode 100644 index e588aa2..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.base.Joiner; -import com.google.common.collect.Queues; -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.tool.HotKeyPathTool; -import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; -import com.jd.platform.hotkey.worker.tool.AsyncPool; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.*; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - - -/** - * 将热key推送到etcd，dashboard监听后留做备份 - * - * @author wuweifeng wrote on 2020-02-24 - * @version 1.0 - */ -@Component -public class EtcdPusher implements IPusher { - @Resource - private IConfigCenter iConfigCenter; - - private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); - - @Override - public void push(HotKeyModel model) { - hotKeyStoreQueue.offer(model); - } - - @PostConstruct - public void uploadToEtcd() { - Map> map = new HashMap<>(); - AsyncPool.asyncDo(() -> { - while (true) { - try { - //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 - List tempModels = new ArrayList<>(); - Queues.drain(hotKeyStoreQueue, tempModels, 1000, 1, TimeUnit.SECONDS); - if (CollectionUtil.isEmpty(tempModels)) { - continue; - } - //区分出来不同的duration，不同的duration往etcd放时方法不一样 - for (int i = 0; i < tempModels.size(); i++) { - HotKeyModel hotKeyModel = tempModels.get(i); - int duration = KeyRuleHolder.getRuleByAppAndKey(hotKeyModel).getDuration(); - //取到value - List keys = map.computeIfAbsent(duration, s -> new ArrayList<>()); - //将该model的key存进去 - keys.add(HotKeyPathTool.keyRecordPath(hotKeyModel)); - } - - for (Integer duration : map.keySet()) { - List list = map.get(duration); - if (CollectionUtil.isEmpty(list)) { - continue; - } - Joiner joiner = Joiner.on(","); - String s = joiner.join(list); - iConfigCenter.putAndGrant(ConfigConstant.hotKeyBatchRecordPath + s, UUID.randomUUID().toString(), duration); - map.get(duration).clear(); - } - - - } catch (Exception e) { - e.printStackTrace(); - } - } - }); - } - - @Override - @Deprecated - public void remove(HotKeyModel model) { - //推送etcd删除 - iConfigCenter.delete(HotKeyPathTool.keyPath(model)); - } - - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java deleted file mode 100644 index 1dad842..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import cn.hutool.core.collection.ConcurrentHashSet; -import cn.hutool.core.util.StrUtil; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import org.springframework.stereotype.Component; - -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - -import static com.jd.platform.hotkey.common.tool.KeyRecordKeyTool.key; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-26 - */ -@Component -@Deprecated -public class KeyCollector { - private Set keySet0 = new ConcurrentHashSet<>(); - private Set keySet1 = new ConcurrentHashSet<>(); - private AtomicLong atomicLong = new AtomicLong(0); - - public Set lockAndGetResult() { - //自增后，对应的map就会停止被写入，等待被读取 - atomicLong.addAndGet(1); - Set set; - if (atomicLong.get() % 2 == 0) { - set = new HashSet<>(keySet1); - keySet1.clear(); - } else { - set = new HashSet<>(keySet0); - keySet0.clear(); - } - return set; - } - - public void collect(HotKeyModel hotKeyModel) { - String key = hotKeyModel.getKey(); - if (StrUtil.isEmpty(key)) { - return; - } - if (atomicLong.get() % 2 == 0) { - keySet0.add(key(hotKeyModel)); - } else { - keySet1.add(key(hotKeyModel)); - } - - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java new file mode 100644 index 0000000..8ee6e11 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java @@ -0,0 +1,46 @@ +package com.jd.platform.hotkey.worker.netty.pusher; + +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.collect.Queues; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.worker.netty.dashboard.DashboardHolder; +import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; +import com.jd.platform.hotkey.worker.tool.AsyncPool; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * 定时批量往dashboard发送热key，供入库 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +@Component +public class KeyUploader { + + @PostConstruct + public void uploadToDashboard() { + AsyncPool.asyncDo(() -> { + while (true) { + try { + //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 + List tempModels = new ArrayList<>(); + Queues.drain(HotkeyTempStore.getQueue(), tempModels, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + + //将热key推到dashboard + DashboardHolder.flushToDashboard(FastJsonUtils.convertObjectToJSON(tempModels)); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java new file mode 100644 index 0000000..c93d18d --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java @@ -0,0 +1,27 @@ +package com.jd.platform.hotkey.worker.netty.pusher.store; + +import com.jd.platform.hotkey.common.model.HotKeyModel; + +import java.util.concurrent.LinkedBlockingQueue; + +/** + * 已热待上报入库的热key集中营 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class HotkeyTempStore { + /** + * 热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + + public static void push(HotKeyModel model) { + hotKeyStoreQueue.offer(model); + } + + public static LinkedBlockingQueue getQueue() { + return hotKeyStoreQueue; + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 9fe62a4..2c60d6e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.worker.starters; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.StrUtil; import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; @@ -13,6 +14,7 @@ import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.worker.cache.CaffeineCacheHolder; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.model.TotalCount; +import com.jd.platform.hotkey.worker.netty.dashboard.NettyClient; import com.jd.platform.hotkey.worker.netty.filter.HotKeyFilter; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; @@ -217,6 +219,29 @@ public class EtcdStarter { } } + /** + * 每隔30秒去获取一下dashboard的地址 + */ + @Scheduled(fixedRate = 30000) + public void fetchDashboardIp() { + try { + //获取DashboardIp + List keyValues = configCenter.getPrefix(ConfigConstant.dashboardPath); + + //是空，给个警告 + if (CollectionUtil.isEmpty(keyValues)) { + logger.warn("very important warn !!! Dashboard ip is null!!!"); + return; + } + + String dashboardIp = keyValues.get(0).getValue().toStringUtf8(); + NettyClient.getInstance().connect(dashboardIp); + + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 校验一下receive的key数量，如果一段时间没变，考虑网络问题，就将worker注册自己到etcd的心跳给断掉30秒，让各client重连一下自己 */ diff --git a/worker/src/test/java/TestBatch.java b/worker/src/test/java/TestBatch.java index 524dcaa..c62653f 100644 --- a/worker/src/test/java/TestBatch.java +++ b/worker/src/test/java/TestBatch.java @@ -1,13 +1,28 @@ +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + /** * @author wuweifeng * @version 1.0 * @date 2020-08-27 */ -//public class TestBatch { -// public static BlockingQueue QUEUE = new LinkedBlockingQueue<>(2000000); -// private static Logger logger = LoggerFactory.getLogger("tttt"); +public class TestBatch { + public static BlockingQueue QUEUE = new LinkedBlockingQueue<>(2000000); + private static Logger logger = LoggerFactory.getLogger("tttt"); + private static int i; + + public static void main(String[] args) throws InterruptedException { + +// for (int j = 0; j < 10; j++) { +// QUEUE.offer("" + j); +// } +// List list = new ArrayList<>(); +// Queues.drain(QUEUE, list, 2, 1, TimeUnit.SECONDS); +// System.out.println(QUEUE); // -// public static void main(String[] args) { // new Thread(new Runnable() { // @Override // public void run() { @@ -17,7 +32,20 @@ // } catch (InterruptedException e) { // e.printStackTrace(); // } -// QUEUE.offer("1"); +// QUEUE.offer("" + i++); +// } +// } +// }).start(); +// new Thread(new Runnable() { +// @Override +// public void run() { +// while (true) { +// try { +// Thread.sleep(1000); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// System.out.println(QUEUE); // } // } // }).start(); @@ -25,12 +53,14 @@ // while (true) { // List list = new ArrayList<>(); // try { -// Queues.drain(QUEUE, list, 1000, 1, TimeUnit.SECONDS); +// System.out.println("size " + QUEUE.size()); +// Queues.drain(QUEUE, list, 2, 1, TimeUnit.SECONDS); +// Thread.sleep(100); // } catch (InterruptedException e) { // e.printStackTrace(); // } // logger.info(list.toString()); // } -// -// } -//} + + } +} -- Gitee From 34910627d0d7e1e1be0191309e8ab4b005ea36cb Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 1 Sep 2020 14:57:25 +0800 Subject: [PATCH 19/42] =?UTF-8?q?worker=E7=9B=B4=E6=8E=A8dashboard?= =?UTF-8?q?=E7=9A=84netty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/common/model/HotKeyModel.java | 6 +- .../hotkey/dashboard/etcd/EtcdMonitor.java | 67 ++----------------- .../dashboard/netty/HotKeyReceiver.java | 54 ++++++++------- .../dashboard/netty/NodesServerHandler.java | 4 +- 4 files changed, 40 insertions(+), 91 deletions(-) diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java index c82323a..11de0cc 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java @@ -23,11 +23,7 @@ public class HotKeyModel extends BaseModel { @Override public String toString() { - return "HotKeyModel{" + - "appName='" + appName + '\'' + - ", keyType=" + keyType + - ", remove=" + remove + - '}'; + return "appName:" + appName + "-key=" + getKey(); } public boolean isRemove() { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 16f7e48..123be53 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -77,59 +77,6 @@ public class EtcdMonitor { }); } - /** - * 监听新来的热key，该key的产生是来自于worker集群推送过来的 - */ - public void watchHotKeyRecord() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyRecordPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - - //提交获取该key的任务到线程池 - threadPoolExecutor.submit(() -> { - EventWrapper eventWrapper = build(event); - - String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyRecordPath, ""); - eventWrapper.setKey(appKey); - - dataHandler.offer(eventWrapper); - }); - - } - }); - } - - /** - * 监听新来的批量热key，该key的产生是来自于worker集群推送过来的 - */ -// public void watchBatchHotKeyRecord() { -// threadPoolExecutor.submit(() -> { -// KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyBatchRecordPath); -// while (watchIterator.hasNext()) { -// Event event = event(watchIterator); -// -// //提交获取该key的任务到线程池 -// threadPoolExecutor.submit(() -> { -// EventWrapper eventWrapper = build(event); -// -// String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyBatchRecordPath, ""); -// String[] keyArray = appKey.split(","); -// for (int i = 0; i < keyArray.length; i++) { -// String key = keyArray[i]; -// if (StrUtil.isEmpty(key)) { -// continue; -// } -// eventWrapper.setKey(key.replace(ConfigConstant.hotKeyRecordPath, "")); -// -// dataHandler.offer(eventWrapper); -// } -// -// }); -// -// } -// }); -// } private EventWrapper build(Event event) { KeyValue kv = event.getKv(); @@ -158,7 +105,7 @@ public class EtcdMonitor { dealHotkey(); //开始监听热key产生 - watchHotKeyRecord(); +// watchHotKeyRecord(); //监听批量发来的热key // watchBatchHotKeyRecord(); @@ -199,21 +146,21 @@ public class EtcdMonitor { */ private void dealHotkey() { threadPoolExecutor.submit(() -> { - try { - while (true) { + while (true) { + try { //获取发来的这个热key，存入本地caffeine，设置过期时间 - HotKeyModel model = HotKeyReceiver.getQueue().take(); + HotKeyModel model = HotKeyReceiver.take(); //将该key放入实时热key本地缓存中 HotKeyReceiver.put(model); //入库record表，TODO 需要改成batch入库 dataHandler.insertRecord(model.getAppName() + "/" + model.getKey(), 0); + } catch (Exception e) { + e.printStackTrace(); } - - } catch (InterruptedException e) { - e.printStackTrace(); } + }); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java index ff2f558..682670e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -5,10 +5,8 @@ import com.github.benmanes.caffeine.cache.Cache; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.dashboard.cache.CaffeineBuilder; -import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyTimely; -import com.jd.platform.hotkey.dashboard.util.PageUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,12 +37,20 @@ public class HotKeyReceiver { private static Logger logger = LoggerFactory.getLogger("HotKeyReceiver"); + /** + * netty收到的先存这里 + */ public static void push(HotKeyModel model) { hotKeyStoreQueue.offer(model); } - public static LinkedBlockingQueue getQueue() { - return hotKeyStoreQueue; + public static HotKeyModel take() { + try { + return hotKeyStoreQueue.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + return null; + } } /** @@ -95,26 +101,26 @@ public class HotKeyReceiver { } - public static void main(String[] args) { - List keyTimelyList = new ArrayList<>(); - for (int i = 0; i < 24; i++) { - KeyTimely keyTimely = new KeyTimely(); - keyTimely.setKey(i + ""); - keyTimely.setCreateTime(new Date()); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - keyTimelyList.add(keyTimely); - } - keyTimelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); - - Page page =PageUtil.pagination(keyTimelyList, 10, 3); - System.out.println(page.getTotal()); - System.out.println(page.getPage()); - System.out.println(page.getRows()); - } +// public static void main(String[] args) { +// List keyTimelyList = new ArrayList<>(); +// for (int i = 0; i < 24; i++) { +// KeyTimely keyTimely = new KeyTimely(); +// keyTimely.setKey(i + ""); +// keyTimely.setCreateTime(new Date()); +// try { +// Thread.sleep(10); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// keyTimelyList.add(keyTimely); +// } +// keyTimelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); +// +// Page page =PageUtil.pagination(keyTimelyList, 10, 3); +// System.out.println(page.getTotal()); +// System.out.println(page.getPage()); +// System.out.println(page.getRows()); +// } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java index 2a2a435..9650c2f 100755 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java @@ -37,8 +37,8 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { FlushUtil.flush(ctx, MsgBuilder.buildByteBuf(hotMsg)); } else if (MessageType.REQUEST_HOT_KEY == msg.getMessageType()) { List list = FastJsonUtils.toList(msg.getBody(), HotKeyModel.class); - for (int i = 0; i < list.size(); i++) { - HotKeyReceiver.push(list.get(0)); + for (HotKeyModel hotKeyModel : list) { + HotKeyReceiver.push(hotKeyModel); } } } catch (Exception e) { -- Gitee From 3a6efb630586a394de05994038385f1c392ba7c9 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 1 Sep 2020 14:57:49 +0800 Subject: [PATCH 20/42] =?UTF-8?q?=E9=87=87=E7=94=A8ChannelGroup=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E4=B9=8B=E5=89=8D=E7=9A=84hashmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 123be53..5b9aebe 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -104,12 +104,6 @@ public class EtcdMonitor { //开始入库 dealHotkey(); - //开始监听热key产生 -// watchHotKeyRecord(); - - //监听批量发来的热key -// watchBatchHotKeyRecord(); - //监听手工创建的key watchHandOperationHotKey(); -- Gitee From 015297c48782789e8ffd8fc2d30a09cca095f3c9 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 1 Sep 2020 16:03:52 +0800 Subject: [PATCH 21/42] =?UTF-8?q?=E9=87=87=E7=94=A8ChannelGroup=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E4=B9=8B=E5=89=8D=E7=9A=84hashmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/netty/HotKeyReceiver.java | 48 ++++++++++--------- .../hotkey/dashboard/service/KeyService.java | 4 -- .../service/impl/KeyServiceImpl.java | 28 +++++------ 3 files changed, 37 insertions(+), 43 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java index 682670e..e8269dd 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -64,7 +64,7 @@ public class HotKeyReceiver { return; } Cache cache = aliveKeyStore.computeIfAbsent(keyRule.getDuration(), s -> CaffeineBuilder.cache(keyRule.getDuration())); - cache.put(hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(), hotKeyModel); + cache.put(appNameKey, hotKeyModel); } /** @@ -73,11 +73,13 @@ public class HotKeyReceiver { public static List list(SearchReq searchReq) { List timelyList = new ArrayList<>(); + long now = System.currentTimeMillis(); + for (Integer duration : aliveKeyStore.keySet()) { Cache cache = aliveKeyStore.get(duration); ConcurrentMap concurrentHashMap = cache.asMap(); for (Map.Entry entry : concurrentHashMap.entrySet()) { - KeyTimely keyTimely = parse((HotKeyModel) entry.getValue()); + KeyTimely keyTimely = parse((HotKeyModel) entry.getValue(), now); if (keyTimely == null) { continue; } @@ -87,18 +89,37 @@ public class HotKeyReceiver { if (searchReq != null) { if (StrUtil.isNotEmpty(searchReq.getApp())) { - timelyList = timelyList.stream().filter(keyTimely -> searchReq.getApp().equals(keyTimely.getAppName())).collect(Collectors.toList()); + timelyList = timelyList.parallelStream().filter(keyTimely -> searchReq.getApp().equals(keyTimely.getAppName())).collect(Collectors.toList()); } if (StrUtil.isNotEmpty(searchReq.getKey())) { - timelyList = timelyList.stream().filter(keyTimely -> searchReq.getKey().equals(keyTimely.getKey())).collect(Collectors.toList()); + timelyList = timelyList.parallelStream().filter(keyTimely -> keyTimely.getKey().startsWith(searchReq.getKey())).collect(Collectors.toList()); } } timelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); + logger.info("query cost :" + (System.currentTimeMillis() - now)); return timelyList; + } + /** + * 将hotkeyModel变成前端需要的对象 + */ + private static KeyTimely parse(HotKeyModel hotKeyModel, long now) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return null; + } + long remainTime = keyRule.getDuration() * 1000 - (now - hotKeyModel.getCreateTime()); + return KeyTimely.aKeyTimely() + .key(hotKeyModel.getKey()) + .val(UUID.randomUUID().toString()) + .appName(hotKeyModel.getAppName()) + .duration(remainTime / 1000) + .ruleDesc(RuleUtil.ruleDesc(appNameKey)) + .createTime(new Date(hotKeyModel.getCreateTime())).build(); } // public static void main(String[] args) { @@ -124,24 +145,7 @@ public class HotKeyReceiver { - /** - * 将hotkeyModel变成前端需要的对象 - */ - private static KeyTimely parse(HotKeyModel hotKeyModel) { - String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); - KeyRule keyRule = RuleUtil.findByKey(appNameKey); - if (keyRule == null) { - return null; - } - long remainTime = keyRule.getDuration() * 1000 - (System.currentTimeMillis() - hotKeyModel.getCreateTime()); - return KeyTimely.aKeyTimely() - .key(hotKeyModel.getKey()) - .val(UUID.randomUUID().toString()) - .appName(hotKeyModel.getAppName()) - .duration(remainTime / 1000) - .ruleDesc(RuleUtil.ruleDesc(appNameKey)) - .createTime(new Date(hotKeyModel.getCreateTime())).build(); - } + /** * 删除实时热key diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java index 785793a..558e814 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java @@ -30,10 +30,6 @@ public interface KeyService { int delKeyByUser(KeyTimely keyTimely); - KeyTimely selectByKey(String key); - - KeyTimely selectByPk(Long key); - Page pageKeyTimely(PageReq page, SearchReq param); PageInfo pageMaxHot(PageReq page, SearchReq param); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 6c54995..23bd130 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -8,6 +8,7 @@ import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; @@ -16,7 +17,6 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.KeyRecord; @@ -55,8 +55,6 @@ public class KeyServiceImpl implements KeyService { @Resource private KeyRecordMapper recordMapper; @Resource - private KeyTimelyMapper keyTimelyMapper; - @Resource private StatisticsMapper statisticsMapper; @Resource private RuleService ruleService; @@ -172,6 +170,12 @@ public class KeyServiceImpl implements KeyService { public int insertKeyByUser(KeyTimely key) { configCenter.putAndGrant(ConfigConstant.hotKeyPath + key.getAppName() + "/" + key.getKey(), System.currentTimeMillis() + "", key.getDuration()); + + HotKeyModel hotKeyModel = new HotKeyModel(); + hotKeyModel.setCreateTime(System.currentTimeMillis()); + hotKeyModel.setAppName(key.getAppName()); + hotKeyModel.setKey(key.getKey()); + HotKeyReceiver.put(hotKeyModel); return logMapper.insertSelective(new ChangeLog(key.getAppName(), Constant.HOTKEY_CHANGE, "", key.getKey(), key.getUpdater(), SystemClock.now() + "")); } @@ -189,32 +193,22 @@ public class KeyServiceImpl implements KeyService { String[] arr = keyTimely.getKey().split("/"); //删除client监听目录的key String etcdKey = ConfigConstant.hotKeyPath + arr[0] + "/" + arr[1]; + + //删除caffeine里的实时key + HotKeyReceiver.delete(arr[0] + "/" + arr[1]); + if (configCenter.get(etcdKey) == null) { //如果手工目录也就是client监听的目录里没有该key，那么就往里面放一个，然后再删掉它，这样client才能监听到删除事件 configCenter.putAndGrant(etcdKey, com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE, 1); } configCenter.delete(etcdKey); - //也删除Record目录下的该key，因为不确定要删的key到底在哪 - String recordKey = ConfigConstant.hotKeyRecordPath + arr[0] + "/" + arr[1]; - configCenter.delete(recordKey); - KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(), "", keyTimely.getUpdater(), SystemClock.now() + "")); } - @Override - public KeyTimely selectByKey(String key) { - return keyTimelyMapper.selectByKey(key); - } - - @Override - public KeyTimely selectByPk(Long id) { - return keyTimelyMapper.selectByPrimaryKey(id); - } - private Map keyDateMap(List statistics, int hours) { Map map = new HashMap<>(10); -- Gitee From c1189ab15a4d43a95af0173de50e12d3315636c1 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 1 Sep 2020 16:19:47 +0800 Subject: [PATCH 22/42] =?UTF-8?q?fix=E5=86=85=E5=AD=98=E5=88=86=E9=A1=B5?= =?UTF-8?q?=20=E5=89=8D=E7=AB=AF=E9=BB=98=E8=AE=A4=E4=BC=A01?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/service/impl/KeyServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 6c54995..e58e33f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -116,7 +116,7 @@ public class KeyServiceImpl implements KeyService { @Override public Page pageKeyTimely(PageReq page, SearchReq param) { List keyTimelies = HotKeyReceiver.list(param); - return PageUtil.pagination(keyTimelies, page.getPageSize(), page.getPageNum()); + return PageUtil.pagination(keyTimelies, page.getPageSize(), page.getPageNum()-1); } -- Gitee From 30cfa8e3ba4654b708966c14a8d46526a3ef1f26 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 1 Sep 2020 17:00:12 +0800 Subject: [PATCH 23/42] =?UTF-8?q?=E4=BF=AE=E6=94=B9js=E5=88=86=E9=A1=B5?= =?UTF-8?q?=E4=BB=8E1=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/controller/KeyController.java | 7 ------- .../jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java | 1 - .../com/jd/platform/hotkey/dashboard/util/PageUtil.java | 3 ++- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index 330e3b4..f3f362d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -122,13 +122,6 @@ public class KeyController extends BaseController { } - @GetMapping("/edit/{id}") - public String edit(@PathVariable("id") Long id, ModelMap modelMap){ - modelMap.put("key", keyService.selectByPk(id)); - return prefix + "/edit"; - } - - @PostMapping("/edit") @ResponseBody public Result editSave(KeyTimely key) { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java index e8269dd..88057c9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -99,7 +99,6 @@ public class HotKeyReceiver { timelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); - logger.info("query cost :" + (System.currentTimeMillis() - now)); return timelyList; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java index e809f3d..cd4a987 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java @@ -4,6 +4,7 @@ import com.google.common.collect.Lists; import com.jd.platform.hotkey.dashboard.common.domain.Page; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; import java.util.List; /** @@ -23,7 +24,7 @@ public class PageUtil { */ public static Page pagination(final List data, final int pageSize, final int pageNum) { if (CollectionUtils.isEmpty(data)) { - return new Page<>(0, 0, null); + return new Page<>(1, 0, new ArrayList<>()); } List> lists = Lists.partition(data, pageSize); int localPageNum = pageNum; -- Gitee From f14e7438633a6d3b1c8ff817fa047c9994fa6737 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 11:53:09 +0800 Subject: [PATCH 24/42] =?UTF-8?q?record=E6=94=B9=E7=94=A8=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/DashboardApplication.java | 6 - .../dashboard/common/domain/EventWrapper.java | 95 -------- .../dashboard/common/domain/IRecord.java | 27 +++ .../dashboard/common/monitor/DataHandler.java | 110 +++------- .../hotkey/dashboard/etcd/EtcdMonitor.java | 77 ++++--- .../dashboard/mapper/KeyTimelyMapper.java | 31 --- .../hotkey/dashboard/model/KeyRecord.java | 8 +- .../hotkey/dashboard/model/KeyTimely.java | 12 - .../service/impl/KeyServiceImpl.java | 3 +- dashboard/src/main/resources/application.yml | 20 +- .../dashboard/mapper/KeyTimelyMapper.xml | 207 ------------------ .../resources/templates/admin/key/list.html | 2 +- .../templates/admin/key/listtimely.html | 2 +- .../java/com/jd/platform/sample/Starter.java | 14 +- 14 files changed, 143 insertions(+), 471 deletions(-) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 074e45d..3070702 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.dashboard; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; @@ -9,7 +8,6 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import javax.annotation.Resource; @EnableAsync @@ -19,8 +17,6 @@ public class DashboardApplication implements CommandLineRunner { private Logger logger = LoggerFactory.getLogger(getClass()); - @Resource - private KeyTimelyMapper timelyMapper; public static void main(String[] args) { try { @@ -33,8 +29,6 @@ public class DashboardApplication implements CommandLineRunner { @Override public void run(String... args) { - int row = timelyMapper.clear(); - logger.info("clear db timely hotKey, effect row : {}",row); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java deleted file mode 100644 index 04d4a83..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.domain; - -import com.ibm.etcd.api.Event; - -import java.io.Serializable; -import java.util.Date; - -/** - * 包装类，用于保存事件最准确的时间 - */ -public class EventWrapper implements Serializable { - - private String key; - - private String value; - - private Event.EventType eventType; - - private Date date; - - private Long ttl; - - private long version; - - private String uuid; - - @Override - public String toString() { - return "EventWrapper{" + - "key='" + key + '\'' + - ", value='" + value + '\'' + - ", eventType=" + eventType + - ", date=" + date + - ", ttl=" + ttl + - ", version=" + version + - ", uuid='" + uuid + '\'' + - '}'; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public long getVersion() { - return version; - } - - public void setVersion(long version) { - this.version = version; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public Event.EventType getEventType() { - return eventType; - } - - public void setEventType(Event.EventType eventType) { - this.eventType = eventType; - } - - public void setDate(Date date) { - this.date = date; - } - - public void setTtl(Long ttl) { - this.ttl = ttl; - } - - public Date getDate() { - return date; - } - - public Long getTtl() { - return ttl; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java new file mode 100644 index 0000000..3585f73 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java @@ -0,0 +1,27 @@ +package com.jd.platform.hotkey.dashboard.common.domain; + +import java.util.Date; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-09-02 + */ +public interface IRecord { + /** + * appName + "/" + key + */ + String appNameKey(); + + /** + * 手工添加的是时间戳13位，worker传过来的是uuid + */ + String value(); + + /** + * 0插入，1删除 + */ + int type(); + + Date createTime(); +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 0b8a50c..bd3a5e0 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -1,25 +1,22 @@ package com.jd.platform.hotkey.dashboard.common.monitor; +import cn.hutool.core.collection.CollectionUtil; import com.alibaba.fastjson.JSON; -import com.ibm.etcd.api.Event; +import com.google.common.collect.Queues; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.IRecord; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; -import com.jd.platform.hotkey.dashboard.util.TwoTuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -27,11 +24,13 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; @Component public class DataHandler { @@ -41,8 +40,6 @@ public class DataHandler { @Resource private KeyRecordMapper keyRecordMapper; @Resource - private KeyTimelyMapper keyTimelyMapper; - @Resource private StatisticsMapper statisticsMapper; @Resource @@ -51,18 +48,17 @@ public class DataHandler { @Resource private IConfigCenter configCenter; - /** * 队列 */ - private BlockingQueue queue = new LinkedBlockingQueue<>(); + private BlockingQueue queue = new LinkedBlockingQueue<>(); /** * 入队 */ - public void offer(EventWrapper eventWrapper) { + public void offer(IRecord record) { try { - queue.put(eventWrapper); + queue.put(record); } catch (InterruptedException e) { e.printStackTrace(); } @@ -70,84 +66,46 @@ public class DataHandler { public void insertRecords() { while (true) { - EventWrapper eventWrapper; - try { - eventWrapper = queue.take(); - } catch (InterruptedException e) { - e.printStackTrace(); - continue; - } try { - TwoTuple twoTuple = handHotKey(eventWrapper); - if (twoTuple == null) { + List records = new ArrayList<>(); + Queues.drain(queue, records, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(records)) { continue; } - KeyRecord keyRecord = twoTuple.getSecond(); - KeyTimely keyTimely = twoTuple.getFirst(); - - if (keyTimely.getUuid() == null) { - keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); - } else { - try { - keyTimelyMapper.saveOrUpdate(keyTimely); - } catch (Exception e) { - log.info("insert timely error",e); + List keyRecordList = new ArrayList<>(); + for (IRecord iRecord : records) { + KeyRecord keyRecord = handHotKey(iRecord); + if (keyRecord != null) { + keyRecordList.add(keyRecord); } } - if (keyRecord != null) { - //插入记录 - keyRecordMapper.insertSelective(keyRecord); - } - } catch (Exception e) { - log.error("eventWrapper:" + eventWrapper); + keyRecordMapper.batchInsert(keyRecordList); + + } catch (InterruptedException e) { e.printStackTrace(); - log.error("handHotKey error ," + e.getCause()); } - - } } - /** - * 插入记录表 - * @param type 0是新增，1是删除 - */ - public void insertRecord(String appKey, int type) { - String source = Constant.SYSTEM; - String rule = RuleUtil.rule(appKey); - KeyRule keyRule = RuleUtil.findByKey(appKey); - //appName+"/"+"key" - String[] arr = appKey.split("/"); - String appName = arr[0]; - String key = arr[1]; - String uuid = UUID.randomUUID().toString(); - KeyRecord keyRecord = new KeyRecord(key, rule, appName, (long)keyRule.getDuration(), source, type, uuid, new Date()); - keyRecord.setRule(rule); - keyRecordMapper.insertSelective(keyRecord); - } - /** * 处理热点key和记录 */ - private TwoTuple handHotKey(EventWrapper eventWrapper) { - Date date = eventWrapper.getDate(); - long ttl = eventWrapper.getTtl(); - Event.EventType eventType = eventWrapper.getEventType(); - String appKey = eventWrapper.getKey(); - String value = eventWrapper.getValue(); + private KeyRecord handHotKey(IRecord record) { + Date date = record.createTime(); + String appKey = record.appNameKey(); + String value = record.value(); //appName+"/"+"key" String[] arr = appKey.split("/"); String appName = arr[0]; String key = arr[1]; String uuid = UUID.randomUUID().toString(); - int type = eventType.getNumber(); + int type = record.type(); //组建成对象，供累计后批量插入、删除 - TwoTuple timelyKeyRecordTwoTuple = new TwoTuple<>(); - if (eventType.equals(Event.EventType.PUT)) { + if (type == 0) { //如果是客户端删除时发出的put指令 if (com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE.equals(value)) { log.info("client remove key event : " + appKey); @@ -155,17 +113,14 @@ public class DataHandler { } //手工添加的是时间戳13位，worker传过来的是uuid String source = value.length() == 13 ? Constant.HAND : Constant.SYSTEM; - timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).val(value).appName(appName).duration(ttl).uuid(appKey).createTime(date).build()); String rule = RuleUtil.rule(appKey); - KeyRecord keyRecord = new KeyRecord(key, rule, appName, ttl, source, type, uuid, date); + KeyRecord keyRecord = new KeyRecord(key, rule, appName, 1, source, type, uuid, date); keyRecord.setRule(rule); - timelyKeyRecordTwoTuple.setSecond(keyRecord); - return timelyKeyRecordTwoTuple; - } else if (eventType.equals(Event.EventType.DELETE)) { - timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).appName(appName).build()); - return timelyKeyRecordTwoTuple; + return keyRecord; + } else { + //是删除 + return null; } - return timelyKeyRecordTwoTuple; } @@ -249,7 +204,6 @@ public class DataHandler { } - /** * 每天根据app的配置清理过期数据 */ @@ -261,13 +215,13 @@ public class DataHandler { for (KeyValue kv : keyValues) { String key = kv.getKey().toStringUtf8(); String ttl = kv.getValue().toStringUtf8(); - String app = key.replace(ConfigConstant.clearCfgPath,""); + String app = key.replace(ConfigConstant.clearCfgPath, ""); Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); summaryMapper.clearExpireData(app, expireDate); keyRecordMapper.clearExpireData(app, expireDate); statisticsMapper.clearExpireData(app, expireDate); } - } catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 5b9aebe..234e447 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -10,7 +10,7 @@ import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.IRecord; import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.Worker; @@ -26,10 +26,7 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -58,7 +55,7 @@ public class EtcdMonitor { private DataHandler dataHandler; - public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(256); + public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(16); /** * 监听新来的热key，该key的产生是来自于手工在控制台添加 @@ -68,28 +65,34 @@ public class EtcdMonitor { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); - EventWrapper eventWrapper = build(event); + KeyValue kv = event.getKv(); + String v = kv.getValue().toStringUtf8(); + Event.EventType eventType = event.getType(); String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyPath, ""); - eventWrapper.setKey(appKey); - dataHandler.offer(eventWrapper); - } - }); - } + dataHandler.offer(new IRecord() { + @Override + public String appNameKey() { + return appKey; + } + @Override + public String value() { + return v; + } - private EventWrapper build(Event event) { - KeyValue kv = event.getKv(); - long ttl = configCenter.timeToLive(kv.getLease()); - String v = kv.getValue().toStringUtf8(); - Event.EventType eventType = event.getType(); - EventWrapper eventWrapper = new EventWrapper(); - eventWrapper.setValue(v); - eventWrapper.setDate(new Date()); - eventWrapper.setTtl(ttl); - eventWrapper.setVersion(kv.getVersion()); - eventWrapper.setEventType(eventType); - return eventWrapper; + @Override + public int type() { + return eventType.getNumber(); + } + + @Override + public Date createTime() { + return new Date(); + } + }); + } + }); } @@ -120,7 +123,7 @@ public class EtcdMonitor { * 每秒去清理一次caffeine */ @Scheduled(fixedRate = 1000) - public void fetchDashboardIp() { + public void cleanCaffeine() { try { HotKeyReceiver.cleanUpCaffeine(); } catch (Exception e) { @@ -148,8 +151,28 @@ public class EtcdMonitor { //将该key放入实时热key本地缓存中 HotKeyReceiver.put(model); - //入库record表，TODO 需要改成batch入库 - dataHandler.insertRecord(model.getAppName() + "/" + model.getKey(), 0); + dataHandler.offer(new IRecord() { + @Override + public String appNameKey() { + return model.getAppName() + "/" + model.getKey(); + } + + @Override + public String value() { + return UUID.randomUUID().toString(); + } + + @Override + public int type() { + return 0; + } + + @Override + public Date createTime() { + return new Date(); + } + }); + } catch (Exception e) { e.printStackTrace(); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java deleted file mode 100644 index 3423da3..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.hotkey.dashboard.mapper; - -import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; -import org.apache.ibatis.annotations.Mapper; - -import java.util.List; - -@Mapper -public interface KeyTimelyMapper { - - int clear(); - - int deleteByKeyAndApp(String key,String appName); - - int insertSelective(KeyTimely key); - - KeyTimely selectByPrimaryKey(Long id); - - KeyTimely selectByKey(String key); - - int updateByKey(KeyTimely key); - - List listKeyTimely(SearchReq param); - - int batchInsert(List list); - - void batchDeleted(List deleteList); - - void saveOrUpdate(KeyTimely keyTimely); -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java index 97f0b4b..e8fdf0f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java @@ -16,7 +16,7 @@ public class KeyRecord implements Serializable { /** * 缓存时间 */ - private Long duration; + private int duration; /** * 来源： SYSTEM 系统探测；USERNAME创建人 @@ -43,7 +43,7 @@ public class KeyRecord implements Serializable { public KeyRecord() { } - public KeyRecord(String key,String val, String appName, Long duration, + public KeyRecord(String key,String val, String appName, int duration, String source, Integer type,String uuid, Date createTime) { this.key = key; this.val = val; @@ -103,11 +103,11 @@ public class KeyRecord implements Serializable { this.val = val; } - public Long getDuration() { + public int getDuration() { return duration; } - public void setDuration(Long duration) { + public void setDuration(int duration) { this.duration = duration; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java index 0dd7907..67b9d0a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java @@ -22,7 +22,6 @@ public class KeyTimely implements Serializable { private Date createTime; - private String uuid; /** * 该rule的描述 */ @@ -44,7 +43,6 @@ public class KeyTimely implements Serializable { public KeyTimely(String key, String val, String appName, Long duration, String uuid, Date createTime) { this.key = key; this.val = val; - this.uuid = uuid; this.appName = appName; this.duration = duration; this.createTime = createTime; @@ -59,7 +57,6 @@ public class KeyTimely implements Serializable { ", val='" + val + '\'' + ", duration=" + duration + ", createTime=" + createTime + - ", uuid='" + uuid + '\'' + ", ruleDesc='" + ruleDesc + '\'' + ", updater='" + updater + '\'' + '}'; @@ -121,14 +118,6 @@ public class KeyTimely implements Serializable { this.createTime = createTime; } - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - public String getUpdater() { return updater; } @@ -201,7 +190,6 @@ public class KeyTimely implements Serializable { keyTimely.setVal(val); keyTimely.setDuration(duration); keyTimely.setCreateTime(createTime); - keyTimely.setUuid(uuid); keyTimely.setRuleDesc(ruleDesc); keyTimely.setUpdater(updater); return keyTimely; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index c268344..09dbd32 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -171,6 +171,7 @@ public class KeyServiceImpl implements KeyService { configCenter.putAndGrant(ConfigConstant.hotKeyPath + key.getAppName() + "/" + key.getKey(), System.currentTimeMillis() + "", key.getDuration()); + //写入本地缓存，实时热key信息 HotKeyModel hotKeyModel = new HotKeyModel(); hotKeyModel.setCreateTime(System.currentTimeMillis()); hotKeyModel.setAppName(key.getAppName()); @@ -203,7 +204,7 @@ public class KeyServiceImpl implements KeyService { } configCenter.delete(etcdKey); - KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, + KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(), "", keyTimely.getUpdater(), SystemClock.now() + "")); diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 1678e74..3e558d0 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -1,5 +1,5 @@ server : - port : 8080 + port : 8081 servlet : context-path : / spring : @@ -27,18 +27,24 @@ spring : time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss datasource: - username: root - password: password - url: jdbc:mysql://localhost:3306/hotkey?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC + username: ${MYSQL_USER:root} + password: ${MYSQL_PASS:Wu@0396110} + url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 driver-class-name: com.mysql.cj.jdbc.Driver + hikari: + max-lifetime: 120000 + idle-timeout: 60000 + connection-timeout: 30000 + maximum-pool-size: 32 + minimum-idle: 10 pagehelper: helperDialect: mysql reasonable: true supportMethodsArguments: true params: count=countSql -mybatis: - configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl etcd: server: ${etcdServer:http://127.0.0.1:2379} diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml deleted file mode 100644 index 04704cf..0000000 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml +++ /dev/null @@ -1,207 +0,0 @@ - - - - - - - - - - - - - - id, key_name, val, uuid, app_name, duration, create_time - - - - - delete from hk_key_timely - where key_name = #{key} and app_name = #{appName} - - - - - delete from hk_key_timely - where (app_name,key_name) in - - (#{item.appName},#{item.key}) - - - - - delete from hk_key_timely where 1 = 1 - - - - - INSERT ignore INTO - hk_key_timely( - key_name, - val, - app_name, - uuid, - duration, - create_time) - VALUES - - (#{k.key}, - #{k.val}, - #{k.appName}, - #{k.uuid}, - #{k.duration}, - #{k.createTime}) - - - - - insert into hk_key_timely - - - id, - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{id,jdbcType=BIGINT}, - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - - - - update hk_key_timely - - - key_name = #{key,jdbcType=VARCHAR}, - - - val = #{val,jdbcType=VARCHAR}, - - - uuid = #{uuid,jdbcType=VARCHAR}, - - - app_name = #{appName,jdbcType=VARCHAR}, - - - duration = #{duration,jdbcType=BIGINT}, - - - create_time = #{createTime,jdbcType=TIMESTAMP}, - - - where key_name = #{key} - - - - - - - - insert into hk_key_timely - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - on duplicate key update - create_time = now() - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/admin/key/list.html b/dashboard/src/main/resources/templates/admin/key/list.html index 27ff05d..6f755c8 100644 --- a/dashboard/src/main/resources/templates/admin/key/list.html +++ b/dashboard/src/main/resources/templates/admin/key/list.html @@ -163,7 +163,7 @@ var options = { { field: 'key', title: 'key'}, { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, - { field: 'duration', title: '缓存时间'}, + { field: 'duration', title: '剩余缓存时间'}, { field: 'source', title: '来源'}, { field: 'type', title: '操作类型', formatter:function (val) { diff --git a/dashboard/src/main/resources/templates/admin/key/listtimely.html b/dashboard/src/main/resources/templates/admin/key/listtimely.html index 37e85be..545caa7 100644 --- a/dashboard/src/main/resources/templates/admin/key/listtimely.html +++ b/dashboard/src/main/resources/templates/admin/key/listtimely.html @@ -162,7 +162,7 @@ var options = { { field: 'key', title: 'key'}, { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, - { field: 'duration', title: '缓存时间'}, + { field: 'duration', title: '剩余缓存时间'}, { field: 'createTime', title: '创建时间', formatter:function (val,row,index) { return changeDateFormat(val); diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index 23bd5e6..d5b22bb 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -58,7 +58,19 @@ public class Starter { builder1.append(UUID.randomUUID()); } - a(); + String s = "prefix"; + System.out.println(System.currentTimeMillis()); + + for (int i = 0; i < 10000; i++) { + if (JdHotKeyStore.getValue(s + i) != null) { + System.out.println("ishot"); + } else { + int j = i; + } + } + System.out.println(System.currentTimeMillis()); + +// a(); // logger.info("begin : " + System.currentTimeMillis()); // iConfigCenter.putAndGrant( "/abc/" + builder1.toString(), UUID.randomUUID().toString(), 60); // for (int i = 0; i < 200000; i++) { -- Gitee From b61da1bbc38d3b53cae30f9c9a28954b27bee306 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 14:25:39 +0800 Subject: [PATCH 25/42] =?UTF-8?q?=E8=B0=83=E5=A4=A7client=E6=9C=80?= =?UTF-8?q?=E5=A4=A7=E7=BC=93=E5=AD=98=E6=95=B0=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/hotkey/client/ClientStarter.java | 9 +-------- .../main/java/com/jd/platform/hotkey/client/Context.java | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java index 1102d8a..27474e7 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.client; -import com.jd.platform.hotkey.client.cache.LocalCache; import com.jd.platform.hotkey.client.callback.ReceiveNewKeySubscribe; import com.jd.platform.hotkey.client.core.eventbus.EventBusCenter; import com.jd.platform.hotkey.client.core.key.PushSchedulerStarter; @@ -40,9 +39,8 @@ public class ClientStarter { public static class Builder { private String appName; private String etcdServer; - private LocalCache localCache; private Long pushPeriod; - private int caffeineSize = 50000; + private int caffeineSize = 200000; public Builder() { } @@ -65,11 +63,6 @@ public class ClientStarter { return this; } - public Builder setLocalCache(LocalCache localCache) { - this.localCache = localCache; - return this; - } - public Builder setPushPeriod(Long pushPeriod) { this.pushPeriod = pushPeriod; return this; diff --git a/client/src/main/java/com/jd/platform/hotkey/client/Context.java b/client/src/main/java/com/jd/platform/hotkey/client/Context.java index f0455a2..fea4106 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/Context.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/Context.java @@ -12,5 +12,5 @@ public class Context { */ public static boolean NEED_RECONNECT = true; - public static int CAFFEINE_SIZE = 50000; + public static int CAFFEINE_SIZE; } -- Gitee From b3738e34a94ec4294dff92b5450b6264fd7590b8 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 14:28:37 +0800 Subject: [PATCH 26/42] =?UTF-8?q?=E8=B0=83=E5=A4=A7client=E6=9C=80?= =?UTF-8?q?=E5=A4=A7=E7=BC=93=E5=AD=98=E6=95=B0=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 3e558d0..95ab885 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -28,7 +28,7 @@ spring : date-format: yyyy-MM-dd HH:mm:ss datasource: username: ${MYSQL_USER:root} - password: ${MYSQL_PASS:Wu@0396110} + password: ${MYSQL_PASS:root} url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 driver-class-name: com.mysql.cj.jdbc.Driver hikari: -- Gitee From 5a3cb79dc71da535267ab0db55e58fe7c97e8f12 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 15:06:11 +0800 Subject: [PATCH 27/42] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 215 +++++++++++++++--- .../common/configcenter/ConfigConstant.java | 4 - .../dashboard/mapper/KeyRecordMapper.java | 1 - .../dashboard/mapper/SummaryMapper.java | 1 - dashboard/src/main/resources/db.sql | 9 +- .../worker/netty/dashboard/NettyClient.java | 2 +- 6 files changed, 185 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index c557dae..2d7a858 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,44 @@ # hotkey +![输入图片说明](https://images.gitee.com/uploads/images/2020/0616/105737_e5b876cd_303698.png "redis热key探测及缓存到JVM (1).png") -正在京东APP后台灰度了几千台机器，等618海量并发检验后再推广使用，暂时不要下载使用它。 +对任意突发性的无法预先感知的热点数据，包括并不限于热点数据（如突发大量请求同一个商品）、热用户（如恶意爬虫刷子）、热接口（突发海量请求同一个接口）等，进行毫秒级精准探测到。然后对这些热数据、热用户等，推送到所有服务端JVM内存中，以大幅减轻对后端数据存储层的冲击，并可以由使用者决定如何分配、使用这些热key（譬如对热商品做本地缓存、对热用户进行拒绝访问、对热接口进行熔断或返回默认值）。这些热数据在整个服务端集群内保持一致性，并且业务隔离，worker端性能强悍。 -该框架历经多次压测，8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒20万，实际压测达到30万以上，CPU平稳支撑，框架无异常。测试详情可去我[CSDN博客](https://blog.csdn.net/tianyaleixiaowu)查看。 +京东APP后台热数据探测框架，历经多次高压压测和2020年京东618大促考验。在上线运行的这段时间内，每天探测的key数量数十亿计，精准捕获了大量爬虫、刷子用户，另准确探测大量热门商品并毫秒级推送到各个服务端内存，大幅降低了热数据对数据层的查询压力，提升了应用性能。 + +该框架历经多次压测，性能指标主要有两个： + +1 探测性能：8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒平稳处理30万以上，实际压测达到37万，CPU平稳支撑，框架无异常。 + +2 推送性能：在高并发写入的同时，对外推送目前性能约平稳推送每秒10-12万次，譬如有1千台server，一台worker上每秒产生了100个热key，那么这1秒会平稳推送100 * 1000 = 10万次，10万次推送会明确在1s内全部送达。如果是写入少，推送多，以纯推送来计数的话，该框架每秒可稳定对外推送40-60万次平稳，80次极限可撑几秒。 + +在真实业务场景中，可用1：1000的比例，即1台worker支撑1000台业务服务端的key探测任务，即可带来极大的数据存储资源节省（如对redis集群的扩充）。测试详情可去我[CSDN博客](https://blog.csdn.net/tianyaleixiaowu)查看。 + +#### 架架架架架架架架架架架构构构构构构构构构构构设设设设设设设设设设计计计计计计计计计计计计 + **见京东零售公众号** +https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw + + **！！！！！！！！！！！！！！！！重要！！！！！！！！！！！！！！！不然看不懂框架原理** + +#### 参与开发 +该框架的开发得到了来自于： + +京东数科-生活服务研发组-李云峰的大力支持，完成了整个dashboard控制台的研发。 + +京东零售-平台业务中心-APP后台架构师 杜晓宇的大力支持，参与并帮助解决了诸多技术难题。 + +京东零售-平台业务中心-PC购物车业务组 任启恒，进行了多次极限压测及功能验证，为框架的健壮性提供了更高的标准。 + +京东零售-技术与数据中心-订单研发组 姜坤坤，进行了漫长和覆盖更加全面的功能验证，协助进行了bug发现和bugfix。 + +京东零售-平台业务中心-业务测试部 郭世儒，配合进行了多次军演模拟压力测试演练，协助验证框架在模拟真实流量时的表现。 + +京东数科-消费金融研发部-账务服务 王小飞，对框架提出了重要的功能新增建议，以及新增功能的协助验证。 + +外部开发者，来自于[lowbMan](https://gitee.com/lowbMan)提供了关于修改proto序列化方式的建议，以及其他有相当价值的优化建议。 + +另有京东集团多个部门提供的意见、建议，和对框架的见解、优化方案。 + +在此一起感谢。 #### 介绍 对任意突发性的无法预先感知的热点请求，包括并不限于热点数据（如突发大量请求同一个商品）、热用户（如爬虫、刷子）、热接口（突发海量请求同一个接口）等，进行毫秒级精准探测到。 @@ -23,64 +59,169 @@ 5 接口、用户维度限流 -6 单机接口、用户维度限流限流 +6 单机接口、用户维度限流 7 集群用户维度限流 8 集群接口维度限流 -#### 尚未完成 -控制台功能缺失如下： +该开源项目战略意义重大，经历百万级并发，参与京东开源中间件项目建设，一直在等你。 + +#### worker端强悍的性能表现 +每10秒打印一行，totalDealCount代表处理过的key总量，可以看到每10秒处理量在270万-310万之间，对应每秒30万左右QPS。 -1.rule的定时保存，当etcd更换时需要能一键导入原有配置 +仅需要很少的机器，即可完成海量key的实时探测计算推送任务。比扩容redis集群规模成本低太多。 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152336_78597937_303698.png "屏幕截图.png") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152249_4ac01178_303698.png "屏幕截图.png") -2.对/jd/count/cartsoa/ 目录下信息进行展示，代表的是每个worker连接的client数量（已完成） -3.对/jd/caffeineSize/ 目录进行展示，里面是每个worker内caffeine缓存的数量 +采用protobuf序列化后性能进一步得到提升。在秒级36万以上时，能稳定在CPU 60%，压测持续时长超过5小时，未见任何异常。30万时，压测时长超过数日，未见任何异常。 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0817/104833_6837a091_303698.png "屏幕截图.png") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0817/104930_c522bd91_303698.png "屏幕截图.png") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0817/105009_132a878b_303698.png "屏幕截图.png") -4.筛选功能，对记录表里做筛选，按时间范围筛选key出现次数大于xx次的数据 +### 界面效果 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/163805_0aa68d4b_303698.png "屏幕截图.png") +### 加我微信拉入群，群里大厂大佬多 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0807/170047_e37c882e_303698.jpeg "1596790826590.jpg") -5.导出功能，将key排重后导出的功能，按时间范围筛选 -6.还有很多 +#### 安装教程 +1. 安装etcd -该开源项目战略意义重大，要经历百万级并发，参与京东开源中间件项目建设，一直在等你。 + 在etcd下载页面下载对应操作系统的etcd，https://github.com/etcd-io/etcd/releases 使用3.4.x以上。相关搭建细节，及常见问题会发布到CSDN博客内。 -#### worker端强悍的性能表现 -每10秒打印一行，totalDealCount代表处理过的key总量，可以看到每10秒处理量在270万-310万之间，对应每秒30万左右QPS。 +2. 启动worker（集群） + 下载并编译好代码，将worker打包为jar，启动即可。如： -仅需要很少的机器，即可完成海量key的实时探测计算推送任务。比扩容redis集群规模成本低太多了。 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152336_78597937_303698.png "屏幕截图.png") -![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152249_4ac01178_303698.png "屏幕截图.png") + **` java -jar $JAVA_OPTS worker-0.0.1-SNAPSHOT.jar --etcd.server=${etcdServer}`** + worker可供配置项如下： +![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/164514_c57d740a_303698.png "屏幕截图.png") + etcdServer为etcd集群的地址，用逗号分隔 -#### 安装教程 + JAVA_OPTS是配置的JVM相关，可根据实际情况配置 + + threadCount为处理key的线程数，不指定时由程序来计算。 + + workerPath代表该worker为哪个应用提供计算服务，譬如不同的应用appName需要用不同的worker进行隔离，以避免资源竞争。 + +3. 启动控制台 + + 下载并编译好dashboard项目，创建数据库并导入resource下db.sql文件。 配置一下application.yml里的数据库相关和etcdServer地址。 + + 启动dashboard项目，访问ip:8081，即可看到界面。 + + 其中节点信息里，即是当前已启动的worker列表。 + + 规则配置就是为各app设置规则的地方，初次使用时需要先添加APP。在用户管理菜单中，添加一个新用户，设置他的APP名字，如sample。之后新添加的这个用户就可以登录dashboard给自己的APP设置规则了，登录密码默认123456。 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/175255_e1b05b4c_303698.png "屏幕截图.png") + + 如图就是一组规则，譬如其中as__开头的热key的规则就是interval-2秒内出现了threshold-10次就认为它是热key，它就会被推送到jvm内存中，并缓存60秒，prefix-true代表前缀匹配。那么在应用中，就可以把一组key，都用as__开头，用来探测。 + +4. client端接入使用 + + 引入client的pom依赖。 + + 在应用启动的地方初始化HotKey，譬如 + +``` +@PostConstruct + +public void initHotkey() { + + ClientStarter.Builder builder = new ClientStarter.Builder(); + ClientStarter starter = builder.setAppName("appName").setEtcdServer("http://1.8.8.4:2379,http://1.1.4.4:2379,http://1.1.1.1:2379").build(); + starter.startPipeline(); +} +``` +其中还可以setCaffeineSize(int size)设置本地缓存最大数量，默认5万，setPushPeriod(Long period)设置批量推送key的间隔时间，默认500ms，该值越小，上报热key越频繁，相应越及时，建议根据实际情况调整，如单机每秒qps10个，那么0.5秒上报一次即可，否则是空跑。该值最小为1，即1ms上报一次。 + +注意： + +如果原有项目里使用了guava，需要升级guava为以下版本，否则过低的guava版本可能发生jar包冲突。或者删除自己项目里的guava的maven依赖，guava升级不会影响原有任何逻辑。 + +``` + + com.google.guava + guava + 28.2-jre + compile + +``` + +有时可能项目里没有直接依赖guava，但是引入的某个pom里引了guava，也需要将guava排除掉。 + + + +使用： + +主要有如下4个方法可供使用 + +boolean JdHotKeyStore.isHotKey(String key) + +Object JdHotKeyStore.get(String key) + +void JdHotKeyStore.smartSet(String key, Object value) + +Object JdHotKeyStore.getValue(String key) + + + +1 boolean isHotKey(String key) ，该方法会返回该key是否是热key，如果是返回true，如果不是返回false，并且会将key上报到探测集群进行数量计算。该方法通常用于判断只需要判断key是否热、不需要缓存value的场景，如刷子用户、接口访问频率等。 + +2 Object get(String key)，该方法返回该key本地缓存的value值，可用于判断是热key后，再去获取本地缓存的value值，通常用于redis热key缓存 + +3 void smartSet(String key, Object value)，方法给热key赋值value，如果是热key，该方法才会赋值，非热key，什么也不做 + +4 Object getValue(String key)，该方法是一个整合方法，相当于isHotKey和get两个方法的整合，该方法直接返回本地缓存的value。 +如果是热key，则存在两种情况，1是返回value，2是返回null。返回null是因为尚未给它set真正的value，返回非null说明已经调用过set方法了，本地缓存value有值了。 +如果不是热key，则返回null，并且将key上报到探测集群进行数量探测。 + + +最佳实践： + +1 判断用户是否是刷子 + + if (JdHotKeyStore.isHotKey(“pin__” + thePin)) { + //限流他，do your job + } +2 判断商品id是否是热点 + + + Object skuInfo = JdHotKeyStore.getValue("skuId__" + skuId); + if(skuInfo == null) { + JdHotKeyStore.smartSet("skuId__" + skuId, theSkuInfo); + } else { + //使用缓存好的value即可 + } + + 或者这样： + + + + if (JdHotKeyStore.isHotKey(key)) { + //注意是get，不是getValue。getValue会获取并上报，get是纯粹的本地获取 + Object skuInfo = JdHotKeyStore.get("skuId__" + skuId); + if(skuInfo == null) { + JdHotKeyStore.smartSet("skuId__" + skuId, theSkuInfo); + } else { + //使用缓存好的value即可 + } + + } -1. xxxx -2. xxxx -3. xxxx +### 测试环境 -#### 使用说明 +我司为方便大家快速接入试用、查看hotkey效果，提供了1台公网etcd机器（16c16g）、2台worker机器（8c12g），供快速接入测试。 -1. xxxx -2. xxxx -3. xxxx -#### 参与贡献 +etcd地址为： **http://open-etcd.jd.com:2000** ，可以在下载项目后，在sample项目、dashboard项目里yml文件修改etcd连接地址为该地址，然后进入界面控制台，即可看到当前连接的worker，之后可以在sample项目里，进行hotkey测试。 -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request +控制台公网测试地址：http://hotkey.tianyalei.com:9001/ 账号admin，密码123456 -#### 码云特技 -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. 码云官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解码云上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是码云最有价值开源项目，是码云综合评定出的优秀开源项目 -5. 码云官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. 码云封面人物是一档用来展示码云会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +测试前，注意在控制台新建自己的app规则。 diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index e115056..2a7eff0 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -41,10 +41,6 @@ public interface ConfigConstant { * 每个app的热key记录放这里，供控制台监听入库用。格式如：jd/records/app1/userA */ String hotKeyRecordPath = "/jd/keyRecords/"; - /** - * 每个app的热key记录放这里，供控制台监听入库用。格式如：jd/batchRecords/ jd/records/app1/userA,jd/records/app1/userB 一个逗号分隔的巨大的key集合 - */ - String hotKeyBatchRecordPath = "/jd/keyBatchRecords/"; /** * caffeine的size */ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java index 56c05ab..e135ff6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.dashboard.mapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java index b12ff0d..0305c4c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java @@ -2,7 +2,6 @@ package com.jd.platform.hotkey.dashboard.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; -import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; import org.apache.ibatis.annotations.Mapper; diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 8cbb0fa..98fbd82 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -31,6 +31,11 @@ CREATE TABLE `hk_user` ( ) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; +-- pwd: 123456 +INSERT INTO `hk_user` VALUES (2, 'admin', 'admin', 'e10adc3949ba59abbe56e057f20f883e', '1888888', 'ADMIN', '', '2020-07-28 14:01:03', 1); + + + DROP TABLE IF EXISTS `hk_key_record`; CREATE TABLE `hk_key_record` ( `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -44,8 +49,7 @@ CREATE TABLE `hk_key_record` ( `uuid` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '防重ID', `rule` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '''' COMMENT '规则', PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引', - INDEX `idx_app_ct`(`app_name`, `create_time`) USING BTREE COMMENT '查询索引' + UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引' ) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; @@ -77,7 +81,6 @@ CREATE TABLE `hk_statistics` ( `create_time` datetime(0) NOT NULL COMMENT '创建时间', PRIMARY KEY (`id`) USING BTREE, UNIQUE INDEX `uniq_uuid`(`uuid`) USING BTREE COMMENT '防重唯一索引' - INDEX `idx_app_ct`(`app`, `create_time`) USING BTREE COMMENT '查询索引' ) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Compact; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java index 47d1ef2..5209c35 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java @@ -69,7 +69,7 @@ public class NettyClient { } catch (Exception e) { DashboardHolder.hasConnected = false; DashboardHolder.channel = null; - e.printStackTrace(); + e.printStackTrace(); } //这一步就阻塞了 -- Gitee From 53f0d4b75bad3861ac532753393f5ce36792129d Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 16:52:46 +0800 Subject: [PATCH 28/42] =?UTF-8?q?=E6=B8=85=E7=90=86=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/client/netty/NettyClient.java | 7 -- .../platform/hotkey/common/coder/Codec.java | 30 --------- .../hotkey/common/coder/MessageDecoder.java | 53 --------------- .../hotkey/common/coder/MessageEncoder.java | 47 -------------- .../hotkey/common/coder/NettyCodec.java | 36 ---------- .../jd/platform/hotkey/worker/cache/Test.java | 64 ------------------ .../hotkey/worker/config/CodecConfig.java | 20 ------ .../hotkey/worker/config/DisruptorConfig.java | 33 ---------- .../worker/disruptor/AbsEventConsumer.java | 54 --------------- .../worker/disruptor/AbsWorkConsumer.java | 54 --------------- .../hotkey/worker/disruptor/BaseEvent.java | 26 -------- .../worker/disruptor/DisruptorBuilder.java | 65 ------------------- .../worker/disruptor/MessageProducer.java | 8 --- .../worker/disruptor/ProducerFactory.java | 50 -------------- .../worker/disruptor/hotkey/HotKeyEvent.java | 18 ----- .../disruptor/hotkey/HotKeyEventConsumer.java | 41 ------------ .../disruptor/hotkey/HotKeyEventProducer.java | 32 --------- .../hotkey/worker/model/ClientInfo.java | 62 ------------------ .../hotkey/worker/mq/IMqMessageReceiver.java | 9 --- .../worker/netty/filter/HotKeyFilter.java | 8 +-- .../worker/netty/filter/KeyCounterFilter.java | 8 +-- .../hotkey/worker/tool/InitConstant.java | 4 -- worker/src/main/resources/application.yml | 1 - 23 files changed, 2 insertions(+), 728 deletions(-) delete mode 100755 common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java delete mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java delete mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java delete mode 100755 common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java index d2a36a4..d8ba892 100755 --- a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java @@ -2,8 +2,6 @@ package com.jd.platform.hotkey.client.netty; import com.jd.platform.hotkey.client.core.worker.WorkerInfoHolder; import com.jd.platform.hotkey.client.log.JdLogger; -import com.jd.platform.hotkey.common.coder.Codec; -import com.jd.platform.hotkey.common.coder.NettyCodec; import com.jd.platform.hotkey.common.tool.Constant; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; @@ -28,9 +26,6 @@ public class NettyClient { private Bootstrap bootstrap; - private Codec codec = new NettyCodec(); - - public static NettyClient getInstance() { return nettyClient; } @@ -56,8 +51,6 @@ public class NettyClient { ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); ch.pipeline() .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) -// .addLast(codec.newEncoder()) -// .addLast(codec.newDecoder()) .addLast(new StringDecoder()) //10秒没消息时，就发心跳包过去 .addLast(new IdleStateHandler(0, 0, 30)) diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java deleted file mode 100755 index c61805b..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jd.platform.hotkey.common.coder; - -import io.netty.channel.ChannelHandler; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public interface Codec { - - ChannelHandler newEncoder(); - - ChannelHandler newDecoder(); -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java deleted file mode 100644 index 24cb0ed..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jd.platform.hotkey.common.coder; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.charset.Charset; -import java.util.List; - -/** - * @author wuweifeng wrote on 2020-01-06 - * @version 1.0 - */ -public class MessageDecoder extends ByteToMessageDecoder { - private Logger logger = LoggerFactory.getLogger(getClass()); - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List out) { - if (byteBuf.readableBytes() < 10 || byteBuf.readableBytes() > Constant.MAX_LENGTH) { - logger.warn("数据包不正确，当前包大小为：" + byteBuf.readableBytes()); - return; - } - - byteBuf.markReaderIndex(); - - HotKeyMsg message = new HotKeyMsg(); - - int magicNumber = byteBuf.readInt(); - if (Constant.MAGIC_NUMBER != magicNumber) { - logger.warn("MAGIC_NUMBER不正确:" + magicNumber); - return; - } - - message.setMagicNumber(byteBuf.readInt()); // 读取魔数 - - MessageType messageType = MessageType.get(byteBuf.readByte()); - if (messageType == null) { - logger.error("messageType is null , byteBuf readByte = " + byteBuf.readByte()); - return; - } - message.setMessageType(messageType); // 读取当前的消息类型 - - int bodyLength = byteBuf.readInt(); // 读取消息体长度和数据 - CharSequence body = byteBuf.readCharSequence(bodyLength, Charset.defaultCharset()); - message.setBody(body.toString()); - out.add(message); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java deleted file mode 100644 index acf2162..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.jd.platform.hotkey.common.coder; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToByteEncoder; - -import java.nio.charset.Charset; - -/** - * @author wuweifeng wrote on 2020-01-06 - * @version 1.0 - */ -public class MessageEncoder extends MessageToByteEncoder { - - @Override - protected void encode(ChannelHandlerContext ctx, HotKeyMsg message, ByteBuf out) { - MessageType messageType = message.getMessageType(); - //非法类型 - if (MessageType.get(messageType.getType()) == null) { - return; - } - // 这里会判断消息类型是不是EMPTY类型，如果是EMPTY类型，则表示当前消息不需要写入到管道中 - if (messageType == MessageType.EMPTY) { - return; - } - //4个byte - out.writeInt(Constant.MAGIC_NUMBER); // 写入当前的魔数 - //1个byte - out.writeByte(message.getMessageType().getType()); // 写入当前消息的类型 - - if (null == message.getBody()) { - out.writeInt(0); // 如果消息体为空，则写入0，表示消息体长度为0 - } else { - System.out.println(message.getBody().length()); - out.writeInt(message.getBody().length()); - out.writeCharSequence(message.getBody(), Charset.defaultCharset()); - } - out.writeBytes(Constant.DELIMITER.getBytes()); - } - - public static void main(String[] args) { - System.out.println(Constant.MAGIC_NUMBER + ""); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java deleted file mode 100755 index 1aeac45..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jd.platform.hotkey.common.coder; - -import io.netty.channel.ChannelHandler; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public class NettyCodec implements Codec { - - @Override - public ChannelHandler newEncoder() { - return new MessageEncoder(); - } - - @Override - public ChannelHandler newDecoder() { - return new MessageDecoder(); - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java deleted file mode 100644 index bf2afa4..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.jd.platform.hotkey.worker.cache; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-04 - */ -@Component -public class Test { - @Resource(name = "hotKeyCache") - private Cache hotCache; - - public static void main(String[] args) { - Map map = new HashMap(); -// map.put("a", "b"); -// System.out.println(map); - Cache cache = Caffeine.newBuilder() - .initialCapacity(1024)//初始大小 - .maximumSize(5000000)//最大数量 - .expireAfterWrite(5, TimeUnit.MINUTES)//过期时间 - .softValues() - .build(); - - cache.put("yuan2012555", 1); -// System.out.println(cache.getIfPresent("yuan2012555")); - - - System.out.println("lwymail163".hashCode() %4); - System.out.println("272551766_m".hashCode() %4); - System.out.println("hanxu123".hashCode() %4); - System.out.println("abc123ab".hashCode() %4); - - System.out.println("hanxu123hanxu".hashCode() %4); - System.out.println(Math.abs("lwy163mail".hashCode() %4)); - - } - -// @PostConstruct -// public void aa() throws InterruptedException { -// Executor executor = Executors.newCachedThreadPool(); -// Cache cache = Caffeine.newBuilder() -// .executor(executor) -// .initialCapacity(1024)//初始大小 -// .maximumSize(5000000)//最大数量 -// .expireAfterWrite(5, TimeUnit.MINUTES)//过期时间 -// .softValues() -// .build(); -// long i = 0; -// while (true) { -// cache.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); -//// hotCache.put("i" + i, i); -// i++; -// } -// } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java deleted file mode 100644 index 0d62c90..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.common.coder.Codec; -import com.jd.platform.hotkey.common.coder.NettyCodec; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * netty的编解码器 - * @author wuweifeng wrote on 2020-02-21 - * @version 1.0 - */ -@Configuration -public class CodecConfig { - - @Bean - public Codec codec() { - return new NettyCodec(); - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java deleted file mode 100644 index 0a92f8f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.Resource; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -@Configuration -public class DisruptorConfig { - @Resource - private IKeyListener iKeyListener; - - @Value("${open.timeout}") - private boolean openTimeOut; - @Value("${thread.count}") - private int threadCount; - -// @Bean -// public MessageProducer messageProducer() { -// InitConstant.openTimeOut = openTimeOut; -// //将实际值赋给static变量 -// InitConstant.threadCount = threadCount; -// -// return ProducerFactory.createHotKeyProducer(iKeyListener); -// } - - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java deleted file mode 100755 index 47839ae..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java +++ /dev/null @@ -1,54 +0,0 @@ -//package com.jd.platform.hotkey.worker.disruptor; -// -//import cn.hutool.core.date.SystemClock; -//import com.jd.platform.hotkey.common.model.BaseModel; -//import com.jd.platform.hotkey.common.tool.Constant; -//import com.jd.platform.hotkey.worker.tool.InitConstant; -//import com.lmax.disruptor.EventHandler; -// -//import java.util.concurrent.atomic.LongAdder; -//import java.util.zip.CRC32; -// -///**各个消费者重复消费 -// * @author wuweifeng wrote on 2019-08-21. -// */ -//public abstract class AbsEventConsumer implements EventHandler { -// -// private int hashIndex; -// -// public static final LongAdder totalDealCount = new LongAdder(); -// //过期的 -// public static final LongAdder expireTotalCount = new LongAdder(); -// -// public AbsEventConsumer(int hashIndex) { -// this.hashIndex = hashIndex; -// } -// -// @Override -// public void onEvent(T t, long l, boolean b) { -// //每个消费者，只处理特定的key。保证相同的key，一定被同一个线程处理 -// BaseModel model = t.getModel(); -// if (model == null || model.getKey() == null) { -// return; -// } -// if (Math.abs(hash(model.getKey())) % Constant.Default_Threads == hashIndex) { -// //5秒前的过时消息就不处理了 -// if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { -// expireTotalCount.increment(); -// return; -// } -// onNewEvent(t); -// -// //处理完毕，将数量加1 -// totalDealCount.increment(); -// } -// } -// -// protected abstract void onNewEvent(T t); -// -// private long hash(String key) { -// CRC32 crc = new CRC32(); -// crc.update(key.getBytes()); -// return crc.getValue(); -// } -//} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java deleted file mode 100755 index bca2e70..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import cn.hutool.core.date.SystemClock; -import com.jd.platform.hotkey.common.model.BaseModel; -import com.jd.platform.hotkey.worker.tool.InitConstant; -import com.lmax.disruptor.WorkHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.atomic.LongAdder; - -/** - * 各个消费者不重复消费 - * - * @author wuweifeng wrote on 2019-08-21. - */ -public abstract class AbsWorkConsumer implements WorkHandler { - - private int hashIndex; - - public static final LongAdder totalDealCount = new LongAdder(); - //过期的 - public static final LongAdder expireTotalCount = new LongAdder(); - - private Logger logger = LoggerFactory.getLogger(getClass()); - - public AbsWorkConsumer(int hashIndex) { - this.hashIndex = hashIndex; - } - - @Override - public void onEvent(T t) { - BaseModel model = t.getModel(); - if (model == null || model.getKey() == null) { - return; - } - //5秒前的过时消息就不处理了 - if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { - expireTotalCount.increment(); - logger.warn("this key timeout : " + model.getKey() + " createAt : " + model.getCreateTime() + - " event at " + SystemClock.now()); - if (InitConstant.openTimeOut) { - return; - } - } - onNewEvent(t); - - //处理完毕，将数量加1 - totalDealCount.increment(); - } - - protected abstract void onNewEvent(T t); - -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java deleted file mode 100644 index e8483fe..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.jd.platform.hotkey.common.model.BaseModel; - -/** - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class BaseEvent { - private T model; - - public BaseEvent(T model) { - this.model = model; - } - - public BaseEvent() { - } - - public T getModel() { - return model; - } - - public void setModel(T model) { - this.model = model; - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java deleted file mode 100644 index 64dc4c0..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.lmax.disruptor.BlockingWaitStrategy; -import com.lmax.disruptor.EventFactory; -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.WorkHandler; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.dsl.ProducerType; - -import java.util.concurrent.Executors; - -/** - * 每个worker重复消费生产者消息 - * - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class DisruptorBuilder { - private int bufferSize; - private EventHandler[] eventHandlers; - private WorkHandler[] workHandlers; - private EventFactory eventFactory; - - public Disruptor build() { - Disruptor disruptor = new Disruptor<>(eventFactory, bufferSize, Executors.defaultThreadFactory(), - ProducerType.SINGLE, new BlockingWaitStrategy()); - - if (eventHandlers != null) { - disruptor.handleEventsWith(eventHandlers); - } - if (workHandlers != null) { - disruptor.handleEventsWithWorkerPool(workHandlers); - } - - disruptor.start(); - return disruptor; - } - - /** - * 每个worker会重复消费 - */ - public DisruptorBuilder setEventHandlers(EventHandler... eventHandlers) { - this.eventHandlers = eventHandlers; - return this; - } - - /** - * 每个worker不会重复消费 - */ - public DisruptorBuilder setWorkerHandlers(WorkHandler... workHandlers) { - this.workHandlers = workHandlers; - return this; - } - - public DisruptorBuilder setEventFactory(EventFactory eventFactory) { - this.eventFactory = eventFactory; - return this; - } - - public DisruptorBuilder setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - return this; - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java deleted file mode 100755 index 63d1421..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -/** - * @author wuweifeng wrote on 2018/4/20. - */ -public interface MessageProducer { - void publish(T t); -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java deleted file mode 100755 index 8854627..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEvent; -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventConsumer; -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventProducer; -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import com.jd.platform.hotkey.worker.tool.CpuNum; -import com.jd.platform.hotkey.worker.tool.InitConstant; -import com.lmax.disruptor.dsl.Disruptor; - -/** - * Disruptor创建producer的地方 - * - * @author wuweifeng wrote on 2019-11-5. - */ -public class ProducerFactory { - - /** - * 创建热key的producer - * - * @return HotKeyEventProducer - */ - public static MessageProducer createHotKeyProducer(IKeyListener iKeyListener) { - int threadCount = CpuNum.workerCount(); - //如果手工指定了线程数，就用手工指定的 - if (InitConstant.threadCount != 0) { - threadCount = InitConstant.threadCount; - } else { - if (threadCount >= 8) { - threadCount = threadCount / 2; - } - } - - HotKeyEventConsumer[] array = new HotKeyEventConsumer[threadCount]; - for (int i = 0; i < threadCount; i++) { - array[i] = new HotKeyEventConsumer(i); - array[i].setKeyListener(iKeyListener); - } - DisruptorBuilder builder = new DisruptorBuilder<>(); - Disruptor disruptor = builder - .setBufferSize(InitConstant.bufferSize * 1024 * 1024) - .setEventFactory(HotKeyEvent::new) -// .setEventHandlers(array) //重复消费的 - .setWorkerHandlers(array) //不重复消费的 - .build(); - - return new HotKeyEventProducer(disruptor); - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java deleted file mode 100755 index 8ec41c3..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.disruptor.BaseEvent; - -/** - * @author wuweifeng wrote on 2019-08-21. - */ -public class HotKeyEvent extends BaseEvent { - - public HotKeyEvent(HotKeyModel hotKeyModel) { - super(hotKeyModel); - } - - public HotKeyEvent() { - - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java deleted file mode 100755 index ca248f5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.disruptor.AbsWorkConsumer; -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import com.jd.platform.hotkey.worker.keylistener.KeyEventOriginal; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * 热key的消费者 - * @author wuweifeng wrote on 2019-12-12. - */ -public class HotKeyEventConsumer extends AbsWorkConsumer { - private Logger logger = LoggerFactory.getLogger(getClass()); - - private IKeyListener iKeyListener; - - public HotKeyEventConsumer(int hashIndex) { - super(hashIndex); - } - - @Override - protected void onNewEvent(HotKeyEvent hotKeyEvent) { - HotKeyModel model = hotKeyEvent.getModel(); - if (iKeyListener == null) { - logger.warn("new key is coming, but no consumer deal this key!"); - return; - } - - if (model.isRemove()) { - iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); - } else { - iKeyListener.newKey(model, KeyEventOriginal.CLIENT); - } - } - - public void setKeyListener(IKeyListener iKeyListener) { - this.iKeyListener = iKeyListener; - } -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java deleted file mode 100755 index 42dae5f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.worker.disruptor.MessageProducer; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; - -/** - * 所有节点发来的消息，都进入这里，然后publish出去，供消费者消费 - * - * @author wuweifeng wrote on 2019/11/6. - */ -public class HotKeyEventProducer implements MessageProducer { - - private Disruptor disruptor; - - public HotKeyEventProducer(Disruptor disruptor) { - this.disruptor = disruptor; - } - - @Override - public void publish(HotKeyEvent hotKeyEvent) { - RingBuffer ringBuffer = disruptor.getRingBuffer(); - long sequence = ringBuffer.next(); - try { - HotKeyEvent event = ringBuffer.get(sequence); - event.setModel(hotKeyEvent.getModel()); - } finally { - ringBuffer.publish(sequence); - } - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java b/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java deleted file mode 100644 index 081652b..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.jd.platform.hotkey.worker.model; - -import io.netty.channel.ChannelHandlerContext; - -/** - * @author wuweifeng wrote on 2019-12-05 - * @version 1.0 - */ -public class ClientInfo { - /** - * 应用名 - */ - private String appName; - /** - * 连接的id - */ - private String channelId; - - private ChannelHandlerContext channelHandlerContext; - - public ClientInfo() { - } - - public ClientInfo(String appName, String channelId, ChannelHandlerContext channelHandlerContext) { - this.appName = appName; - this.channelId = channelId; - this.channelHandlerContext = channelHandlerContext; - } - - @Override - public String toString() { - return "ClientInfo{" + - "appName='" + appName + '\'' + - ", channelId='" + channelId + '\'' + - ", channelHandlerContext=" + channelHandlerContext + - '}'; - } - - public ChannelHandlerContext getChannelHandlerContext() { - return channelHandlerContext; - } - - public void setChannelHandlerContext(ChannelHandlerContext channelHandlerContext) { - this.channelHandlerContext = channelHandlerContext; - } - - public String getAppName() { - return appName; - } - - public void setAppName(String appName) { - this.appName = appName; - } - - public String getChannelId() { - return channelId; - } - - public void setChannelId(String channelId) { - this.channelId = channelId; - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java b/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java deleted file mode 100644 index 155a476..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.jd.platform.hotkey.worker.mq; - -/** - * @author wuweifeng wrote on 2019-12-12 - * @version 1.0 - */ -public interface IMqMessageReceiver { - void receive(String msg); -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 0a1521a..2640fd7 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -7,7 +7,6 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.keydispatcher.KeyProducer; -import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -27,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; */ @Component @Order(3) -public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { +public class HotKeyFilter implements INettyMsgFilter { @Resource private KeyProducer keyProducer; @@ -48,11 +47,6 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { return true; } - @Override - public void receive(String msg) { - publishMsg(msg, null); - } - private void publishMsg(String message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, HotKeyModel.class); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index d8eae48..b268156 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -8,7 +8,6 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.counter.KeyCountItem; -import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; import com.jd.platform.hotkey.worker.tool.InitConstant; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -28,7 +27,7 @@ import static com.jd.platform.hotkey.worker.counter.CounterConfig.COUNTER_QUEUE; */ @Component @Order(4) -public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { +public class KeyCounterFilter implements INettyMsgFilter { private Logger logger = LoggerFactory.getLogger(getClass()); /** @@ -52,11 +51,6 @@ public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { return true; } - @Override - public void receive(String msg) { - publishMsg("", msg, null); - } - private void publishMsg(String appName, String message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, KeyCountModel.class); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java index f9702f2..f603c1c 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java @@ -13,10 +13,6 @@ public class InitConstant { //单位是百万 public static int bufferSize = 2; - public static boolean openTimeOut; - - public static int threadCount; - public static final LongAdder expireTotalCount = new LongAdder(); public static final LongAdder totalDealCount = new LongAdder(); diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index 3d33600..1708974 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -5,7 +5,6 @@ netty: local: address: ${localAddress:} #有些获取到的ip不能用，需要手工配worker的地址 open: - timeout: ${openTimeOut:true} monitor: ${openMonitor:false} #开启持续无key发送监控，如果持续1分钟没发来key，就断开和etcd的连接，之后重建和客户端连接 thread: count: ${threadCount:0} -- Gitee From a571a713a9528a29f4eb1668bef507f73ce03aad Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 3 Sep 2020 14:11:44 +0800 Subject: [PATCH 29/42] =?UTF-8?q?=E5=A4=A7=E6=94=B9=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=BA=94=E7=94=A8=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 6 + .../dashboard/DashboardApplication.java | 2 +- .../biz/controller/AppCfgController.java | 58 +++++++ .../controller/ChangLogController.java | 4 +- .../{ => biz}/controller/ClearController.java | 4 +- .../{ => biz}/controller/KeyController.java | 11 +- .../{ => biz}/controller/RuleController.java | 6 +- .../{ => biz}/controller/UserController.java | 4 +- .../controller/WorkerController.java | 4 +- .../{ => biz}/mapper/ChangeLogMapper.java | 2 +- .../{ => biz}/mapper/KeyRecordMapper.java | 3 +- .../{ => biz}/mapper/KeyTimelyMapper.java | 2 +- .../{ => biz}/mapper/RulesMapper.java | 2 +- .../{ => biz}/mapper/StatisticsMapper.java | 2 +- .../{ => biz}/mapper/SummaryMapper.java | 3 +- .../{ => biz}/mapper/UserMapper.java | 2 +- .../{ => biz}/mapper/WorkerMapper.java | 2 +- .../dashboard/biz/service/AppCfgService.java | 21 +++ .../{ => biz}/service/ChangeLogService.java | 2 +- .../{ => biz}/service/ClearService.java | 2 +- .../{ => biz}/service/KeyService.java | 2 +- .../{ => biz}/service/RuleService.java | 2 +- .../{ => biz}/service/UserService.java | 2 +- .../{ => biz}/service/WorkerService.java | 2 +- .../biz/service/impl/AppCfgServiceImpl.java | 84 +++++++++ .../service/impl/ChangeLogServiceImpl.java | 6 +- .../service/impl/ClearServiceImpl.java | 6 +- .../service/impl/KeyServiceImpl.java | 14 +- .../service/impl/RuleServiceImpl.java | 10 +- .../service/impl/UserServiceImpl.java | 6 +- .../service/impl/WorkerServiceImpl.java | 8 +- .../dashboard/common/base/BaseController.java | 3 +- .../dashboard/common/domain/Constant.java | 1 + .../common/domain/PushMsgWrapper.java | 48 ++++++ .../dashboard/common/domain/vo/AppCfgVo.java | 113 +++++++++++++ .../common/domain/vo/ClearCfgVo.java | 4 +- .../common/domain/vo/HotKeyLineChartVo.java | 3 +- .../common/domain/vo/LineChartVo.java | 3 +- .../common/domain/vo/RuleLineChartVo.java | 4 +- .../dashboard/common/monitor/DataHandler.java | 118 +++++++++---- .../dashboard/common/monitor/PushHandler.java | 100 +++++++++++ .../common/monitor/SlidingWindow.java | 120 +++++++++++++ .../dashboard/controller/TestController.java | 21 --- .../dashboard/erp/ErpUimInterceptor.java | 2 +- .../{common/config => etcd}/EtcdConfig.java | 2 +- .../hotkey/dashboard/etcd/EtcdMonitor.java | 36 ++-- .../dashboard/interceptor/JwtInterceptor.java | 2 - .../hotkey/dashboard/model/KeyRecord.java | 4 +- .../hotkey/dashboard/util/CommonUtil.java | 53 +----- .../{ => biz}/mapper/ChangeLogMapper.xml | 2 +- .../{ => biz}/mapper/KeyRecordMapper.xml | 2 +- .../{ => biz}/mapper/KeyTimelyMapper.xml | 2 +- .../{ => biz}/mapper/RulesMapper.xml | 2 +- .../{ => biz}/mapper/StatisticsMapper.xml | 2 +- .../{ => biz}/mapper/SummaryMapper.xml | 2 +- .../dashboard/{ => biz}/mapper/UserMapper.xml | 2 +- .../{ => biz}/mapper/WorkerMapper.xml | 2 +- .../static/admin/common/js/appcfg-edit.js | 25 +++ .../templates/admin/appcfg/edit.html | 86 ++++++++++ .../templates/admin/appcfg/list.html | 160 ++++++++++++++++++ .../templates/admin/common/html/leftMenu.html | 4 +- 61 files changed, 1010 insertions(+), 202 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/ChangLogController.java (90%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/ClearController.java (93%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/KeyController.java (93%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/RuleController.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/UserController.java (97%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/WorkerController.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/ChangeLogMapper.java (87%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/KeyRecordMapper.java (85%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/KeyTimelyMapper.java (92%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/RulesMapper.java (83%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/StatisticsMapper.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/SummaryMapper.java (84%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/UserMapper.java (91%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/WorkerMapper.java (91%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/ChangeLogService.java (91%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/ClearService.java (90%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/KeyService.java (95%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/RuleService.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/UserService.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/WorkerService.java (93%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/ChangeLogServiceImpl.java (84%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/ClearServiceImpl.java (93%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/KeyServiceImpl.java (95%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/RuleServiceImpl.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/UserServiceImpl.java (96%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/WorkerServiceImpl.java (95%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{common/config => etcd}/EtcdConfig.java (93%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/ChangeLogMapper.xml (97%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/KeyRecordMapper.xml (98%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/KeyTimelyMapper.xml (98%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/RulesMapper.xml (95%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/StatisticsMapper.xml (97%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/SummaryMapper.xml (96%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/UserMapper.xml (98%) rename dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/WorkerMapper.xml (98%) create mode 100644 dashboard/src/main/resources/static/admin/common/js/appcfg-edit.js create mode 100644 dashboard/src/main/resources/templates/admin/appcfg/edit.html create mode 100644 dashboard/src/main/resources/templates/admin/appcfg/list.html diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index e115056..250d12c 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -68,6 +68,12 @@ public interface ConfigConstant { * time unit : day */ String clearCfgPath = "/jd/clearCfg/"; + + /** + * app配置 + */ + String appCfgPath = "/jd/appCfg/"; + /** * 控制台启动的netty端口 */ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 074e45d..7d4c8f2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,7 +1,7 @@ package com.jd.platform.hotkey.dashboard; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.KeyTimelyMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java new file mode 100644 index 0000000..10fc824 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java @@ -0,0 +1,58 @@ +package com.jd.platform.hotkey.dashboard.biz.controller; + +import com.jd.platform.hotkey.dashboard.common.base.BaseController; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import com.jd.platform.hotkey.dashboard.biz.service.AppCfgService; +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + + +@Controller +@RequestMapping("/appCfg") +public class AppCfgController extends BaseController { + + + private String prefix = "admin/appcfg"; + + + @Resource + private AppCfgService appCfgService; + + @GetMapping("/view") + public String view(ModelMap modelMap){ + modelMap.put("title", Constant.APP_CFG_VIEW); + return prefix + "/list"; + } + + + @PostMapping("/list") + @ResponseBody + public Page list(PageReq page, String app){ + return appCfgService.pageAppCfgVo(page, app); + } + + + @GetMapping("/edit/{app}") + public String edit(@PathVariable("app") String app, ModelMap modelMap){ + modelMap.put("appCfg", appCfgService.selectAppCfgVo(app)); + return prefix + "/edit"; + } + + @PostMapping("/save") + @ResponseBody + public Result save(AppCfgVo cfg){ + cfg.setModifier(userName()); + appCfgService.saveAppCfgVo(cfg); + return Result.success(); + } + + +} + diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ChangLogController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ChangLogController.java similarity index 90% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ChangLogController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ChangLogController.java index e9a1788..1e18dc2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ChangLogController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ChangLogController.java @@ -1,11 +1,11 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.*; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.model.ChangeLog; -import com.jd.platform.hotkey.dashboard.service.ChangeLogService; +import com.jd.platform.hotkey.dashboard.biz.service.ChangeLogService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ClearController.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ClearController.java index 0ce81f0..e6ce6b4 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ClearController.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.Constant; @@ -6,7 +6,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; -import com.jd.platform.hotkey.dashboard.service.ClearService; +import com.jd.platform.hotkey.dashboard.biz.service.ClearService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index f3f362d..8c74507 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; @@ -14,7 +14,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; -import com.jd.platform.hotkey.dashboard.service.KeyService; +import com.jd.platform.hotkey.dashboard.biz.service.KeyService; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.ExcelUtil; import org.springframework.stereotype.Controller; @@ -122,6 +122,13 @@ public class KeyController extends BaseController { } + /*@GetMapping("/edit/{id}") + public String edit(@PathVariable("id") Long id, ModelMap modelMap){ + modelMap.put("key", keyService.selectByPk(id)); + return prefix + "/edit"; + }*/ + + @PostMapping("/edit") @ResponseBody public Result editSave(KeyTimely key) { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java index e1869c9..7ce52bb 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java @@ -1,9 +1,8 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONTokener; -import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; @@ -14,9 +13,8 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.model.Rule; import com.jd.platform.hotkey.dashboard.model.Rules; -import com.jd.platform.hotkey.dashboard.service.RuleService; +import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java similarity index 97% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java index bc571ba..69ad035 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import java.util.*; @@ -11,7 +11,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/WorkerController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/WorkerController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java index 4e9471e..bab4207 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/WorkerController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java @@ -1,11 +1,11 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.*; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.model.Worker; -import com.jd.platform.hotkey.dashboard.service.WorkerService; +import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.java similarity index 87% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.java index c36f871..a1ceb20 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.ChangeLog; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java similarity index 85% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java index 56c05ab..dde2c25 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java @@ -1,6 +1,5 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java similarity index 92% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java index 3423da3..c82f78b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyTimely; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.java similarity index 83% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.java index 1c6835c..3c1c894 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.model.Rules; import org.apache.ibatis.annotations.Mapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.java index daf848e..a657342 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.java similarity index 84% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.java index b12ff0d..99882f7 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.java @@ -1,8 +1,7 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; -import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; import org.apache.ibatis.annotations.Mapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java index c8d72ee..257a27d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.User; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.java index 501316b..29befc2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Worker; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java new file mode 100644 index 0000000..fa814b8 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java @@ -0,0 +1,21 @@ +package com.jd.platform.hotkey.dashboard.biz.service; + +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearService + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:51 + */ +public interface AppCfgService { + + Page pageAppCfgVo(PageReq page, String app); + + AppCfgVo selectAppCfgVo(String app); + + void saveAppCfgVo(AppCfgVo cfg); + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ChangeLogService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ChangeLogService.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ChangeLogService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ChangeLogService.java index 498f4e4..cfed249 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ChangeLogService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ChangeLogService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ClearService.java similarity index 90% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ClearService.java index 5266ef3..383fa9c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ClearService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java similarity index 95% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java index 558e814..d8908f9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.Page; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/RuleService.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/RuleService.java index 5580df1..1a55b99 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/RuleService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/UserService.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/UserService.java index 71f57ea..b4bb868 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/UserService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/WorkerService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/WorkerService.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/WorkerService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/WorkerService.java index 4995172..398dae8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/WorkerService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/WorkerService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java new file mode 100644 index 0000000..2afa872 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -0,0 +1,84 @@ +package com.jd.platform.hotkey.dashboard.biz.service.impl; + +import com.alibaba.fastjson.JSON; +import com.github.pagehelper.util.StringUtil; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import com.jd.platform.hotkey.dashboard.biz.service.AppCfgService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; +import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; +import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; +import com.jd.platform.hotkey.dashboard.util.PageUtil; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; + +/** + * @ProjectName: hotkey + * @ClassName: AppCfgServiceImpl + * @Author: liyunfeng31 + * @Date: 2020/9/2 9:57 + */ +@Service +public class AppCfgServiceImpl implements AppCfgService { + + + @Resource + private IConfigCenter configCenter; + + @Resource + private UserService userService; + + + @Override + public Page pageAppCfgVo(PageReq page, String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + List cfgVos = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + String key = kv.getKey().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + configCenter.put(key, JSON.toJSONString(new AppCfgVo(key))); + continue; + } + AppCfgVo vo = JSON.parseObject(v, AppCfgVo.class); + vo.setVersion(kv.getModRevision()); + String k = key.replace(ConfigConstant.appCfgPath,""); + if(StringUtils.isEmpty(app)){ + cfgVos.add(vo); + }else{ + if(k.equals(app)){ cfgVos.add(vo); } + } + } + return PageUtil.pagination(cfgVos,page.getPageSize(),page.getPageNum()-1); + } + + @Override + public AppCfgVo selectAppCfgVo(String app) { + KeyValue kv = configCenter.getKv(ConfigConstant.appCfgPath + app); + if(kv == null || kv.getValue() == null){ + AppCfgVo ap = new AppCfgVo(app); + configCenter.put(ConfigConstant.appCfgPath + app, JSON.toJSONString(ap)); + return ap; + } + String v = kv.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(v, AppCfgVo.class); + cfg.setVersion(kv.getModRevision()); + return cfg; + } + + @Override + public void saveAppCfgVo(AppCfgVo cfg) { + cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnThreshold())); + PushHandler.appCfgMap.put(cfg.getApp(), cfg); + configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ChangeLogServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ChangeLogServiceImpl.java similarity index 84% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ChangeLogServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ChangeLogServiceImpl.java index 50e11af..5fa90a5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ChangeLogServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ChangeLogServiceImpl.java @@ -1,12 +1,12 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; -import com.jd.platform.hotkey.dashboard.service.ChangeLogService; +import com.jd.platform.hotkey.dashboard.biz.service.ChangeLogService; import org.springframework.stereotype.Service; import javax.annotation.Resource; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ClearServiceImpl.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ClearServiceImpl.java index 8227a95..462e5c8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ClearServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; @@ -8,8 +8,8 @@ import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; -import com.jd.platform.hotkey.dashboard.service.ClearService; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.ClearService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java similarity index 95% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java index c268344..9c4ecaa 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import cn.hutool.core.date.SystemClock; @@ -15,16 +15,16 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; -import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.ChangeLogMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.KeyRecordMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.StatisticsMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; -import com.jd.platform.hotkey.dashboard.service.KeyService; -import com.jd.platform.hotkey.dashboard.service.RuleService; +import com.jd.platform.hotkey.dashboard.biz.service.KeyService; +import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.PageUtil; @@ -203,7 +203,7 @@ public class KeyServiceImpl implements KeyService { } configCenter.delete(etcdKey); - KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, + KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(), "", keyTimely.getUpdater(), SystemClock.now() + "")); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java index 9dd493e..26f38ec 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import cn.hutool.core.lang.UUID; import com.alibaba.fastjson.JSON; @@ -11,11 +11,11 @@ import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; -import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; -import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.ChangeLogMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.RulesMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.*; -import com.jd.platform.hotkey.dashboard.service.RuleService; +import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/UserServiceImpl.java similarity index 96% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/UserServiceImpl.java index c03ea2e..f5da4a8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/UserServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; @@ -8,9 +8,9 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.mapper.UserMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.UserMapper; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/WorkerServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/WorkerServiceImpl.java similarity index 95% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/WorkerServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/WorkerServiceImpl.java index 4c133ee..d6270a1 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/WorkerServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/WorkerServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import cn.hutool.core.date.SystemClock; import com.alibaba.fastjson.JSON; @@ -10,11 +10,11 @@ import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; -import com.jd.platform.hotkey.dashboard.mapper.WorkerMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.ChangeLogMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.WorkerMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.Worker; -import com.jd.platform.hotkey.dashboard.service.WorkerService; +import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index 7aa346f..e4065ec 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -8,8 +8,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; import org.springframework.beans.propertyeditors.CustomDateEditor; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index 952aa2a..d47ca2e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -37,6 +37,7 @@ public class Constant { public static final String CLEAR_VIEW = "数据清理"; + public static final String APP_CFG_VIEW = "应用配置"; public static final int MAX_DAY_RANGE = 3; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java new file mode 100644 index 0000000..937baf5 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java @@ -0,0 +1,48 @@ +package com.jd.platform.hotkey.dashboard.common.domain; + +import cn.hutool.core.date.SystemClock; + +import java.io.Serializable; + +/** + * 报警消息包装类，用于保存事件最准确的时间 + */ +public class PushMsgWrapper implements Serializable { + + private String app; + + private Long date; + + private String msg; + + public PushMsgWrapper(String app) { + this.app = app; + this.date = SystemClock.now(); + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public Long getDate() { + return date; + } + + public void setDate(Long date) { + this.date = date; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } +} + + diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java new file mode 100644 index 0000000..3e6d4fe --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -0,0 +1,113 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; + +import java.io.Serializable; + +/** + * @ProjectName: hotkey + * @ClassName: AppCfgVo + * @Author: liyunfeng31 + * @Date: 2020/9/2 10:29 + */ +public class AppCfgVo implements Serializable { + + private String app; + + /** + * 数据保存时长 + */ + private Integer dataTtl; + + /** + * 警报周期 + */ + private Integer warnPeriod; + + /** + * 警报阈值 + */ + private Integer warnThreshold; + + /** + * 版本 + */ + private Long version; + + /* + * 最后修改人 + */ + private String modifier; + + private SlidingWindow window; + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public Integer getDataTtl() { + return dataTtl; + } + + public void setDataTtl(Integer dataTtl) { + this.dataTtl = dataTtl; + } + + public Integer getWarnPeriod() { + return warnPeriod; + } + + public void setWarnPeriod(Integer warnPeriod) { + this.warnPeriod = warnPeriod; + } + + public Integer getWarnThreshold() { + return warnThreshold; + } + + public void setWarnThreshold(Integer warnThreshold) { + this.warnThreshold = warnThreshold; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } + + public String getModifier() { + return modifier; + } + + public void setModifier(String modifier) { + this.modifier = modifier; + } + + public SlidingWindow getWindow() { + return window; + } + + public void setWindow(SlidingWindow window) { + this.window = window; + } + + public AppCfgVo() { + } + + public AppCfgVo(String app) { + this.app = app; + this.dataTtl = 30; + this.warnPeriod = 10*60; + this.warnThreshold = 1000; + this.version = 0L; + this.modifier = "SYSTEM"; + this.window = new SlidingWindow(warnPeriod/60,warnThreshold); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java index 14b872b..65df755 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java @@ -1,5 +1,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; + /** * @ProjectName: hotkey * @ClassName: ClearCfgVo @@ -7,7 +9,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; * @Author: liyunfeng31 * @Date: 2020/8/3 9:54 */ -public class ClearCfgVo { +public class ClearCfgVo implements Serializable { private String app; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java index 3392fc2..71171a6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Set; @@ -7,7 +8,7 @@ import java.util.Set; /** * @author liyunfeng31 */ -public class HotKeyLineChartVo { +public class HotKeyLineChartVo implements Serializable { private List xAxis; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java index 273ce59..3179c5f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Set; @@ -7,7 +8,7 @@ import java.util.Set; /** * @author liyunfeng31 */ -public class LineChartVo { +public class LineChartVo implements Serializable { private Set legend; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java index 31ba530..8f6fbc5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java @@ -1,9 +1,11 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; + /** * @author liyunfeng31 */ -public class RuleLineChartVo { +public class RuleLineChartVo implements Serializable { private String rule; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 0b8a50c..29743cb 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -2,24 +2,30 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import com.alibaba.fastjson.JSON; +import com.google.common.collect.Queues; import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; -import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import com.jd.platform.hotkey.dashboard.biz.mapper.KeyRecordMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.KeyTimelyMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import com.jd.platform.hotkey.dashboard.util.TwoTuple; +import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -27,11 +33,13 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; @Component public class DataHandler { @@ -50,19 +58,29 @@ public class DataHandler { @Resource private IConfigCenter configCenter; + @Resource + private PushHandler pushHandler; + + + private static final Integer CACHE_SIZE = 100; + + private static final Integer CLEAR_CACHE_INTERVAL = 10; /** * 队列 */ - private BlockingQueue queue = new LinkedBlockingQueue<>(); + private BlockingQueue EVENT_QUEUE = new LinkedBlockingQueue<>(); + + private BlockingQueue RECORD_QUEUE = new LinkedBlockingQueue<>(); + /** * 入队 */ public void offer(EventWrapper eventWrapper) { try { - queue.put(eventWrapper); + EVENT_QUEUE.put(eventWrapper); } catch (InterruptedException e) { e.printStackTrace(); } @@ -72,7 +90,7 @@ public class DataHandler { while (true) { EventWrapper eventWrapper; try { - eventWrapper = queue.take(); + eventWrapper = EVENT_QUEUE.take(); } catch (InterruptedException e) { e.printStackTrace(); continue; @@ -104,28 +122,7 @@ public class DataHandler { e.printStackTrace(); log.error("handHotKey error ," + e.getCause()); } - - } - - } - - /** - * 插入记录表 - * @param type 0是新增，1是删除 - */ - public void insertRecord(String appKey, int type) { - String source = Constant.SYSTEM; - String rule = RuleUtil.rule(appKey); - KeyRule keyRule = RuleUtil.findByKey(appKey); - //appName+"/"+"key" - String[] arr = appKey.split("/"); - String appName = arr[0]; - String key = arr[1]; - String uuid = UUID.randomUUID().toString(); - KeyRecord keyRecord = new KeyRecord(key, rule, appName, (long)keyRule.getDuration(), source, type, uuid, new Date()); - keyRecord.setRule(rule); - keyRecordMapper.insertSelective(keyRecord); } @@ -157,7 +154,7 @@ public class DataHandler { String source = value.length() == 13 ? Constant.HAND : Constant.SYSTEM; timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).val(value).appName(appName).duration(ttl).uuid(appKey).createTime(date).build()); String rule = RuleUtil.rule(appKey); - KeyRecord keyRecord = new KeyRecord(key, rule, appName, ttl, source, type, uuid, date); + KeyRecord keyRecord = new KeyRecord(key, rule, appName, (int)ttl, source, type, uuid, date); keyRecord.setRule(rule); timelyKeyRecordTwoTuple.setSecond(keyRecord); return timelyKeyRecordTwoTuple; @@ -257,12 +254,12 @@ public class DataHandler { public void clearExpireData() { try { LocalDateTime now = LocalDateTime.now(); - List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); for (KeyValue kv : keyValues) { - String key = kv.getKey().toStringUtf8(); - String ttl = kv.getValue().toStringUtf8(); - String app = key.replace(ConfigConstant.clearCfgPath,""); - Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); + String val = kv.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); + String app = cfg.getApp(); + Date expireDate = DateUtil.ldtToDate(now.minusDays(cfg.getDataTtl())); summaryMapper.clearExpireData(app, expireDate); keyRecordMapper.clearExpireData(app, expireDate); statisticsMapper.clearExpireData(app, expireDate); @@ -274,4 +271,57 @@ public class DataHandler { } + public void dealHotKey() { + while (true) { + try { + //获取发来的这个热key，存入本地caffeine，设置过期时间 + HotKeyModel model = HotKeyReceiver.take(); + //将该key放入实时热key本地缓存中 + if(model != null){ + HotKeyReceiver.put(model); + putRecord(model.getAppName(), model.getKey(), 0); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + public void dealKeyRecord(){ + List data = new ArrayList<>(CACHE_SIZE); + while (true) { + try { + //清空data + data.clear(); + //将缓存中的数据放入data，规则：缓存数据超过缓存大小的0.75或者时间超过10s + Queues.drain(RECORD_QUEUE, data, CACHE_SIZE * 3 >> 2, CLEAR_CACHE_INTERVAL, TimeUnit.SECONDS); + //执行批量入库 + if (ObjectUtils.isNotEmpty(data)) { + keyRecordMapper.batchInsert(data); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + + + public void putRecord(String app, String key, int type) throws InterruptedException { + String appKey =app + "/" + key; + String source = Constant.SYSTEM; + String rule = RuleUtil.rule(appKey); + KeyRule keyRule = RuleUtil.findByKey(appKey); + String uuid = UUID.randomUUID().toString(); + assert keyRule != null; + KeyRecord keyRecord = new KeyRecord(key, rule, app, keyRule.getDuration(), source, type, uuid, new Date()); + keyRecord.setRule(rule); + RECORD_QUEUE.put(keyRecord); + SlidingWindow wd = PushHandler.appCfgMap.get(app).getWindow(); + int count = wd.addCount(1); + if(count > 0){ + pushHandler.offer(new PushMsgWrapper(app) ); + } + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java new file mode 100644 index 0000000..7ce5b7b --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -0,0 +1,100 @@ +package com.jd.platform.hotkey.dashboard.common.monitor; + +import com.alibaba.fastjson.JSON; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; +import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import org.apache.commons.collections4.CollectionUtils; + +import javax.annotation.Resource; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * @ProjectName: hotkey + * @ClassName: PushHandler + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/9/3 10:47 + */ +public class PushHandler { + + private static final long interval = 10*60*1000L; + + public static Map appCfgMap = new ConcurrentHashMap<>(); + + private static Map appIntervalMap = new ConcurrentHashMap<>(); + + private static final BlockingQueue MSG_QUEUE = new LinkedBlockingQueue<>(); + + @Resource + private IConfigCenter configCenter; + @Resource + private UserService userService; + + + /** + * 入队 + */ + public void offer(PushMsgWrapper msgWrapper) { + try { + MSG_QUEUE.put(msgWrapper); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + + public void pushWarnMsg() { + initAppCfgMap(); + while (true){ + try { + PushMsgWrapper msgWrapper = MSG_QUEUE.take(); + String warnApp = msgWrapper.getApp(); + Long msgTime = msgWrapper.getDate(); + boolean send = check(warnApp, msgTime); + if(send){ doPush(); } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + + private static synchronized boolean check(String warnApp, Long msgTime){ + Long maxTime = appIntervalMap.get(warnApp); + if(maxTime == null){ + appIntervalMap.put(warnApp,msgTime+interval); + return true; + }else{ + if(msgTime > maxTime){ + appIntervalMap.put(warnApp,msgTime+interval); + return true; + } + } + return false; + } + + private static void doPush() { + } + + + private void initAppCfgMap() { + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + if(CollectionUtils.isEmpty(keyValues)){ + List apps = userService.listApp(); + for (String ap : apps) { + AppCfgVo cfg = new AppCfgVo(ap); + appCfgMap.put(ap,cfg); + configCenter.put(ConfigConstant.appCfgPath + ap, JSON.toJSONString(cfg)); + } + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java new file mode 100644 index 0000000..5543623 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java @@ -0,0 +1,120 @@ +package com.jd.platform.hotkey.dashboard.common.monitor; + + +import cn.hutool.core.date.SystemClock; + +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 滑动窗口。该窗口同样的key都是单线程计算。 + */ +public class SlidingWindow { + /** + * 循环队列，就是装多个窗口用，该数量是windowSize的2倍 + */ + private AtomicInteger[] timeSlices; + /** + * 队列的总长度 + */ + private int timeSliceSize; + /** + * 每个时间片的时长，以毫秒为单位 + */ + private int timeMillisPerSlice; + /** + * 共有多少个时间片（即窗口长度） + */ + private int windowSize; + /** + * 在一个完整窗口期内允许通过的最大阈值 + */ + private int threshold; + /** + * 该滑窗的起始创建时间，也就是第一个数据 + */ + private long beginTimestamp; + /** + * 最后一个数据的时间戳 + */ + private long lastAddTimestamp; + + public static void main(String[] args) { + //1秒一个时间片，窗口共5个 + SlidingWindow window = new SlidingWindow(10, 20); + + CyclicBarrier cyclicBarrier = new CyclicBarrier(10); + for (int i = 0; i < 10; i++) { + new Thread(() -> { + try { + cyclicBarrier.await(); + } catch (InterruptedException | BrokenBarrierException e) { + e.printStackTrace(); + } + int hot = window.addCount(2); + System.out.println(hot); + }).start(); + } + } + + + public SlidingWindow(int windowSize, int threshold) { + this.timeMillisPerSlice = 60 * 1000; + this.windowSize = windowSize; + this.threshold = threshold; + // 保证存储在至少两个window + this.timeSliceSize = windowSize * 2; + reset(); + } + + /** + * 初始化 + */ + private void reset() { + beginTimestamp = SystemClock.now(); + //窗口个数 + AtomicInteger[] localTimeSlices = new AtomicInteger[timeSliceSize]; + for (int i = 0; i < timeSliceSize; i++) { + localTimeSlices[i] = new AtomicInteger(0); + } + timeSlices = localTimeSlices; + } + + + private int locationIndex() { + long now = SystemClock.now(); + if (now - lastAddTimestamp > timeMillisPerSlice * windowSize) { reset(); } + int index = (int) (((now - beginTimestamp) / timeMillisPerSlice) % timeSliceSize); + return Math.max(index, 0); + } + + + public synchronized int addCount(int count) { + int index = locationIndex(); + clearFromIndex(index); + int sum = 0; + // 在当前时间片里继续+1 + sum += timeSlices[index].addAndGet(count); + //加上前面几个时间片 + for (int i = 1; i < windowSize; i++) { + sum += timeSlices[(index - i + timeSliceSize) % timeSliceSize].get(); + } + lastAddTimestamp = SystemClock.now(); + if(sum >= threshold){ + return sum; + } + return 0; + } + + private void clearFromIndex(int index) { + for (int i = 1; i <= windowSize; i++) { + int j = index + i; + if (j >= windowSize * 2) { + j -= windowSize * 2; + } + timeSlices[j].set(0); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java deleted file mode 100644 index 5d42394..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.jd.platform.hotkey.dashboard.controller; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-07-16 - */ -//@RestController -//public class TestController { -// @Resource -// private IConfigCenter configCenter; -// -// @RequestMapping("delete") -// public Object delete(String key) { -//// configCenter.putAndGrant(ConfigConstant.hotKeyRecordPath + "sample" + "/" + key, com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE, 1); -// configCenter.putAndGrant(ConfigConstant.hotKeyRecordPath + "bbb" + "/" + key, "", 60); -//// configCenter.delete(ConfigConstant.hotKeyPath + "sample" + "/" + key); -// -// return "1"; -// } -//} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java index beabf4e..2bd7653 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java @@ -3,7 +3,7 @@ package com.jd.platform.hotkey.dashboard.erp; import com.jd.common.springmvc.interceptor.SpringSSOInterceptor; import com.jd.common.web.LoginContext; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.Cookie; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/EtcdConfig.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdConfig.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/EtcdConfig.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdConfig.java index 0f585b6..edb0708 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/EtcdConfig.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdConfig.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.common.config; +package com.jd.platform.hotkey.dashboard.etcd; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.configcenter.etcd.JdEtcdBuilder; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 5b9aebe..44d82b9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -12,10 +12,11 @@ import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; import com.jd.platform.hotkey.dashboard.model.Worker; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; -import com.jd.platform.hotkey.dashboard.service.WorkerService; +import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import io.grpc.StatusRuntimeException; @@ -57,6 +58,9 @@ public class EtcdMonitor { @Resource private DataHandler dataHandler; + @Resource + private PushHandler pushHandler; + public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(256); @@ -102,7 +106,7 @@ public class EtcdMonitor { insertRecords(); //开始入库 - dealHotkey(); + dealHotKey(); //监听手工创建的key watchHandOperationHotKey(); @@ -114,6 +118,8 @@ public class EtcdMonitor { //观察热key访问次数和总访问次数，并做统计 watchHitCount(); + + pushWarn(); } /** @@ -138,23 +144,13 @@ public class EtcdMonitor { /** * 开始消费各worker发来的热key */ - private void dealHotkey() { + private void dealHotKey() { threadPoolExecutor.submit(() -> { - while (true) { - try { - //获取发来的这个热key，存入本地caffeine，设置过期时间 - HotKeyModel model = HotKeyReceiver.take(); - - //将该key放入实时热key本地缓存中 - HotKeyReceiver.put(model); - - //入库record表，TODO 需要改成batch入库 - dataHandler.insertRecord(model.getAppName() + "/" + model.getKey(), 0); - } catch (Exception e) { - e.printStackTrace(); - } - } + dataHandler.dealHotKey(); + }); + threadPoolExecutor.submit(() -> { + dataHandler.dealKeyRecord(); }); } @@ -274,4 +270,8 @@ public class EtcdMonitor { } + + private void pushWarn() { + threadPoolExecutor.submit(() ->pushHandler.pushWarnMsg()); + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 173a465..8fcd977 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -4,14 +4,12 @@ package com.jd.platform.hotkey.dashboard.interceptor; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; import org.springframework.http.HttpMethod; import org.springframework.util.StringUtils; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; -import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java index 97f0b4b..54f0386 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java @@ -43,12 +43,12 @@ public class KeyRecord implements Serializable { public KeyRecord() { } - public KeyRecord(String key,String val, String appName, Long duration, + public KeyRecord(String key,String val, String appName, Integer duration, String source, Integer type,String uuid, Date createTime) { this.key = key; this.val = val; this.appName = appName; - this.duration = duration; + this.duration = duration.longValue(); this.source = source; this.type = type; this.uuid = uuid; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 21488ac..425d875 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -54,10 +54,6 @@ public class CommonUtil { } - public static String keyName(String k) { - int index = k.lastIndexOf("/"); - return k.substring(index + 1); - } public static String encoder(String text) { @@ -81,49 +77,6 @@ public class CommonUtil { } - /** - * 拼装数据 - * - * @param list list-data - * @param startTime 开始时间 - * @param size 格子数 - * @param type 类型 1分钟 2小时 - * @return vo - */ - public static HotKeyLineChartVo assembleData(List list, LocalDateTime startTime, int size, int type) { - Set set = new TreeSet<>(); - boolean isHour = type == 1; - String suffix = isHour ? "60" : "24"; - String pattern = isHour ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; - Map map = new HashMap<>(10); - Map> listMap = listGroup(list); -// log.info("按照rule分组以后的listMap--> {}", JSON.toJSONString(listMap)); - for (Map.Entry> m : listMap.entrySet()) { - int start = DateUtil.reviseTime(startTime, 0, type); - map.put(m.getKey(), new int[size]); - int[] data = map.get(m.getKey()); - int tmp = 0; - for (int i = 0; i < size; i++) { - if (String.valueOf(start).endsWith(suffix)) { - LocalDateTime tmpTime = DateUtil.strToLdt((start - 1) + "", pattern); - start = DateUtil.reviseTime(tmpTime, 1, type); - } -// log.info("start--> {}, tmp---> {} ", start, tmp); - set.add(DateUtil.strToLdt(start + "", pattern).toString().replace("T", " ")); - Statistics st = m.getValue().get(tmp); - int val = isHour ? st.getMinutes() : st.getHours(); - if (start != val) { - data[i] = 0; - } else { - tmp++; - data[i] = st.getCount(); - } - start++; - } - } - return new HotKeyLineChartVo(new ArrayList<>(set), map); - } - /** * 分组 @@ -132,10 +85,8 @@ public class CommonUtil { * @return map */ private static Map> listGroup(List list) { - if (Constant.VERSION != 1) { - return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); - } - return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); + return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); + // return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); } /** diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml similarity index 97% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml index 90b3b5d..9a1c828 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml similarity index 98% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml index 94037b7..5f347bf 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml similarity index 98% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml index 04704cf..eb743fb 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.xml similarity index 95% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.xml index 505fa97..5686331 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.xml similarity index 97% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.xml index ac7f464..c409f64 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml similarity index 96% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml index bd04545..de74e50 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml similarity index 98% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml index bacdd87..9fb891c 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml similarity index 98% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml index 5be300c..6597dc6 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/dashboard/src/main/resources/static/admin/common/js/appcfg-edit.js b/dashboard/src/main/resources/static/admin/common/js/appcfg-edit.js new file mode 100644 index 0000000..f2ba469 --- /dev/null +++ b/dashboard/src/main/resources/static/admin/common/js/appcfg-edit.js @@ -0,0 +1,25 @@ +$("#form-edit").validate({ + submitHandler:function(form){ + edit(); + } +}); + +function edit() { + let dataFormJson=$("#form-edit").serialize(); + $.ajax({ + cache : true, + type : "POST", + url : "/appCfg/save", + data : dataFormJson, + headers: { + "Authorization":getCookie("token") + }, + async : false, + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); + }, + success : function(data) { + $.operate.saveSuccess(data); + } + }); +} diff --git a/dashboard/src/main/resources/templates/admin/appcfg/edit.html b/dashboard/src/main/resources/templates/admin/appcfg/edit.html new file mode 100644 index 0000000..fdc26cd --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/appcfg/edit.html @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ + +
+
+
+
+
+
+
+
+
+ + +
+ +
+ + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/appcfg/list.html b/dashboard/src/main/resources/templates/admin/appcfg/list.html new file mode 100644 index 0000000..51b46f2 --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/appcfg/list.html @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + +
+
+ +
+ +
+
+ +
+
+
搜索条件
+
+ +
+
+
+
+ + +
+ +
+
+
+
+ + +
+
+
表单
+
+ + + +
+
+ + + +
+
+ +
+ +
+
+
+
+ +
+ +
+
+
+ + + +
+ +
+ + + + + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 668341d..0cf7527 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -60,10 +60,10 @@ 变更记录 - - 数据清理配置 + 应用配置 -- Gitee From 89c140e13fd63817e13ea90d51ae1bec0142f622 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 3 Sep 2020 17:32:44 +0800 Subject: [PATCH 30/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E6=B3=A8?= =?UTF-8?q?=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/service/impl/AppCfgServiceImpl.java | 6 ++-- .../dashboard/common/monitor/PushHandler.java | 36 +++++++++++++++++-- .../common/monitor/SlidingWindow.java | 2 +- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index 2afa872..83416ae 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -34,9 +34,6 @@ public class AppCfgServiceImpl implements AppCfgService { @Resource private IConfigCenter configCenter; - @Resource - private UserService userService; - @Override public Page pageAppCfgVo(PageReq page, String app) { @@ -75,6 +72,9 @@ public class AppCfgServiceImpl implements AppCfgService { return cfg; } + /** + * todo 多节点问题 待完善 + */ @Override public void saveAppCfgVo(AppCfgVo cfg) { cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnThreshold())); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java index 7ce5b7b..bdf85a3 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -25,12 +25,24 @@ import java.util.concurrent.LinkedBlockingQueue; */ public class PushHandler { + /** + * 拦截重复报警间隔 10分钟 + */ private static final long interval = 10*60*1000L; + /** + * app-config map + */ public static Map appCfgMap = new ConcurrentHashMap<>(); + /** + * app-time 用于app存储报警时间 做拦截 + */ private static Map appIntervalMap = new ConcurrentHashMap<>(); + /** + * 超过阈值的热点记录次数 放到队列里 + */ private static final BlockingQueue MSG_QUEUE = new LinkedBlockingQueue<>(); @Resource @@ -50,8 +62,11 @@ public class PushHandler { } } - + /** + * 启动线程处理警报消息队列 + */ public void pushWarnMsg() { + // 初始化MAP到内存 initAppCfgMap(); while (true){ try { @@ -67,7 +82,13 @@ public class PushHandler { } - private static synchronized boolean check(String warnApp, Long msgTime){ + /** + * 防止重复发送警报 + * @param warnApp app + * @param msgTime time + * @return result + */ + private synchronized boolean check(String warnApp, Long msgTime){ Long maxTime = appIntervalMap.get(warnApp); if(maxTime == null){ appIntervalMap.put(warnApp,msgTime+interval); @@ -81,10 +102,14 @@ public class PushHandler { return false; } + // todo 执行报警 private static void doPush() { } + /** + * 初始化cfgMap和滑动窗口 + */ private void initAppCfgMap() { List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); if(CollectionUtils.isEmpty(keyValues)){ @@ -94,7 +119,14 @@ public class PushHandler { appCfgMap.put(ap,cfg); configCenter.put(ConfigConstant.appCfgPath + ap, JSON.toJSONString(cfg)); } + }else{ + for (KeyValue keyValue : keyValues) { + String val = keyValue.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); + appCfgMap.put(cfg.getApp(),cfg); + } } + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java index 5543623..040d945 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java @@ -8,7 +8,7 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicInteger; /** - * 滑动窗口。该窗口同样的key都是单线程计算。 + * 滑动窗口 */ public class SlidingWindow { /** -- Gitee From ed2c8151e68c6a3dee535c36e9ae2f896948d4e5 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 3 Sep 2020 19:33:52 +0800 Subject: [PATCH 31/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E6=8A=A5?= =?UTF-8?q?=E8=AD=A6=E5=BC=80=E5=85=B3=20todo=20=E5=A4=9A=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=86=85=E5=AD=98=E5=90=8C=E6=AD=A5=20worker=E7=9B=91?= =?UTF-8?q?=E6=8E=A7=E6=8A=A5=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/DashboardApplication.java | 12 +- .../dashboard/biz/mapper/KeyTimelyMapper.java | 31 ---- .../biz/service/impl/AppCfgServiceImpl.java | 2 - .../dashboard/common/domain/vo/AppCfgVo.java | 17 ++ .../dashboard/common/monitor/DataHandler.java | 70 ++------ .../dashboard/common/monitor/PushHandler.java | 22 ++- .../hotkey/dashboard/etcd/EtcdMonitor.java | 41 +---- .../dashboard/biz/mapper/KeyTimelyMapper.xml | 162 ------------------ .../templates/admin/appcfg/edit.html | 9 +- .../templates/admin/appcfg/list.html | 6 + 10 files changed, 66 insertions(+), 306 deletions(-) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 3a8357a..3d9f9f5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -3,21 +3,16 @@ package com.jd.platform.hotkey.dashboard; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import javax.annotation.Resource; @EnableAsync @EnableScheduling @SpringBootApplication -public class DashboardApplication implements CommandLineRunner { - - private Logger logger = LoggerFactory.getLogger(getClass()); - +public class DashboardApplication{ public static void main(String[] args) { try { @@ -27,9 +22,4 @@ public class DashboardApplication implements CommandLineRunner { } } - - @Override - public void run(String... args) { - } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java deleted file mode 100644 index c82f78b..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.hotkey.dashboard.biz.mapper; - -import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; -import org.apache.ibatis.annotations.Mapper; - -import java.util.List; - -@Mapper -public interface KeyTimelyMapper { - - int clear(); - - int deleteByKeyAndApp(String key,String appName); - - int insertSelective(KeyTimely key); - - KeyTimely selectByPrimaryKey(Long id); - - KeyTimely selectByKey(String key); - - int updateByKey(KeyTimely key); - - List listKeyTimely(SearchReq param); - - int batchInsert(List list); - - void batchDeleted(List deleteList); - - void saveOrUpdate(KeyTimely keyTimely); -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index 83416ae..a250db7 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -9,11 +9,9 @@ import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import com.jd.platform.hotkey.dashboard.biz.service.AppCfgService; -import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; import com.jd.platform.hotkey.dashboard.util.PageUtil; -import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java index 3e6d4fe..6f2b877 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -1,7 +1,9 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; +import java.beans.Transient; import java.io.Serializable; /** @@ -29,6 +31,11 @@ public class AppCfgVo implements Serializable { */ private Integer warnThreshold; + /** + * 警报开关 1开启 0关闭 + */ + private Integer warn; + /** * 版本 */ @@ -39,6 +46,7 @@ public class AppCfgVo implements Serializable { */ private String modifier; + @JsonIgnore private SlidingWindow window; public String getApp() { @@ -73,6 +81,14 @@ public class AppCfgVo implements Serializable { this.warnThreshold = warnThreshold; } + public Integer getWarn() { + return warn; + } + + public void setWarn(Integer warn) { + this.warn = warn; + } + public Long getVersion() { return version; } @@ -106,6 +122,7 @@ public class AppCfgVo implements Serializable { this.warnPeriod = 10*60; this.warnThreshold = 1000; this.version = 0L; + this.warn = 1; this.modifier = "SYSTEM"; this.window = new SlidingWindow(warnPeriod/60,warnThreshold); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 61a0233..4d92458 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -9,18 +9,12 @@ import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.IRecord; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import com.jd.platform.hotkey.dashboard.biz.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.biz.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.biz.mapper.StatisticsMapper; import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; @@ -28,8 +22,6 @@ import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; -import com.jd.platform.hotkey.dashboard.util.TwoTuple; -import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -52,8 +44,7 @@ public class DataHandler { @Resource private KeyRecordMapper keyRecordMapper; - @Resource - private KeyTimelyMapper keyTimelyMapper; + @Resource private StatisticsMapper statisticsMapper; @@ -62,28 +53,26 @@ public class DataHandler { @Resource private IConfigCenter configCenter; + @Resource private PushHandler pushHandler; private static final Integer CACHE_SIZE = 100; - private static final Integer CLEAR_CACHE_INTERVAL = 10; /** * 队列 */ + private BlockingQueue RECORD_QUEUE = new LinkedBlockingQueue<>(); - private BlockingQueue RECORD_QUEUE = new LinkedBlockingQueue<>(); - - private BlockingQueue queue = new LinkedBlockingQueue<>(); /** * 入队 */ public void offer(IRecord record) { try { - queue.put(record); + RECORD_QUEUE.put(record); } catch (InterruptedException e) { e.printStackTrace(); } @@ -93,7 +82,7 @@ public class DataHandler { while (true) { try { List records = new ArrayList<>(); - Queues.drain(queue, records, 1000, 1, TimeUnit.SECONDS); + Queues.drain(RECORD_QUEUE, records, CACHE_SIZE * 3 >> 2, 1, TimeUnit.SECONDS); if (CollectionUtil.isEmpty(records)) { continue; } @@ -262,25 +251,7 @@ public class DataHandler { //将该key放入实时热key本地缓存中 if(model != null){ HotKeyReceiver.put(model); - putRecord(model.getAppName(), model.getKey(), 0); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - public void dealKeyRecord(){ - List data = new ArrayList<>(CACHE_SIZE); - while (true) { - try { - //清空data - data.clear(); - //将缓存中的数据放入data，规则：缓存数据超过缓存大小的0.75或者时间超过10s - Queues.drain(RECORD_QUEUE, data, CACHE_SIZE * 3 >> 2, CLEAR_CACHE_INTERVAL, TimeUnit.SECONDS); - //执行批量入库 - if (ObjectUtils.isNotEmpty(data)) { - keyRecordMapper.batchInsert(data); + putRecord(model.getAppName(), model.getKey()); } } catch (Exception e) { e.printStackTrace(); @@ -289,22 +260,19 @@ public class DataHandler { } - - public void putRecord(String app, String key, int type) throws InterruptedException { - String appKey =app + "/" + key; - String source = Constant.SYSTEM; - String rule = RuleUtil.rule(appKey); - KeyRule keyRule = RuleUtil.findByKey(appKey); - String uuid = UUID.randomUUID().toString(); - assert keyRule != null; - KeyRecord keyRecord = new KeyRecord(key, rule, app, keyRule.getDuration(), source, type, uuid, new Date()); - keyRecord.setRule(rule); - RECORD_QUEUE.put(keyRecord); - SlidingWindow wd = PushHandler.appCfgMap.get(app).getWindow(); - int count = wd.addCount(1); - if(count > 0){ - pushHandler.offer(new PushMsgWrapper(app) ); - } + public void putRecord(String app, String key) throws InterruptedException { + this.offer(new IRecord() { + @Override + public String appNameKey() { return app + "/" + key; } + @Override + public String value() { return UUID.randomUUID().toString(); } + @Override + public int type() { return 0; } + @Override + public Date createTime() { return new Date(); } + }); + // 监控和推送 + pushHandler.monitorAndPush(app); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java index bdf85a3..073cd9f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -8,6 +8,7 @@ import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.util.List; @@ -19,10 +20,12 @@ import java.util.concurrent.LinkedBlockingQueue; /** * @ProjectName: hotkey * @ClassName: PushHandler - * @Description: TODO(一句话描述该类的功能) + * @Description: 处理推送 * @Author: liyunfeng31 * @Date: 2020/9/3 10:47 */ + +@Component public class PushHandler { /** @@ -52,16 +55,20 @@ public class PushHandler { /** - * 入队 + * 监控和入队 */ - public void offer(PushMsgWrapper msgWrapper) { - try { - MSG_QUEUE.put(msgWrapper); - } catch (InterruptedException e) { - e.printStackTrace(); + public void monitorAndPush(String app) throws InterruptedException { + AppCfgVo cfg = PushHandler.appCfgMap.get(app); + if(cfg.getWarn().equals(1)){ + SlidingWindow wd = cfg.getWindow(); + int count = wd.addCount(1); + if(count > 0){ + MSG_QUEUE.put(new PushMsgWrapper(app)); + } } } + /** * 启动线程处理警报消息队列 */ @@ -126,7 +133,6 @@ public class PushHandler { appCfgMap.put(cfg.getApp(),cfg); } } - } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index a686cd2..6714da9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -6,7 +6,6 @@ import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.client.kv.KvClient; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; @@ -121,8 +120,6 @@ public class EtcdMonitor { //观察热key访问次数和总访问次数，并做统计 watchHitCount(); - - pushWarn(); } /** @@ -151,37 +148,10 @@ public class EtcdMonitor { threadPoolExecutor.submit(() -> { dataHandler.dealHotKey(); }); - threadPoolExecutor.submit(() -> { - dataHandler.dealKeyRecord(); - dataHandler.offer(new IRecord() { - @Override - public String appNameKey() { - return model.getAppName() + "/" + model.getKey(); - } - - @Override - public String value() { - return UUID.randomUUID().toString(); - } - - @Override - public int type() { - return 0; - } - - @Override - public Date createTime() { - return new Date(); - } - }); - - } catch (Exception e) { - e.printStackTrace(); - } - } - + pushHandler.pushWarnMsg(); }); + } /** @@ -197,7 +167,7 @@ public class EtcdMonitor { Event event = event(watchIterator); try { - log.info("---------watch rule change---------"); + log.info("---------watch rule change---------"+event.getType().toString()); //全量拉取rule信息 fetchRuleFromEtcd(); } catch (Exception e) { @@ -299,9 +269,4 @@ public class EtcdMonitor { return watchIterator.next().getEvents().get(0); } - - - private void pushWarn() { - threadPoolExecutor.submit(() ->pushHandler.pushWarnMsg()); - } } diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml deleted file mode 100644 index 4566a39..0000000 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyTimelyMapper.xml +++ /dev/null @@ -1,162 +0,0 @@ - - - INSERT ignore INTO - hk_key_timely( - key_name, - val, - app_name, - uuid, - duration, - create_time) - VALUES - - (#{k.key}, - #{k.val}, - #{k.appName}, - #{k.uuid}, - #{k.duration}, - #{k.createTime}) - - - - - insert into hk_key_timely - - - id, - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{id,jdbcType=BIGINT}, - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - - - - update hk_key_timely - - - key_name = #{key,jdbcType=VARCHAR}, - - - val = #{val,jdbcType=VARCHAR}, - - - uuid = #{uuid,jdbcType=VARCHAR}, - - - app_name = #{appName,jdbcType=VARCHAR}, - - - duration = #{duration,jdbcType=BIGINT}, - - - create_time = #{createTime,jdbcType=TIMESTAMP}, - - - where key_name = #{key} - - - - - - - - insert into hk_key_timely - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - on duplicate key update - create_time = now() - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/admin/appcfg/edit.html b/dashboard/src/main/resources/templates/admin/appcfg/edit.html index fdc26cd..fb768ca 100644 --- a/dashboard/src/main/resources/templates/admin/appcfg/edit.html +++ b/dashboard/src/main/resources/templates/admin/appcfg/edit.html @@ -34,7 +34,7 @@
- +
@@ -50,9 +50,12 @@
- +
- +
diff --git a/dashboard/src/main/resources/templates/admin/appcfg/list.html b/dashboard/src/main/resources/templates/admin/appcfg/list.html index 51b46f2..19da89f 100644 --- a/dashboard/src/main/resources/templates/admin/appcfg/list.html +++ b/dashboard/src/main/resources/templates/admin/appcfg/list.html @@ -105,6 +105,12 @@ { field: 'warnPeriod', title: '警报周期(秒)',align:"center"}, { field: 'warnThreshold', title: '警报阈值',align:"center"}, { field: 'modifier', title: '修改人',align:"center"}, + { field: 'warn', title: '是否开启警报', + formatter: function (value) { + if(value === 1){ return "开启";} + else{ return "关闭";} + } + }, { field: 'version', title: '版本',align:"center"}, { title: '操作', -- Gitee From b5d31dd633fdf425719973341f5dfa15023f2096 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 3 Sep 2020 20:51:31 +0800 Subject: [PATCH 32/42] =?UTF-8?q?=E5=88=A0disruptor=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index a548184..e11fe7a 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -15,7 +15,6 @@ 1.8 4.1.42.Final - 3.4.2 1.1.7.3 1.2.70 0.0.16 @@ -40,13 +39,6 @@ netty-all ${netty4.version} - - - com.lmax - disruptor - ${disruptor.version} - - com.alibaba fastjson -- Gitee From efaf9254eace24cd34c0f8b6c85ed98f08797607 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 4 Sep 2020 16:41:53 +0800 Subject: [PATCH 33/42] =?UTF-8?q?=E5=88=A0disruptor=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/sample/Starter.java | 113 +++++------------- sample/src/main/resources/application.yml | 10 +- 2 files changed, 35 insertions(+), 88 deletions(-) diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index d5b22bb..43baaae 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -1,7 +1,5 @@ package com.jd.platform.sample; -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.base.Joiner; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.client.ClientStarter; import com.jd.platform.hotkey.client.callback.JdHotKeyStore; @@ -14,10 +12,6 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; /** * @author wuweifeng wrote on 2020-01-14 @@ -30,103 +24,52 @@ public class Starter { @Value("${spring.application.name}") private String appName; + @Value("${total.count}") + private int count; + @Value("${total.sleep}") + private int sleep; + @Value("${total.period}") + private long period; + @Resource private IConfigCenter iConfigCenter; - private Logger logger = LoggerFactory.getLogger(getClass()); - - private AtomicLong counter = new AtomicLong(); + private Logger logger = LoggerFactory.getLogger(getClass()); @PostConstruct public void init() { ClientStarter.Builder builder = new ClientStarter.Builder(); - ClientStarter starter = builder.setAppName(appName).setEtcdServer(etcd).build(); + ClientStarter starter = builder.setAppName(appName).setEtcdServer(etcd).setPushPeriod(period).build(); starter.startPipeline(); List list = iConfigCenter.getPrefix("/jd/workers/sample/host"); for (KeyValue keyValue : list) { System.out.println(keyValue.getKey() + keyValue.getValue().toStringUtf8()); } - //监听巨大的key -// watchBigKey(); - - ExecutorService executorService = Executors.newFixedThreadPool(128); - - - StringBuilder builder1 = new StringBuilder(); - for (int i = 0; i < 10000; i++) { - builder1.append(UUID.randomUUID()); - } - - String s = "prefix"; - System.out.println(System.currentTimeMillis()); - for (int i = 0; i < 10000; i++) { - if (JdHotKeyStore.getValue(s + i) != null) { - System.out.println("ishot"); - } else { - int j = i; - } + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + e.printStackTrace(); } - System.out.println(System.currentTimeMillis()); - -// a(); -// logger.info("begin : " + System.currentTimeMillis()); -// iConfigCenter.putAndGrant( "/abc/" + builder1.toString(), UUID.randomUUID().toString(), 60); -// for (int i = 0; i < 200000; i++) { -// final int j = i; -// executorService.submit(() -> { -// iConfigCenter.putAndGrant( ConfigConstant.hotKeyRecordPath + "abc/" + j, UUID.randomUUID().toString(), 10000); -// counter.incrementAndGet(); -// if (counter.get() % 5000 == 0) { -// logger.info("counter:" + counter.get()); -// } -// }); -// } -// a(); - } -// private void watchBigKey() { -// CompletableFuture.runAsync(() -> { -// KvClient.WatchIterator watchIterator = iConfigCenter.watchPrefix("/abc"); -// //如果有新事件，即rule的变更，就重新拉取所有的信息 -// while (watchIterator.hasNext()) { -// //这句必须写，next会让他卡住，除非真的有新rule变更 -// Event event = watchIterator.next().getEvents().get(0); -// double m = ObjectSizeCalculator.getObjectSize(event.getKv().getKey().toStringUtf8()); -// logger.error("length:" + m); -// } -// -// }); -// } - - private void a() { - System.out.println("begin"); - while (true) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (JdHotKeyStore.getValue("a") != null) { - System.out.println("ishot"); - } else { - JdHotKeyStore.smartSet("a", 1); -// logger.error("smartSet"); + new Thread(() -> { + long i1 = 0; + while (true) { + if (JdHotKeyStore.getValue(i1 + "") == null) { + i1++; + if (i1 % count == 0) { + try { + Thread.sleep(sleep); + logger.info("number: " + i1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } } - } + }).start(); } - - public static void main(String[] args) { - Joiner joiner = Joiner.on(","); - String s = "1"; - List list = CollectionUtil.list(true, s); - String j = joiner.join(list); - System.out.println(j); - - String[] ss = j.split(","); - System.out.println(ss); - } } diff --git a/sample/src/main/resources/application.yml b/sample/src/main/resources/application.yml index 6424c32..a8e2a8c 100644 --- a/sample/src/main/resources/application.yml +++ b/sample/src/main/resources/application.yml @@ -1,11 +1,15 @@ #etcd的地址，如有多个用逗号分隔 etcd: server: ${etcdServer:https://127.0.0.1:2379} -# server: http://10.170.161.91:2379 - # server: http://open-etcd.jd.com:2000 +# server: ${etcdServer:http://10.170.161.91:2379｝ +# server: http://open-etcd.jd.com:2000 +total: + count: ${count:40000} + sleep: ${sleep:1000} + period: ${period:10} spring: application: - name: sample + name: ${name:sample} ###############################---redis---############################## # redis: # host: ${REDIS_HOST:127.0.0.1} -- Gitee From e93bd12dfd037a09c315a3f936455f3e8bd968b5 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Sat, 5 Sep 2020 14:50:59 +0800 Subject: [PATCH 34/42] =?UTF-8?q?=E6=89=B9=E9=87=8F=E6=94=B91=E4=B8=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/common/monitor/DataHandler.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 4d92458..8216cdd 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -4,19 +4,17 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import cn.hutool.core.collection.CollectionUtil; import com.alibaba.fastjson.JSON; import com.google.common.collect.Queues; -import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.dashboard.biz.mapper.KeyRecordMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.IRecord; -import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; -import com.jd.platform.hotkey.dashboard.biz.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.biz.mapper.StatisticsMapper; -import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; @@ -58,7 +56,7 @@ public class DataHandler { private PushHandler pushHandler; - private static final Integer CACHE_SIZE = 100; + private static final Integer CACHE_SIZE = 10000; /** @@ -82,7 +80,7 @@ public class DataHandler { while (true) { try { List records = new ArrayList<>(); - Queues.drain(RECORD_QUEUE, records, CACHE_SIZE * 3 >> 2, 1, TimeUnit.SECONDS); + Queues.drain(RECORD_QUEUE, records, CACHE_SIZE, 1, TimeUnit.SECONDS); if (CollectionUtil.isEmpty(records)) { continue; } -- Gitee From 524a01f42bdf2227f370ee3323ece6a4b6822130 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Mon, 7 Sep 2020 10:08:00 +0800 Subject: [PATCH 35/42] rollback version for rule linechart --- .../com/jd/platform/hotkey/dashboard/util/CommonUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 425d875..4ec40fc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -85,8 +85,8 @@ public class CommonUtil { * @return map */ private static Map> listGroup(List list) { - return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); - // return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); + // return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); + return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); } /** -- Gitee From 7c4cedce0d6384071c3a630cb1e7d7fa4e3740b8 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Mon, 14 Sep 2020 14:36:03 +0800 Subject: [PATCH 36/42] =?UTF-8?q?fix=20=20=E6=BB=91=E5=8A=A8=E7=AA=97?= =?UTF-8?q?=E5=8F=A3=E5=8F=8D=E5=BA=8F=E5=88=97=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/domain/vo/AppCfgVo.java | 3 ++- .../dashboard/common/monitor/SlidingWindow.java | 16 ++++++++++++++++ .../admin/assets/js/bootstrap/js/base_list.js | 2 +- .../src/main/resources/templates/admin/main.html | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java index 6f2b877..8124945 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import com.alibaba.fastjson.JSON; import com.fasterxml.jackson.annotation.JsonIgnore; import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; @@ -46,7 +47,7 @@ public class AppCfgVo implements Serializable { */ private String modifier; - @JsonIgnore + private SlidingWindow window; public String getApp() { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java index 040d945..00998c8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java @@ -117,4 +117,20 @@ public class SlidingWindow { } } + + public int getWindowSize() { + return windowSize; + } + + public void setWindowSize(int windowSize) { + this.windowSize = windowSize; + } + + public int getThreshold() { + return threshold; + } + + public void setThreshold(int threshold) { + this.threshold = threshold; + } } diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 01763f2..13c97e4 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -46,7 +46,7 @@ }, onLoadError: function(status){ //加载失败时执行 let token = getCookie("token"); - if(status === 500 && ( token == "undefined" || token =="")){ + if(status === 1000 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index ee7b7fe..75e714a 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -179,7 +179,7 @@ }, error: function(result){ //加载失败时执行 let token = getCookie("token"); - if(result.status == 500 && ( token == "undefined" || token =="")){ + if(result.status == 1000 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); -- Gitee From c14b378c589fa9047427dbcb46fa78c240d9f771 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 14:34:26 +0800 Subject: [PATCH 37/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 4 +++ .../platform/hotkey/common/tool/Constant.java | 2 +- .../worker/keylistener/KeyListener.java | 8 ++++- .../hotkey/worker/starters/EtcdStarter.java | 30 +++++++++++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 3d22e8c..009a107 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -58,6 +58,10 @@ public interface ConfigConstant { * 存放客户端hotKey访问次数和总访问次数的path */ String keyHitCountPath = "/jd/keyHitCount/"; + /** + * 是否开启日志 + */ + String logToggle = "/jd/logOn"; /** * 清理历史数据的配置的path diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 5c74754..81a40ec 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -31,6 +31,6 @@ public class Constant { public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; //单次包最大2M - public static int MAX_LENGTH = 2 * 1024 * 1024; + public static int MAX_LENGTH = 4 * 1024 * 1024; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java index e941eb6..f944394 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java @@ -9,6 +9,7 @@ import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.worker.cache.CaffeineCacheHolder; import com.jd.platform.hotkey.worker.netty.pusher.IPusher; import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; +import com.jd.platform.hotkey.worker.starters.EtcdStarter; import com.jd.platform.hotkey.worker.tool.SlidingWindow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +76,12 @@ public class KeyListener implements IKeyListener { //开启推送 hotKeyModel.setCreateTime(SystemClock.now()); - logger.info(NEW_KEY_EVENT + hotKeyModel.getKey()); + + //当开关打开时，打印日志。大促时关闭日志，就不打印了 + if (EtcdStarter.LOGGER_ON) { + logger.info(NEW_KEY_EVENT + hotKeyModel.getKey()); + } + //分别推送到各client和etcd for (IPusher pusher : iPushers) { pusher.push(hotKeyModel); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 2c60d6e..152af11 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -46,6 +46,11 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.*; */ @Component public class EtcdStarter { + /** + * 是否开启日志 + */ + public static boolean LOGGER_ON = true; + private Logger logger = LoggerFactory.getLogger(getClass()); @Resource @@ -100,6 +105,31 @@ public class EtcdStarter { return !DEFAULT_PATH.equals(workerPath); } + @PostConstruct + public void watchLog() { + AsyncPool.asyncDo(() -> { + try { + String loggerOn = configCenter.get(ConfigConstant.logToggle); + LOGGER_ON = "true".equals(loggerOn) || "1".equals(loggerOn); + } catch (StatusRuntimeException ex) { + logger.error(ETCD_DOWN); + } + + KvClient.WatchIterator watchIterator = configCenter.watch(ConfigConstant.logToggle); + while (watchIterator.hasNext()) { + WatchUpdate watchUpdate = watchIterator.next(); + List eventList = watchUpdate.getEvents(); + + KeyValue keyValue = eventList.get(0).getKv(); + logger.info("log toggle changed : " + keyValue); + + String value = keyValue.getValue().toStringUtf8(); + LOGGER_ON = "true".equals(value) || "1".equals(value); + } + }); + + } + /** * 启动回调监听器，监听rule变化 */ -- Gitee From 9ebe1e822eaf471bed80cae66be4c4404d659665 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 14:44:49 +0800 Subject: [PATCH 38/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/core/worker/WorkerInfoHolder.java | 5 +-- .../hotkey/worker/config/DisruptorConfig.java | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java b/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java index 72471a9..c12182f 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java @@ -57,10 +57,11 @@ public class WorkerInfoHolder { } public static Channel chooseChannel(String key) { - if (StrUtil.isEmpty(key) || WORKER_HOLDER.size() == 0) { + int size = WORKER_HOLDER.size(); + if (StrUtil.isEmpty(key) || size == 0) { return null; } - int index = Math.abs(key.hashCode() % WORKER_HOLDER.size()); + int index = Math.abs(key.hashCode() % size); return WORKER_HOLDER.get(index).channel; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java new file mode 100644 index 0000000..0a92f8f --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java @@ -0,0 +1,33 @@ +package com.jd.platform.hotkey.worker.config; + +import com.jd.platform.hotkey.worker.keylistener.IKeyListener; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.Resource; + +/** + * @author wuweifeng wrote on 2019-12-11 + * @version 1.0 + */ +@Configuration +public class DisruptorConfig { + @Resource + private IKeyListener iKeyListener; + + @Value("${open.timeout}") + private boolean openTimeOut; + @Value("${thread.count}") + private int threadCount; + +// @Bean +// public MessageProducer messageProducer() { +// InitConstant.openTimeOut = openTimeOut; +// //将实际值赋给static变量 +// InitConstant.threadCount = threadCount; +// +// return ProducerFactory.createHotKeyProducer(iKeyListener); +// } + + +} -- Gitee From beefca189542cb6bef4d879c91aff4b02c4eadfb Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 14:59:18 +0800 Subject: [PATCH 39/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/worker/config/DisruptorConfig.java | 33 ------------------- .../worker/netty/filter/KeyCounterFilter.java | 6 +++- .../netty/server/NodesServerHandler.java | 3 +- .../hotkey/worker/tool/InitConstant.java | 4 +++ 4 files changed, 11 insertions(+), 35 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java deleted file mode 100644 index 0a92f8f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.Resource; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -@Configuration -public class DisruptorConfig { - @Resource - private IKeyListener iKeyListener; - - @Value("${open.timeout}") - private boolean openTimeOut; - @Value("${thread.count}") - private int threadCount; - -// @Bean -// public MessageProducer messageProducer() { -// InitConstant.openTimeOut = openTimeOut; -// //将实际值赋给static变量 -// InitConstant.threadCount = threadCount; -// -// return ProducerFactory.createHotKeyProducer(iKeyListener); -// } - - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index b268156..424a9fc 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.worker.netty.filter; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.SystemClock; import cn.hutool.core.util.StrUtil; import com.jd.platform.hotkey.common.model.HotKeyMsg; @@ -51,9 +52,12 @@ public class KeyCounterFilter implements INettyMsgFilter { return true; } + private void publishMsg(String appName, String message, ChannelHandlerContext ctx) { - //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, KeyCountModel.class); + if (CollectionUtil.isEmpty(models)) { + return; + } long timeOut = SystemClock.now() - models.get(0).getCreateTime(); //超时5秒以上的就不处理了，因为client是每10秒发送一次，所以最迟15秒以后的就不处理了 if (timeOut > InitConstant.timeOut + 10000) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java index d6b5fcc..8aa16f5 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java @@ -46,7 +46,8 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { } @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); logger.error("some thing is error , " + cause.getMessage()); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java index f603c1c..f9702f2 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java @@ -13,6 +13,10 @@ public class InitConstant { //单位是百万 public static int bufferSize = 2; + public static boolean openTimeOut; + + public static int threadCount; + public static final LongAdder expireTotalCount = new LongAdder(); public static final LongAdder totalDealCount = new LongAdder(); -- Gitee From 0c55cee3a8c0877124815738f1c5afbfbbc2d9b9 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 15:58:33 +0800 Subject: [PATCH 40/42] merge erp --- .../platform/hotkey/dashboard/DashboardApplication.java | 2 -- .../hotkey/dashboard/biz/controller/KeyController.java | 2 +- .../hotkey/dashboard/biz/controller/RuleController.java | 2 +- .../hotkey/dashboard/biz/controller/UserController.java | 8 ++++---- .../hotkey/dashboard/biz/controller/WorkerController.java | 4 ++-- .../hotkey/dashboard/biz/mapper/ChangeLogMapper.xml | 2 +- .../platform/hotkey/dashboard/biz/mapper/UserMapper.xml | 2 +- .../platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml | 2 +- .../src/main/resources/templates/admin/appcfg/edit.html | 4 ++-- .../src/main/resources/templates/admin/clear/edit.html | 4 ++-- .../resources/templates/admin/common/html/leftMenu.html | 2 +- 11 files changed, 16 insertions(+), 18 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 3d9f9f5..e1431c9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,8 +1,6 @@ package com.jd.platform.hotkey.dashboard; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index 8c74507..281c26a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -34,7 +34,7 @@ import java.util.List; @Controller @RequestMapping("/key") public class KeyController extends BaseController { - + private String prefix = "admin/key"; @Resource diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java index 7ce52bb..ce8600d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java @@ -29,7 +29,7 @@ import java.util.List; @Controller @RequestMapping("/rule") public class RuleController extends BaseController { - + @Resource private RuleService ruleService; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java index 69ad035..8711eb4 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java @@ -1,17 +1,15 @@ package com.jd.platform.hotkey.dashboard.biz.controller; -import java.util.*; - import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; -import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; @@ -23,6 +21,8 @@ import javax.annotation.Resource; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.util.HashMap; +import java.util.Map; @Controller diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java index bab4207..aedd57c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java @@ -16,7 +16,7 @@ import javax.annotation.Resource; @Controller @RequestMapping("/worker") public class WorkerController extends BaseController { - + private String prefix = "admin/worker"; @Resource @@ -63,7 +63,7 @@ public class WorkerController extends BaseController { modelMap.put("worker", workerService.selectByKey(key.replace("_","/"))); return prefix + "/edit"; } - + @PostMapping("/edit") @ResponseBody diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml index 9a1c828..9654fee 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml @@ -16,7 +16,7 @@ id, biz_id, biz_type, from_str, to_str, app_name,update_user, create_time,uuid - select + select from hk_user where id = #{id,jdbcType=INTEGER} diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml index 6597dc6..09e72b2 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml @@ -16,7 +16,7 @@ + + + and token = #{token} + and flag = #{flag} + and CREATED_BY = #{createdBy} + and CREATED_TIME = #{createdTime} + and UPDATED_BY = #{updatedBy} + and UPDATED_TIME = #{updatedTime} + + + + + + + insert into biz_access_token + + token, + flag, + CREATED_BY, + CREATED_TIME, + UPDATED_BY, + UPDATED_TIME, + + + #{token}, + #{flag}, + #{createdBy}, + #{createdTime}, + #{updatedBy}, + #{updatedTime}, + + + + + update biz_access_token + + token = #{token}, + flag = #{flag}, + CREATED_BY = #{createdBy}, + CREATED_TIME = #{createdTime}, + UPDATED_BY = #{updatedBy}, + UPDATED_TIME = #{updatedTime}, + + where id = #{id} + + + + delete from biz_access_token where id = #{id} + + \ No newline at end of file diff --git a/dashboard/src/main/resources/jsf-consumer.xml b/dashboard/src/main/resources/jsf-consumer.xml new file mode 100644 index 0000000..acf8a7b --- /dev/null +++ b/dashboard/src/main/resources/jsf-consumer.xml @@ -0,0 +1,13 @@ + + + + + + + \ No newline at end of file -- Gitee From 405ee53ecef6852323352297637f5169902a3572 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Mon, 19 Oct 2020 16:35:45 +0800 Subject: [PATCH 42/42] =?UTF-8?q?=E5=92=9A=E5=92=9A=E6=8A=A5=E8=AD=A6?= =?UTF-8?q?=E6=94=B9=E4=B8=BAhttp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/DashboardApplication.java | 18 +- .../biz/service/impl/AppCfgServiceImpl.java | 2 +- .../common/domain/PushMsgWrapper.java | 16 +- .../dashboard/common/domain/vo/AppCfgVo.java | 30 +- .../dashboard/common/monitor/PushHandler.java | 46 +- .../common/monitor/SlidingWindow.java | 53 +- .../warn/dongdong/AccessTokenTask.java | 24 +- .../dongdong/DDOpenAPIResultCodeEnum.java | 151 ----- .../warn/dongdong/DongDongApiManager.java | 197 +++---- .../warn/dongdong/DongDongProperties.java | 18 - .../dashboard/warn/dongdong/DongDongUtil.java | 114 ++++ .../dongdong/jsf/annotation/EnableJsf.java | 56 -- .../dongdong/jsf/annotation/JsfConsumer.java | 94 ---- .../jsf/annotation/JsfConsumerMethod.java | 90 --- .../dongdong/jsf/annotation/JsfParameter.java | 33 -- .../dongdong/jsf/annotation/JsfProvider.java | 71 --- .../jsf/annotation/JsfProviderMethod.java | 57 -- .../dongdong/jsf/annotation/JsfProviders.java | 27 - .../warn/dongdong/jsf/annotation/Server.java | 63 --- .../config/JsfBeanFactoryPostProcessor.java | 515 ------------------ .../jsf/config/JsfComponentRegistrar.java | 74 --- ...sfConsumerAnnotationBeanPostProcessor.java | 65 --- .../config/JsfConsumerInjectionMetadata.java | 25 - .../warn/dongdong/jsf/config/NameUtils.java | 55 -- dashboard/src/main/resources/application.yml | 6 +- dashboard/src/main/resources/db.sql | 21 + .../templates/admin/appcfg/edit.html | 12 +- .../templates/admin/appcfg/list.html | 3 +- 28 files changed, 344 insertions(+), 1592 deletions(-) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DDOpenAPIResultCodeEnum.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongProperties.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongUtil.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/EnableJsf.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumer.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumerMethod.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfParameter.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProvider.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviderMethod.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviders.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/Server.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfBeanFactoryPostProcessor.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfComponentRegistrar.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerAnnotationBeanPostProcessor.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerInjectionMetadata.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/NameUtils.java diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 52f0755..f5f97ee 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -17,7 +17,7 @@ import java.util.List; @EnableAsync @EnableScheduling @SpringBootApplication -public class DashboardApplication implements CommandLineRunner { +public class DashboardApplication{ @Autowired private DongDongApiManager apiManager; @@ -30,20 +30,4 @@ public class DashboardApplication implements CommandLineRunner { } } - @Resource - PushHandler pushHandler; - - - @Override - public void run(String... args) throws Exception { - Thread.sleep(5000); - // pushHandler.monitorAndPush("test"); - apiManager.refreshAccessSignature(); - Thread.sleep(1000); - List list = new ArrayList<>(); - list.add("liyunfeng31"); - list.add("wuweifeng10"); - // String title, String content, List erps, String extendStr) { - apiManager.push("title","this is content",list,"this is extend"); - } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index a250db7..82ba42a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -75,7 +75,7 @@ public class AppCfgServiceImpl implements AppCfgService { */ @Override public void saveAppCfgVo(AppCfgVo cfg) { - cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnThreshold())); + cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnMin(),cfg.getWarnMax())); PushHandler.appCfgMap.put(cfg.getApp(), cfg); configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java index 937baf5..9b6b788 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java @@ -13,10 +13,16 @@ public class PushMsgWrapper implements Serializable { private Long date; + /** + * -1小于最小阈值; 1超过最大阈值 + */ + private Integer count; + private String msg; - public PushMsgWrapper(String app) { + public PushMsgWrapper(String app, int count) { this.app = app; + this.count = count; this.date = SystemClock.now(); } @@ -28,6 +34,14 @@ public class PushMsgWrapper implements Serializable { this.app = app; } + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } + public Long getDate() { return date; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java index 8124945..1efe322 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -28,9 +28,14 @@ public class AppCfgVo implements Serializable { private Integer warnPeriod; /** - * 警报阈值 + * 警报阈值最小值 */ - private Integer warnThreshold; + private Integer warnMin; + + /** + * 警报阈值最大值 + */ + private Integer warnMax; /** * 警报开关 1开启 0关闭 @@ -74,12 +79,20 @@ public class AppCfgVo implements Serializable { this.warnPeriod = warnPeriod; } - public Integer getWarnThreshold() { - return warnThreshold; + public Integer getWarnMin() { + return warnMin; + } + + public void setWarnMin(Integer warnMin) { + this.warnMin = warnMin; + } + + public Integer getWarnMax() { + return warnMax; } - public void setWarnThreshold(Integer warnThreshold) { - this.warnThreshold = warnThreshold; + public void setWarnMax(Integer warnMax) { + this.warnMax = warnMax; } public Integer getWarn() { @@ -121,11 +134,12 @@ public class AppCfgVo implements Serializable { this.app = app; this.dataTtl = 30; this.warnPeriod = 10*60; - this.warnThreshold = 1000; + this.warnMin = -1; + this.warnMax = 1000; this.version = 0L; this.warn = 1; this.modifier = "SYSTEM"; - this.window = new SlidingWindow(warnPeriod/60,warnThreshold); + this.window = new SlidingWindow(warnPeriod/60,warnMin,warnMax); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java index 4531ba3..91135bb 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -7,6 +7,7 @@ import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import com.jd.platform.hotkey.dashboard.warn.dongdong.DongDongApiManager; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -32,10 +33,18 @@ import java.util.concurrent.LinkedBlockingQueue; @Component public class PushHandler { + private Logger logger = LoggerFactory.getLogger(getClass()); + + + /** + * 报警标题 + */ + private static final String TITLE = "hotKey异常提醒"; + /** * 拦截重复报警间隔 10分钟 */ - private static final long interval = 10*60*1000L; + private static final long INTERVAL = 10*60*1000L; /** * app-config map @@ -52,11 +61,15 @@ public class PushHandler { */ private static final BlockingQueue MSG_QUEUE = new LinkedBlockingQueue<>(); + @Resource private IConfigCenter configCenter; + @Resource private UserService userService; + @Resource + private DongDongApiManager apiManager; /** * 监控和入队 @@ -66,9 +79,8 @@ public class PushHandler { if(cfg.getWarn().equals(1)){ SlidingWindow wd = cfg.getWindow(); int count = wd.addCount(1); - if(count > 0){ - MSG_QUEUE.put(new PushMsgWrapper(app)); - } + if(count == 0){ return; } + MSG_QUEUE.put(new PushMsgWrapper(app, count)); } } @@ -79,16 +91,21 @@ public class PushHandler { public void pushWarnMsg() { // 初始化MAP到内存 initAppCfgMap(); - System.out.println("initAppCfgMap-result-> "+JSON.toJSONString(appCfgMap)); + logger.info("initAppCfgMap success AppCfgMap:{}",JSON.toJSONString(appCfgMap)); + while (true){ try { PushMsgWrapper msgWrapper = MSG_QUEUE.take(); String warnApp = msgWrapper.getApp(); Long msgTime = msgWrapper.getDate(); boolean send = check(warnApp, msgTime); - if(send){ doPush(); } + if(send){ + Integer ct = msgWrapper.getCount(); + String content = ct == -1 ? warnApp+"热点记录频率低于最小阈值，请注意！" : warnApp+"热点记录频率高于最大阈值，请注意！"; + apiManager.push(TITLE,content); + } } catch (InterruptedException e) { - e.printStackTrace(); + logger.info("pushWarnMsg thread error , msg :{}", e.getMessage()); } } } @@ -103,27 +120,26 @@ public class PushHandler { private synchronized boolean check(String warnApp, Long msgTime){ Long maxTime = appIntervalMap.get(warnApp); if(maxTime == null){ - appIntervalMap.put(warnApp,msgTime+interval); + appIntervalMap.put(warnApp,msgTime+INTERVAL); return true; }else{ if(msgTime > maxTime){ - appIntervalMap.put(warnApp,msgTime+interval); + appIntervalMap.put(warnApp,msgTime+INTERVAL); return true; } } return false; } - // todo 执行报警 - private static void doPush() { - } - - private Logger logger = LoggerFactory.getLogger(getClass()); /** * 初始化cfgMap和滑动窗口 */ private void initAppCfgMap() { + /** + * 为了加入最小值 + */ + configCenter.delete(ConfigConstant.appCfgPath); List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); if(CollectionUtils.isEmpty(keyValues) || keyValues.size()==1){ List apps = userService.listApp(); @@ -142,8 +158,6 @@ public class PushHandler { appCfgMap.put(cfg.getApp(),cfg); } } - - } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java index 00998c8..9dcbcec 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java @@ -30,7 +30,12 @@ public class SlidingWindow { /** * 在一个完整窗口期内允许通过的最大阈值 */ - private int threshold; + private int max; + + /** + * 在一个完整窗口期内最小阈值 + */ + private int min; /** * 该滑窗的起始创建时间，也就是第一个数据 */ @@ -40,29 +45,12 @@ public class SlidingWindow { */ private long lastAddTimestamp; - public static void main(String[] args) { - //1秒一个时间片，窗口共5个 - SlidingWindow window = new SlidingWindow(10, 20); - - CyclicBarrier cyclicBarrier = new CyclicBarrier(10); - for (int i = 0; i < 10; i++) { - new Thread(() -> { - try { - cyclicBarrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { - e.printStackTrace(); - } - int hot = window.addCount(2); - System.out.println(hot); - }).start(); - } - } - - public SlidingWindow(int windowSize, int threshold) { + public SlidingWindow(int windowSize, int min, int max) { this.timeMillisPerSlice = 60 * 1000; this.windowSize = windowSize; - this.threshold = threshold; + this.max = max; + this.min = min; // 保证存储在至少两个window this.timeSliceSize = windowSize * 2; reset(); @@ -101,10 +89,13 @@ public class SlidingWindow { sum += timeSlices[(index - i + timeSliceSize) % timeSliceSize].get(); } lastAddTimestamp = SystemClock.now(); - if(sum >= threshold){ + if(sum >= max){ return sum; + }else if(sum < min){ + return -1; + }else{ + return 0; } - return 0; } private void clearFromIndex(int index) { @@ -126,11 +117,19 @@ public class SlidingWindow { this.windowSize = windowSize; } - public int getThreshold() { - return threshold; + public int getMax() { + return max; + } + + public void setMax(int max) { + this.max = max; + } + + public int getMin() { + return min; } - public void setThreshold(int threshold) { - this.threshold = threshold; + public void setMin(int min) { + this.min = min; } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/AccessTokenTask.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/AccessTokenTask.java index 88c3fb9..6b40e8b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/AccessTokenTask.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/AccessTokenTask.java @@ -1,15 +1,31 @@ package com.jd.platform.hotkey.dashboard.warn.dongdong; -import org.springframework.beans.factory.annotation.Autowired; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import javax.annotation.Resource; + +/** + * @author liyunfeng31 + */ @Component("accessTokenTask") public class AccessTokenTask { - @Autowired - private DongDongApiManager dongDongApiManager; + + @Resource + private DongDongApiManager apiManager; + + + private Logger log = LoggerFactory.getLogger(getClass()); + + @Scheduled(cron = "0 0 0/1 ? * *") public void updateAccessToken(){ - dongDongApiManager.refreshAccessSignature(); + try { + apiManager.refreshAccessSignature(); + }catch (Exception e){ + log.error("refreshAccessSignature error:",e); + } } } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DDOpenAPIResultCodeEnum.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DDOpenAPIResultCodeEnum.java deleted file mode 100644 index 6407d6b..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DDOpenAPIResultCodeEnum.java +++ /dev/null @@ -1,151 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong; - -public enum DDOpenAPIResultCodeEnum { - - MSG_SUCCESS(230001,"消息发送成功"), - MSG_COMM_EXCEEDED_FREQUENT(230002,"消息发送过快，请稍后再试"), - MSG_COMM_SYS_BUSY(230003,"系统繁忙，请稍后再试"), - MSG_COMM_VALIDATE_PASS(230004,"消息校验通过"), - MSG_COMM_SEND_EXCEPTION(230005,"消息发送处理异常"), - MSG_COMM_SEND_ACK_NULL(230006,"消息发送回执为空"), - MSG_COMM_SEND_ACK_FAILED(230007,"消息发送回执校验失败"), - MSG_HEADER_FROM_INVALID(230011,"消息发送字段无效"), - MSG_HEADER_TO_INVALID(230012,"消息接收字段无效"), - MSG_HEADER_VER_ILLEGAL(230013,"消息版本号不支持"), - MSG_HEADER_TYPE_ILLEGAL(230014,"消息类型不支持"), - MSG_HEADER_AID_INVALID(230015,"消息AID无效"), - MSG_HEADER_AID_SUC(230016,"消息AID校验通过"), - MSG_HEADER_EXCEPTION(230017,"消息头部异常"), - MSG_HEADER_CLIENT_ILLEGAL(230018,"消息客户端类型不支持"), - MSG_HEADER_APP_ILLEGAL(230019,"消息投递app不匹配"), - MSG_BODY_NULL(230021,"消息体为空"), - MSG_BODY_TYPE_ILLEGAL(230022,"消息体类型非法"), - MSG_BODY_SUBTYPE_ILLEGAL(230023,"消息体子类型非法"), - MSG_BODY_FORMAT_ILLEGAL(230024,"消息体格式非法"), - MSG_SIGN_SUCCESS(230031,"获取访问签名信息成功"), - MSG_SIGN_SUC_VER_EXPIRING(230032,"获取访问签名信息成功，但APP本地版本号即将过期"), - MSG_SIGN_FAILED_VER_EXPIRED(230033,"获取访问签名信息失败，APP本地版本号已过期"), - MSG_SIGN_FAILED_ILLEGAL(230034,"获取访问签名信息失败，APP签名凭证非法"), - MSG_SIGN_FAILED_NOT_WHITE_LIST(230035,"获取访问签名信息失败，非白名单IP地址"), - MSG_SIGN_FAILED_EXCEEDED_FREQUENT(230036,"获取访问签名信息太频繁，请稍后再试"), - MSG_SIGN_FAILED_NULL(230037,"签名信息为空"), - MSG_SIGN_FAILED_ACCID_DUPLICATED(230038,"签名信息accessId重复"), - MSG_SIGN_FAILED_VER_ILLEGAL(230039,"签名信息版本非法"), - MSG_SIGN_FAILED_TOKEN_ILLEGAL(230040,"签名信息accessToken非法"), - MSG_SIGN_FAILED_EXCEPTION(230041,"获取签名信息异常"), - MSG_SIGN_FAILED_ACCID_INVALID(230042,"签名信息accessId无效"), - MSG_SIGN_FAILED_ASPID_INVALID(230043,"签名信息aspId无效"), - MSG_SIGN_FAILED_HOST_INVALID(230044,"访问主机无效"), - MSG_SIGN_FAILED_ASPINFO_NOT_READY(230045,"asp信息未完成配置"), - MSG_SIGN_FAILED_ASPINFO_SECRET_NULL(230046,"asp配置凭证为空"), - MSG_SIGN_FAILED_ASPINFO_ACCESSTOKEN_NULL(230047,"asp配置accessToken为空"), - MSG_SIGN_FAILED_HTTP_FORBIDDEN(230048,"禁止非https访问"), - MSG_NOTICE_SUC(230070,"通知消息发送成功"), - MSG_NOTICE_ILLEGAL(230071,"通知消息非法"), - MSG_NOTICE_EXCEED_MAX(230072,"通知接受者超出范围"), - MSG_NOTICE_RECEIVE_EMPTY(230073,"通知接受者列表为空"), - MSG_NOTICE_SEND_FAILED(230074,"发送通知消息失败"), - MSG_NOTICE_SEND_EXCEPTION(230075,"发送通知消息失败"), - MSG_DATA_REQ_SUC(230100,"请求数据服务操作成功"), - MSG_DATA_APP_ILLEGAL(230101,"请求数据app不合法"), - MSG_DATA_REQ_PARAM_ILLEGAL(230102,"请求数据参数不合法"), - MSG_DATA_RESP_FAILED(230103,"请求数据服务返回失败"), - MSG_DATA_RESP_NULL(230104,"请求数据服务范围空值"), - MSG_DATA_REQ_EXCEPTION(230105,"请求数据处理异常"), - MSG_DATA_REQ_VER_ILLEGAL(230106,"请求数据版本号非法"), - MSG_DATA_REQ_LIST_EXCEEDED(230107,"请求数据列表长度超出范围"), - MSG_DATA_RESP_UNSUPPORTED(230108,"请求返回数据不支持"), - MSG_DATA_RESP_APP_ILLEGAL(230109,"新增appid格式不合法"), - MSG_DATA_RESP_APP_EXISTED(230110,"新增appid已存在"), - MSG_SOA_REQUEST_SUC(230200,"SOA请求成功"), - MSG_SOA_APPID_EXEPTION(230201,"appId异常"), - MSG_SOA_DATATYPE_EXEPTION(230202,"dataType异常"), - MSG_SOA_ACTION_EXEPTION(230203,"action异常"), - MSG_SOA_PARAM_NULL(230204,"param参数空"), - MSG_SOA_REQUEST_NULL(230205,"soa request空"), - MSG_SOA_REQUEST_EXCEPTION(230206,"soa request异常"), - MSG_SOA_RESULT_NULL(230207,"请求返回为空"), - MSG_SOA_RESULT_INVALID(230208,"请求返回校验不通过"), - MSG_SOA_REQUEST_FAILED(230209,"请求返回操作失败"), - MSG_SOA_SERVICE_NOT_EXIST(230210,"请求SOA服务不存在"), - MSG_SOA_REQ_PASS(230211,"SOA请求校验通过"), - MSG_SOA_REQ_HTTP_CHARSET_EXCEPTION(230212,"SOA HTTP请求不支持的字符集"), - MSG_SOA_REQ_HTTP_IO_EXCEPTION(230213,"SOA HTTP请求IO异常"), - MSG_SOA_REQ_HTTP_PARSE_EXCEPTION(230214,"SOA HTTP请求解析异常"), - MSG_SOA_REQ_HTTP_ENCODING_EXCEPTION(230215,"SOA HTTP请求编码异常"), - MSG_SOA_RESULT_AES_DECODE_EXCEPTION(230216,"SOA请求AES解析异常"), - MSG_SOA_REQ_HTTP_URL_NULL(230217,"SOA HTTP请求URL为空"), - MSG_SOA_REQ_HTTP_RESP_FAILED(230218,"SOA HTTP请求返回校验失败"), - MSG_CHAT_MORE_OPERATE_NEEDED(230301,"需要更多的操作"), - MSG_CHAT_WAITER_MISSED(230302,"没有配置客服"), - MSG_CHAT_SHUNT_SUCCESS(230303,"分流成功"), - MSG_CHAT_RISK_NO_WAITER_SERVICE(230304,"客服暂时不能为您提供服务"), - MSG_CHAT_RISK_CAPTCHA_INPUT(230305,"请输入验证码"), - MSG_CHAT_RISK_CAPTCHA_MISMATCH(230306,"验证码错误，请重新输入"), - MSG_CHAT_RISK_CAPTCHA_MISMATCH_EXCEEDED(230307,"验证码错误次数超过限制"), - MSG_CHAT_RISK_CAPTCHA_MATCHED(230308,"验证码回答正确"), - MSG_CHAT_RISK_CAPTCHA_VOICE_INPUT(230309,"请输入语音验证码"), - MSG_CHAT_RISK_CAPTCHA_VOICE_MISMATCH(230310,"风控语音验证码错误，请重新输入验证码"), - MSG_CHAT_RISK_PHONE_NOT_BOUND(230311,"用户未绑定电话"), - MSG_CHAT_RISK_LANG_FAILED(230312,"语言验证失败，重新验证"), - MSG_CHAT_RISK_CAPTCHA_VOICE_PRESS(230313,"为了保护您的账号安全，请点击按钮进行语音验证"), - MSG_CHAT_WAITER_OFFLINE(230314,"分配不在工作时间或客服不在线"), - MSG_CHAT_SEND_FAILED(230315,"消息发送失败"), - MSG_CHAT_RISK_NETWORK_EXCEPTION(230316,"网络异常，请重新输入验证码"), - MSG_CHAT_REPEAT_CUSTOMER(230317,"消息重复发送 顾客消息"), - MSG_CHAT_REPEAT_WAITER(230318,"消息重复发送 客服消息"), - MSG_CHAT_LEAVE_SUCCESS(230319,"留言成功"), - MSG_CHAT_LEAVE_FAILED(230320,"留言失败"), - MSG_CHAT_RISK_TIPS(230321,"风险提示"), - MSG_CHAT_ACK_UNKNOWN(230322,"未知返回chat_message类型"), - MSG_CHAT_ACK_CODE_NOT_SUPPORT(230323,"不支持的返回code"), - MSG_INVITE_FAILED(230351,"邀评失败"), - MSG_INVITE_EVALUATED(230352,"已评价"), - MSG_INVITE_DUPLICATED(230353,"已邀评"), - MSG_INVITE_CUSTOMER_OFFLINE(230354,"顾客已经离线"), - MSG_INVITE_UNKNOWN(230355,"未知邀评状态"); - - private int code; - private String description; - - DDOpenAPIResultCodeEnum(int code, String description) { - this.code = code; - this.description = description; - } - - /** - * Getter for property 'code'. - * - * @return Value for property 'code'. - */ - public int getCode() { - return code; - } - - /** - * Setter for property 'code'. - * - * @param code Value to set for property 'code'. - */ - public void setCode(int code) { - this.code = code; - } - - /** - * Getter for property 'description'. - * - * @return Value for property 'description'. - */ - public String getDescription() { - return description; - } - - /** - * Setter for property 'description'. - * - * @param description Value to set for property 'description'. - */ - public void setDescription(String description) { - this.description = description; - } -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongApiManager.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongApiManager.java index 0b637cd..ad7ffbc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongApiManager.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongApiManager.java @@ -1,25 +1,18 @@ package com.jd.platform.hotkey.dashboard.warn.dongdong; -import com.alibaba.fastjson.JSONObject; import com.github.rholder.retry.*; -import com.jd.dd.open.gw.api.GrantService; -import com.jd.dd.open.gw.api.MessagePushService; -import com.jd.dd.open.gw.api.domain.*; -import com.jd.fastjson.JSON; -import com.jd.jsf.gd.util.DateUtils; import com.jd.platform.hotkey.dashboard.biz.service.IBizAccessTokenService; import com.jd.platform.hotkey.dashboard.model.BizAccessToken; -import com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation.JsfConsumer; -import com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation.JsfParameter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thymeleaf.util.StringUtils; +import javax.annotation.Resource; import java.util.Date; import java.util.List; -import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -28,71 +21,57 @@ import java.util.concurrent.TimeUnit; * @author lihongliang32 * @date 2020/5/20 5:53 下午 */ -@Slf4j -@Service("dongdongAPIManager") +@Service public class DongDongApiManager { - @JsfConsumer(id="dongdongGrantService",alias="${dongdong.alias}", timeout="${dongdong.timeout}", - parameters= { @JsfParameter(key="token", keyValue="${dongdong.token}", hide=true)}) - private GrantService grantService; - @JsfConsumer(id="dongdongMessagePushService",alias="${dongdong.alias}", timeout="${dongdong.timeout}", - parameters= { @JsfParameter(key="token", keyValue="${dongdong.token}", hide=true)}) - private MessagePushService messagePushService; - - @Value("${jsf.dongdong.aspId}") + @Value("${dongdong.aspId:110.200.0000002.266}") private String aspId; - @Value("${jsf.dongdong.secret}") + @Value("${dongdong.secret:e2b05aa43959ddb6d37bd056b0243bc9}") private String secret; - @Value("${jsf.dongdong.noticeId}") + @Value("${dongdong.noticeId:~hotkey}") private String noticeId; + @Resource + private DongDongUtil dongUtil; + + + private Logger log = LoggerFactory.getLogger(getClass()); + + + private static final String VERSION="4.3"; + @Autowired private IBizAccessTokenService accessTokenService; public boolean refreshAccessSignature() { Callable callable = () -> { - AppSigInfo appSigInfo = new AppSigInfo(); - appSigInfo.setAspid(aspId); - appSigInfo.setSecret(secret); - appSigInfo.setVersion("4.3"); - AccessSignatureResult accessSignatureResult = this.grantService.refreshAccessSignature(appSigInfo); - if (accessSignatureResult.getCode() == DDOpenAPIResultCodeEnum.MSG_SIGN_SUCCESS.getCode()) { - String accessToken = accessSignatureResult.getAccessToken(); - //请求成功 - log.info("DongdongOpenAPIManagerImpl::refreshAccessSignature success! accessToken={}", accessToken); - try{ - List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); - if(tokens.size()>0){ - BizAccessToken token = tokens.get(0); - token.setToken(accessToken); - token.setUpdatedBy("system"); - token.setUpdatedTime(new Date()); - log.info("updateBizAccessToken={}", accessToken); - - // accessTokenService.updateBizAccessToken(token); - }else{ - BizAccessToken token=new BizAccessToken(); - token.setToken(accessToken); - token.setCreatedBy("system"); - token.setCreatedTime(new Date()); - log.info("insertBizAccessToken={}", accessToken); - - // accessTokenService.insertBizAccessToken(token); - } - }catch (Exception e){ - log.error("accessToken保存失败："+e.getMessage()); - return false; + try { + String accessToken = dongUtil.grant(); + List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); + if(tokens.size()>0){ + BizAccessToken token = tokens.get(0); + token.setToken(accessToken); + token.setUpdatedBy("system"); + token.setUpdatedTime(new Date()); + log.info("updateBizAccessToken={}", accessToken); + accessTokenService.updateBizAccessToken(token); + }else{ + BizAccessToken token=new BizAccessToken(); + token.setToken(accessToken); + token.setCreatedBy("system"); + token.setCreatedTime(new Date()); + log.info("insertBizAccessToken={}", accessToken); + accessTokenService.insertBizAccessToken(token); } return true; + }catch (Exception e){ + return false; } - log.error("DongdongOpenAPIManagerImpl::refreshAccessSignature failure! code={}, reason={}", accessSignatureResult.getCode(), accessSignatureResult.getErrmsg()); - return false; }; boolean result = false; Retryer retryer = RetryerBuilder.newBuilder() -// .retryIfResult(Predicates.alwaysFalse()) .retryIfRuntimeException() .withWaitStrategy(WaitStrategies.fibonacciWait(100, 2, TimeUnit.MINUTES)) .withStopStrategy(StopStrategies.stopAfterAttempt(5)) @@ -100,72 +79,56 @@ public class DongDongApiManager { try { result = retryer.call(callable); } catch (RetryException | ExecutionException e) { - // It will stop after attempting to retry 3 times and throw a RetryException log.error("DongdongOpenAPIManagerImpl::refreshAccessSignature Exception!", e); } return result; } - public boolean push(String title, String content, List erps, String extendStr) { - boolean result; + + public boolean push(String title, String content) { + List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); + if(tokens.size() == 0){ + log.error("没有查询到有效的token！"); + return false; + } + String accessToken = tokens.get(0).getToken(); + if(StringUtils.isEmpty(accessToken)){ + return false; + } try { - List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); - if(tokens.size() == 0){ - log.error("没有查询到有效的token！"); - return false; - } - String accessToken = tokens.get(0).getToken(); - if (StringUtils.isNotBlank(accessToken)) { - AccessSignature as = new AccessSignature(); - as.setAccessid(UUID.randomUUID().toString()); - as.setAccessToken(accessToken);//授权后获取的字段 - as.setAspid(aspId);//需要申请，与授权使用相同aspid - as.setVersion("4.3"); - as.setTimestamp(System.currentTimeMillis()); - - //构造消息参数 - Message msg = new Message(); - JSONObject json = new JSONObject(); - json.put("type", "notice_message"); - json.put("ver", "4.3"); - json.put("title", title);//长度限制50 - json.put("content", content);//目前暂不支持换行，长度限制1500 - json.put("noticeId", noticeId);// ~加原来的应用标识 如原来的 ump -> ~ump，申请邮件反馈中会给出相应的noticeId - json.put("toTerminal", 7);//企业版支持(推所有终端传 7 ，非所有端请联系我们确认) - json.put("sla", 1);//是否需要离线投递，申请通知时需要配置,0在线投递，1离线投递 - json.put("app", "ee");//代表发送的用户群体，ee代表国内ERP，泰国th.ee - json.put("tos", erps);//一次最大500个 - - JSONObject extend = new JSONObject(); - if(StringUtils.isEmpty(extendStr)) { - extend.put("url", "https://fangzhou.jd.com");//点击通知可跳转的url，url中含有中文时，请将中文进行urlencode编码 - }else{ - extend.put("url", extendStr);//点击通知可跳转的url，url中含有中文时，请将中文进行urlencode编码 - } - //extend.put("pic", pic);//图片暂不支持 - json.put("extend", extend); - -// json.put("extend", extend); - msg.setJsonMsg(json.toJSONString()); - MessagePushResult messagePushResult = messagePushService.push(as, msg); - if (messagePushResult.getCode() == 230070) { - //请求成功 - log.info("DongdongOpenAPIManagerImpl::push success. to erps={}", JSON.toJSONString(erps)); - result = true; - } else { - //失败操作 - log.error("DongdongOpenAPIManagerImpl::push can not push message.code={} reason={}", messagePushResult.getCode(), messagePushResult.getErrmsg()); - - result= false; - } - } else { - log.error("DongdongOpenAPIManagerImpl::push can not get dongdong.accessToken from JimDB"); - result = false; - } - } catch (Exception e) { - log.error("DongdongOpenAPIManagerImpl::push Exception happened!", e); - result = false; + return dongUtil.push(title,content,accessToken); + }catch (Exception e){ + return false; } - return result; + } + + + + public String getAspId() { + return aspId; + } + + public void setAspId(String aspId) { + this.aspId = aspId; + } + + public String getSecret() { + return secret; + } + + public void setSecret(String secret) { + this.secret = secret; + } + + public String getNoticeId() { + return noticeId; + } + + public void setNoticeId(String noticeId) { + this.noticeId = noticeId; + } + + public static String getVERSION() { + return VERSION; } } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongProperties.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongProperties.java deleted file mode 100644 index ffd251d..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongProperties.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong; - -import com.jd.platform.hotkey.dashboard.autoconfigure.AbstractProperties; -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Data -@Component -@ConfigurationProperties(prefix = "dongdong") -public class DongDongProperties extends AbstractProperties { - private String alias; - private String token; - private String timeout; - private String appId; - private String secret; - private String noticeId; -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongUtil.java new file mode 100644 index 0000000..a756923 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/DongDongUtil.java @@ -0,0 +1,114 @@ +package com.jd.platform.hotkey.dashboard.warn.dongdong; + +import cn.hutool.json.JSONObject; +import com.alibaba.fastjson.JSON; +import com.jd.dd.open.gw.api.domain.AccessSignatureResult; +import com.jd.dd.open.gw.api.domain.MessagePushResult; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * @author liyunfeng31 + */ +@Component +public class DongDongUtil { + + @Value("${dongdong.aspId:110.200.0000002.266}") + private String aspId; + + @Value("${dongdong.secret:e2b05aa43959ddb6d37bd056b0243bc9}") + private String secret; + + @Value("${dongdong.noticeId:~hotkey}") + private String noticeId; + + + private final String VERSION="4.3"; + + + public String grant() throws Exception{ + String dogUrl = "http://dog-ee.jd.com/grant"; + List params = new ArrayList<>(); + params.add(new BasicNameValuePair("aspid", aspId)); + params.add(new BasicNameValuePair("secret", secret)); + params.add(new BasicNameValuePair("version", VERSION)); + + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpPost httpPost = new HttpPost(dogUrl); + //传参 + httpPost.setEntity(new UrlEncodedFormEntity(params)); + + CloseableHttpResponse response = httpclient.execute(httpPost); + String res = EntityUtils.toString(response.getEntity()); + + //返回结果对象AccessSignatureResult + AccessSignatureResult result = JSON.parseObject(res, AccessSignatureResult.class); + if (result.getCode() == 230031) { + //请求成功 + return result.getAccessToken(); + } + else { + //失败操作 + return String.valueOf(result.getCode()); + } + + } + + public boolean push(String title,String content,String accessToken) throws Exception{ + String dogUrl = "http://dog-ee.jd.com/push"; + List params = new ArrayList<>(); + params.add(new BasicNameValuePair("accessid", UUID.randomUUID().toString())); + params.add(new BasicNameValuePair("accessToken", accessToken)); + params.add(new BasicNameValuePair("aspid", aspId));//需要申请 + params.add(new BasicNameValuePair("version", VERSION)); + params.add(new BasicNameValuePair("timestamp", String.valueOf(System.currentTimeMillis()))); + + //消息相关参数 + JSONObject json = new JSONObject(); + json.put("type", "notice_message"); + json.put("ver", VERSION); + json.put("title", title); + json.put("content", content); + json.put("noticeId", noticeId); + json.put("toTerminal", 7); + json.put("sla", 0); + json.put("app", "ee"); + //接收方 + List pins = new ArrayList<>(); + pins.add("wuweifeng10"); + pins.add("liyunfeng31"); + pins.add("erp"); + json.put("tos", pins); + + JSONObject extend = new JSONObject(); + extend.put("url", "http://hotkey.jd.com"); + extend.put("pic", "https://img14.360buyimg.com/imagetools/jfs/t1/144156/7/8194/7263/5f5b34baEdfd90a49/a15144ab447bfbf2.png"); + json.put("extend", extend); + params.add(new BasicNameValuePair("jsonMsg",JSON.toJSONString(json))); + + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpPost httpPost = new HttpPost(dogUrl); + //传参 + httpPost.setEntity(new UrlEncodedFormEntity(params)); + httpPost.setEntity(new UrlEncodedFormEntity(params,"UTF-8")); + CloseableHttpResponse response = httpclient.execute(httpPost); + String res = EntityUtils.toString(response.getEntity()); + System.out.println(""+ JSON.toJSONString(res)); + //返回结果对象AccessSignatureResult + MessagePushResult result = JSON.parseObject(res, MessagePushResult.class); + return result.getCode() == 230070; + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/EnableJsf.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/EnableJsf.java deleted file mode 100644 index ffbed40..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/EnableJsf.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -import com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.config.JsfComponentRegistrar; -import org.springframework.context.annotation.Import; - -import java.lang.annotation.*; - - -@Target({ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Inherited -@Documented -@Import(JsfComponentRegistrar.class) -public @interface EnableJsf { - /** - * 注册中心 i.jsf.jd.com - * - * @return - */ - String index() default ""; - - /** - * 设置 {@link #tokenFile()} 到 security.token.client.fileUrl - *

- * 是否开启消费端认证 - * true|false - * @return - */ - String isConsumerOpenSecurity() default ""; - - /** - * 设置 {@link #tokenFile()} 到 security.token.server.fileUrl - *

- * 是否开启服务端认证 - * true|false - * @return - */ - String isProviderOpenSecurity() default ""; - - /** - * token 文件 - * - * @return - */ - String tokenFile() default ""; - - /** - * security.isOpen.provider 的值

- * 0 关闭

- * 1 只监听，所有服务使用者都可访问该服务

- * 2 对未授权的服务使用者进行阻断

- * - * @return - */ - String providerSecurityLevel() default ""; -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumer.java deleted file mode 100644 index 7f2ccf6..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumer.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - - - -import java.lang.annotation.*; - -/** - * Date: 2016/11/9 12:30 - * @author: wangyulie@jd.com - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD, ElementType.METHOD}) -public @interface JsfConsumer { - - /** - * Spring的BeanId - * - * @return - */ - String id() default ""; - - /** - * 服务别名分组信息 - * - * @return the string - */ - String alias(); - - /** - * 调用的协议 - * - * @return the string - */ - String protocol() default com.jd.jsf.gd.util.Constants.DEFAULT_PROTOCOL; - - /** - * 直连地址，配置了此地址就不再从注册中心获取。参见：JSF客户端用户手册#直连调用 - * - * @return the string - */ - String url() default ""; - - /** - * 过滤器实现链。 List - * - * @return - */ - String[] filterRef() default {}; - - /** - * 是否注册到注册中心 - * - * @return - */ - String register() default "true"; - - /** - * 是否从注册中心订阅 - * - * @return - */ - String subscribe() default "true"; - - /** - * 调用超时，选填 - * - * @return the int - */ - String timeout() default "5000"; - - - /** - * 失败后重试次数，选填 - * - * @return the int - */ - int retries() default com.jd.jsf.gd.util.Constants.DEFAULT_RETRIES_TIME; - - /** - * 序列化方式 - * - * @return the string - */ - String serialization() default com.jd.jsf.gd.util.Constants.DEFAULT_CODEC; - - /** - * 参数 - * - * @return - */ - JsfParameter[] parameters() default {}; - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumerMethod.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumerMethod.java deleted file mode 100644 index dc7361d..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfConsumerMethod.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -/** - * Date: 2016/11/28 16:09 - * Email: wangyulie@jd.com - */ -public @interface JsfConsumerMethod { - /** - * 方法名称（不支持重载方法） - * - * @return - */ - String name(); - - /** - * 是否校验参数，支持JSR303，参见JSF用户手册#参数校验 - * - * @return - */ - boolean validation() default false; - - /** - * 该方法的最大可并行执行请求数 - * - * @return - */ - int concurrents() default -1; - - /** - * 是否启动Mock实现 - * - * @return - */ - boolean mock() default false; - - /** - * 是否开启结果缓存。如果开启需要指定cacheref - * - * @return - */ - boolean cache() default false; - - /** - * 压缩算法（启动后是否压缩还取决于数据包大小） - * - * @return - */ - String compress() default ""; - - /** - * 方法调用超时时间，单位毫秒 - * - * @return - */ - String timeout() default ""; - - /** - * @return - */ - int retries() default -1; - - /** - * 是否异步调用 - * - * @return - */ - boolean async() default false; - - /** - * 方法的返回事件监听器实例，多个用逗号分隔。需要async为true时使用。List - * - * @return - */ - String[] onreturnRef() default ""; - - /** - * 目标参数（机房/分组等）索引，从0开始计数 - * - * @return - */ - int dstParam() default -1; - - /** - * 参数 - * - * @return - */ - JsfParameter[] parameters() default {}; - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfParameter.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfParameter.java deleted file mode 100644 index 15e349f..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfParameter.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -import java.lang.annotation.*; - -/** - * Date: 2016/11/28 15:53 - * Email: wangyulie@jd.com - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.ANNOTATION_TYPE}) -public @interface JsfParameter { - /** - * 参数配置关键字 - * - * @return - */ - String key(); - - /** - * 参数配置值 - * - * @return - */ - String keyValue(); - - /** - * 是否为隐藏配置。是的话，key自动加上"."作为前缀，且业务代码不能获取到，只能从filter里取到 - * - * @return - */ - boolean hide() default false; -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProvider.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProvider.java deleted file mode 100644 index b4d8135..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProvider.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -import com.jd.jsf.gd.util.Constants; - -import java.lang.annotation.*; - -/** - * Date: 2016/11/3 12:03 - * Email: wangyulie@jd.com - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE}) -public @interface JsfProvider { - - /** - * alias名字, 不同的alias使用同一份server - * 每一个 interfaceClass 都会有一份对应的 alias， 总共生成的provder个数是 - * interfaceClass.length + alias.length - * - * @return - */ - String[] alias(); - - /** - * 接口类型 - *

- * 不同的 interfaceClass 使用不同的server - *

- * 默认单接口的类可以不指定，如果有多个接口， 请手工指定 - * - * @return - */ - Class[] interfaceClass() default {}; - - /** - * 注册到的服务端，必填 - * - * @return the server [ ] - */ - Server[] server() default {@Server}; - - /** - * 过滤器实现链。 List - * - * @return - */ - String[] filterRef() default {}; - - /** - * 是否注册到注册中心 - * - * @return the boolean - */ - String register() default "true"; - - /** - * 是否动态发布服务 - * - * @return the boolean - */ - String dynamic() default "true"; - - /** - * 服务端权重 - * - * @return the int - */ - String weight() default Constants.DEFAULT_PROVIDER_WEIGHT + ""; - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviderMethod.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviderMethod.java deleted file mode 100644 index 353ce3e..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviderMethod.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -/** - * Date: 2016/11/28 16:09 - * Email: wangyulie@jd.com - */ -public @interface JsfProviderMethod { - /** - * 方法名称（不支持重载方法） - * - * @return - */ - String name(); - - /** - * 是否校验参数，支持JSR303，参见JSF用户手册#参数校验 - * - * @return - */ - boolean validation() default false; - - /** - * 该方法的最大可并行执行请求数 - * - * @return - */ - int concurrents() default -1; - - /** - * 是否启动Mock实现 - * - * @return - */ - boolean mock() default false; - - /** - * 是否开启结果缓存。如果开启需要指定cacheref - * - * @return - */ - boolean cache() default false; - - /** - * 压缩算法（启动后是否压缩还取决于数据包大小） - * - * @return - */ - String compress() default ""; - - /** - * 参数 - * - * @return - */ - JsfParameter[] parameters() default {}; - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviders.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviders.java deleted file mode 100644 index 715f7e7..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/JsfProviders.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -import java.lang.annotation.*; - -/** - * Date: 2016/11/3 22:30 - * Email: wangyulie@jd.com - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) -public @interface JsfProviders { - JsfProvider[] value(); - - /** - * 参数 - * - * @return - */ - JsfParameter[] parameters() default {}; - - /** - * 方法设置 - * @return - */ - JsfProviderMethod[] methods() default {}; -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/Server.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/Server.java deleted file mode 100644 index 06e29c8..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/annotation/Server.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation; - -import com.jd.jsf.gd.util.Constants; - -import java.lang.annotation.*; - -/** - * Date: 2016/11/8 18:04 - * Email: wangyulie@jd.com - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.ANNOTATION_TYPE}) -public @interface Server { - - /** - * 默认服务数量 - */ - int serverNum() default 1; - - /** - * 协议名，必填 - * - * @return the string - */ - String protocol() default Constants.DEFAULT_PROTOCOL; - - /** - * 主机地址，选填 - * - * @return the string - */ - String host() default ""; - - /** - * 端口地址，选填 - * - * @return the int - */ - int port() default Constants.DEFAULT_SERVER_PORT; - - /** - * 业务线程池大小，选填 - * - * @return the int - */ - int threads() default Constants.DEFAULT_SERVER_BIZ_THREADS; - - /** - * 线程池类型，选填 - * - * @return the string - */ - String threadpool() default Constants.THREADPOOL_TYPE_CACHED; - - /** - * 参数 - * - * @return - */ - JsfParameter[] parameters() default {}; - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfBeanFactoryPostProcessor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfBeanFactoryPostProcessor.java deleted file mode 100644 index e27c046..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfBeanFactoryPostProcessor.java +++ /dev/null @@ -1,515 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.config; - -import com.jd.jsf.gd.config.RegistryConfig; -import com.jd.jsf.gd.config.spring.ConsumerBean; -import com.jd.jsf.gd.config.spring.ProviderBean; -import com.jd.jsf.gd.config.spring.ServerBean; -import com.jd.jsf.gd.filter.AbstractFilter; -import com.jd.jsf.gd.filter.ExcludeFilter; -import com.jd.jsf.gd.util.ClassLoaderUtils; -import com.jd.jsf.gd.util.Constants; -import com.jd.jsf.gd.util.JSFContext; -import com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.aop.TargetSource; -import org.springframework.aop.framework.ProxyFactory; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.BeanFactoryAware; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; -import org.springframework.beans.factory.annotation.InjectionMetadata; -import org.springframework.beans.factory.config.*; -import org.springframework.beans.factory.support.*; -import org.springframework.context.EnvironmentAware; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.BridgeMethodResolver; -import org.springframework.core.Ordered; -import org.springframework.core.PriorityOrdered; -import org.springframework.core.env.Environment; -import org.springframework.util.ClassUtils; -import org.springframework.util.ReflectionUtils; - -import java.beans.PropertyDescriptor; -import java.lang.reflect.*; -import java.util.*; - -/** - * Date: 2016/11/4 15:27 - * Email: wangyulie@jd.com - */ -public class JsfBeanFactoryPostProcessor implements BeanDefinitionRegistryPostProcessor, BeanFactoryAware, PriorityOrdered, EnvironmentAware { - - private Logger logger = LoggerFactory.getLogger(getClass()); - - static final String JSF_CONSUMER_INJECTION_METADATA_BEAN = "JSF_CONSUMER_INJECTION_METADATA_BEAN"; - - private Environment environment; - private ConfigurableListableBeanFactory beanFactory; - - private JsfConsumerInjectionMetadata jsfConsumerInjectionMetadata = new JsfConsumerInjectionMetadata(); - - private final String registryIndex; - private final String consumerOpenSecurity; - private final String providerOpenSecurity; - private final String tokenFile; - private final String providerSecurityLevel; - - - public JsfBeanFactoryPostProcessor() { - this(null, null, null, null, null); - } - - public JsfBeanFactoryPostProcessor(String registryIndex, String isConsumerOpenSecurity, String isProviderOpenSecurity, String tokenFile, String providerSecurityLevel) { - this.registryIndex = registryIndex; - this.consumerOpenSecurity = isConsumerOpenSecurity; - this.providerOpenSecurity = isProviderOpenSecurity; - this.tokenFile = tokenFile; - this.providerSecurityLevel = providerSecurityLevel; - } - - @Override - public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) throws BeansException { - // not need impl - } - - @Override - public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) throws BeansException { - - final String registryConfig = buildRegistryConfig(registry); - - // 注册配置单例 - beanFactory.registerSingleton(JSF_CONSUMER_INJECTION_METADATA_BEAN, jsfConsumerInjectionMetadata); - - initContextParam(); - - for (String beanDefinitionName : registry.getBeanDefinitionNames()) { - BeanDefinition beanDefinition = registry.getBeanDefinition(beanDefinitionName); - String className = beanDefinition.getBeanClassName(); - if (className == null) { - continue; - } - try { - Class clazz = Class.forName(className); - parseConsumer(clazz, registry, registryConfig); - parseProvider(clazz, beanDefinitionName, registry, registryConfig); - } catch (ClassNotFoundException e) { - logger.warn("ClassNotFoundException className:{}", className); - } - } - } - - private void initContextParam() { - if (consumerOpenSecurity == null && providerOpenSecurity == null) { - // 未开启安全配置 - return; - } - if (tokenFile == null) { - throw new IllegalArgumentException("tokenFile must not be null"); - } - - String aTokenFile = environment.resolveRequiredPlaceholders(tokenFile); - - if (providerOpenSecurity != null) { - String aProviderOpenSecurity = environment.resolveRequiredPlaceholders(providerOpenSecurity); - if (Boolean.valueOf(aProviderOpenSecurity)) { - JSFContext.putGlobalVal(Constants.SECURITY_TOKEN_SERVER_FILE_URL, aTokenFile); - if (providerSecurityLevel != null) { - String aProviderSecurityLevel = environment.resolveRequiredPlaceholders(providerSecurityLevel); - JSFContext.putGlobalVal(Constants.SECURITY_IS_OPEN_PROVIDER, Integer.valueOf(aProviderSecurityLevel).toString()); - } - } - } - if (consumerOpenSecurity != null) { - String aConsumerOpenSecurity = environment.resolveRequiredPlaceholders(consumerOpenSecurity); - if (Boolean.valueOf(aConsumerOpenSecurity)) { - JSFContext.putGlobalVal(Constants.SECURITY_TOKEN_CLIENT_FILE_URL, aTokenFile); - } - } - } - - private String buildRegistryConfig(BeanDefinitionRegistry registry) { - if (registryIndex == null) { - return null; - } - BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(RegistryConfig.class); - builder.addPropertyValue("index", registryIndex); - - return BeanDefinitionReaderUtils.registerWithGeneratedName(builder.getBeanDefinition(), registry); - } - - private ManagedList buildManagedList(String beanName) { - ManagedList list = new ManagedList(); - list.add(new RuntimeBeanReference(beanName)); - return list; - } - - /** - * Obtain a lazily resolving resource proxy for the given name and type, - * delegating to {@link #getResource} on demand once a method call comes in. - * - * @param element the descriptor for the annotated field/method - * @param requestingBeanName the name of the requesting bean - * @return the resource object (never {@code null}) - * @see #getResource - * @see Lazy - * @since 4.2 - */ - private Object buildLazyResourceProxy(final JsfConsumerInjectedElement element, final String requestingBeanName) { - TargetSource ts = new TargetSource() { - @Override - public Class getTargetClass() { - return element.lookupType; - } - - @Override - public boolean isStatic() { - return false; - } - - @Override - public Object getTarget() { - return getResource(element, requestingBeanName); - } - - @Override - public void releaseTarget(Object target) { - } - }; - ProxyFactory pf = new ProxyFactory(); - pf.setTargetSource(ts); - if (element.lookupType.isInterface()) { - pf.addInterface(element.lookupType); - } - ClassLoader classLoader = (this.beanFactory != null ? - this.beanFactory.getBeanClassLoader() : null); - return pf.getProxy(classLoader); - } - - /** - * Obtain the resource object for the given name and type. - * - * @param element the descriptor for the annotated field/method - * @param requestingBeanName the name of the requesting bean - * @return the resource object (never {@code null}) - * @throws BeansException if we failed to obtain the target resource - */ - private Object getResource(JsfConsumerInjectedElement element, String requestingBeanName) throws BeansException { - if (this.beanFactory == null) { - throw new NoSuchBeanDefinitionException(element.lookupType, - "No resource factory configured - specify the 'resourceFactory' property"); - } - return autowireResource(this.beanFactory, element, requestingBeanName); - } - - /** - * Obtain a resource object for the given name and type through autowiring - * based on the given factory. - * - * @param factory the factory to autowire against - * @param element the descriptor for the annotated field/method - * @param requestingBeanName the name of the requesting bean - * @return the resource object (never {@code null}) - * @throws BeansException if we failed to obtain the target resource - */ - private Object autowireResource(BeanFactory factory, JsfConsumerInjectedElement element, String requestingBeanName) - throws BeansException { - - String name = element.name; - Object resource = factory.getBean(name, element.lookupType); - Set autowiredBeanNames = Collections.singleton(name); - - if (factory instanceof ConfigurableBeanFactory) { - ConfigurableBeanFactory beanFactory = (ConfigurableBeanFactory) factory; - for (String autowiredBeanName : autowiredBeanNames) { - if (beanFactory.containsBean(autowiredBeanName)) { - beanFactory.registerDependentBean(autowiredBeanName, requestingBeanName); - } - } - } - - return resource; - } - - - private void parseConsumer(Class clazz, BeanDefinitionRegistry registry, String registryConfig) { - List list = findJsfConsumerAnnotation(clazz); - if (list.isEmpty()) { - return; - } - - // 注册到配置中心去 - jsfConsumerInjectionMetadata.register(clazz, new InjectionMetadata(clazz, list)); - - // 循环生成BeanDefinition - for (InjectionMetadata.InjectedElement injectedElement : list) { - JsfConsumerInjectedElement element = ((JsfConsumerInjectedElement) injectedElement); - Class consumerClazz = element.lookupType; - JsfConsumer jsfConsumer = element.jsfConsumer; - - String consumerBeanName = NameUtils.checkConsumerBeanName(consumerClazz, jsfConsumer.alias()); - if (consumerBeanName != null) { - element.name = consumerBeanName; - } else { - consumerBeanName = NameUtils.buildConsumerBeanName(consumerClazz, jsfConsumer.alias(), jsfConsumer.id()); - - element.name = consumerBeanName; - - BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(ConsumerBean.class); - builder.addPropertyValue("interfaceId", consumerClazz.getName()); - builder.addPropertyValue("protocol", jsfConsumer.protocol()); - builder.addPropertyValue("alias", jsfConsumer.alias()); - builder.addPropertyValue("timeout", jsfConsumer.timeout()); - builder.addPropertyValue("serialization", jsfConsumer.serialization()); - builder.addPropertyValue("url", jsfConsumer.url().length() == 0 ? null : jsfConsumer.url()); - builder.addPropertyValue("retries", jsfConsumer.retries()); - builder.addPropertyValue("register", jsfConsumer.register()); - builder.addPropertyValue("subscribe", jsfConsumer.subscribe()); - - if (registryConfig != null) { - builder.addPropertyValue("registry", buildManagedList(registryConfig)); - } - - if (jsfConsumer.filterRef().length > 0) { - builder.addPropertyValue("filter", buildFilterList(registry, jsfConsumer.filterRef())); - } - - if (jsfConsumer.parameters().length > 0) { - ManagedMap parameters = new ManagedMap(); - for (JsfParameter jsfParameter : jsfConsumer.parameters()) { - String key = jsfParameter.key(); - if (jsfParameter.hide()) { - key = "." + key; - } - String value = jsfParameter.keyValue(); - parameters.put(key, new TypedStringValue(value, String.class)); - } - builder.addPropertyValue("parameters", parameters); - } - - registry.registerBeanDefinition(consumerBeanName, builder.getBeanDefinition()); - } - } - } - - private ManagedList buildFilterList(BeanDefinitionRegistry registry, String[] filterRef) { - ManagedList list = new ManagedList(); - for (String filter : filterRef) { - if (filter.startsWith("-")) { - list.add(new ExcludeFilter(filter)); - } else { - final BeanDefinition fd = registry.getBeanDefinition(filter); - if (!registry.containsBeanDefinition(filter)) { - throw new IllegalArgumentException(String.format("custom filter:%s not found in spring context", filter)); - } - if (fd.isSingleton()) { - logger.warn("If custom filter:\"{}\" used by multiple provider/consumer," + - " you need to set attribute scope=\"property\"!", filter); - } - - if (fd.getBeanClassName() != null) { - try { - if (!AbstractFilter.class.isAssignableFrom(ClassLoaderUtils.forName(fd.getBeanClassName()))) { - throw new IllegalArgumentException("Failed to set " - + filter + ", cause by type of \"" + filter + "\" is " + fd.getBeanClassName() - + ", not " + AbstractFilter.class.getName()); - } - } catch (ClassNotFoundException e) { - } - } - list.add(new RuntimeBeanReference(filter)); - } - } - return list; - } - - - private List findJsfConsumerAnnotation(Class clazz) { - List list = new ArrayList(); - // FIELD - list.addAll(findJsfConsumerAnnotationFromField(clazz)); - // SET - list.addAll(findJsfConsumerAnnotationFromSetMethod(clazz)); - return list; - } - - private List findJsfConsumerAnnotationFromSetMethod(final Class clazz) { - final List list = new ArrayList(); - - ReflectionUtils.doWithLocalMethods(clazz, new ReflectionUtils.MethodCallback() { - @Override - public void doWith(Method method) throws IllegalArgumentException, IllegalAccessException { - Method bridgedMethod = BridgeMethodResolver.findBridgedMethod(method); - if (!BridgeMethodResolver.isVisibilityBridgeMethodPair(method, bridgedMethod)) { - return; - } - if (method.equals(ClassUtils.getMostSpecificMethod(method, clazz))) { - if (bridgedMethod.isAnnotationPresent(JsfConsumer.class)) { - if (Modifier.isStatic(method.getModifiers())) { - throw new IllegalStateException("@JsfConsumer annotation is not supported on static methods"); - } - Class[] paramTypes = method.getParameterTypes(); - if (paramTypes.length != 1) { - throw new IllegalStateException("@JsfConsumer annotation requires a single-arg method: " + method); - } - PropertyDescriptor pd = BeanUtils.findPropertyForMethod(bridgedMethod, clazz); - list.add(new JsfConsumerInjectedElement(method, bridgedMethod, pd)); - } - } - } - }); - return list; - } - - private List findJsfConsumerAnnotationFromField(Class clazz) { - final List list = new ArrayList(); - - ReflectionUtils.doWithLocalFields(clazz, new ReflectionUtils.FieldCallback() { - @Override - public void doWith(Field field) throws IllegalArgumentException, IllegalAccessException { - if (field.isAnnotationPresent(JsfConsumer.class)) { - if (Modifier.isStatic(field.getModifiers())) { - throw new IllegalStateException("@JsfConsumer annotation is not supported on static fields"); - } - list.add(new JsfConsumerInjectedElement(field, field, null)); - } - } - }); - return list; - } - - private ManagedList registerServer(Class interfaceId, Server[] servers, BeanDefinitionRegistry registry) { - if (servers.length == 0) { - return null; - } - ManagedList list = new ManagedList(); - for (Server server : servers) { - for (int i = 0; i < server.serverNum(); i++) { - String serverName = NameUtils.buildServerBeanName(interfaceId); - BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(ServerBean.class); - builder.addPropertyValue("protocol", server.protocol()); - registry.registerBeanDefinition(serverName, builder.getBeanDefinition()); - list.add(new RuntimeBeanReference(serverName)); - } - } - return list; - } - - - private void parseProvider(Class implClazz, String beanDefinitionName, BeanDefinitionRegistry registry, String registryConfig) { - List list = findProviderAnnotation(implClazz); - if (list.isEmpty()) { - return; - } - for (JsfProvider jsfProvider : list) { - // 同一个 jsfProvider 里同一个interfaceClass 不同alias共享同一份 server - // 不同 jsfProvider， 不同 interfaceClass 都是独立的 server - Class[] interfaceIds = findInterfaceId(implClazz, jsfProvider); - for (Class interfaceId : interfaceIds) { - ManagedList serverList = registerServer(interfaceId, jsfProvider.server(), registry); - String[] aliasArr = jsfProvider.alias(); - for (String alias : aliasArr) { - String providerBeanName = NameUtils.buildProviderBeanName(interfaceId); - BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(ProviderBean.class); - builder.addPropertyReference("ref", beanDefinitionName); - builder.addPropertyValue("interfaceId", interfaceId.getName()); - builder.addPropertyValue("alias", alias); - builder.addPropertyValue("server", serverList); - builder.addPropertyValue("register", jsfProvider.register()); - builder.addPropertyValue("dynamic", jsfProvider.dynamic()); - builder.addPropertyValue("weight", jsfProvider.weight()); - - if (registryConfig != null) { - builder.addPropertyValue("registry", buildManagedList(registryConfig)); - } - - if (jsfProvider.filterRef().length > 0) { - builder.addPropertyValue("filter", buildFilterList(registry, jsfProvider.filterRef())); - } - - registry.registerBeanDefinition(providerBeanName, builder.getBeanDefinition()); - } - } - } - } - - - private Class[] findInterfaceId(Class implClazz, JsfProvider jsfProvider) { - if (jsfProvider.interfaceClass().length == 0) { - Class[] interfaces = implClazz.getInterfaces(); - if (interfaces == null || interfaces.length == 0) { - throw new IllegalStateException("Failed to export remote service class " + implClazz.getName() - + ", cause: the service class unimplemented any interfaces."); - } else if (interfaces.length == 1) { - return interfaces; - } else { - throw new IllegalStateException("Failed to export remote service class " + implClazz.getName() - + ", cause: the service class implemented more than one interfaces. " + - "you must defined at @JsfProvider.interfaceClass"); - } - } else { - return jsfProvider.interfaceClass(); - } - } - - - private List findProviderAnnotation(Class clazz) { - List list = new ArrayList(); - JsfProviders jsfProviders = clazz.getAnnotation(JsfProviders.class); - if (jsfProviders != null) { - list.addAll(Arrays.asList(jsfProviders.value())); - } - JsfProvider annotation = clazz.getAnnotation(JsfProvider.class); - if (annotation != null) { - list.add(annotation); - } - return list; - } - - @Override - public int getOrder() { - return Ordered.HIGHEST_PRECEDENCE; - } - - @Override - public void setBeanFactory(BeanFactory beanFactory) throws BeansException { - if (!(beanFactory instanceof ConfigurableListableBeanFactory)) { - throw new IllegalArgumentException( - "JsfConsumerAnnotationBeanPostProcessor requires a ConfigurableListableBeanFactory"); - } - this.beanFactory = (ConfigurableListableBeanFactory) beanFactory; - } - - @Override - public void setEnvironment(Environment environment) { - this.environment = environment; - } - - private class JsfConsumerInjectedElement extends InjectionMetadata.InjectedElement { - - private final Class lookupType; - - private final boolean lazyLookup; - - private final JsfConsumer jsfConsumer; - - private String name; - - JsfConsumerInjectedElement(Member member, AnnotatedElement ae, PropertyDescriptor pd) { - super(member, pd); - this.jsfConsumer = ae.getAnnotation(JsfConsumer.class); - this.lookupType = getResourceType(); - Lazy lazy = ae.getAnnotation(Lazy.class); - this.lazyLookup = (lazy != null && lazy.value()); - } - - @Override - protected Object getResourceToInject(Object target, String requestingBeanName) { - return (this.lazyLookup ? buildLazyResourceProxy(this, requestingBeanName) : - getResource(this, requestingBeanName)); - } - - } - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfComponentRegistrar.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfComponentRegistrar.java deleted file mode 100644 index 17f017c..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfComponentRegistrar.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.config; - -import com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.annotation.EnableJsf; -import org.springframework.beans.factory.config.BeanDefinition; -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.support.BeanDefinitionReaderUtils; -import org.springframework.beans.factory.support.BeanDefinitionRegistry; -import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; -import org.springframework.core.annotation.AnnotationAttributes; -import org.springframework.core.type.AnnotationMetadata; -import org.springframework.util.StringUtils; - -import static org.springframework.beans.factory.support.BeanDefinitionBuilder.rootBeanDefinition; - -/** - * Date: 2018-12-26 14:30 - * Email: wangyulie@jd.com - */ -public class JsfComponentRegistrar implements ImportBeanDefinitionRegistrar { - - @Override - public void registerBeanDefinitions(AnnotationMetadata metadata, BeanDefinitionRegistry registry) { - registerJsfBeanFactoryPostProcessor(metadata, registry); - registerJsfConsumerAnnotationBeanPostProcessor(registry); - } - - private void registerJsfConsumerAnnotationBeanPostProcessor(BeanDefinitionRegistry registry) { - beanRegistrar(registry, JsfConsumerAnnotationBeanPostProcessor.class, null); - } - - private void registerJsfBeanFactoryPostProcessor(AnnotationMetadata metadata, BeanDefinitionRegistry registry) { - - final Object[] constructorArg = getConstructorArg(metadata); - beanRegistrar(registry, JsfBeanFactoryPostProcessor.class, constructorArg); - - } - - private void beanRegistrar(BeanDefinitionRegistry registry, Class clazz, Object[] constructorArg) { - BeanDefinitionBuilder builder = rootBeanDefinition(clazz); - - if (constructorArg != null) { - for (Object arg : constructorArg) { - builder.addConstructorArgValue(arg); - } - } - - builder.setRole(BeanDefinition.ROLE_INFRASTRUCTURE); - - BeanDefinitionReaderUtils.registerWithGeneratedName(builder.getBeanDefinition(), registry); - } - - private Object[] getConstructorArg(AnnotationMetadata metadata) { - AnnotationAttributes attributes = AnnotationAttributes.fromMap(metadata.getAnnotationAttributes(EnableJsf.class.getName())); - - String index = emptyToNull(attributes.getString("index")); - - String isConsumerOpenSecurity = emptyToNull(attributes.getString("isConsumerOpenSecurity")); - - String isProviderOpenSecurity = emptyToNull(attributes.getString("isProviderOpenSecurity")); - String tokenFile = emptyToNull(attributes.getString("tokenFile")); - String providerSecurityLevel = emptyToNull(attributes.getString("providerSecurityLevel")); - - return new Object[]{index, isConsumerOpenSecurity, isProviderOpenSecurity, tokenFile, providerSecurityLevel}; - } - - - private static String emptyToNull(String source) { - if (StringUtils.isEmpty(source)) { - return null; - } - return source; - } - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerAnnotationBeanPostProcessor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerAnnotationBeanPostProcessor.java deleted file mode 100644 index c364c11..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerAnnotationBeanPostProcessor.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.config; - -import org.springframework.beans.BeansException; -import org.springframework.beans.PropertyValues; -import org.springframework.beans.factory.BeanCreationException; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.BeanFactoryAware; -import org.springframework.beans.factory.annotation.InjectionMetadata; -import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; -import org.springframework.beans.factory.config.InstantiationAwareBeanPostProcessorAdapter; -import org.springframework.beans.factory.support.MergedBeanDefinitionPostProcessor; -import org.springframework.beans.factory.support.RootBeanDefinition; -import org.springframework.core.PriorityOrdered; - -import java.beans.PropertyDescriptor; - -/** - * Date: 2016/11/10 10:30 - * Email: wangyulie@jd.com - */ -public class JsfConsumerAnnotationBeanPostProcessor extends InstantiationAwareBeanPostProcessorAdapter implements - MergedBeanDefinitionPostProcessor, PriorityOrdered, BeanFactoryAware { - - private ConfigurableListableBeanFactory beanFactory; - - @Override - public int getOrder() { - return HIGHEST_PRECEDENCE; - } - - @Override - public PropertyValues postProcessPropertyValues(PropertyValues pvs, PropertyDescriptor[] pds, Object bean, String beanName) throws BeansException { - InjectionMetadata metadata = findJsfConsumerMetadata(bean.getClass()); - if (metadata != null) { - try { - metadata.inject(bean, beanName, pvs); - } catch (Throwable ex) { - throw new BeanCreationException(beanName, "Injection of resource dependencies failed", ex); - } - } - return pvs; - } - - @Override - public void postProcessMergedBeanDefinition(RootBeanDefinition beanDefinition, Class beanType, String beanName) { - InjectionMetadata metadata = findJsfConsumerMetadata(beanType); - if (metadata != null) { - metadata.checkConfigMembers(beanDefinition); - } - } - - private InjectionMetadata findJsfConsumerMetadata(Class beanType) { - JsfConsumerInjectionMetadata injectionMetadata = ((JsfConsumerInjectionMetadata) beanFactory.getSingleton(JsfBeanFactoryPostProcessor.JSF_CONSUMER_INJECTION_METADATA_BEAN)); - return injectionMetadata.findMetadata(beanType); - } - - @Override - public void setBeanFactory(BeanFactory beanFactory) throws BeansException { - if (!(beanFactory instanceof ConfigurableListableBeanFactory)) { - throw new IllegalArgumentException( - "JsfConsumerAnnotationBeanPostProcessor requires a ConfigurableListableBeanFactory"); - } - this.beanFactory = (ConfigurableListableBeanFactory) beanFactory; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerInjectionMetadata.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerInjectionMetadata.java deleted file mode 100644 index 3bcf1aa..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/JsfConsumerInjectionMetadata.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.config; - -import org.springframework.beans.factory.annotation.InjectionMetadata; - -import java.util.HashMap; -import java.util.Map; - -/** - * Date: 2016/11/15 11:17 - * Email: wangyulie@jd.com - */ -class JsfConsumerInjectionMetadata { - - private Map clazzMap = new HashMap(); - - void register(Class clazz, InjectionMetadata injectionMetadata) { - this.clazzMap.put(clazz, injectionMetadata); - } - - - InjectionMetadata findMetadata(Class clazz) { - return clazzMap.get(clazz); - } - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/NameUtils.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/NameUtils.java deleted file mode 100644 index 60d773d..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/dongdong/jsf/config/NameUtils.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn.dongdong.jsf.config; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; - - -class NameUtils { - - private static AtomicLong providerId = new AtomicLong(); - - private static AtomicLong serverId = new AtomicLong(); - - private static AtomicLong consumerId = new AtomicLong(); - - private static Map> consumerBeanMap = new ConcurrentHashMap>(); - - static String checkConsumerBeanName(Class consumerClazz, String alias) { - Map map = consumerBeanMap.get(consumerClazz); - return map == null ? null : map.get(alias); - } - - static String buildConsumerBeanName(Class consumerClazz, String alias, String id) { - String beanName; - if (id == null || id.trim().isEmpty()) { - beanName = lowerFirst(consumerClazz.getSimpleName() + "_CONSUMER_" + consumerId.incrementAndGet()); - } else { - beanName = id; - } - addConsumerBeanName(consumerClazz, alias, beanName); - return beanName; - } - - private static void addConsumerBeanName(Class consumerClazz, String alias, String beanName) { - Map map = consumerBeanMap.get(consumerClazz); - if (map == null) { - map = new ConcurrentHashMap(); - consumerBeanMap.put(consumerClazz, map); - } - map.put(alias, beanName); - } - - static String buildServerBeanName(Class interfaceId) { - return lowerFirst(interfaceId.getSimpleName() + "_SERVER_" + serverId.incrementAndGet()); - } - - static String buildProviderBeanName(Class interfaceId) { - return lowerFirst(interfaceId.getSimpleName() + "_JSF_" + providerId.incrementAndGet()); - } - - private static String lowerFirst(String name) { - return name.substring(0, 1).toLowerCase() + name.substring(1); - } - -} diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 099165f..40115d8 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -27,9 +27,9 @@ spring : time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss datasource: - username: root - password: JRTEST - url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 + username: zxc + password: zxc4851225 + url: jdbc:mysql://rm-bp1f55o24g3im17w8yo.mysql.rds.aliyuncs.com:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT driver-class-name: com.mysql.cj.jdbc.Driver pagehelper: helperDialect: mysql diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 98fbd82..7497750 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -1,3 +1,8 @@ +-- !!! 注意设置sql model 否则可能sql报错 ！！！ +-- 查询你的sql_model参数：select @@global.sql_mode; 发现ONLY_FULL_GROUP_BY 则会导致报错 +-- 解决方式：set @@global.sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' +-- 详情查阅：https://www.cnblogs.com/hjhsblogs/p/11079356.html + DROP TABLE IF EXISTS `hk_change_log`; CREATE TABLE `hk_change_log` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -120,3 +125,19 @@ CREATE TABLE `hk_summary` ( ) ENGINE = InnoDB AUTO_INCREMENT = 18 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = '汇总表' ROW_FORMAT = Compact; + +/* 请确认以下SQL符合您的变更需求，务必确认无误后再提交执行 */ + +CREATE TABLE `biz_access_token` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `token` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'token', + `flag` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'flag', + `CREATED_BY` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '创建人', + `CREATED_TIME` datetime NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '创建时间', + `UPDATED_BY` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '修改人', + `UPDATED_TIME` datetime NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`) +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = '咚咚token表' ROW_FORMAT = Compact; + + + diff --git a/dashboard/src/main/resources/templates/admin/appcfg/edit.html b/dashboard/src/main/resources/templates/admin/appcfg/edit.html index fb768ca..e536499 100644 --- a/dashboard/src/main/resources/templates/admin/appcfg/edit.html +++ b/dashboard/src/main/resources/templates/admin/appcfg/edit.html @@ -38,15 +38,21 @@
- +
- +
- + +
+
+
+ +
+
diff --git a/dashboard/src/main/resources/templates/admin/appcfg/list.html b/dashboard/src/main/resources/templates/admin/appcfg/list.html index 19da89f..c26b982 100644 --- a/dashboard/src/main/resources/templates/admin/appcfg/list.html +++ b/dashboard/src/main/resources/templates/admin/appcfg/list.html @@ -103,7 +103,8 @@ { field: 'app', title: 'app',width:200,align:"center"}, { field: 'dataTtl', title: '数据保存时长(天)',align:"center"}, { field: 'warnPeriod', title: '警报周期(秒)',align:"center"}, - { field: 'warnThreshold', title: '警报阈值',align:"center"}, + { field: 'warnMax', title: '警报最大阈值',align:"center"}, + { field: 'warnMin', title: '警报最小阈值',align:"center"}, { field: 'modifier', title: '修改人',align:"center"}, { field: 'warn', title: '是否开启警报', formatter: function (value) { -- Gitee