From d0e1951a79f2717bc4559b1b5895506f079829a2 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 10 Jun 2020 11:33:12 +0800 Subject: [PATCH 001/156] 1 --- .../jd/platform/hotkey/worker/disruptor/ProducerFactory.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java index 150d609..8854627 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java @@ -25,6 +25,10 @@ public class ProducerFactory { //如果手工指定了线程数，就用手工指定的 if (InitConstant.threadCount != 0) { threadCount = InitConstant.threadCount; + } else { + if (threadCount >= 8) { + threadCount = threadCount / 2; + } } HotKeyEventConsumer[] array = new HotKeyEventConsumer[threadCount]; -- Gitee From 43c39ac67f9faef4182e379b4568e300ec8ef105 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 10 Jun 2020 13:34:11 +0800 Subject: [PATCH 002/156] =?UTF-8?q?=E8=B6=85=E6=97=B6=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/worker/disruptor/AbsWorkConsumer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java index a86a716..bca2e70 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java @@ -4,6 +4,8 @@ import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.BaseModel; import com.jd.platform.hotkey.worker.tool.InitConstant; import com.lmax.disruptor.WorkHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.LongAdder; @@ -20,6 +22,8 @@ public abstract class AbsWorkConsumer implements WorkHandle //过期的 public static final LongAdder expireTotalCount = new LongAdder(); + private Logger logger = LoggerFactory.getLogger(getClass()); + public AbsWorkConsumer(int hashIndex) { this.hashIndex = hashIndex; } @@ -33,6 +37,8 @@ public abstract class AbsWorkConsumer implements WorkHandle //5秒前的过时消息就不处理了 if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { expireTotalCount.increment(); + logger.warn("this key timeout : " + model.getKey() + " createAt : " + model.getCreateTime() + + " event at " + SystemClock.now()); if (InitConstant.openTimeOut) { return; } -- Gitee From 481b4318872b3325343fd26e7894d4529e270ce5 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 10 Jun 2020 15:24:46 +0800 Subject: [PATCH 003/156] 1 --- .../hotkey/worker/tool/SystemClock.java | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/tool/SystemClock.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/SystemClock.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/SystemClock.java deleted file mode 100644 index cf5758c..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/SystemClock.java +++ /dev/null @@ -1,49 +0,0 @@ -//package com.jd.platform.hotkey.worker.tool; -// -//import java.util.concurrent.Executors; -//import java.util.concurrent.ScheduledExecutorService; -//import java.util.concurrent.TimeUnit; -//import java.util.concurrent.atomic.AtomicLong; -// -///** -// * 用于解决高并发下System.currentTimeMillis卡顿 -// * @author lry -// */ -//public class SystemClock { -// -// private final int period; -// -// private final AtomicLong now; -// -// private static final SystemClock INSTANCE = new SystemClock(1); -// -// private SystemClock(int period) { -// this.period = period; -// this.now = new AtomicLong(System.currentTimeMillis()); -// scheduleClockUpdating(); -// } -// -// private static SystemClock instance() { -// return INSTANCE; -// } -// -// private void scheduleClockUpdating() { -// ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(runnable -> { -// Thread thread = new Thread(runnable, "System Clock"); -// thread.setDaemon(true); -// return thread; -// }); -// scheduler.scheduleAtFixedRate(() -> now.set(System.currentTimeMillis()), period, period, TimeUnit.MILLISECONDS); -// } -// -// private long currentTimeMillis() { -// return now.get(); -// } -// -// /** -// * 用来替换原来的System.currentTimeMillis() -// */ -// public static long now() { -// return instance().currentTimeMillis(); -// } -//} \ No newline at end of file -- Gitee From 5bdf545de3cec533eae7a163345662525e0f67c6 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 10 Jun 2020 19:14:46 +0800 Subject: [PATCH 004/156] =?UTF-8?q?=E5=8A=A0=E4=B8=8A=E5=BC=BA=E6=9C=89?= =?UTF-8?q?=E5=8A=9B=E7=9A=84=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../worker/keylistener/KeyListener.java | 16 +++- .../hotkey/worker/tool}/SafeExecute.java | 1 + worker/src/test/java/TestBlockQueue.java | 88 +++++++++++++++++++ 3 files changed, 102 insertions(+), 3 deletions(-) rename worker/src/main/java/{ => com/jd/platform/hotkey/worker/tool}/SafeExecute.java (93%) create mode 100644 worker/src/test/java/TestBlockQueue.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java index 67efff4..15592cc 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java @@ -36,9 +36,6 @@ public class KeyListener implements IKeyListener { private static final String NEW_KEY_EVENT = "new key created event, key : "; private static final String DELETE_KEY_EVENT = "key delete event key : "; - //兴许将来用的上 - private static final Object LOCK = new Object(); - @Override public void newKey(HotKeyModel hotKeyModel, KeyEventOriginal original) { //cache里的key @@ -49,6 +46,19 @@ public class KeyListener implements IKeyListener { return; } + //********** watch here ************// + //该方法会被InitConstant.threadCount个线程同时调用，存在多线程问题 + //下面的那句addCount是加了锁的，代表给Key累加数量时是原子性的，不会发生多加、少加的情况，到了设定的阈值一定会hot + //譬如阈值是2，如果多个线程累加，在没hot前，hot的状态肯定是对的，譬如thread1 加1，thread2加1，那么thread2会hot返回true，开启推送 + //但是极端情况下，譬如阈值是10，当前是9，thread1走到这里时，加1，返回true，thread2也走到这里，加1，此时是11，返回true，问题来了 + //该key会走下面的else两次，也就是2次推送。 + //所以出现问题的原因是hotCache.getIfPresent(key)这一句在并发情况下，没return掉，放了两个key+1到addCount这一步时，会有问题 + //测试代码在TestBlockQueue类，直接运行可以看到会同时hot + + //那么该问题用解决吗，NO，不需要解决，1 首先要发生的条件极其苛刻，很难触发，以京东这样高的并发量，线上我也没见过触发连续2次推送同一个key的 + //2 即便触发了，后果也是可以接受的，2次推送而已，毫无影响，客户端无感知。但是如果非要解决，就要对slidingWindow实例加锁了，必然有一些开销 + + //所以只要保证key数量不多计算就可以，少计算了没事。因为热key必然频率高，漏计几次没事。但非热key，多计算了，被干成了热key就不对了 SlidingWindow slidingWindow = checkWindow(hotKeyModel, key); //看看hot没 boolean hot = slidingWindow.addCount(hotKeyModel.getCount()); diff --git a/worker/src/main/java/SafeExecute.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/SafeExecute.java similarity index 93% rename from worker/src/main/java/SafeExecute.java rename to worker/src/main/java/com/jd/platform/hotkey/worker/tool/SafeExecute.java index 179fbfa..12e52aa 100644 --- a/worker/src/main/java/SafeExecute.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/SafeExecute.java @@ -1,3 +1,4 @@ +package com.jd.platform.hotkey.worker.tool; import java.util.function.Consumer; diff --git a/worker/src/test/java/TestBlockQueue.java b/worker/src/test/java/TestBlockQueue.java new file mode 100644 index 0000000..7dc0ff6 --- /dev/null +++ b/worker/src/test/java/TestBlockQueue.java @@ -0,0 +1,88 @@ +//package com.jd.platform.hotkey.worker.tool; +// +//import com.github.benmanes.caffeine.cache.Cache; +//import com.github.benmanes.caffeine.cache.Caffeine; +// +//import java.util.concurrent.*; +// +///** +// * @author wuweifeng +// * @version 1.0 +// * @date 2020-06-10 +// */ +//public class TestBlockQueue { +// public static BlockingQueue QUEUE = new LinkedBlockingQueue<>(2000000); +// private static Cache cache = buildAllKeyCache(); +// private static Cache hotCache = buildAllKeyCache(); +// private static ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); +// +// public static void main(String[] args) throws InterruptedException { +// +// for (int i = 0; i < 8; i++) { +// TestBlockQueue testBlockQueue = new TestBlockQueue(); +// threadPoolExecutor.submit(testBlockQueue::beginConsume); +// } +// +// Thread.sleep(1000); +// while (true) { +// QUEUE.put("abc"); +// } +// +// } +// +// public void beginConsume() { +// while (true) { +// try { +// String key = QUEUE.take(); +// deal(key); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// +// } +// } +// +// public void deal(String key) { +// Object o = hotCache.getIfPresent(key); +// if (o != null) { +// return; +// } +// SlidingWindow slidingWindow = checkWindow(key); +// //看看hot没 +// boolean hot = slidingWindow.addCount(1); +// +// if (!hot) { +// //如果没hot，重新put，cache会自动刷新过期时间 +// cache.put(key, slidingWindow); +// } else { +// hotCache.put(key, 1); +// cache.invalidate(key); +// System.out.println("key Hot " + System.currentTimeMillis() + " " + slidingWindow); +// } +// } +// +// private SlidingWindow checkWindow(String key) { +// //取该key的滑窗 +// SlidingWindow slidingWindow = (SlidingWindow) cache.getIfPresent(key); +// //考虑在某个APP的rule变化后，清空该APP所有key +// if (slidingWindow == null) { +// //是个新key，获取它的规则 +// slidingWindow = new SlidingWindow(2, 20); +// +// } +// return slidingWindow; +// } +// +// /** +// * 构建所有来的要缓存的key cache +// */ +// public static Cache buildAllKeyCache() { +// //老版本jdk1.8.0_20之前，caffeine默认的forkJoinPool在及其密集的淘汰过期时，会有forkJoinPool报错。建议用新版jdk +// return Caffeine.newBuilder() +// .initialCapacity(8192)//初始大小 +// .maximumSize(5000000)//最大数量 +// .expireAfterWrite(5, TimeUnit.SECONDS)//过期时间 +// .softValues() +// .build(); +// } +//} -- Gitee From afa429d5830f0e3323ed3de4f5dd9925c41bc69a Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 11 Jun 2020 00:13:55 +0800 Subject: [PATCH 005/156] =?UTF-8?q?1=E5=88=A0=E9=99=A4=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E7=83=AD=E7=82=B9val=E5=88=97=EF=BC=8C2=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E6=9C=80=E7=83=ADkey=E5=BC=B9=E6=A1=86=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=EF=BC=8C3=E6=9C=80=E7=83=ADkey=E6=97=B6=E9=97=B4=E7=A9=BA?= =?UTF-8?q?=E9=97=B4=E9=BB=98=E8=AE=A4=E5=80=BC=E4=BC=98=E5=8C=96-5?= =?UTF-8?q?=E5=B0=8F=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/templates/admin/key/listmaxhot.html | 9 +++++++-- .../main/resources/templates/admin/key/listtimely.html | 3 +-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html index c927a58..c4ddcec 100644 --- a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html +++ b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html @@ -50,9 +50,9 @@
- + - +
@@ -133,6 +133,11 @@ $(function(){ forceParse: 0, minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 }); + + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); }) function exportHot() { diff --git a/dashboard/src/main/resources/templates/admin/key/listtimely.html b/dashboard/src/main/resources/templates/admin/key/listtimely.html index fb9a722..2d76199 100644 --- a/dashboard/src/main/resources/templates/admin/key/listtimely.html +++ b/dashboard/src/main/resources/templates/admin/key/listtimely.html @@ -163,7 +163,6 @@ var options = { { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, { field: 'duration', title: '缓存时间'}, - { field: 'val', title: 'VAL'}, { field: 'createTime', title: '创建时间', formatter:function (val,row,index) { return changeDateFormat(val); @@ -218,7 +217,7 @@ $(function(){ }) function maxHotKeyView(){ - $.modal.open("最热key" , "/key/viewMaxHot",500,500); + $.modal.open("最热key" , "/key/viewMaxHot",650,520); } -- Gitee From 14ae19123ce1a1e8b7beb758f5829d17c45fc345 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 11 Jun 2020 09:59:35 +0800 Subject: [PATCH 006/156] update README.md. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb0af04..7d10476 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # hotkey -正在京东APP后台灰度内测中，小幅度压测过，3台worker每秒接收16万个key，持续10几个小时，大概处理了60多亿个key，平稳无异常，cpu15%无大波动。 +正在京东APP后台灰度了几千台机器，等618海量并发检验后再推广使用，暂时不要下载使用它。 -单机8c8g压测数据在16万QPS左右，cpu70%，16c32g单机20万QPS，cpu40%，测试详情可去我CSDN博客看详情。 +该框架历经多次压测，8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒20万，实际压测达到30万以上，CPU平稳支撑，框架无异常。测试详情可去我CSDN博客查看。 #### 介绍 对任意突发性的无法预先感知的热点请求，包括并不限于热点数据（如突发大量请求同一个商品）、热用户（如爬虫、刷子）、热接口（突发海量请求同一个接口）等，进行毫秒级精准探测到。 -- Gitee From 6c9b13555f2f9fab2a5c995e43cfcff2b25cbf88 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 11 Jun 2020 10:32:18 +0800 Subject: [PATCH 007/156] 1 --- .../java/com/jd/platform/sample/Cache.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/sample/src/main/java/com/jd/platform/sample/Cache.java b/sample/src/main/java/com/jd/platform/sample/Cache.java index b9f5914..82887a9 100644 --- a/sample/src/main/java/com/jd/platform/sample/Cache.java +++ b/sample/src/main/java/com/jd/platform/sample/Cache.java @@ -20,14 +20,54 @@ public class Cache { return redisTemplate.opsForValue().get(key); } + //最佳实践： + // + //1 判断用户是否是刷子 + // + // if (JdHotKeyStore.isHotKey(“pin__” + thePin)) { + // //限流他，do your job + // } + //2 判断商品id是否是热点 + // + // + // + // Object skuInfo = JdHotKeyStore.getValue("skuId__" + skuId); + // if(skuInfo == null) { + // + // JdHotKeyStore.smartSet("skuId__" + skuId, theSkuInfo); + // } else { + // + // //使用缓存好的value即可 + // + // } + // + // 或者这样： + // + // + // + // if (JdHotKeyStore.isHotKey(key)) { + // //注意是get，不是getValue。getValue会获取并上报，get是纯粹的本地获取 + // + // Object skuInfo = JdHotKeyStore.get("skuId__" + skuId); + // if(skuInfo == null) { + // + // JdHotKeyStore.smartSet("skuId__" + skuId, theSkuInfo); + // } else { + // + // //使用缓存好的value即可 + // + // } + // + // } + public String get(String key) { //如果已经缓存过了 - if (JdHotKeyStore.isValueCached(key)) { + if (JdHotKeyStore.getValue(key) != null) { System.out.println("1"); return JdHotKeyStore.getValue(key).toString(); } else { String value = getFromRedis(key); - JdHotKeyStore.setValue(key, value); + JdHotKeyStore.smartSet(key, value); return value; } } -- Gitee From 83eb71d819794b13e01c021953417c38773fab04 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 11 Jun 2020 15:11:07 +0800 Subject: [PATCH 008/156] update README.md. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7d10476..0e5702e 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ 6.还有很多 -本人不会前端，只能靠外援来做页面。有兴趣参与开发的联系我qq 272551766。 该开源项目战略意义重大，要经历百万级并发，参与京东开源中间件项目建设，一直在等你。 -- Gitee From 9dc7ee21f3f3e2f39193f540d56dd46ea3c95810 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 11 Jun 2020 15:24:01 +0800 Subject: [PATCH 009/156] update README.md. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 0e5702e..3f380c3 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,12 @@ 该开源项目战略意义重大，要经历百万级并发，参与京东开源中间件项目建设，一直在等你。 +#### 强悍的性能表现 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152200_331d5991_303698.png "屏幕截图.png") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152336_78597937_303698.png "屏幕截图.png") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152249_4ac01178_303698.png "屏幕截图.png") + + #### 安装教程 1. xxxx -- Gitee From f51959f15e434d2f747ede5b3626bb5bcf21643a Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 11 Jun 2020 15:28:53 +0800 Subject: [PATCH 010/156] update README.md. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3f380c3..8b05f5d 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,10 @@ 该开源项目战略意义重大，要经历百万级并发，参与京东开源中间件项目建设，一直在等你。 -#### 强悍的性能表现 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152200_331d5991_303698.png "屏幕截图.png") +#### worker端强悍的性能表现 +每10秒打印一行，totalDealCount代表处理过的key总量，可以看到每10秒处理量在270万-310万之间，对应每秒30万左右QPS。 + +仅需要很少的机器，即可完成海量key的实时探测计算推送任务。比扩容redis集群规模成本低太多了。 ![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152336_78597937_303698.png "屏幕截图.png") ![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152249_4ac01178_303698.png "屏幕截图.png") -- Gitee From e9bc9b824eb620fd8e01ea13242e7a9b8f3f6ac6 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 11 Jun 2020 16:50:53 +0800 Subject: [PATCH 011/156] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b05f5d..c557dae 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ 正在京东APP后台灰度了几千台机器，等618海量并发检验后再推广使用，暂时不要下载使用它。 -该框架历经多次压测，8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒20万，实际压测达到30万以上，CPU平稳支撑，框架无异常。测试详情可去我CSDN博客查看。 +该框架历经多次压测，8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒20万，实际压测达到30万以上，CPU平稳支撑，框架无异常。测试详情可去我[CSDN博客](https://blog.csdn.net/tianyaleixiaowu)查看。 #### 介绍 对任意突发性的无法预先感知的热点请求，包括并不限于热点数据（如突发大量请求同一个商品）、热用户（如爬虫、刷子）、热接口（突发海量请求同一个接口）等，进行毫秒级精准探测到。 -- Gitee From 403bce89eee08511a7da68805a0b431bf8b00a23 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 11 Jun 2020 18:40:00 +0800 Subject: [PATCH 012/156] =?UTF-8?q?=E9=A6=96=E9=A1=B5=E5=A4=A7=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/domain/req/ChartReq.java | 4 +- .../common/domain/req/SearchReq.java | 12 +- .../dashboard/common/monitor/DataHandler.java | 4 +- .../dashboard/common/monitor/EtcdMonitor.java | 2 +- .../dashboard/controller/RuleController.java | 9 + .../hotkey/dashboard/service/RuleService.java | 4 + .../service/impl/KeyServiceImpl.java | 190 ++++++++++-------- .../service/impl/RuleServiceImpl.java | 20 +- .../hotkey/dashboard/util/CommonUtil.java | 102 ++++++++-- .../hotkey/dashboard/util/DateUtil.java | 57 ++++-- .../admin/assets/js/bootstrap/js/base_list.js | 4 +- .../main/resources/templates/admin/main.html | 164 ++++++++++++--- .../resources/templates/admin/mainbak.html | 106 ++++++++++ 13 files changed, 530 insertions(+), 148 deletions(-) create mode 100644 dashboard/src/main/resources/templates/admin/mainbak.html diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java index 6b0b4f1..5154e43 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java @@ -28,8 +28,8 @@ public class ChartReq implements Serializable { } public ChartReq(LocalDateTime st, LocalDateTime et, Integer limit) { - this.startTime = DateUtil.localDateTimeToDate(st); - this.endTime = DateUtil.localDateTimeToDate(et); + this.startTime = DateUtil.ldtToDate(st); + this.endTime = DateUtil.ldtToDate(et); this.limit = limit; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index 4e7a462..57b7f0a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -6,6 +6,7 @@ import com.jd.platform.hotkey.dashboard.util.DateUtil; import java.io.Serializable; import java.time.LocalDateTime; import java.util.Date; +import java.util.List; /** * @ProjectName: hotkey @@ -30,6 +31,8 @@ public class SearchReq implements Serializable { private String key; + private String rules; + public SearchReq() { } @@ -81,9 +84,16 @@ public class SearchReq implements Serializable { this.key = key; } + public String getRules() { + return rules; + } + + public void setRules(String rules) { + this.rules = rules; + } public SearchReq(LocalDateTime st) { - this.startTime = DateUtil.localDateTimeToDate(st); + this.startTime = DateUtil.ldtToDate(st); this.endTime = new Date(); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 5ac7073..4308467 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -136,7 +136,7 @@ public class DataHandler { try { // 每小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtil.localDateTimeToDate(now); + Date nowTime = DateUtil.ldtToDate(now); int day = DateUtil.nowDay(now); int hour = DateUtil.nowHour(now); @@ -165,7 +165,7 @@ public class DataHandler { try { // 每分钟小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtil.localDateTimeToDate(now); + Date nowTime = DateUtil.ldtToDate(now); int day = DateUtil.nowDay(now); int hour = DateUtil.nowHour(now); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 26e5d39..680d6d5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -36,7 +36,7 @@ import java.util.concurrent.CompletableFuture; * @Date: 2020/4/18 18:29 */ @SuppressWarnings("ALL") -@Component +//@Component public class EtcdMonitor { private static Logger log = LoggerFactory.getLogger(EtcdMonitor.class); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index ba80e85..bd5a66a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -14,6 +14,7 @@ import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; +import java.util.List; @Controller @@ -107,5 +108,13 @@ public class RuleController extends BaseController { return "admin/rule/view"; } + + @PostMapping("/listRules") + @ResponseBody + public List rules(){ + List info = ruleService.listRules(null); + return info; + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java index 8c6ca10..ebd974e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java @@ -6,6 +6,8 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRule; import com.jd.platform.hotkey.dashboard.model.Rules; +import java.util.List; + /** * @ProjectName: hotkey * @ClassName: RuleService @@ -27,4 +29,6 @@ public interface RuleService { PageInfo pageKeyRule(PageReq page, SearchReq param); int save(Rules rules); + + List listRules(String app); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 9201180..2bf3a06 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.service.impl; +import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; @@ -11,8 +12,6 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; -import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; -import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.ReceiveCountMapper; @@ -54,82 +53,45 @@ public class KeyServiceImpl implements KeyService { @Resource private StatisticsMapper statisticsMapper; - - public HotKeyLineChartVo ruleLineChart2(SearchReq req) { - int type = req.getType(); - if(req.getEndTime() == null){ - req.setEndTime(new Date()); - } - switch (type){ - case 1: - req.setStartTime(DateUtil.preMinus(30)); - List list = statisticsList(); - System.out.println("30 min"); - break; - case 2: - req.setStartTime(DateUtil.preDays(1)); - System.out.println("24 hours"); - break; - case 3: - req.setStartTime(DateUtil.preDays(7)); - System.out.println("7 days"); - break; - default: - System.out.println("============="); - } - return null; + public static void main(String[] args) { + HotKeyLineChartVo result = new KeyServiceImpl().ruleLineChart(new SearchReq()); + System.out.println(result); } @Override public HotKeyLineChartVo ruleLineChart(SearchReq req) { - int type = req.getType(); + System.out.println("req--> "+JSON.toJSONString(req)); + int type = 1; if(req.getEndTime() == null){ req.setEndTime(new Date()); } switch (type){ case 1: req.setStartTime(DateUtil.preMinus(30)); - List list = statisticsList(); - Map map = new HashMap<>(10); - Map> listMap = list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); - for (Map.Entry> m : listMap.entrySet()) { - int start = 1; - map.put(m.getKey(),new int[30]); - int[] data = map.get(m.getKey()); - int tmp = 0; - for (int i = 0; i < 30; i++) { - Statistics st; - try { - st = m.getValue().get(tmp); - if(String.valueOf(start).endsWith("24")){ start = start + 77; } - if(start != st.getHours()){ - data[i] = 0; - }else{ - tmp ++; - data[i] = st.getCount(); - } - start++; - }catch (Exception e){ - data[i] = 0; - } - } - } - - System.out.println("30 min"); - break; + LocalDateTime startTime = DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS); + List list = statisticsList(req.getRules()); + HotKeyLineChartVo vo = CommonUtil.assembleData(list, startTime, 30,1); + System.out.println(JSON.toJSONString(vo)); + return vo; case 2: req.setStartTime(DateUtil.preDays(1)); - System.out.println("24 hours"); - break; + LocalDateTime startTime2 = DateUtil.strToLdt("20063022",DateUtil.PATTERN_HOUR); + List list2 = statisticsList1(); + HotKeyLineChartVo vo2 = CommonUtil.assembleData(list2, startTime2, 24,2); + System.out.println(JSON.toJSONString(vo2)); + return vo2; case 3: req.setStartTime(DateUtil.preDays(7)); - System.out.println("7 days"); - break; + List list3 = statisticsListDay(); + System.out.println(JSON.toJSONString(list3)); + LocalDateTime startTime3 = DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR); + HotKeyLineChartVo vo3 = CommonUtil.assembleData(list3, startTime3, 28,2); + return vo3; default: System.out.println("============="); } - return null; + return new HotKeyLineChartVo(null,null); } @@ -317,32 +279,75 @@ public class KeyServiceImpl implements KeyService { } - private List statisticsList(){ + private static List statisticsList(String strings){ Random rd = new Random(); List list = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - Statistics st = new Statistics(); - st.setApp("rule1"); - st.setKeyName("key1"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - st.setMinutes(2006052140+i); - if(String.valueOf(st.getMinutes()).endsWith("60")){ - st.setMinutes(st.getMinutes()+1); + + if(strings.contains("428k2")){ + return list; + } + System.out.println("strings--> "+JSON.toJSONString(strings)); + + if(strings.contains("k21") && !strings.contains("k22") ){ + System.out.println("strings.contains(21)--> "+strings.contains("k21")); + + for (int i = 0; i < 30 ; i++) { + if(i == 12 || i ==13){ + }else{ + Statistics st = new Statistics(); + st.setApp("rule1"); + st.setKeyName("key1"); + st.setCount(rd.nextInt(100)); + st.setBizType(1); + // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); + // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); + int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); + // System.out.println(time); + st.setMinutes(time); + list.add(st); + } } - list.add(st); + return list; } - List list2 = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - Statistics st2 = new Statistics(); - st2.setApp("rule2"); - st2.setKeyName("key2"); - st2.setCount(rd.nextInt(100)); - st2.setBizType(1); - st2.setMinutes(2006052140+i); - list2.add(st2); + + + if(strings.contains("k21") && strings.contains("k22")){ + System.out.println("strings.contains(21 + 22)--> "+strings.contains("k21")); + + for (int i = 0; i < 30 ; i++) { + if(i == 12 || i ==13){ + }else{ + Statistics st = new Statistics(); + st.setApp("rule1"); + st.setKeyName("key1"); + st.setCount(rd.nextInt(100)); + st.setBizType(1); + // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); + // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); + int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); + // System.out.println(time); + st.setMinutes(time); + list.add(st); + } + } + List list2 = new ArrayList<>(); + for (int i = 0; i < 30 ; i++) { + if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ + + }else{ + Statistics st2 = new Statistics(); + st2.setApp("rule2"); + st2.setKeyName("key2"); + st2.setCount(rd.nextInt(100)); + st2.setBizType(1); + st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); + list2.add(st2); + } + } + list.addAll(list2); + return list; } - list.addAll(list2); + return list; } @@ -353,14 +358,33 @@ public class KeyServiceImpl implements KeyService { Statistics st = new Statistics(); st.setApp("rule1"); st.setKeyName("key1"); - st.setCount(rd.nextInt(100)); + st.setCount(rd.nextInt(1000)); st.setBizType(1); - st.setHours(20060500+i); + st.setHours(DateUtil.reviseTime(DateUtil.strToLdt("20063022",DateUtil.PATTERN_HOUR),i,2)); + list.add(st); } return list; } + private List statisticsListDay(){ + Random rd = new Random(); + List list = new ArrayList<>(); + for (int j = 0; j < 7; j++) { + for (int i = 0; i < 24 ; i++) { + Statistics st = new Statistics(); + st.setApp("rule"); + st.setKeyName("key"); + st.setCount(rd.nextInt(5000)); + st.setBizType(1); + st.setDays(DateUtil.reviseTime(DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR),j,3)); + st.setHours(DateUtil.reviseTime(DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR).plusDays(j), i,2)); + list.add(st); + } + } + return list; + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 6d5a4fd..8688fba 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -100,7 +100,6 @@ public class RuleServiceImpl implements RuleService { List keyValues = configCenter.getPrefix(prefix); List rules = new ArrayList<>(); for (KeyValue kv : keyValues) { - String v = kv.getValue().toStringUtf8(); if(StringUtil.isEmpty(v)){ continue; @@ -129,4 +128,23 @@ public class RuleServiceImpl implements RuleService { configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); return 1; } + + @Override + public List listRules(String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); + List rules = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + continue; + } + String key = kv.getKey().toStringUtf8(); + String appKey = key.replace(ConfigConstant.rulePath,""); + List rs = JSON.parseArray(v, Rule.class); + for (Rule r : rs) { + rules.add(r.getKey()); + } + } + return rules; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 42c4667..75e3809 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -1,32 +1,40 @@ package com.jd.platform.hotkey.dashboard.util; +import com.alibaba.fastjson.JSON; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; +import com.jd.platform.hotkey.dashboard.model.Statistics; + import java.io.UnsupportedEncodingException; -import java.util.Base64; +import java.time.LocalDateTime; +import java.util.*; +import java.util.stream.Collectors; public class CommonUtil { /** * 获取父级Key + * * @param key key * @return string */ - public static String parentK(String key){ - if(key.endsWith("/")){ - key = key.substring(0,key.length()-1); + public static String parentK(String key) { + if (key.endsWith("/")) { + key = key.substring(0, key.length() - 1); } int index = key.lastIndexOf("/"); - return key.substring(0,index+1); + return key.substring(0, index + 1); } /** * 获取AppName + * * @param k k * @return str */ - public static String appName(String k){ + public static String appName(String k) { String[] arr = k.split("/"); - for (int i = 0; i < arr.length ; i++) { - if(i == 3){ + for (int i = 0; i < arr.length; i++) { + if (i == 3) { return arr[i]; } } @@ -34,14 +42,12 @@ public class CommonUtil { } - - public static String keyName(String k){ + public static String keyName(String k) { int index = k.lastIndexOf("/"); - return k.substring(index+1); + return k.substring(index + 1); } - public static String encoder(String text) { try { return Base64.getEncoder().encodeToString(text.getBytes("utf-8")); @@ -53,9 +59,9 @@ public class CommonUtil { public static String decoder(String text) { - byte[] bytes=Base64.getDecoder().decode(text); + byte[] bytes = Base64.getDecoder().decode(text); try { - return new String(bytes,"utf-8"); + return new String(bytes, "utf-8"); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } @@ -63,4 +69,72 @@ public class CommonUtil { } + /** + * 拼装数据 + * @param list list-data + * @param startTime 开始时间 + * @param size 格子数 + * @param type 类型 1分钟 2小时 + * @return vo + */ + public static HotKeyLineChartVo assembleData(List list, LocalDateTime startTime, int size, int type) { + Set set = new TreeSet<>(); + boolean isHour = type == 1; + boolean sevenDays = size == 28; + String suffix = isHour ? "60" : "24"; + String pattern = isHour ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; + Map map = new HashMap<>(10); + Map> listMap = listGroup(list, sevenDays); + for (Map.Entry> m : listMap.entrySet()) { + int start = DateUtil.reviseTime(startTime, 0, type); + map.put(m.getKey(), new int[size]); + int[] data = map.get(m.getKey()); + int tmp = 0; + for (int i = 0; i < size; i++) { + if (String.valueOf(start).endsWith(suffix)) { + LocalDateTime tmpTime = DateUtil.strToLdt((start - 1) + "", pattern); + start = DateUtil.reviseTime(tmpTime, 1, type); + } + set.add(DateUtil.strToLdt(start + "", pattern).toString().replace("T", " ")); + Statistics st = m.getValue().get(tmp); + int val = isHour ? st.getMinutes() : st.getHours(); + if (start != val) { + data[i] = 0; + } else { + tmp++; + data[i] = st.getCount(); + } + if (sevenDays) { + start += 6; + } else { + start++; + } + + } + } + System.out.println("-------"); + + System.out.println(JSON.toJSONString(new HotKeyLineChartVo(new ArrayList<>(set), map))); + return new HotKeyLineChartVo(new ArrayList<>(set), map); + } + + + /** + * 分组 + * @param list list + * @param sevenDays 是否7天 + * @return map + */ + private static Map> listGroup(List list, boolean sevenDays){ + return list.stream().filter(x -> { + if(sevenDays){ + String hs = String.valueOf(x.getHours()); + return Integer.parseInt(hs.substring(hs.length() - 2)) % 6 == 0; + }else{ + return true; + } + }).collect(Collectors.groupingBy(Statistics::getKeyName)); + } + + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index 8864fe6..401e6fe 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -2,6 +2,7 @@ package com.jd.platform.hotkey.dashboard.util; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -13,13 +14,20 @@ import java.util.Date; public class DateUtil { - private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + public static final String PATTERN_MINUS="yyMMddHHmm"; + + public static final String PATTERN_HOUR="yyMMddHH"; + + public static final String PATTERN_DAY="yyMMdd"; - public static final DateTimeFormatter TIME_FORMAT1 = DateTimeFormatter.ofPattern("yyMMddHHmm"); + private static final DateTimeFormatter FORMAT_MINUS = DateTimeFormatter.ofPattern(PATTERN_MINUS); - public static final DateTimeFormatter TIME_FORMAT2 = DateTimeFormatter.ofPattern("yyMMddHH"); + private static final DateTimeFormatter FORMAT_HOUR = DateTimeFormatter.ofPattern(PATTERN_HOUR); + + private static final DateTimeFormatter FORMAT_DAY = DateTimeFormatter.ofPattern(PATTERN_DAY); + + private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - public static final DateTimeFormatter TIME_FORMAT3 = DateTimeFormatter.ofPattern("yyMMddHH"); public static Date strToDate(String str){ try { @@ -30,37 +38,62 @@ public class DateUtil { return null; } + public static LocalDateTime strToLdt(String str, String pattern){ + return LocalDateTime.parse(str, DateTimeFormatter.ofPattern(pattern)); + } + - public static Date localDateTimeToDate(LocalDateTime localDateTime){ + public static int reviseTime(LocalDateTime time, int diff, int type){ + switch (type){ + case 1: + return Integer.parseInt(FORMAT_MINUS.format(time.plusMinutes(diff))); + case 2: + return Integer.parseInt(FORMAT_HOUR.format(time.plusHours(diff))); + case 3: + return Integer.parseInt(FORMAT_DAY.format(time.plusDays(diff))); + default: + } + return 0; + } + + + + + public static Date ldtToDate(LocalDateTime localDateTime){ return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); } + public static LocalDateTime dateToLdt(Date date){ + return LocalDateTime.ofInstant( date.toInstant(), ZoneId.systemDefault()); + } + + public static int nowMinus(LocalDateTime now){ - return Integer.parseInt(now.format(TIME_FORMAT1)) ; + return Integer.parseInt(now.format(FORMAT_MINUS)) ; } public static int nowHour(LocalDateTime now){ - return Integer.parseInt(now.format(TIME_FORMAT2)); + return Integer.parseInt(now.format(FORMAT_HOUR)); } - public static int nowDay(LocalDateTime now){ return Integer.parseInt(now.format(TIME_FORMAT3));} + public static int nowDay(LocalDateTime now){ return Integer.parseInt(now.format(FORMAT_DAY));} public static int preHours(LocalDateTime now, int hours){ - return Integer.parseInt(now.minusHours(hours).format(TIME_FORMAT2)); + return Integer.parseInt(now.minusHours(hours).format(FORMAT_HOUR)); } public static Date preTime(int hours){ - return localDateTimeToDate(LocalDateTime.now().minusHours(hours)); + return ldtToDate(LocalDateTime.now().minusHours(hours)); } public static Date preMinus(int minus){ - return localDateTimeToDate(LocalDateTime.now().minusMinutes(minus)); + return ldtToDate(LocalDateTime.now().minusMinutes(minus)); } public static Date preDays(int days){ - return localDateTimeToDate(LocalDateTime.now().minusDays(days)); + return ldtToDate(LocalDateTime.now().minusDays(days)); } } diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 9ced405..ebc6aaa 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -495,8 +495,8 @@ modal_status = { function getCookie(cname){ let token = window.localStorage.getItem('token'); let time = window.localStorage.getItem('time'); - console.log("token---> "+token); - console.log("存入的时间---> "+time+" 当前的时间---> "+Date.now()); + // console.log("token---> "+token); + // console.log("存入的时间---> "+time+" 当前的时间---> "+Date.now()); if(token != null && token !==""){ let time = window.localStorage.getItem('time'); if(Date.now()-time>7*24*360000){ diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index 96da0de..8f48754 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -1,29 +1,97 @@ - - + + + + +
-
-
hotKey
-
+
+
+
+
+
+
+ + + + + + + + + + + + + + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
-
+ + + diff --git a/dashboard/src/main/resources/templates/admin/mainbak.html b/dashboard/src/main/resources/templates/admin/mainbak.html new file mode 100644 index 0000000..96da0de --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/mainbak.html @@ -0,0 +1,106 @@ + + + + + + + +
+
+
hotKey
+
+ +
+
+ + + + + + + + + + + -- Gitee From d393a336fc2d3bfb035db95511524e6772a63b02 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 12 Jun 2020 17:11:09 +0800 Subject: [PATCH 013/156] =?UTF-8?q?=E8=AE=BE=E7=BD=AEnetty=E7=9A=84worker?= =?UTF-8?q?=E7=9A=84=E7=BA=BF=E7=A8=8B=E6=95=B0=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/worker/cache/CaffeineBuilder.java | 22 ++++++++++++++++--- .../platform/hotkey/worker/tool/CpuNum.java | 10 ++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java index 6c60f17..8eb1b9f 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java @@ -3,6 +3,8 @@ package com.jd.platform.hotkey.worker.cache; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** @@ -10,6 +12,7 @@ import java.util.concurrent.TimeUnit; * @version 1.0 */ public class CaffeineBuilder { + private static ExecutorService executorService = Executors.newFixedThreadPool(4); /** * 构建所有来的要缓存的key cache */ @@ -17,8 +20,9 @@ public class CaffeineBuilder { //老版本jdk1.8.0_20之前，caffeine默认的forkJoinPool在及其密集的淘汰过期时，会有forkJoinPool报错。建议用新版jdk return Caffeine.newBuilder() .initialCapacity(8192)//初始大小 - .maximumSize(5000000)//最大数量 + .maximumSize(20000000)//最大数量 .expireAfterWrite(1, TimeUnit.MINUTES)//过期时间 + .executor(executorService) .softValues() .build(); } @@ -29,14 +33,26 @@ public class CaffeineBuilder { public static Cache buildRecentHotKeyCache() { return Caffeine.newBuilder() .initialCapacity(256)//初始大小 - .maximumSize(50000)//最大数量 + .maximumSize(500000)//最大数量 .expireAfterWrite(10, TimeUnit.SECONDS)//过期时间 + .executor(executorService) .softValues() .build(); } public static void main(String[] args) { - System.out.println(System.getProperty("java.version")); +// Cache cache = buildAllKeyCache(); +// //开启上传worker信息 +// ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); +// scheduledExecutorService.scheduleAtFixedRate(() -> { +// System.out.println(cache.asMap().size()); +// +// }, 0, 5, TimeUnit.SECONDS); +// +// long i = 0; +// while (true) { +// cache.put(i++ + "", i); +// } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java index c886bfe..4c9a735 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java @@ -11,11 +11,15 @@ public class CpuNum { * netty worker线程数量. cpu密集型 */ public static int workerCount() { + int count = 1; if (isNewerVersion()) { - return Runtime.getRuntime().availableProcessors(); - } else { - return 4; + count = Runtime.getRuntime().availableProcessors(); } + if (count >= 4) { + count = count / 2; + } + + return count; } public static void main(String[] args) { -- Gitee From 3714257e26e755ebd5895d0a972785850598d8a1 Mon Sep 17 00:00:00 2001 From: lihongliang32 Date: Mon, 15 Jun 2020 15:59:16 +0800 Subject: [PATCH 014/156] =?UTF-8?q?=E9=9B=86=E6=88=90erp=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/pom.xml | 5 ++ .../autoconfigure/AbstractProperties.java | 18 ++++++ .../dashboard/controller/UserController.java | 37 +++++++++--- .../dashboard/erp/ErpConfiguration.java | 24 ++++++++ .../hotkey/dashboard/erp/ErpProperties.java | 30 ++++++++++ .../dashboard/erp/ErpUimInterceptor.java | 58 +++++++++++++++++++ .../interceptor/MyWebAppConfigurer.java | 8 ++- .../hotkey/dashboard/mapper/UserMapper.java | 5 +- .../hotkey/dashboard/service/UserService.java | 5 ++ .../service/impl/UserServiceImpl.java | 46 +++++++++++++++ .../hotkey/dashboard/util/JwtTokenUtil.java | 15 ++++- dashboard/src/main/resources/application.yml | 13 ++++- .../hotkey/dashboard/mapper/UserMapper.xml | 18 +++++- .../templates/admin/common/html/topMenu.html | 4 +- .../main/resources/templates/admin/index.html | 17 ++++++ 15 files changed, 284 insertions(+), 19 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/autoconfigure/AbstractProperties.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java diff --git a/dashboard/pom.xml b/dashboard/pom.xml index 41dd9cd..30eacfa 100644 --- a/dashboard/pom.xml +++ b/dashboard/pom.xml @@ -81,6 +81,11 @@ poi-ooxml 4.0.1 + + com.jd.common + sso-uim-spring + 1.2.0-SNAPSHOT + diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/autoconfigure/AbstractProperties.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/autoconfigure/AbstractProperties.java new file mode 100644 index 0000000..2350067 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/autoconfigure/AbstractProperties.java @@ -0,0 +1,18 @@ +package com.jd.platform.hotkey.dashboard.autoconfigure; + +/** + * User: fuxueliang + * Date: 16/8/25 + * Email: fuxueliang@jd.com + */ +public abstract class AbstractProperties { + private boolean enabled = true; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index cc7f6e5..71eb59c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -1,9 +1,6 @@ package com.jd.platform.hotkey.dashboard.controller; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; @@ -45,10 +42,10 @@ public class UserController extends BaseController { public Result login(User param, HttpServletResponse response) { User user = userService.findByNameAndPwd(param); if(user == null) return Result.error(ResultEnum.PWD_ERROR); - String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), user.getRole(), user.getAppName()); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), user.getRole(), user.getAppName(), user.getNickName()); Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); cookie.setMaxAge(3600*24*7); - cookie.setDomain("localhost"); +// cookie.setDomain("localhost"); cookie.setPath("/"); response.addCookie(cookie); Map map = new HashMap<>(2); @@ -90,8 +87,18 @@ public class UserController extends BaseController { @GetMapping("/LoginOut") - public String LoginOut(){ - return "redirect:/admin/user/login"; + public String LoginOut(HttpServletRequest request, HttpServletResponse response){ + Cookie[] cookies = request.getCookies(); + for(Cookie cookie : cookies){ + if("token".equals(cookie.getName())){ + Cookie tempCookie = new Cookie(cookie.getName(), null); + tempCookie.setPath("/");//路径要相同 + tempCookie.setMaxAge(0);//生命周期设置为0 + response.addCookie(tempCookie); + break; + } + } + return "redirect:/user/login"; } @@ -177,5 +184,19 @@ public class UserController extends BaseController { return "redirect:/error/500"; } + @PostMapping("getUserName") + @ResponseBody + public String getUserName(HttpServletRequest request, HttpServletResponse response){ + /*Cookie[] cookies = request.getCookies(); + String userName = ""; + for(Cookie cookie : cookies){ + if("erp".equals(cookie.getName())){ + userName = cookie.getValue(); + break; + } + } + return userName;*/ + return userName(); + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java new file mode 100644 index 0000000..6b12a85 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java @@ -0,0 +1,24 @@ +package com.jd.platform.hotkey.dashboard.erp; + + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ErpConfiguration { + + @Autowired + ErpProperties erpProperties; + + @Bean + public ErpUimInterceptor erpUimInterceptor() { + ErpUimInterceptor springSSOInterceptor = new ErpUimInterceptor(); + springSSOInterceptor.setSsoAppKey(erpProperties.getSsoAppKey()); + springSSOInterceptor.setSsoAppToken(erpProperties.getSsoAppToken()); + springSSOInterceptor.setLoginUrl(erpProperties.getLoginUrl()); + springSSOInterceptor.setExcludePath(erpProperties.getExcludePath()); + springSSOInterceptor.setSsoAppUrl(erpProperties.getSsoAppUrl()); + return springSSOInterceptor; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java new file mode 100644 index 0000000..6d9c119 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java @@ -0,0 +1,30 @@ +package com.jd.platform.hotkey.dashboard.erp; + +import com.jd.platform.hotkey.dashboard.autoconfigure.AbstractProperties; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * User: fuxueliang + * Date: 16/8/17 + * Email: fuxueliang@jd.com + */ +@Data +@Component +@ConfigurationProperties(prefix = "erp") +public class ErpProperties extends AbstractProperties { + + private String excludePath; + + private String loginUrl; + + private String ssoAppUrl; + + private String ssoAppKey; + + private String ssoAppToken; + + private Boolean enabled; + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java new file mode 100644 index 0000000..d506a10 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java @@ -0,0 +1,58 @@ +package com.jd.platform.hotkey.dashboard.erp; + +import com.jd.common.springmvc.interceptor.SpringSSOInterceptor; +import com.jd.common.web.LoginContext; +import com.jd.platform.hotkey.dashboard.model.User; +import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + + +@Slf4j +public class ErpUimInterceptor extends SpringSSOInterceptor { + + @Autowired + private UserService userService; + + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + LoginContext loginContext = LoginContext.getLoginContext(); + if (null == loginContext) { + super.preHandle(request, response, handler); + } + loginContext = LoginContext.getLoginContext(); + if (null == loginContext) { + return true; + } + /*Cookie cookie = new Cookie("erp", loginContext.getNick()); + cookie.setMaxAge(3600*24*7); + cookie.setPath("/"); + response.addCookie(cookie);*/ + String pin = loginContext.getPin(); + String nickName = loginContext.getNick(); + String orgName = loginContext.getOrgName(); + String orgId = loginContext.getOrgId(); + String mobile = loginContext.getMobile(); + String email = loginContext.getEmail(); + User user = new User(); + user.setAppName("test"); + user.setUserName(pin); + user.setNickName(nickName); + user.setPhone(mobile); + Cookie cookie = userService.loginErpUser(user); + Cookie[] cookies = request.getCookies(); + if(cookies != null){ + for(Cookie tempCookie : cookies){ + if("token".equals(tempCookie.getName()) && tempCookie.getValue() != null){ + return true; + } + } + } + response.addCookie(cookie); + return true; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java index 135b277..dc8306e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java @@ -1,18 +1,22 @@ package com.jd.platform.hotkey.dashboard.interceptor; +import com.jd.platform.hotkey.dashboard.erp.ErpUimInterceptor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.Ordered; import org.springframework.web.servlet.config.annotation.*; - @Configuration public class MyWebAppConfigurer implements WebMvcConfigurer { + @Autowired + ErpUimInterceptor erpUimInterceptor; @Override public void addInterceptors(InterceptorRegistry registry) { //拦截路径可自行配置多个 可用 ，分隔开 InterceptorRegistration registration = registry.addInterceptor(new JwtInterceptor()).addPathPatterns("/**"); - registration.excludePathPatterns("/user/login","/error","/static/**","/main/**"); + registry.addInterceptor(erpUimInterceptor); + registration.excludePathPatterns("/user/login","/user/getUserName","/error","/static/**","/main/**"); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java index 14d4b6c..91038ba 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java @@ -3,6 +3,7 @@ package com.jd.platform.hotkey.dashboard.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.User; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; import java.util.List; @@ -10,7 +11,7 @@ import java.util.List; public interface UserMapper { int deleteByPrimaryKey(Integer id); - int insertSelective(User record); + int insertSelective(User user); User selectByPrimaryKey(Integer id); @@ -22,4 +23,6 @@ public interface UserMapper { List listApp(); + List selectHkUserList(User user); + } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java index 8d374b4..ae01777 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java @@ -5,6 +5,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.User; +import javax.servlet.http.Cookie; import java.util.List; /** @@ -22,6 +23,10 @@ public interface UserService { int insertUser(User user); + int insertUserByErp(User user); + + Cookie loginErpUser(User user); + int deleteByPrimaryKey(int id); User selectByPrimaryKey(int id); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index b77f695..848a132 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -11,11 +11,14 @@ import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.mapper.UserMapper; import com.jd.platform.hotkey.dashboard.model.User; import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.DigestUtils; import org.springframework.util.StringUtils; import javax.annotation.Resource; +import javax.servlet.http.Cookie; import java.util.Date; import java.util.HashSet; import java.util.List; @@ -31,6 +34,8 @@ import java.util.Set; @Service public class UserServiceImpl implements UserService { + @Value("${erp.defaultPwd}") + private String defaultPwd; @Resource private UserMapper userMapper; @Resource @@ -63,6 +68,47 @@ public class UserServiceImpl implements UserService { return userMapper.insertSelective(user); } + @Override + public int insertUserByErp(User user) { + User userParam = new User(); + userParam.setUserName(user.getUserName()); + List users = userMapper.selectHkUserList(userParam); + if(users.size() == 0){ + user.setCreateTime(new Date()); + user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); + int ret = userMapper.insertSelective(user); + System.out.println(user.getId()); + return ret; + } + return 0; + } + + @Override + public Cookie loginErpUser(User user){ + User userParam = new User(); + userParam.setUserName(user.getUserName()); + List users = userMapper.selectHkUserList(userParam); + if(users.size() == 0){ + user.setCreateTime(new Date()); + user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); + userMapper.insertSelective(user); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), null, user.getAppName(), user.getNickName()); + Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); + cookie.setMaxAge(3600*24*7); + //cookie.setDomain("localhost"); + cookie.setPath("/"); + return cookie; + }else{ + user =users.get(0); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), null, user.getAppName(), user.getNickName()); + Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); + cookie.setMaxAge(3600*24*7); + //cookie.setDomain("localhost"); + cookie.setPath("/"); + return cookie; + } + } + @Override public int deleteByPrimaryKey(int id) { return userMapper.deleteByPrimaryKey(id); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java index 57c111e..f02c036 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java @@ -8,6 +8,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.crypto.spec.SecretKeySpec; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; import javax.xml.bind.DatatypeConverter; import java.security.Key; import java.text.SimpleDateFormat; @@ -47,7 +49,7 @@ public class JwtTokenUtil { * @param role * @return */ - public static String createJWT(Integer userId, String username, String role, String appName) { + public static String createJWT(Integer userId, String username, String role, String appName, String nickName) { try { // 使用HS256加密算法 SignatureAlgorithm signatureAlgorithm = SignatureAlgorithm.HS256; @@ -63,6 +65,7 @@ public class JwtTokenUtil { .claim("userId", encryId) .claim("role", role) .claim("appName", appName) + .claim("nickName", nickName) .setSubject(username) // 代表这个JWT的主体，即它的所有人 // .setIssuer(audience.getClientId()) // 代表这个JWT的签发主体； .setIssuedAt(new Date()) // 是一个时间戳，代表这个JWT的签发时间； @@ -147,6 +150,14 @@ public class JwtTokenUtil { return parseJWT(token).getExpiration().before(new Date()); } - + public static String getAuthHeader(HttpServletRequest request){ + Cookie[] cookies = request.getCookies(); + for(Cookie cookie : cookies){ + if("token".equals(cookie.getName())){ + return cookie.getValue(); + } + } + return null; + } } diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 9e852a2..1678e74 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -28,8 +28,8 @@ spring : date-format: yyyy-MM-dd HH:mm:ss datasource: username: root - password: JRTEST - url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC + password: password + url: jdbc:mysql://localhost:3306/hotkey?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC driver-class-name: com.mysql.cj.jdbc.Driver pagehelper: helperDialect: mysql @@ -42,4 +42,11 @@ mybatis: etcd: server: ${etcdServer:http://127.0.0.1:2379} - +erp: + enabled: false + ssoAppKey: fangzhou + ssoAppToken: 3f48d7ade61940f2ba22079ebc273fb8 + loginUrl: https://ssa.jd.com/sso/login + excludePath: /login,/assets,/bootstarp,/common,/htmlformat,/images,/jn,/jquery,/user/login,login + ssoAppUrl: http://ssa.jd.com + defaultPwd: 123456 diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml index 87b2328..b1b238a 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml @@ -21,11 +21,27 @@ from hk_user where id = #{id,jdbcType=INTEGER} + delete from hk_user where id = #{id,jdbcType=INTEGER} - + insert into hk_user diff --git a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html index 8715107..ff59bd7 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html @@ -120,7 +120,7 @@ - + diff --git a/dashboard/src/main/resources/templates/admin/index.html b/dashboard/src/main/resources/templates/admin/index.html index f677710..861ee36 100644 --- a/dashboard/src/main/resources/templates/admin/index.html +++ b/dashboard/src/main/resources/templates/admin/index.html @@ -60,6 +60,23 @@ height: calc(100vh - 139px);overflow-x: hidden; $(this).addClass("menuSelected"); }); + $(document).ready(function () { + $.ajax({ + type : "POST", + url: "/user/getUserName", + /*headers: { + "Authorization":getCookie("token") + },*/ + async : false, + error : function(request) { + $.modal.alertError("系统错误"); + }, + success : function(data) { + $("#userName").html(data); + } + }); + }); + -- Gitee From c1e94128789ec09c20d91d2b55cb63c928c5ed45 Mon Sep 17 00:00:00 2001 From: lihongliang32 Date: Mon, 15 Jun 2020 16:13:52 +0800 Subject: [PATCH 015/156] =?UTF-8?q?=E9=9B=86=E6=88=90erp=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/common/base/BaseController.java | 6 +++--- .../hotkey/dashboard/controller/UserController.java | 2 +- .../hotkey/dashboard/interceptor/JwtInterceptor.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index edf7053..b4448f2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -29,7 +29,7 @@ public class BaseController { public String userName(){ - final String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + final String authHeader = JwtTokenUtil.getAuthHeader(request); final String token = authHeader.substring(2); return JwtTokenUtil.getUsername(token); } @@ -37,7 +37,7 @@ public class BaseController { public SearchReq param(String text){ - String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + String authHeader = JwtTokenUtil.getAuthHeader(request); SearchReq dto = JSON.parseObject(text, SearchReq.class); if(dto == null){ dto = new SearchReq(); } dto.setAppName(JwtTokenUtil.getAppName(authHeader.substring(2))); @@ -45,7 +45,7 @@ public class BaseController { } public SearchReq param2(SearchReq dto){ - String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + String authHeader = JwtTokenUtil.getAuthHeader(request); if(dto == null){ dto = new SearchReq(); } dto.setAppName(JwtTokenUtil.getAppName(authHeader.substring(2))); return dto; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index 71eb59c..b66b463 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -74,7 +74,7 @@ public class UserController extends BaseController { @ResponseBody @PostMapping("/info") public User info(HttpServletRequest request){ - String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + String authHeader = JwtTokenUtil.getAuthHeader(request); User userPower = JwtTokenUtil.userPower(authHeader.substring(2)); String appName = userPower.getAppName(); String role = userPower.getRole(); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 9f3fe55..811bf59 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -29,7 +29,7 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ } String url = request.getRequestURI(); if(isAjaxRequest){ - final String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + final String authHeader = JwtTokenUtil.getAuthHeader(request); if (StringUtils.isEmpty(authHeader) || !authHeader.startsWith(JwtTokenUtil.TOKEN_PREFIX)) { // response.sendRedirect("login"); @@ -41,9 +41,9 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ final String token = authHeader.substring(2); Claims claims = JwtTokenUtil.parseJWT(token); String role = claims.get("role", String.class); - if(role.equals("ADMIN") || role.equals("APPADMIN")){ + /*if(role.equals("ADMIN") || role.equals("APPADMIN")){ return true; - } + }*/ // appUser只读 if(url.contains("view")||url.contains("list")){ return true; -- Gitee From 1d0a2a88228e204a9d5e79e6720e56b467c352b1 Mon Sep 17 00:00:00 2001 From: lihongliang32 Date: Mon, 15 Jun 2020 17:07:57 +0800 Subject: [PATCH 016/156] =?UTF-8?q?=E9=9B=86=E6=88=90erp=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java | 1 + .../platform/hotkey/dashboard/interceptor/JwtInterceptor.java | 3 ++- .../hotkey/dashboard/service/impl/UserServiceImpl.java | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java index d506a10..e61dd41 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java @@ -43,6 +43,7 @@ public class ErpUimInterceptor extends SpringSSOInterceptor { user.setUserName(pin); user.setNickName(nickName); user.setPhone(mobile); + user.setRole(""); Cookie cookie = userService.loginErpUser(user); Cookie[] cookies = request.getCookies(); if(cookies != null){ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 811bf59..1e842f8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -48,7 +48,8 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ if(url.contains("view")||url.contains("list")){ return true; } - throw new BizException(ResultEnum.NO_PERMISSION); + + //throw new BizException(ResultEnum.NO_PERMISSION); // throw new RuntimeException("NO_PERMISSION"); // response.setStatus(HttpServletResponse.SC_FORBIDDEN); // return false; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index 848a132..dab819c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -92,7 +92,7 @@ public class UserServiceImpl implements UserService { user.setCreateTime(new Date()); user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); userMapper.insertSelective(user); - String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), null, user.getAppName(), user.getNickName()); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), "", user.getAppName(), user.getNickName()); Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); cookie.setMaxAge(3600*24*7); //cookie.setDomain("localhost"); @@ -100,7 +100,7 @@ public class UserServiceImpl implements UserService { return cookie; }else{ user =users.get(0); - String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), null, user.getAppName(), user.getNickName()); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), "", user.getAppName(), user.getNickName()); Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); cookie.setMaxAge(3600*24*7); //cookie.setDomain("localhost"); -- Gitee From 9ea3e1a1351213cdc014c60b1db708a5d3133258 Mon Sep 17 00:00:00 2001 From: lihongliang32 Date: Mon, 15 Jun 2020 17:34:14 +0800 Subject: [PATCH 017/156] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=A6=96=E9=A1=B5?= =?UTF-8?q?=E8=B7=B3=E8=BD=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/templates/admin/common/html/leftMenu.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index fa5ae9b..25343bc 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -3,7 +3,8 @@ @@ -123,7 +123,7 @@ - - --> + -- Gitee From bd80641a087ff15c4a7d1397cd828923938a1e35 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 17 Jun 2020 12:11:34 +0800 Subject: [PATCH 029/156] =?UTF-8?q?=E9=A6=96=E9=A1=B5=E5=A4=A7=E5=B1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/templates/admin/main.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index d72d0a2..5e60d90 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -169,7 +169,7 @@ }; //加载数据 initLegend(option); - loadData(option, "", {"type":1}); + loadData(option, "", {"type":5}); // 使用刚指定的配置项和数据显示图表。 myChart.setOption(option); -- Gitee From e8e80c6847387e8de7deedd1c6ce7548438eb510 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 17 Jun 2020 12:12:23 +0800 Subject: [PATCH 030/156] =?UTF-8?q?=E5=88=A4=E6=96=ADJava=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E8=8E=B7=E5=8F=96docker=E5=86=85cpu=E6=A0=B8=E6=95=B0?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/templates/admin/main.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index d72d0a2..5e60d90 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -169,7 +169,7 @@ }; //加载数据 initLegend(option); - loadData(option, "", {"type":1}); + loadData(option, "", {"type":5}); // 使用刚指定的配置项和数据显示图表。 myChart.setOption(option); -- Gitee From b61d4acab26ade151b8236e82247def7adb4e906 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Wed, 17 Jun 2020 13:37:43 +0800 Subject: [PATCH 031/156] =?UTF-8?q?=E4=BF=AE=E6=94=B9etcd=E8=BF=94?= =?UTF-8?q?=E5=9B=9Erule=20=E4=BD=BF=E4=B9=8B=E5=92=8CruleUtil=E5=AD=98?= =?UTF-8?q?=E5=85=A5=E7=9A=84=E6=A0=BC=E5=BC=8F=E4=B8=80=E8=87=B4=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E5=B1=95=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/service/impl/RuleServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 8688fba..624d706 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -142,7 +142,7 @@ public class RuleServiceImpl implements RuleService { String appKey = key.replace(ConfigConstant.rulePath,""); List rs = JSON.parseArray(v, Rule.class); for (Rule r : rs) { - rules.add(r.getKey()); + rules.add(appKey+"-"+r.getKey()); } } return rules; -- Gitee From cd2f45271bf5f5dc050b56c5335708d9dfbb5c6f Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 17 Jun 2020 14:09:45 +0800 Subject: [PATCH 032/156] 1 --- .../platform/hotkey/dashboard/service/impl/RuleServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 8688fba..624d706 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -142,7 +142,7 @@ public class RuleServiceImpl implements RuleService { String appKey = key.replace(ConfigConstant.rulePath,""); List rs = JSON.parseArray(v, Rule.class); for (Rule r : rs) { - rules.add(r.getKey()); + rules.add(appKey+"-"+r.getKey()); } } return rules; -- Gitee From ad22bdbe59aad71fc5f6d01e91e1eaba0ab65cd1 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 17 Jun 2020 14:44:55 +0800 Subject: [PATCH 033/156] 1 --- .../common/domain/req/SearchReq.java | 1 - .../dashboard/common/monitor/DataHandler.java | 1 - .../dashboard/controller/RuleController.java | 3 +-- .../dashboard/controller/UserController.java | 11 +++++----- .../dashboard/mapper/StatisticsMapper.java | 1 - .../hotkey/dashboard/mapper/UserMapper.java | 4 +++- .../hotkey/dashboard/service/RuleService.java | 1 - .../service/impl/KeyServiceImpl.java | 1 - .../hotkey/dashboard/util/DateUtil.java | 1 - .../hotkey/dashboard/mapper/UserMapper.xml | 18 ++++++++++++++++- .../templates/admin/common/html/leftMenu.html | 2 +- .../templates/admin/common/html/topMenu.html | 20 +++++++++---------- 12 files changed, 37 insertions(+), 27 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index 7bb37ef..2ea2d5f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -6,7 +6,6 @@ import com.jd.platform.hotkey.dashboard.util.DateUtil; import java.io.Serializable; import java.time.LocalDateTime; import java.util.Date; -import java.util.List; /** * @ProjectName: hotkey diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index bc94342..0538405 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -4,7 +4,6 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index bd5a66a..5e8a52d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -4,9 +4,8 @@ import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; -import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; -import com.jd.platform.hotkey.dashboard.model.KeyRule; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; import org.springframework.stereotype.Controller; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index cc7f6e5..80a48c7 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -1,17 +1,12 @@ package com.jd.platform.hotkey.dashboard.controller; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; -import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.User; import com.jd.platform.hotkey.dashboard.service.UserService; @@ -25,6 +20,10 @@ import javax.annotation.Resource; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Controller diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java index fc5c79d..dabdfc5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.dashboard.mapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java index 14d4b6c..cc6ca7a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java @@ -10,7 +10,7 @@ import java.util.List; public interface UserMapper { int deleteByPrimaryKey(Integer id); - int insertSelective(User record); + int insertSelective(User user); User selectByPrimaryKey(Integer id); @@ -22,4 +22,6 @@ public interface UserMapper { List listApp(); + List selectHkUserList(User user); + } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java index ebd974e..f06e426 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java @@ -3,7 +3,6 @@ package com.jd.platform.hotkey.dashboard.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyRule; import com.jd.platform.hotkey.dashboard.model.Rules; import java.util.List; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index d32e630..348e2bf 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.dashboard.service.impl; -import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index bb49094..dbe252f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -2,7 +2,6 @@ package com.jd.platform.hotkey.dashboard.util; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml index 87b2328..b1b238a 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml @@ -21,11 +21,27 @@ from hk_user where id = #{id,jdbcType=INTEGER} + delete from hk_user where id = #{id,jdbcType=INTEGER} - + insert into hk_user diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index b1aea2e..3864a2d 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -3,7 +3,7 @@
diff --git a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html index 8715107..354cc91 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html @@ -89,24 +89,24 @@
    -
  •   Berlin  - 85 ; 15km/h +
  •   使命  +  -  科技引领生活
    • -
  •   Yogyakarta  - 85 ; Tonight- 72 °F (22.2 °C) +
  •   愿景  +  -  成为全球最值得信赖的企业
    • -
  •   Sttugart  - 85 ; 15km/h +
  •   核心价值观  +  -   客户为先 诚信 协作 感恩 拼搏 担当
    • -
  •   Muchen  +
@@ -123,7 +123,7 @@
- + @@ -74,15 +74,15 @@ // 基于准备好的dom，初始化echarts实例 var myChart = echarts.init(document.getElementById('ruleLineChart')); - // 指定图表的配置项和数据 + // 指定图表的配置项和数据 var option = { xAxis: { type: 'category', data: ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] }, - // color:['#4472C5'], + // color:['#4472C5'], legend: { - data:['key21','key22','key23','key24','key25','key26','key27','key28','key29','key30','key31'], + data:['key21','key22'], itemWidth:80, itemHeight:25, itemGap:18, @@ -118,52 +118,6 @@ name: 'key22', data: [350, 255, 310, 494, 189, 333, 620], type: 'line' - }, - { - name: 'key23', - data: [310, 415, 350, 714, 219, 300, 500], - type: 'line' - }, - { - name: 'key24', - data: [380, 385, 370, 654, 569, 303, 620], - type: 'line' - }, - { - name: 'key25', - data: [320, 265, 300, 574, 389, 500, 700], - type: 'line' - }, - { - name: 'key26', - data: [390, 485, 320, 604, 429, 363, 620], - type: 'line' - }, - { - name: 'key27', - data: [300, 515, 580, 514, 259, 510, 430], - type: 'line' - }, - { - name: 'key28', - data: [280, 415, 380, 404, 429, 363, 480], - type: 'line' - }, - { - name: 'key29', - data: [450, 555, 320, 314, 559, 370, 460], - type: 'line' - } - , - { - name: 'key30', - data: [280, 615, 660, 444, 429, 363, 410], - type: 'line' - }, - { - name: 'key31', - data: [400, 555, 600, 333, 559, 310, 560], - type: 'line' } ] }; @@ -188,11 +142,11 @@ }, success : function(data) { console.log(data) - let selected_item = {}; + // let selected_item = {}; for (let i = 0; i < data.length; i++) { - selected_item[data[i]] = i === 0; + // selected_item[data[i]] = i === 0; option.legend.data.push(data[i]); - option.legend.selected = selected_item; + // option.legend.selected = selected_item; } } }); -- Gitee From e0a1c0688e541aa0a5bba256dcb79102d6e3cdf8 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 23 Jun 2020 12:41:22 +0800 Subject: [PATCH 049/156] =?UTF-8?q?=E7=BC=96=E8=BE=91=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/static/admin/common/js/user-editPwd.js | 3 +++ .../src/main/resources/templates/admin/user/editPwd.html | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js b/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js index 91ef833..eedeb56 100644 --- a/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js +++ b/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js @@ -38,6 +38,9 @@ function edit() { url : "/user/editPwd", data : dataFormJson, async : false, + headers: { + "Authorization":getCookie("token") + }, error : function(request) { $.modal.alertError("系统错误"); }, diff --git a/dashboard/src/main/resources/templates/admin/user/editPwd.html b/dashboard/src/main/resources/templates/admin/user/editPwd.html index ccb9c3e..085cdf5 100644 --- a/dashboard/src/main/resources/templates/admin/user/editPwd.html +++ b/dashboard/src/main/resources/templates/admin/user/editPwd.html @@ -25,18 +25,18 @@
-
+
- +
- +
@@ -71,7 +71,7 @@ - + -- Gitee From 7694cdcf45a024ec91a646d44f1aaaf4bfa77c7d Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 23 Jun 2020 12:42:48 +0800 Subject: [PATCH 050/156] =?UTF-8?q?=E7=BC=96=E8=BE=91=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../static/admin/common/js/user-editPwd.js | 25 ++++++++------- .../templates/admin/user/editPwd.html | 32 +++++++++---------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js b/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js index 91ef833..fdbb9cb 100644 --- a/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js +++ b/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js @@ -11,19 +11,19 @@ $("#form-edit").validate({ maxlength: 20, equalTo:"#pwd" } - + }, messages: { - password: { - required: "请输入新密码", - minlength: "密码不能小于6个字符", - maxlength: "密码不能大于20个字符" - }, - confirm: { - required: "请再次输入新密码", - equalTo: "两次密码输入不一致" - } - }, + password: { + required: "请输入新密码", + minlength: "密码不能小于6个字符", + maxlength: "密码不能大于20个字符" + }, + confirm: { + required: "请再次输入新密码", + equalTo: "两次密码输入不一致" + } + }, submitHandler:function(form){ edit(); } @@ -38,6 +38,9 @@ function edit() { url : "/user/editPwd", data : dataFormJson, async : false, + headers: { + "Authorization":getCookie("token") + }, error : function(request) { $.modal.alertError("系统错误"); }, diff --git a/dashboard/src/main/resources/templates/admin/user/editPwd.html b/dashboard/src/main/resources/templates/admin/user/editPwd.html index ccb9c3e..6d5a9a1 100644 --- a/dashboard/src/main/resources/templates/admin/user/editPwd.html +++ b/dashboard/src/main/resources/templates/admin/user/editPwd.html @@ -1,17 +1,17 @@ - + - - - - - - - - - + + + + + + + + + @@ -25,18 +25,18 @@
- +
- +
- +
@@ -61,7 +61,7 @@
- +
@@ -71,7 +71,7 @@ - + -- Gitee From e6503aa8b5546976550e37266edde52d0393c93d Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 23 Jun 2020 16:37:03 +0800 Subject: [PATCH 051/156] =?UTF-8?q?Record=E8=A1=A8=E5=8A=A0rule=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/mapper/KeyRecordMapper.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml index a18ac6d..87f83c7 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml @@ -49,9 +49,9 @@ duration, - + type, @@ -81,9 +81,9 @@ #{duration,jdbcType=BIGINT}, - + #{type,jdbcType=INTEGER}, -- Gitee From f78bc1b8034a299bb9b3b3c056d270c03b81270e Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Wed, 24 Jun 2020 23:07:39 +0800 Subject: [PATCH 052/156] =?UTF-8?q?=E6=9D=83=E9=99=90=E5=8F=98=E6=9B=B4?= =?UTF-8?q?=EF=BC=8C=E6=A0=B7=E5=BC=8F=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/base/BaseController.java | 20 ++-- .../dashboard/common/domain/Constant.java | 16 +++ .../dashboard/controller/KeyController.java | 19 +++- .../dashboard/controller/RuleController.java | 36 ++----- .../dashboard/controller/UserController.java | 13 +-- .../dashboard/interceptor/JwtInterceptor.java | 28 ++--- .../service/impl/RuleServiceImpl.java | 6 +- .../hotkey/dashboard/util/JwtTokenUtil.java | 7 +- .../templates/admin/common/html/footer.html | 20 ++-- .../resources/templates/admin/rule/add.html | 100 ------------------ .../resources/templates/admin/rule/edit.html | 99 ----------------- .../resources/templates/admin/rule/view.html | 75 +++++-------- .../resources/templates/admin/user/list.html | 13 ++- 13 files changed, 118 insertions(+), 334 deletions(-) delete mode 100644 dashboard/src/main/resources/templates/admin/rule/add.html delete mode 100644 dashboard/src/main/resources/templates/admin/rule/edit.html diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index b4448f2..2b54b13 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -4,8 +4,10 @@ import java.text.SimpleDateFormat; import java.util.Date; import com.alibaba.fastjson.JSON; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; +import io.jsonwebtoken.Claims; import org.springframework.beans.propertyeditors.CustomDateEditor; import org.springframework.web.bind.WebDataBinder; @@ -27,6 +29,18 @@ public class BaseController { binder.registerCustomEditor(Date.class, new CustomDateEditor(dateFormat, true)); } + public boolean checkApp(String app){ + String authHeader = JwtTokenUtil.getAuthHeader(request); + assert authHeader != null; + Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); + String role = claims.get("role",String.class); + if(role.equals(Constant.ADMIN)){ + return true; + } + String appName = claims.get("appName",String.class); + return appName.equals(app); + } + public String userName(){ final String authHeader = JwtTokenUtil.getAuthHeader(request); @@ -44,11 +58,5 @@ public class BaseController { return dto; } - public SearchReq param2(SearchReq dto){ - String authHeader = JwtTokenUtil.getAuthHeader(request); - if(dto == null){ dto = new SearchReq(); } - dto.setAppName(JwtTokenUtil.getAppName(authHeader.substring(2))); - return dto; - } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index 8ff0af7..de02283 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -37,6 +37,22 @@ public class Constant { public static final int VERSION = 1; + public static final String ADMIN = "ADMIN"; + + public static final String APP_UER = "APPUER"; + + public static final String POST = "POST"; + + + public static final String VIEW = "view"; + + + public static final String LIST = "list"; + + + public static final String INFO = "info"; + + public static final List HEAD = new ArrayList<>(); static { HEAD.add("热点key"); HEAD.add("次数"); HEAD.add("所属APP");} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index b61ce04..4042e93 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -11,6 +11,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; +import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; @@ -76,7 +77,7 @@ public class KeyController extends BaseController { @PostMapping("/list") @ResponseBody public Page list(PageReq page, SearchReq searchReq){ - PageInfo info = keyService.pageKeyRecord(page, param2(searchReq)); + PageInfo info = keyService.pageKeyRecord(page, searchReq); return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); } @@ -90,7 +91,7 @@ public class KeyController extends BaseController { @PostMapping("/listTimely") @ResponseBody public Page listTimely(PageReq page, SearchReq searchReq){ - PageInfo info = keyService.pageKeyTimely(page, param2(searchReq)); + PageInfo info = keyService.pageKeyTimely(page, searchReq); return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); } @@ -118,6 +119,9 @@ public class KeyController extends BaseController { @PostMapping("/add") @ResponseBody public Result add(KeyTimely key){ + if(!checkApp(key.getAppName())){ + return Result.error(ResultEnum.NO_PERMISSION); + } key.setType(0); key.setSource(userName()); int b = keyService.insertKeyByUser(key); @@ -127,6 +131,10 @@ public class KeyController extends BaseController { @PostMapping("/remove") @ResponseBody public Result remove(String key){ + String[] arr = key.split("/"); + if(!checkApp(arr[0])){ + return Result.error(ResultEnum.NO_PERMISSION); + } int b = keyService.delKeyByUser(new KeyTimely(key,userName())); return b == 0 ? Result.fail():Result.success(); } @@ -141,8 +149,11 @@ public class KeyController extends BaseController { @PostMapping("/edit") @ResponseBody - public Result editSave(KeyTimely keyTimely) { - return Result.success(keyService.updateKeyByUser(keyTimely)); + public Result editSave(KeyTimely key) { + if(!checkApp(key.getAppName())){ + return Result.error(ResultEnum.NO_PERMISSION); + } + return Result.success(keyService.updateKeyByUser(key)); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index 5e8a52d..c884980 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -6,6 +6,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; import org.springframework.stereotype.Controller; @@ -26,14 +27,8 @@ public class RuleController extends BaseController { - @GetMapping("/view2") - public String view2(ModelMap modelMap){ - modelMap.put("title", Constant.RULE_CONFIG_VIEW); - return "admin/rule/view"; - } - @GetMapping("/viewDetail") - public String view3(ModelMap modelMap){ + public String viewDetail(ModelMap modelMap){ modelMap.put("title", Constant.RULE_CONFIG_VIEW); return "admin/rule/jn"; } @@ -45,26 +40,13 @@ public class RuleController extends BaseController { } - @PostMapping("/add") - @ResponseBody - public Result add(Rules rule){ - rule.setUpdateUser(userName()); - int b = ruleService.add(rule); - return b == 0 ? Result.fail():Result.success(); - } - - - @PostMapping("/update") - @ResponseBody - public Result update(Rules rules){ - rules.setUpdateUser(userName()); - int b = ruleService.updateRule(rules); - return b == 0 ? Result.fail():Result.success(); - } @PostMapping("/save") @ResponseBody public Result save(Rules rules){ + if(!checkApp(rules.getApp())){ + return Result.error(ResultEnum.NO_PERMISSION); + } rules.setUpdateUser(userName()); int b = ruleService.save(rules); return b == 0 ? Result.fail():Result.success(); @@ -74,6 +56,9 @@ public class RuleController extends BaseController { @PostMapping("/remove") @ResponseBody public Result remove(String key){ + if(!checkApp(key)){ + return Result.error(ResultEnum.NO_PERMISSION); + } int b = ruleService.delRule(key, userName()); return b == 0 ? Result.fail():Result.success(); } @@ -87,7 +72,7 @@ public class RuleController extends BaseController { @PostMapping("/list") @ResponseBody - public Page list2(PageReq page, String searchText){ + public Page list(PageReq page, String searchText){ page.setPageSize(30); PageInfo info = ruleService.pageKeyRule(page, param(searchText)); return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); @@ -111,8 +96,7 @@ public class RuleController extends BaseController { @PostMapping("/listRules") @ResponseBody public List rules(){ - List info = ruleService.listRules(null); - return info; + return ruleService.listRules(null); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index b66b463..b6c3d6e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -14,6 +14,7 @@ import com.jd.platform.hotkey.dashboard.model.User; import com.jd.platform.hotkey.dashboard.service.UserService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; +import io.jsonwebtoken.Claims; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; @@ -41,11 +42,10 @@ public class UserController extends BaseController { @ResponseBody public Result login(User param, HttpServletResponse response) { User user = userService.findByNameAndPwd(param); - if(user == null) return Result.error(ResultEnum.PWD_ERROR); + if(user == null) { return Result.error(ResultEnum.PWD_ERROR); } String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), user.getRole(), user.getAppName(), user.getNickName()); Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); cookie.setMaxAge(3600*24*7); -// cookie.setDomain("localhost"); cookie.setPath("/"); response.addCookie(cookie); Map map = new HashMap<>(2); @@ -75,10 +75,11 @@ public class UserController extends BaseController { @PostMapping("/info") public User info(HttpServletRequest request){ String authHeader = JwtTokenUtil.getAuthHeader(request); - User userPower = JwtTokenUtil.userPower(authHeader.substring(2)); - String appName = userPower.getAppName(); - String role = userPower.getRole(); - if(role.equals("ADMIN")){ + assert authHeader != null; + Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); + String role = claims.get("role",String.class); + String appName = claims.get("appName",String.class); + if(role.equals(Constant.ADMIN)){ List apps = userService.listApp(); return new User(role,apps); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index a5258b6..aa7c29f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.interceptor; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; @@ -20,39 +21,24 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ response.setStatus(HttpServletResponse.SC_OK); return true; } - //判断是否为ajax请求，默认不是 - boolean isAjaxRequest = false; - if(!StringUtils.isEmpty(request.getHeader("x-requested-with")) - && request.getHeader("x-requested-with").equals("XMLHttpRequest") - && request.getMethod().equals("POST")){ - isAjaxRequest = true; - } - String url = request.getRequestURI(); - if(isAjaxRequest){ + String header = request.getHeader("x-requested-with"); + if(!StringUtils.isEmpty(header) && "XMLHttpRequest".endsWith(header) && request.getMethod().equals(Constant.POST)){ final String authHeader = JwtTokenUtil.getAuthHeader(request); if (StringUtils.isEmpty(authHeader) || !authHeader.startsWith(JwtTokenUtil.TOKEN_PREFIX)) { - // response.sendRedirect("login"); throw new BizException(ResultEnum.NO_LOGIN); - // response.setStatus(HttpServletResponse.SC_FORBIDDEN); - // return false; - // throw new RuntimeException("NO_LOGIN"); - } + } final String token = authHeader.substring(2); Claims claims = JwtTokenUtil.parseJWT(token); String role = claims.get("role", String.class); - if(role.equals("ADMIN") || role.equals("APPADMIN")){ + if(role.equals(Constant.ADMIN)){ return true; } - // appUser只读 - if(url.contains("view")||url.contains("list")||url.contains("info")){ + String url = request.getRequestURI(); + if(url.contains(Constant.VIEW)||url.contains(Constant.LIST)||url.contains(Constant.INFO)){ return true; } - //throw new BizException(ResultEnum.NO_PERMISSION); - // throw new RuntimeException("NO_PERMISSION"); - // response.setStatus(HttpServletResponse.SC_FORBIDDEN); - // return false; } return true; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 624d706..8b99137 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -95,9 +95,7 @@ public class RuleServiceImpl implements RuleService { @Override public PageInfo pageKeyRule(PageReq page, SearchReq param) { - String app = param.getAppName(); - String prefix = StringUtil.isEmpty(app) ? ConfigConstant.rulePath : ConfigConstant.rulePath + app; - List keyValues = configCenter.getPrefix(prefix); + List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); List rules = new ArrayList<>(); for (KeyValue kv : keyValues) { String v = kv.getValue().toStringUtf8(); @@ -124,7 +122,7 @@ public class RuleServiceImpl implements RuleService { // } String to = JSON.toJSONString(rules); // logMapper.insertSelective(new ChangeLog(app, 1, from, to, -// rules.getUpdateUser(), app, SystemClock.nowDate())); +// rules.getUpdateUser(), app, SystemClock.nowDate())); configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); return 1; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java index f02c036..42f7843 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/JwtTokenUtil.java @@ -134,11 +134,8 @@ public class JwtTokenUtil { } - public static User userPower(String token){ - Claims claims = parseJWT(token); - String role = claims.get("role",String.class); - String appName = claims.get("appName",String.class); - return new User(role,appName); + public static Claims claims(String token){ + return parseJWT(token); } /** diff --git a/dashboard/src/main/resources/templates/admin/common/html/footer.html b/dashboard/src/main/resources/templates/admin/common/html/footer.html index fbffa9a..a6cc3ff 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/footer.html +++ b/dashboard/src/main/resources/templates/admin/common/html/footer.html @@ -1,14 +1,18 @@
- -
diff --git a/dashboard/src/main/resources/templates/admin/rule/add.html b/dashboard/src/main/resources/templates/admin/rule/add.html deleted file mode 100644 index 2a8e783..0000000 --- a/dashboard/src/main/resources/templates/admin/rule/add.html +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
- -
- -
- -
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
-
- -
-
- - -
-
- -
-
-
-
-
-
- - -
- -
- - - - - - - - - - - - - - - - diff --git a/dashboard/src/main/resources/templates/admin/rule/edit.html b/dashboard/src/main/resources/templates/admin/rule/edit.html deleted file mode 100644 index 1b36a46..0000000 --- a/dashboard/src/main/resources/templates/admin/rule/edit.html +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
- - -
- -
- -
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
-
- -
- -
- -
-
-
-
- - -
-
-
-
-
-
-
-
-
- - -
- -
- - - - - - - - - - - - - - diff --git a/dashboard/src/main/resources/templates/admin/rule/view.html b/dashboard/src/main/resources/templates/admin/rule/view.html index c98e4e4..4982947 100644 --- a/dashboard/src/main/resources/templates/admin/rule/view.html +++ b/dashboard/src/main/resources/templates/admin/rule/view.html @@ -15,58 +15,33 @@
- -
- - - - - - - - -
-
-
-
- - - -
- - -
-
-
- - - -
-
-
- -
- -
+
+
+
+
+
+ +
- - - - - + +
+
+
+
+            + key-(*代表任意以key为前缀),   prefix-是否前缀,   interval-间隔时间(秒), + threshold-阈值,   duration-缓存时间(秒),默认60 +
+
+
+ +
+
+ +
+
diff --git a/dashboard/src/main/resources/templates/admin/user/list.html b/dashboard/src/main/resources/templates/admin/user/list.html index 1b295ec..3c2df58 100644 --- a/dashboard/src/main/resources/templates/admin/user/list.html +++ b/dashboard/src/main/resources/templates/admin/user/list.html @@ -23,10 +23,6 @@
- -
-
-
@@ -153,9 +149,16 @@ var options = { { field: 'phone', title: '手机号' + }, + { + field: 'createTime', + title: '创建时间', + formatter:function (val) { + return changeDateFormat(val); + } },{ title: '操作', - formatter: function (value, row, index) { + formatter: function (value, row) { var id = row.id; var actions = []; actions.push('编辑 '); -- Gitee From ad18c3f764682661c3d4e3e5c790a977a25859a6 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 25 Jun 2020 18:54:19 +0800 Subject: [PATCH 053/156] =?UTF-8?q?=E8=A7=84=E5=88=99=E6=A0=A1=E9=AA=8Cjso?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/controller/RuleController.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index c884980..c1b3e19 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.controller; +import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; @@ -7,6 +8,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; +import com.jd.platform.hotkey.dashboard.model.Rule; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; import org.springframework.stereotype.Controller; @@ -17,6 +19,9 @@ import javax.annotation.Resource; import java.util.List; +/** + * @author liyunfeng31 + */ @Controller @RequestMapping("/rule") public class RuleController extends BaseController { @@ -47,12 +52,16 @@ public class RuleController extends BaseController { if(!checkApp(rules.getApp())){ return Result.error(ResultEnum.NO_PERMISSION); } + if(!checkRule(rules.getRules())){ + return Result.error(ResultEnum.PARAM_ERROR); + } rules.setUpdateUser(userName()); int b = ruleService.save(rules); return b == 0 ? Result.fail():Result.success(); } + @PostMapping("/remove") @ResponseBody public Result remove(String key){ @@ -99,5 +108,21 @@ public class RuleController extends BaseController { return ruleService.listRules(null); } + + /** + * 校验是否合法 + * @param rules rules + * @return boolean + */ + private boolean checkRule(String rules) { + try { + JSON.parseArray(rules, Rule.class); + }catch(Exception e){ + return false; + } + return true; + } + + } -- Gitee From 2031a63728c3417dd43d3a0c478093207ecdc2b4 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 25 Jun 2020 19:04:21 +0800 Subject: [PATCH 054/156] =?UTF-8?q?=E6=94=BE=E5=BC=80app=20=E4=B8=8B?= =?UTF-8?q?=E6=8B=89=E6=A1=86=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/controller/UserController.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index b6c3d6e..586ef74 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -78,12 +78,7 @@ public class UserController extends BaseController { assert authHeader != null; Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); String role = claims.get("role",String.class); - String appName = claims.get("appName",String.class); - if(role.equals(Constant.ADMIN)){ - List apps = userService.listApp(); - return new User(role,apps); - } - return new User(role, Collections.singletonList(appName)); + return new User(role, userService.listApp()); } -- Gitee From 02bf76fb9d7974dcd9c7f8e0ffc0ba5c9b088d18 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 26 Jun 2020 19:57:12 +0800 Subject: [PATCH 055/156] =?UTF-8?q?=E5=85=A8=E5=B1=80=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=92=8C=E9=A1=B5=E9=9D=A2=E8=BF=94=E5=9B=9E=E5=A4=84=E7=90=86?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/base/BaseController.java | 15 ++++---- .../common/config/MyExceptionHandler.java | 34 +++++++++++++++++++ .../dashboard/common/domain/Result.java | 6 +++- .../dashboard/common/eunm/ResultEnum.java | 18 +++++----- .../dashboard/common/ex/BizException.java | 2 +- .../dashboard/common/monitor/DataHandler.java | 2 ++ .../dashboard/controller/KeyController.java | 12 ++----- .../dashboard/controller/RuleController.java | 22 +++++------- .../dashboard/controller/UserController.java | 3 +- .../platform/hotkey/dashboard/model/User.java | 9 +++-- .../hotkey/dashboard/service/RuleService.java | 2 +- .../service/impl/RuleServiceImpl.java | 11 ++++-- .../hotkey/dashboard/util/CommonUtil.java | 4 +++ .../dashboard/mapper/KeyRecordMapper.xml | 4 +-- .../dashboard/mapper/StatisticsMapper.xml | 16 ++++----- .../admin/assets/js/bootstrap/js/base_list.js | 6 +--- .../static/admin/jn/js/jquery.json.js | 2 +- .../resources/templates/admin/key/list.html | 14 +++++--- .../templates/admin/key/listmaxhot.html | 16 ++++++--- .../templates/admin/key/listtimely.html | 15 +++++--- .../resources/templates/admin/rule/jn.html | 2 -- .../resources/templates/admin/rule/list.html | 14 +++++--- .../resources/templates/admin/rule/view.html | 29 ++++++++++------ 23 files changed, 167 insertions(+), 91 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index 2b54b13..d21a237 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -6,6 +6,8 @@ import java.util.Date; import com.alibaba.fastjson.JSON; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; +import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; import org.springframework.beans.propertyeditors.CustomDateEditor; @@ -29,16 +31,17 @@ public class BaseController { binder.registerCustomEditor(Date.class, new CustomDateEditor(dateFormat, true)); } - public boolean checkApp(String app){ + public void checkApp(String app){ String authHeader = JwtTokenUtil.getAuthHeader(request); assert authHeader != null; Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); String role = claims.get("role",String.class); - if(role.equals(Constant.ADMIN)){ - return true; + if(!role.equals(Constant.ADMIN)){ + String appName = claims.get("appName",String.class); + if(!appName.equals(app)){ + throw new BizException(ResultEnum.NO_PERMISSION); + } } - String appName = claims.get("appName",String.class); - return appName.equals(app); } @@ -54,7 +57,7 @@ public class BaseController { String authHeader = JwtTokenUtil.getAuthHeader(request); SearchReq dto = JSON.parseObject(text, SearchReq.class); if(dto == null){ dto = new SearchReq(); } - dto.setAppName(JwtTokenUtil.getAppName(authHeader.substring(2))); + // dto.setAppName(JwtTokenUtil.getAppName(authHeader.substring(2))); return dto; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java new file mode 100644 index 0000000..5b3175e --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java @@ -0,0 +1,34 @@ +package com.jd.platform.hotkey.dashboard.common.config; + +import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; +import com.jd.platform.hotkey.dashboard.common.ex.BizException; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; + +import javax.servlet.http.HttpServletResponse; + + +/** + * @author liyunfeng31 + */ +@ControllerAdvice +public class MyExceptionHandler { + + + @ExceptionHandler(value = BizException.class) + @ResponseBody + public Result bizExceptionHandler(BizException e, HttpServletResponse resp){ + resp.setStatus(e.getCode()); + return Result.error(e.getCode(),e.getMsg()); + + } + + + @ExceptionHandler(value =Exception.class) + @ResponseBody + public Result exceptionHandler(Exception e){ + return Result.error(ResultEnum.BIZ_ERROR); + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Result.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Result.java index ea1df6b..70db062 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Result.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Result.java @@ -30,7 +30,7 @@ public class Result implements Serializable { public Result(ResultEnum resultEnum) { this.code = resultEnum.getCode(); - this.msg = resultEnum.getMessage(); + this.msg = resultEnum.getMsg(); } public static Result success(){ @@ -45,6 +45,10 @@ public class Result implements Serializable { return new Result<>(ResultEnum.NO_CHANGE); } + public static Result error(int code, String msg){ + return new Result<>(code,msg); + } + public static Result error(){ return new Result<>(ResultEnum.BIZ_ERROR); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/eunm/ResultEnum.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/eunm/ResultEnum.java index 133746f..9aab5ed 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/eunm/ResultEnum.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/eunm/ResultEnum.java @@ -24,16 +24,18 @@ public enum ResultEnum { CONFLICT_ERROR(1007, "用户名/手机号已存在"), - TIME_RANGE_LARGE(1008, "查询时间过大"); + ILLEGAL_JSON_ARR(1008, "非法的json数组"), + + TIME_RANGE_LARGE(1022, "查询时间过大"); private int code; - private String message; + private String msg; - ResultEnum(int code, String message) { + ResultEnum(int code, String msg) { this.code = code; - this.message = message; + this.msg = msg; } public int getCode() { @@ -44,11 +46,11 @@ public enum ResultEnum { this.code = code; } - public String getMessage() { - return message; + public String getMsg() { + return msg; } - public void setMessage(String message) { - this.message = message; + public void setMsg(String msg) { + this.msg = msg; } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/ex/BizException.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/ex/BizException.java index cd1c1e2..6453a68 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/ex/BizException.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/ex/BizException.java @@ -15,7 +15,7 @@ public class BizException extends RuntimeException { public BizException(ResultEnum result) { this.code = result.getCode(); - this.msg = result.getMessage(); + this.msg = result.getMsg(); } public int getCode() { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index d33df52..9bc20f2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -159,6 +159,7 @@ public class DataHandler { if (statistics.size() != 0) { statistics.forEach(x -> { x.setBizType(6); + x.setRule(x.getRule()); x.setCreateTime(nowTime); x.setDays(day); x.setHours(hour); @@ -193,6 +194,7 @@ public class DataHandler { } records.forEach(x -> { x.setBizType(5); + x.setRule(x.getRule()); x.setCreateTime(nowTime); x.setDays(day); x.setHours(hour); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index 4042e93..176e670 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -119,9 +119,7 @@ public class KeyController extends BaseController { @PostMapping("/add") @ResponseBody public Result add(KeyTimely key){ - if(!checkApp(key.getAppName())){ - return Result.error(ResultEnum.NO_PERMISSION); - } + checkApp(key.getAppName()); key.setType(0); key.setSource(userName()); int b = keyService.insertKeyByUser(key); @@ -132,9 +130,7 @@ public class KeyController extends BaseController { @ResponseBody public Result remove(String key){ String[] arr = key.split("/"); - if(!checkApp(arr[0])){ - return Result.error(ResultEnum.NO_PERMISSION); - } + checkApp(arr[0]); int b = keyService.delKeyByUser(new KeyTimely(key,userName())); return b == 0 ? Result.fail():Result.success(); } @@ -150,9 +146,7 @@ public class KeyController extends BaseController { @PostMapping("/edit") @ResponseBody public Result editSave(KeyTimely key) { - if(!checkApp(key.getAppName())){ - return Result.error(ResultEnum.NO_PERMISSION); - } + checkApp(key.getAppName()); return Result.success(keyService.updateKeyByUser(key)); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index c1b3e19..c8f4fde 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -8,6 +8,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; +import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.model.Rule; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; @@ -49,12 +50,8 @@ public class RuleController extends BaseController { @PostMapping("/save") @ResponseBody public Result save(Rules rules){ - if(!checkApp(rules.getApp())){ - return Result.error(ResultEnum.NO_PERMISSION); - } - if(!checkRule(rules.getRules())){ - return Result.error(ResultEnum.PARAM_ERROR); - } + checkApp(rules.getApp()); + checkRule(rules.getRules()); rules.setUpdateUser(userName()); int b = ruleService.save(rules); return b == 0 ? Result.fail():Result.success(); @@ -65,9 +62,7 @@ public class RuleController extends BaseController { @PostMapping("/remove") @ResponseBody public Result remove(String key){ - if(!checkApp(key)){ - return Result.error(ResultEnum.NO_PERMISSION); - } + checkApp(key); int b = ruleService.delRule(key, userName()); return b == 0 ? Result.fail():Result.success(); } @@ -81,9 +76,9 @@ public class RuleController extends BaseController { @PostMapping("/list") @ResponseBody - public Page list(PageReq page, String searchText){ + public Page list(PageReq page, String appName){ page.setPageSize(30); - PageInfo info = ruleService.pageKeyRule(page, param(searchText)); + PageInfo info = ruleService.pageKeyRule(page, appName); return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); } @@ -114,13 +109,12 @@ public class RuleController extends BaseController { * @param rules rules * @return boolean */ - private boolean checkRule(String rules) { + private void checkRule(String rules) { try { JSON.parseArray(rules, Rule.class); }catch(Exception e){ - return false; + throw new BizException(ResultEnum.ILLEGAL_JSON_ARR); } - return true; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index 586ef74..9c3c251 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -78,7 +78,8 @@ public class UserController extends BaseController { assert authHeader != null; Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); String role = claims.get("role",String.class); - return new User(role, userService.listApp()); + String appName = claims.get("appName",String.class); + return new User(role, userService.listApp(),appName); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/User.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/User.java index 1644036..79ed8cc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/User.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/User.java @@ -38,8 +38,13 @@ public class User { this.appNames = appNames; } - public User(Integer id, String userName, String role, String appName) { - this.id = id; + public User(String role, List appNames, String app) { + this.role = role; + this.appNames = appNames; + this.appName = app; + } + + public User(String userName, String role, String appName) { this.userName = userName; this.role = role; this.appName = appName; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java index f06e426..d07a728 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java @@ -25,7 +25,7 @@ public interface RuleService { int delRule(String key, String updater); - PageInfo pageKeyRule(PageReq page, SearchReq param); + PageInfo pageKeyRule(PageReq page, String appName); int save(Rules rules); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 8b99137..2a85103 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -15,6 +15,7 @@ import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.Rule; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -94,7 +95,7 @@ public class RuleServiceImpl implements RuleService { } @Override - public PageInfo pageKeyRule(PageReq page, SearchReq param) { + public PageInfo pageKeyRule(PageReq page, String appName) { List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); List rules = new ArrayList<>(); for (KeyValue kv : keyValues) { @@ -104,7 +105,13 @@ public class RuleServiceImpl implements RuleService { } String key = kv.getKey().toStringUtf8(); String k = key.replace(ConfigConstant.rulePath,""); - rules.add(new Rules(k, v)); + if(StringUtils.isEmpty(appName)){ + rules.add(new Rules(k, v)); + }else{ + if(k.equals(appName)){ + rules.add(new Rules(k, v)); + } + } } return new PageInfo<>(rules); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 784c639..373ddec 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.util; import com.alibaba.fastjson.JSON; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; import com.jd.platform.hotkey.dashboard.model.Statistics; @@ -124,6 +125,9 @@ public class CommonUtil { * @return map */ private static Map> listGroup(List list){ + if(Constant.VERSION != 1){ + return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); + } return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); } diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml index 87f83c7..5c3fb6c 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml @@ -178,13 +178,13 @@ = #{startTime} and create_time < #{endTime}]]> - GROUP BY app_name, key_name ORDER BY count desc limit 1000 + GROUP BY app_name, key_name ORDER BY count desc limit 100 - select id, key_name, count, app, days, hours, minutes, biz_type, uuid, create_time + select id, key_name, count, app,rule, days, hours, minutes, biz_type, uuid, create_time from hk_statistics where 1=1 @@ -26,7 +27,7 @@ - delete - from hk_statistics + delete from hk_statistics where biz_type = #{type} \ No newline at end of file diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 9ced405..bf8c13a 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -78,7 +78,6 @@ search[field.name] = field.value; }); var params = $("#dataTable").bootstrapTable("getOptions"); - console.log(params.searchText); params.queryParams=function(params){ search.pageSize= params.pageSize; search.pageNum=params.pageNumber; @@ -494,9 +493,6 @@ modal_status = { function getCookie(cname){ let token = window.localStorage.getItem('token'); - let time = window.localStorage.getItem('time'); - console.log("token---> "+token); - console.log("存入的时间---> "+time+" 当前的时间---> "+Date.now()); if(token != null && token !==""){ let time = window.localStorage.getItem('time'); if(Date.now()-time>7*24*360000){ @@ -509,7 +505,7 @@ function getCookie(cname){ let ca = document.cookie.split(';'); for(let i=0; i" + apps[i] + ""); + let appName = data.appName; + for (let i = 0; i < apps.length; i++) { + let app = apps[i]; + if(app === appName){ + $("#apps").append(""); + }else{ + $("#apps").append(""); + } } } }); diff --git a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html index c4ddcec..3352370 100644 --- a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html +++ b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html @@ -105,19 +105,25 @@ $(function(){ "Authorization":getCookie("token") }, async : false, - error : function(request) { - $.modal.alertError("系统错误"); + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); }, success : function(data) { console.log(data) - $("#apps").append(""); let apps = data.appNames; - for (var i = 0; i < apps.length; i++) { - $("#apps").append(""); + let appName = data.appName; + for (let i = 0; i < apps.length; i++) { + let app = apps[i]; + if(app === appName){ + $("#apps").append(""); + }else{ + $("#apps").append(""); + } } } }); + var oTab=$.table.oTableInit(options); oTab.Init(); diff --git a/dashboard/src/main/resources/templates/admin/key/listtimely.html b/dashboard/src/main/resources/templates/admin/key/listtimely.html index 2d76199..8970cd7 100644 --- a/dashboard/src/main/resources/templates/admin/key/listtimely.html +++ b/dashboard/src/main/resources/templates/admin/key/listtimely.html @@ -187,18 +187,25 @@ $(function(){ "Authorization":getCookie("token") }, async : false, - error : function(request) { - $.modal.alertError("系统错误"); + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); }, success : function(data) { console.log(data) let apps = data.appNames; - for (var i = 0; i < apps.length; i++) { - $("#apps").append(""); + let appName = data.appName; + for (let i = 0; i < apps.length; i++) { + let app = apps[i]; + if(app === appName){ + $("#apps").append(""); + }else{ + $("#apps").append(""); + } } } }); + var oTab=$.table.oTableInit(options); oTab.Init(); diff --git a/dashboard/src/main/resources/templates/admin/rule/jn.html b/dashboard/src/main/resources/templates/admin/rule/jn.html index 387876c..33f3417 100644 --- a/dashboard/src/main/resources/templates/admin/rule/jn.html +++ b/dashboard/src/main/resources/templates/admin/rule/jn.html @@ -14,8 +14,6 @@ - - diff --git a/dashboard/src/main/resources/templates/admin/rule/list.html b/dashboard/src/main/resources/templates/admin/rule/list.html index b316a63..88cf035 100644 --- a/dashboard/src/main/resources/templates/admin/rule/list.html +++ b/dashboard/src/main/resources/templates/admin/rule/list.html @@ -140,14 +140,20 @@ "Authorization":getCookie("token") }, async : false, - error : function(request) { - $.modal.alertError("系统错误"); + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); }, success : function(data) { console.log(data) let apps = data.appNames; - for (var i = 0; i < apps.length; i++) { - $("#apps").append(""); + let appName = data.appName; + for (let i = 0; i < apps.length; i++) { + let app = apps[i]; + if(app === appName){ + $("#apps").append(""); + }else{ + $("#apps").append(""); + } } } }); diff --git a/dashboard/src/main/resources/templates/admin/rule/view.html b/dashboard/src/main/resources/templates/admin/rule/view.html index 4982947..1cfda91 100644 --- a/dashboard/src/main/resources/templates/admin/rule/view.html +++ b/dashboard/src/main/resources/templates/admin/rule/view.html @@ -72,17 +72,26 @@ "Authorization":getCookie("token") }, async : false, - error : function(request) { - $.modal.alertError("系统错误"); + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); }, success : function(data) { - console.log(data) let apps = data.appNames; - for (var i = 0; i < apps.length; i++) { - if(apps[i] == initApp){ - $("#jnApp").append(""); + let appName = data.appName; + for (let i = 0; i < apps.length; i++) { + let app = apps[i]; + if(initApp === 1){ + if(app === appName){ + $("#jnApp").append(""); + }else{ + $("#jnApp").append(""); + } }else{ - $("#jnApp").append(""); + if(apps[i] === initApp){ + $("#jnApp").append(""); + }else{ + $("#jnApp").append(""); + } } } } @@ -110,7 +119,6 @@ $.modal.alertError("系统错误"); }, success : function(data) { - console.log(data) layer.alert(data.msg) } }); @@ -127,11 +135,10 @@ }, data:{"app":app,"rules":rules}, async : false, - error : function(request) { - $.modal.alertError("系统错误"); + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); }, success : function(data) { - console.log(data) layer.alert(data.msg) $("#recordSearch", parent.document.body).trigger("click"); parent.layer.closeAll() -- Gitee From 1428fe68459fe0c21ffe09554869834e89d3cbd0 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 26 Jun 2020 22:30:45 +0800 Subject: [PATCH 056/156] =?UTF-8?q?=E6=8A=98=E7=BA=BF=E5=9B=BE=E6=95=B0?= =?UTF-8?q?=E7=BB=84=E5=8F=98map=E9=98=B2=E6=AD=A2=E8=B6=8A=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/domain/vo/HotKeyLineChartVo.java | 25 +++++++++ .../common/domain/vo/LineChartVo.java | 28 ++++++++++ .../service/impl/KeyServiceImpl.java | 40 ++++++++++---- .../hotkey/dashboard/util/CommonUtil.java | 55 ++++++++++++++++++- .../hotkey/dashboard/util/DateUtil.java | 5 ++ 5 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java index ce4945e..5f4768b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java @@ -2,6 +2,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; import java.util.List; import java.util.Map; +import java.util.Set; /** * @author liyunfeng31 @@ -12,6 +13,9 @@ public class HotKeyLineChartVo { private Map series; + public HotKeyLineChartVo() { + } + public HotKeyLineChartVo(List xAxis, Map series) { this.xAxis = xAxis; this.series = series; @@ -24,4 +28,25 @@ public class HotKeyLineChartVo { public Map getSeries() { return series; } + + + private Set xAxis2; + + private Map> series2; + + public Set getxAxis2() { + return xAxis2; + } + + public void setxAxis2(Set xAxis2) { + this.xAxis2 = xAxis2; + } + + public Map> getSeries2() { + return series2; + } + + public void setSeries2(Map> series2) { + this.series2 = series2; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java new file mode 100644 index 0000000..2ae730a --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java @@ -0,0 +1,28 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author liyunfeng31 + */ +public class LineChartVo { + + private Set xAxis; + + private Map> series; + + public LineChartVo(Set xAxis, Map> series) { + this.xAxis = xAxis; + this.series = series; + } + + public Set getxAxis() { + return xAxis; + } + + public Map> getSeries() { + return series; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 348e2bf..f1bab02 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.service.impl; +import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; @@ -26,6 +27,7 @@ import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.time.Duration; import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; @@ -67,18 +69,30 @@ public class KeyServiceImpl implements KeyService { LocalDateTime startTime = now.minusMinutes(31); req.setStartTime(DateUtil.ldtToDate(startTime)); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.assembleData(list, startTime, 30,1); + try { + return CommonUtil.processData(startTime,now,list,true); + }catch (Exception e){ + return CommonUtil.assembleData(list, startTime, 30,1); + } }else if(type == 6){ LocalDateTime startTime2 = now.minusHours(25); req.setStartTime(DateUtil.ldtToDate(startTime2)); List list2 = statisticsMapper.listOrderByTime(req); - return CommonUtil.assembleData(list2, startTime2, 24,2); + try { + return CommonUtil.processData(startTime2,now,list2,false); + }catch (Exception e){ + return CommonUtil.assembleData(list2, startTime2, 24,2); + } }else{ LocalDateTime startTime3 = now.minusDays(7).minusHours(1); req.setStartTime(DateUtil.ldtToDate(startTime3)); req.setType(6); List list3 = statisticsMapper.listOrderByTime(req); - return CommonUtil.assembleData(list3, startTime3, 7*24,2); + try { + return CommonUtil.processData(startTime3,now,list3,false); + }catch (Exception e){ + return CommonUtil.assembleData(list3, startTime3, 7*24,2); + } } } @@ -265,11 +279,14 @@ public class KeyServiceImpl implements KeyService { } - private List statisticsList(){ + private static List statisticsList(){ Random rd = new Random(); List records = new ArrayList<>(); - statisticsMapper.clear(1); + // statisticsMapper.clear(1); for (int i = 0; i < 35; i++) { + if(i== 2||i== 3||i== 4||i== 25||i== 12||i== 13||i== 14||i== 30||i==32||i==33||i== 34){ + continue; + } // 每分钟小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now().minusMinutes(35-i); Date nowTime = DateUtil.ldtToDate(now); @@ -301,7 +318,11 @@ public class KeyServiceImpl implements KeyService { s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); records.add(s11); } - statisticsMapper.batchInsert(records); + + if(1==1){ + return records; + } + // statisticsMapper.batchInsert(records); List list = new ArrayList<>(); for (int i = 0; i < 30 ; i++) { if(i == 12 || i ==13){ @@ -316,7 +337,7 @@ public class KeyServiceImpl implements KeyService { int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); // System.out.println(time); st.setMinutes(time); - list.add(st); + // list.add(st); } } List list2 = new ArrayList<>(); @@ -333,7 +354,7 @@ public class KeyServiceImpl implements KeyService { list2.add(st2); } } - list.addAll(list2); + // list.addAll(list2); return list; } @@ -378,7 +399,7 @@ public class KeyServiceImpl implements KeyService { Statistics s33 = new Statistics(); s33.setKeyName("k33"); s33.setApp("key33APP"); - s33.setBizType(6 ); + s33.setBizType(6); s33.setCreateTime(nowTime); s33.setDays(day); s33.setCount(rd.nextInt(300)); @@ -424,7 +445,6 @@ public class KeyServiceImpl implements KeyService { return records; } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 373ddec..c352204 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -3,12 +3,13 @@ package com.jd.platform.hotkey.dashboard.util; import com.alibaba.fastjson.JSON; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; -import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; +import com.jd.platform.hotkey.dashboard.common.domain.vo.LineChartVo; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; +import java.time.Duration; import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; @@ -131,5 +132,57 @@ public class CommonUtil { return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); } + /** + * 分组 + * @param list list + * @return map + */ + private static Map> listGroupByTime(List list, boolean isMinute){ + if(isMinute){ + return list.stream().collect(Collectors.groupingBy(Statistics::getMinutes)); + } + return list.stream().collect(Collectors.groupingBy(Statistics::getHours)); + } + + + /** + * 处理数据 + * @param st 开始时间 + * @param et 结束时间 + * @param list 数据 + * @param isMinute 类型 + * @return vo + */ + public static HotKeyLineChartVo processData(LocalDateTime st, LocalDateTime et, List list, boolean isMinute){ + Set set = new TreeSet<>(); + Duration duration = Duration.between(st,et); + long passTime = isMinute ? duration.toMinutes() : duration.toHours(); + Map timeCountMap = new TreeMap<>(); + String pattern = isMinute ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; + for (int i = 0; i < passTime; i++) { + int time = DateUtil.reviseTime(st, i, isMinute ? 1:2); + set.add(DateUtil.formatTime(time, pattern)); + timeCountMap.put(time,null); + } + Map> ruleStatsMap = listGroup(list); + Map> ruleDataMap = new HashMap<>(ruleStatsMap.size()); + ruleStatsMap.forEach((rule,statisticsList)->{ + Map> timeStatsMap = listGroupByTime(statisticsList, isMinute); + timeCountMap.forEach((k,v)->{ + if(timeStatsMap.get(k) == null){ + timeCountMap.put(k,0); + }else{ + timeCountMap.put(k,timeStatsMap.get(k).get(0).getCount()); + } + }); + ruleDataMap.put(rule, new ArrayList<>(timeCountMap.values())); + }); + + HotKeyLineChartVo vo = new HotKeyLineChartVo(); + vo.setxAxis2(set); + vo.setSeries2(ruleDataMap); + return vo; + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index dbe252f..c049d78 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -99,4 +99,9 @@ public class DateUtil { public static Date preDays(int days){ return ldtToDate(LocalDateTime.now().minusDays(days)); } + + public static String formatTime(int time,String pattern){ + return strToLdt(time+"", pattern).toString().replace("T", " "); + } + } -- Gitee From 936561d8cb8e4749383486a0e8ec0e5d21e6cf68 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Sun, 28 Jun 2020 12:03:02 +0800 Subject: [PATCH 057/156] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E7=83=AD=E7=82=B9?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E6=AC=A1=E6=95=B0=E5=92=8C=E6=80=BB=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=E6=AC=A1=E6=95=B0=E7=9A=84=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/client/ClientStarter.java | 1 + .../hotkey/client/callback/JdHotKeyStore.java | 48 ++++---- .../client/core/key/DefaultKeyHandler.java | 24 +++- .../hotkey/client/core/key/IKeyCollector.java | 14 ++- .../hotkey/client/core/key/IKeyHandler.java | 12 -- .../hotkey/client/core/key/IKeyPusher.java | 9 ++ .../client/core/key/KeyHandlerFactory.java | 22 ++-- .../hotkey/client/core/key/KeyHotModel.java | 32 +++++ .../client/core/key/NettyKeyPusher.java | 37 +++++- .../client/core/key/PushSchedulerStarter.java | 22 +++- .../client/core/key/TurnCountCollector.java | 115 ++++++++++++++++++ .../client/core/key/TurnKeyCollector.java | 19 ++- .../client/core/rule/KeyRuleHolder.java | 15 ++- .../hotkey/common/model/HotKeyMsg.java | 15 +++ .../hotkey/common/model/KeyCountModel.java | 67 ++++++++++ .../common/model/typeenum/MessageType.java | 1 + .../platform/hotkey/common/tool/Constant.java | 7 ++ .../platform/hotkey/common/tool/TwoTuple.java | 23 ++++ .../java/com/jd/platform/sample/Cache.java | 7 +- .../disruptor/hotkey/HotKeyEventProducer.java | 39 ------ .../worker/netty/filter/HotKeyFilter.java | 2 +- .../worker/netty/filter/KeyCounterFilter.java | 66 ++++++++++ 22 files changed, 482 insertions(+), 115 deletions(-) delete mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java create mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java create mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java index f6267a3..1102d8a 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java @@ -98,6 +98,7 @@ public class ClientStarter { EtcdConfigFactory.buildConfigCenter(etcdServer); //开始定时推送 PushSchedulerStarter.startPusher(pushPeriod); + PushSchedulerStarter.startCountPusher(10); //开启worker重连器 WorkerRetryConnector.retryConnectWorkers(); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java index 9bf7d35..ee43413 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java @@ -3,6 +3,8 @@ package com.jd.platform.hotkey.client.callback; import com.jd.platform.hotkey.client.cache.CacheFactory; import com.jd.platform.hotkey.client.cache.LocalCache; import com.jd.platform.hotkey.client.core.key.HotKeyPusher; +import com.jd.platform.hotkey.client.core.key.KeyHandlerFactory; +import com.jd.platform.hotkey.client.core.key.KeyHotModel; import com.jd.platform.hotkey.common.model.typeenum.KeyType; import com.jd.platform.hotkey.common.tool.Constant; @@ -20,11 +22,13 @@ public class JdHotKeyStore { if (!inRule(key)) { return false; } - Object value = getValueSimple(key); - if (value == null) { + boolean isHot = isHot(key); + if (!isHot) { HotKeyPusher.push(key, null); } - return value != null; + //统计计数 + KeyHandlerFactory.getCounter().collect(new KeyHotModel(key, isHot)); + return isHot; } /** @@ -43,7 +47,9 @@ public class JdHotKeyStore { * 判断是否是热key，如果是热key，则给value赋值 */ public static void smartSet(String key, Object value) { - setValue(key, value); + if (isHot(key)) { + setValueDirectly(key, value); + } } /** @@ -58,6 +64,8 @@ public class JdHotKeyStore { if (value == null) { HotKeyPusher.push(key, keyType); } + //统计计数 + KeyHandlerFactory.getCounter().collect(new KeyHotModel(key, value != null)); //如果是默认值，也返回null if(value instanceof Integer && Constant.MAGIC_NUMBER == (int) value) { return null; @@ -76,16 +84,10 @@ public class JdHotKeyStore { return getCache(key).get(key); } - public static void setValue(String key, Object value) { - if (isHot(key)) { - setValueDirectly(key, value); - } - } - /** * 纯粹的本地缓存，无需该key是热key */ - public static void setValueDirectly(String key, Object value) { + static void setValueDirectly(String key, Object value) { getCache(key).set(key, value); } @@ -98,28 +100,28 @@ public class JdHotKeyStore { /** * 判断是否是热key。适用于只需要判断key，而不需要value的场景 */ - public static boolean isHot(String key) { + static boolean isHot(String key) { return getValueSimple(key) != null; } /** * 判断是否已经缓存过了。 */ - public static boolean isValueCached(String key, KeyType keyType) { - Object value = getValue(key, keyType); - //如果value不为null且不为默认值 - if (value == null) { - return false; - } - return !(value instanceof Integer) || Constant.MAGIC_NUMBER != (int) value; - } +// private static boolean isValueCached(String key, KeyType keyType) { +// Object value = getValue(key, keyType); +// //如果value不为null且不为默认值 +// if (value == null) { +// return false; +// } +// return !(value instanceof Integer) || Constant.MAGIC_NUMBER != (int) value; +// } /** * 判断某key的value是否已经缓存过了 */ - public static boolean isValueCached(String key) { - return isValueCached(key, KeyType.REDIS_KEY); - } +// public static boolean isValueCached(String key) { +// return isValueCached(key, KeyType.REDIS_KEY); +// } private static LocalCache getCache(String key) { return CacheFactory.getNonNullCache(key); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java index bfc0a8d..33b71f3 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java @@ -1,17 +1,29 @@ package com.jd.platform.hotkey.client.core.key; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; + /** * @author wuweifeng wrote on 2020-02-25 * @version 1.0 */ -public class DefaultKeyHandler implements IKeyHandler { - @Override +public class DefaultKeyHandler { + private IKeyPusher iKeyPusher = new NettyKeyPusher(); + + private IKeyCollector iKeyCollector = new TurnKeyCollector(); + + private IKeyCollector iKeyCounter = new TurnCountCollector(); + + public IKeyPusher keyPusher() { - return new NettyKeyPusher(); + return iKeyPusher; + } + + public IKeyCollector keyCollector() { + return iKeyCollector; } - @Override - public IKeyCollector keyCollector() { - return new TurnKeyCollector(); + public IKeyCollector keyCounter() { + return iKeyCounter; } } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java index e53de7f..c2a5fa2 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java @@ -1,7 +1,5 @@ package com.jd.platform.hotkey.client.core.key; -import com.jd.platform.hotkey.common.model.HotKeyModel; - import java.util.List; /** @@ -9,10 +7,16 @@ import java.util.List; * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ -public interface IKeyCollector { - List lockAndGetResult(); +public interface IKeyCollector { + /** + * 锁定后的返回值 + */ + List lockAndGetResult(); - void collect(HotKeyModel hotKeyModel); + /** + * 输入的参数 + */ + void collect(T t); void finishOnce(); } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java deleted file mode 100644 index 64b33c2..0000000 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.jd.platform.hotkey.client.core.key; - -/** - * @author wuweifeng wrote on 2020-02-25 - * @version 1.0 - */ -public interface IKeyHandler { - - IKeyPusher keyPusher(); - - IKeyCollector keyCollector(); -} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java index 7d70b57..c8c706f 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.client.core.key; import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; import java.util.List; @@ -10,5 +11,13 @@ import java.util.List; * @version 1.0 */ public interface IKeyPusher { + /** + * 发送待测key + */ void send(String appName, List list); + + /** + * 发送热key访问量 + */ + void sendCount(String appName, List list); } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java index 0453446..f9fd24d 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java @@ -1,25 +1,27 @@ package com.jd.platform.hotkey.client.core.key; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; + /** * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ public class KeyHandlerFactory { - private static final IKeyHandler iKeyHandler = new DefaultKeyHandler(); - - private static final IKeyPusher iKeyPusher = iKeyHandler.keyPusher(); - - private static final IKeyCollector iKeyCollector = iKeyHandler.keyCollector(); - + private static final DefaultKeyHandler iKeyHandler = new DefaultKeyHandler(); private KeyHandlerFactory() { } - public synchronized static IKeyPusher getPusher() { - return iKeyPusher; + public static IKeyPusher getPusher() { + return iKeyHandler.keyPusher(); + } + + public static IKeyCollector getCollector() { + return iKeyHandler.keyCollector(); } - public synchronized static IKeyCollector getCollector() { - return iKeyCollector; + public static IKeyCollector getCounter() { + return iKeyHandler.keyCounter(); } } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java new file mode 100644 index 0000000..96df161 --- /dev/null +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java @@ -0,0 +1,32 @@ +package com.jd.platform.hotkey.client.core.key; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class KeyHotModel { + private String key; + private boolean isHot; + + public KeyHotModel(String key, boolean isHot) { + this.key = key; + this.isHot = isHot; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public boolean isHot() { + return isHot; + } + + public void setHot(boolean hot) { + isHot = hot; + } +} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index fca5cf2..983b5a8 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -4,6 +4,7 @@ import com.jd.platform.hotkey.client.core.worker.WorkerInfoHolder; import com.jd.platform.hotkey.client.log.JdLogger; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; @@ -35,8 +36,8 @@ public class NettyKeyPusher implements IKeyPusher { continue; } - map.computeIfAbsent(channel, k -> new ArrayList<>()); - map.get(channel).add(model); + List newList = map.computeIfAbsent(channel, k -> new ArrayList<>()); + newList.add(model); } for (Channel channel : map.keySet()) { @@ -56,4 +57,36 @@ public class NettyKeyPusher implements IKeyPusher { } + @Override + public void sendCount(String appName, List list) { + //积攒了10秒的数量，按照hash分发到不同的worker + long now = System.currentTimeMillis(); + Map> map = new HashMap<>(); + for(KeyCountModel model : list) { + model.setCreateTime(now); + Channel channel = WorkerInfoHolder.chooseChannel(model.getRuleKey()); + if (channel == null) { + continue; + } + + List newList = map.computeIfAbsent(channel, k -> new ArrayList<>()); + newList.add(model); + } + + for (Channel channel : map.keySet()) { + try { + List batch = map.get(channel); + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))); + } catch (Exception e) { + try { + InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); + JdLogger.error(getClass(),"flush error " + insocket.getAddress().getHostAddress()); + } catch (Exception ex) { + JdLogger.error(getClass(),"flush error"); + } + + } + } + } + } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java index a77439e..b5eda05 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java @@ -1,6 +1,8 @@ package com.jd.platform.hotkey.client.core.key; import com.jd.platform.hotkey.client.Context; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -13,16 +15,34 @@ import java.util.concurrent.TimeUnit; */ public class PushSchedulerStarter { + /** + * 每0.5秒推送一次待测key + */ public static void startPusher(Long period) { if (period == null || period <= 0) { period = 500L; } ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); scheduledExecutorService.scheduleAtFixedRate(() -> { - IKeyCollector collectHK = KeyHandlerFactory.getCollector(); + IKeyCollector collectHK = KeyHandlerFactory.getCollector(); KeyHandlerFactory.getPusher().send(Context.APP_NAME, collectHK.lockAndGetResult()); collectHK.finishOnce(); },0, period, TimeUnit.MILLISECONDS); } + /** + * 每10秒推送一次数量统计 + */ + public static void startCountPusher(Integer period) { + if (period == null || period <= 0) { + period = 10; + } + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.scheduleAtFixedRate(() -> { + IKeyCollector collectHK = KeyHandlerFactory.getCounter(); + KeyHandlerFactory.getPusher().sendCount(Context.APP_NAME, collectHK.lockAndGetResult()); + collectHK.finishOnce(); + },0, period, TimeUnit.SECONDS); + } + } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java new file mode 100644 index 0000000..329ec7c --- /dev/null +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java @@ -0,0 +1,115 @@ +package com.jd.platform.hotkey.client.core.key; + +import cn.hutool.core.util.StrUtil; +import com.jd.platform.hotkey.client.core.rule.KeyRuleHolder; +import com.jd.platform.hotkey.common.model.KeyCountModel; +import com.jd.platform.hotkey.common.tool.Constant; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 热点数量统计 + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class TurnCountCollector implements IKeyCollector { + /** + * 存储格式为： + * pin_20200624091024 -> 10 + * pin_20200624091025 -> 20 + *

+ * sku_20200624091024 -> 123 + * sku_20200624091025 -> 142 + */ + private ConcurrentHashMap HIT_MAP_0 = new ConcurrentHashMap<>(512); + private ConcurrentHashMap HIT_MAP_1 = new ConcurrentHashMap<>(512); + + private static final String FORMAT = "yyyy-MM-dd HH:mm:ss"; + + private AtomicLong atomicLong = new AtomicLong(0); + + @Override + public List lockAndGetResult() { + //自增后，对应的map就会停止被写入，等待被读取 + atomicLong.addAndGet(1); + + List list; + if (atomicLong.get() % 2 == 0) { + list = get(HIT_MAP_1); + HIT_MAP_1.clear(); + } else { + list = get(HIT_MAP_0); + HIT_MAP_0.clear(); + } + return list; + } + + /** + * 每10秒上报一次最近10秒的数据，并且删掉相应的key + */ + private List get(ConcurrentHashMap map) { + List list = new ArrayList<>(); + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + HitCount hitCount = entry.getValue(); + KeyCountModel keyCountModel = new KeyCountModel(); + keyCountModel.setTotalHitCount(hitCount.totalHitCount.get()); + keyCountModel.setRuleKey(key); + keyCountModel.setHotHitCount(hitCount.hotHitCount.get()); + list.add(keyCountModel); + } + return list; + } + + @Override + public void collect(KeyHotModel keyHotModel) { + if (atomicLong.get() % 2 == 0) { + put(keyHotModel.getKey(), keyHotModel.isHot(), HIT_MAP_0); + } else { + put(keyHotModel.getKey(), keyHotModel.isHot(), HIT_MAP_1); + } + } + + @Override + public void finishOnce() { + + } + + public void put(String key, boolean isHot, ConcurrentHashMap map) { + //如key是pin_的前缀，则存储pin_ + String rule = KeyRuleHolder.rule(key); + //不在规则内的不处理 + if (StrUtil.isEmpty(rule)) { + return; + } + String nowTime = nowTime(); + + //rule + 分隔符 + 2020-10-23 21:11:22 + String mapKey = rule + Constant.COUNT_DELIMITER + nowTime; + //该方法线程安全 + HitCount hitCount = map.computeIfAbsent(mapKey, v -> new HitCount()); + if (isHot) { + hitCount.hotHitCount.incrementAndGet(); + } + hitCount.totalHitCount.incrementAndGet(); + } + + private String nowTime() { + Date nowTime = new Date(System.currentTimeMillis()); + SimpleDateFormat sdFormatter = new SimpleDateFormat(FORMAT); + return sdFormatter.format(nowTime); + } + + private class HitCount { + private AtomicInteger hotHitCount = new AtomicInteger(); + private AtomicInteger totalHitCount = new AtomicInteger(); + } +} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java index 87e9031..353938c 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java @@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong; * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ -public class TurnKeyCollector implements IKeyCollector { +public class TurnKeyCollector implements IKeyCollector { private ConcurrentHashMap map0 = new ConcurrentHashMap<>(); private ConcurrentHashMap map1 = new ConcurrentHashMap<>(); @@ -49,21 +49,15 @@ public class TurnKeyCollector implements IKeyCollector { return; } if (atomicLong.get() % 2 == 0) { - HotKeyModel model = map0.get(key); - if (model == null) { - map0.put(key, hotKeyModel); - } else { + //不存在时返回null并将key-value放入，已有相同key时，返回该key对应的value，并且不覆盖 + HotKeyModel model = map0.putIfAbsent(key, hotKeyModel); + if (model != null) { model.setCount(model.getCount() + hotKeyModel.getCount()); - map0.put(key, model); } - } else { - HotKeyModel model = map1.get(key); - if (model == null) { - map1.put(key, hotKeyModel); - } else { + HotKeyModel model = map1.putIfAbsent(key, hotKeyModel); + if (model != null) { model.setCount(model.getCount() + hotKeyModel.getCount()); - map1.put(key, model); } } @@ -73,4 +67,5 @@ public class TurnKeyCollector implements IKeyCollector { public void finishOnce() { } + } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java b/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java index 79dd504..4397d63 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java @@ -101,6 +101,19 @@ public class KeyRuleHolder { } + /** + * 判断该key命中了哪个rule + */ + public static String rule(String key) { + String rule = ""; + for (KeyRule keyRule : KEY_RULES) { + if (key.equals(keyRule.getKey()) || (keyRule.isPrefix() && key.startsWith(keyRule.getKey())) || "*".equals(keyRule.getKey())) { + return keyRule.getKey(); + } + } + return rule; + } + /** * 判断key是否在配置的要探测的规则内 */ @@ -121,7 +134,7 @@ public class KeyRuleHolder { @Subscribe public void ruleChange(KeyRuleInfoChangeEvent event) { - JdLogger.info(getClass(),"new rules info is :" + event.getKeyRules()); + JdLogger.info(getClass(), "new rules info is :" + event.getKeyRules()); List ruleList = event.getKeyRules(); if (ruleList == null) { return; diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java index 4e197f3..a1caf1a 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java @@ -10,11 +10,18 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; public class HotKeyMsg { private int magicNumber; + private String appName; + private MessageType messageType; private String body; public HotKeyMsg(MessageType messageType, String body) { + this(null, messageType, body); + } + + public HotKeyMsg(String appName, MessageType messageType, String body) { + this.appName = appName; this.messageType = messageType; this.body = body; } @@ -31,6 +38,14 @@ public class HotKeyMsg { '}'; } + public String getAppName() { + return appName; + } + + public void setAppName(String appName) { + this.appName = appName; + } + public int getMagicNumber() { return magicNumber; } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java b/common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java new file mode 100644 index 0000000..09f3563 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java @@ -0,0 +1,67 @@ +package com.jd.platform.hotkey.common.model; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class KeyCountModel { + /** + * 对应的规则名，如 pin_2020-08-09 11:32:43 + */ + private String ruleKey; + /** + * 总访问次数 + */ + private int totalHitCount; + /** + * 热后访问次数 + */ + private int hotHitCount; + /** + * 发送时的时间 + */ + private long createTime; + + @Override + public String toString() { + return "KeyCountModel{" + + "ruleKey='" + ruleKey + '\'' + + ", totalHitCount=" + totalHitCount + + ", hotHitCount=" + hotHitCount + + ", createTime=" + createTime + + '}'; + } + + public long getCreateTime() { + return createTime; + } + + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public String getRuleKey() { + return ruleKey; + } + + public void setRuleKey(String ruleKey) { + this.ruleKey = ruleKey; + } + + public int getTotalHitCount() { + return totalHitCount; + } + + public void setTotalHitCount(int totalHitCount) { + this.totalHitCount = totalHitCount; + } + + public int getHotHitCount() { + return hotHitCount; + } + + public void setHotHitCount(int hotHitCount) { + this.hotHitCount = hotHitCount; + } +} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java index 57daa2b..cd3f983 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java @@ -8,6 +8,7 @@ public enum MessageType { APP_NAME((byte) 1), REQUEST_NEW_KEY((byte) 2), RESPONSE_NEW_KEY((byte) 3), + REQUEST_HIT_COUNT((byte) 7), PING((byte) 4), PONG((byte) 5), EMPTY((byte) 6); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 9e0cc85..0c9bf86 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -14,7 +14,14 @@ public class Constant { public static int Default_Threads = 4; public static int MAGIC_NUMBER = 0x12fcf76; + /** + * netty的分隔符 + */ public static String DELIMITER = "$(* *)$"; + /** + * 数量统计时，rule+时间 组成key用的分隔符 + */ + public static String COUNT_DELIMITER = "#**#"; //单次包最大2M public static int MAX_LENGTH = 2 * 1024 * 1024; diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java new file mode 100644 index 0000000..9aa4051 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java @@ -0,0 +1,23 @@ +package com.jd.platform.hotkey.common.tool; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class TwoTuple { + + public final A first; + + public final B second; + + public TwoTuple(A a, B b){ + first = a; + second = b; + } + + public String toString(){ + return "(" + first + ", " + second + ")"; + } + +} \ No newline at end of file diff --git a/sample/src/main/java/com/jd/platform/sample/Cache.java b/sample/src/main/java/com/jd/platform/sample/Cache.java index 82887a9..7bc1443 100644 --- a/sample/src/main/java/com/jd/platform/sample/Cache.java +++ b/sample/src/main/java/com/jd/platform/sample/Cache.java @@ -61,10 +61,11 @@ public class Cache { // } public String get(String key) { + Object object = JdHotKeyStore.getValue(key); //如果已经缓存过了 - if (JdHotKeyStore.getValue(key) != null) { - System.out.println("1"); - return JdHotKeyStore.getValue(key).toString(); + if (object != null) { + System.out.println("is hot key"); + return object.toString(); } else { String value = getFromRedis(key); JdHotKeyStore.smartSet(key, value); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java index cdfd38b..42dae5f 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java @@ -29,43 +29,4 @@ public class HotKeyEventProducer implements MessageProducer { } } - //nioEventLoopGroup-3-63" #206 prio=10 os_prio=0 tid=0x00007ff498077000 nid=0x6a6 runnable [0x00007ff422f44000] - // java.lang.Thread.State: TIMED_WAITING (parking) - // at sun.misc.Unsafe.park(Native Method) - // at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) - // at com.lmax.disruptor.SingleProducerSequencer.next(SingleProducerSequencer.java:137) - // at com.lmax.disruptor.SingleProducerSequencer.next(SingleProducerSequencer.java:110) - // at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) - // at com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventProducer.publish(HotKeyEventProducer.java:23) - // at com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventProducer.publish(HotKeyEventProducer.java:12) - // at com.jd.platform.hotkey.worker.netty.filter.HotKeyFilter.publishMsg(HotKeyFilter.java:46) - // at com.jd.platform.hotkey.worker.netty.filter.HotKeyFilter.chain(HotKeyFilter.java:31) - // at com.jd.platform.hotkey.worker.netty.server.NodesServerHandler.channelRead0(NodesServerHandler.java:38) - // at com.jd.platform.hotkey.worker.netty.server.NodesServerHandler.channelRead0(NodesServerHandler.java:21) - // at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) - // at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) - // at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) - // at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) - // at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326) - // at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) - // at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) - // at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) - // at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) - // at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) - // at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) - // at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700) - // at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:635) - // at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:552) - // at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) - // at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) - // at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) - // at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) - // at java.lang.Thread.run(Thread.java:745) } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 073b71a..080a9d9 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicLong; /** - * 热key消息，包括从netty来的和mq来的。收到消息，都发到disruptor去 + * 热key消息，包括从netty来的和mq来的。收到消息，都发到队列去 * * @author wuweifeng wrote on 2019-12-11 * @version 1.0 diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java new file mode 100644 index 0000000..9d319d1 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -0,0 +1,66 @@ +package com.jd.platform.hotkey.worker.netty.filter; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.KeyCountModel; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.worker.keydispatcher.KeyProducer; +import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.List; + +/** + * 对热key访问次数和总访问次数进行累计 + * @author wuweifeng wrote on 2020-6-24 + * @version 1.0 + */ +@Component +@Order(4) +public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { + @Resource + private KeyProducer keyProducer; + + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { + if (MessageType.REQUEST_HIT_COUNT == message.getMessageType()) { + + publishMsg(message.getAppName(), message.getBody(), ctx); + + return false; + } + + return true; + } + + @Override + public void receive(String msg) { + publishMsg("", msg, null); + } + + private void publishMsg(String appName, String message, ChannelHandlerContext ctx) { + //老版的用的单个HotKeyModel，新版用的数组 + List models = FastJsonUtils.toList(message, KeyCountModel.class); +// for (HotKeyModel model : models) { +// //白名单key不处理 +// if (WhiteListHolder.contains(model.getKey())) { +// continue; +// } +// long timeOut = SystemClock.now() - model.getCreateTime(); +// if (timeOut > 1000) { +// logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); +// } +// keyProducer.push(model); +// } + + } + +} \ No newline at end of file -- Gitee From f971108ca53cc7ff568b1cafa10f475e4e0103dd Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Sun, 28 Jun 2020 13:33:38 +0800 Subject: [PATCH 058/156] =?UTF-8?q?=E8=A7=84=E5=88=99=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E5=92=8C=E8=A7=84=E5=88=99=E6=A0=A1=E9=AA=8C=E5=9C=A8JS?= =?UTF-8?q?=E5=8A=A0=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/templates/admin/key/list.html | 4 ++++ .../resources/templates/admin/key/listmaxhot.html | 4 ++++ .../resources/templates/admin/key/listtimely.html | 4 ++++ .../main/resources/templates/admin/rule/list.html | 14 +++++--------- .../main/resources/templates/admin/rule/view.html | 12 +++++++++++- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/dashboard/src/main/resources/templates/admin/key/list.html b/dashboard/src/main/resources/templates/admin/key/list.html index f4dc90b..6060e84 100644 --- a/dashboard/src/main/resources/templates/admin/key/list.html +++ b/dashboard/src/main/resources/templates/admin/key/list.html @@ -189,6 +189,10 @@ $(function(){ }, success : function(data) { console.log(data) + let role = data.role; + if(role === "ADMIN"){ + $("#apps").append(""); + } let apps = data.appNames; let appName = data.appName; for (let i = 0; i < apps.length; i++) { diff --git a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html index 3352370..c203968 100644 --- a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html +++ b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html @@ -110,6 +110,10 @@ $(function(){ }, success : function(data) { console.log(data) + let role = data.role; + if(role === "ADMIN"){ + $("#apps").append(""); + } let apps = data.appNames; let appName = data.appName; for (let i = 0; i < apps.length; i++) { diff --git a/dashboard/src/main/resources/templates/admin/key/listtimely.html b/dashboard/src/main/resources/templates/admin/key/listtimely.html index 8970cd7..70db0e0 100644 --- a/dashboard/src/main/resources/templates/admin/key/listtimely.html +++ b/dashboard/src/main/resources/templates/admin/key/listtimely.html @@ -192,6 +192,10 @@ $(function(){ }, success : function(data) { console.log(data) + let role = data.role; + if(role === "ADMIN"){ + $("#apps").append(""); + } let apps = data.appNames; let appName = data.appName; for (let i = 0; i < apps.length; i++) { diff --git a/dashboard/src/main/resources/templates/admin/rule/list.html b/dashboard/src/main/resources/templates/admin/rule/list.html index 88cf035..66a3f05 100644 --- a/dashboard/src/main/resources/templates/admin/rule/list.html +++ b/dashboard/src/main/resources/templates/admin/rule/list.html @@ -4,16 +4,11 @@ - - - + + + + + + +

+
+ +
+ +
+
+ +
+
+
搜索条件
+
+ +
+
+
+
+ +
+ + + + +
+
+
+ +
+ + + + +
+
+
+ + +
+ +
+
+
+
+ + + +
+ +
+ +
+
+
+ + + +
+ +
+ + + + + + + + + + + + + + -- Gitee From 238bac57b5840e8a14e61edab9d200ef40459d4f Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Wed, 15 Jul 2020 20:37:11 +0800 Subject: [PATCH 107/156] fix build --- .../com/jd/platform/hotkey/dashboard/util/CommonUtil.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 15e8d7f..21488ac 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory; import org.springframework.util.StringUtils; import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; import java.time.Duration; import java.time.LocalDateTime; import java.util.*; @@ -236,8 +237,9 @@ public class CommonUtil { String uuid = app + "-" + rule + hitTime; return Summary.aSummary().indexName(rule).rule(rule).app(app) - .val1(totalCount).val2(hitCount) + .val1(totalCount).val2(hitCount).val3(BigDecimal.ZERO) .days(day).hours(hour).minutes(minus).seconds((int) seconds) - .bizType(1).uuid(uuid).build(); + .bizType(1).uuid(uuid).createTime(new Date()).build(); } + } -- Gitee From 80849cebb23446c3a5cee50f14068f69233a114b Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 16 Jul 2020 20:36:28 +0800 Subject: [PATCH 108/156] =?UTF-8?q?=E8=AE=BE=E7=BD=AEclient=E5=88=A0?= =?UTF-8?q?=E9=99=A4key=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/callback/AbsReceiveNewKey.java | 117 ++++++++++-------- .../callback/DefaultNewKeyListener.java | 35 +++++- .../hotkey/client/callback/JdHotKeyStore.java | 3 + .../callback/ReceiveNewKeySubscribe.java | 2 +- .../hotkey/client/core/key/HotKeyPusher.java | 8 +- .../hotkey/client/core/key/KeyRemover.java | 21 ++++ .../hotkey/client/etcd/EtcdStarter.java | 35 +++--- .../platform/hotkey/common/tool/Constant.java | 5 + 8 files changed, 150 insertions(+), 76 deletions(-) create mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyRemover.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java index 924532f..e3013be 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java @@ -1,52 +1,65 @@ -package com.jd.platform.hotkey.client.callback; - -import com.jd.platform.hotkey.client.log.JdLogger; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.model.typeenum.KeyType; - -/** - * @author wuweifeng wrote on 2020-02-24 - * @version 1.0 - */ -public abstract class AbsReceiveNewKey implements ReceiveNewKeyListener { - - - @Override - public void newKey(HotKeyModel hotKeyModel) { - long now = System.currentTimeMillis(); - //如果key到达时已经过去5秒了，记录一下。手工删除key时，没有CreateTime - if (hotKeyModel.getCreateTime() != 0 && Math.abs(now - hotKeyModel.getCreateTime()) > 1000) { - JdLogger.warn(getClass(), "the key comes too late : " + hotKeyModel.getKey() + " now " + - + now + " keyCreateAt " + hotKeyModel.getCreateTime()); - } - if (hotKeyModel.isRemove()) { - deleteKey(hotKeyModel.getKey(), hotKeyModel.getKeyType(), hotKeyModel.getCreateTime()); - } else { - //已经是热key了，又推过来同样的热key，做个日志记录，并刷新一下 - if (JdHotKeyStore.isHot(hotKeyModel.getKey())) { - JdLogger.warn(getClass(), "receive repeat hot key ：" + hotKeyModel.getKey() + " at " + now); - - //可能存在瞬间的该value过期的情况，所以要判空 - ValueModel valueModel = JdHotKeyStore.getValueSimple(hotKeyModel.getKey()); - if (valueModel != null) { - valueModel.setCreateTime(System.currentTimeMillis()); - } else { - //else大概率走不到 - valueModel = ValueModel.defaultValue(hotKeyModel.getKey()); - } - if (valueModel != null) { - //刷新过期时间 - JdHotKeyStore.setValueDirectly(hotKeyModel.getKey(), valueModel); - } - } else { - //不是重复热key时，新建热key - addKey(hotKeyModel.getKey(), hotKeyModel.getKeyType(), hotKeyModel.getCreateTime()); - } - } - - } - - abstract void addKey(String key, KeyType keyType, long createTime); - - abstract void deleteKey(String key, KeyType keyType, long createTime); -} +//package com.jd.platform.hotkey.client.callback; +// +//import com.jd.platform.hotkey.client.log.JdLogger; +//import com.jd.platform.hotkey.common.model.HotKeyModel; +//import com.jd.platform.hotkey.common.model.typeenum.KeyType; +// +///** +// * @author wuweifeng wrote on 2020-02-24 +// * @version 1.0 +// */ +//public abstract class AbsReceiveNewKey implements ReceiveNewKeyListener { +// +// +// @Override +// public void newKey(HotKeyModel hotKeyModel) { +// long now = System.currentTimeMillis(); +// //如果key到达时已经过去5秒了，记录一下。手工删除key时，没有CreateTime +// if (hotKeyModel.getCreateTime() != 0 && Math.abs(now - hotKeyModel.getCreateTime()) > 1000) { +// JdLogger.warn(getClass(), "the key comes too late : " + hotKeyModel.getKey() + " now " + +// + now + " keyCreateAt " + hotKeyModel.getCreateTime()); +// } +// if (hotKeyModel.isRemove()) { +// deleteKey(hotKeyModel.getKey(), hotKeyModel.getKeyType(), hotKeyModel.getCreateTime()); +// } else { +// //已经是热key了，又推过来同样的热key，做个日志记录，并刷新一下 +// if (JdHotKeyStore.isHot(hotKeyModel.getKey())) { +// JdLogger.warn(getClass(), "receive repeat hot key ：" + hotKeyModel.getKey() + " at " + now); +// +// //可能存在瞬间的该value过期的情况，所以要判空 +//// ValueModel valueModel = JdHotKeyStore.getValueSimple(hotKeyModel.getKey()); +//// if (valueModel != null) { +//// valueModel.setCreateTime(System.currentTimeMillis()); +//// } else { +//// //else大概率走不到 +//// valueModel = ValueModel.defaultValue(hotKeyModel.getKey()); +//// } +// //已经是热key了，重新发过来了一样的，就将value初始化,客户端需要重新reset the value +// //why my idea is so kakaka 怎么办，用baidu还卡不卡呢还是卡啊范围欧尼潍坊 +// ValueModel valueModel = ValueModel.defaultValue(hotKeyModel.getKey()); +// if (valueModel != null) { +// valueModel.setValue(valueModel); +// //刷新过期时间 +// JdHotKeyStore.setValueDirectly(hotKeyModel.getKey(), valueModel); +// } +// } else { +// //不是重复热key时，新建热key +// addKey(hotKeyModel.getKey(), hotKeyModel.getKeyType(), hotKeyModel.getCreateTime()); +// } +// } +// +// } +// +// abstract void addKey(String key, KeyType keyType, long createTime); +// +// abstract void deleteKey(String key, KeyType keyType, long createTime); +// +// protected void addNewKey(HotKeyModel hotKeyModel) { +// ValueModel valueModel = ValueModel.defaultValue(hotKeyModel.getKey()); +// if (valueModel != null) { +// valueModel.setValue(valueModel); +// //刷新过期时间 +// JdHotKeyStore.setValueDirectly(hotKeyModel.getKey(), valueModel); +// } +// } +//} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java index bd4002f..9b4f3cf 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java @@ -1,27 +1,50 @@ package com.jd.platform.hotkey.client.callback; import com.jd.platform.hotkey.client.cache.CacheFactory; -import com.jd.platform.hotkey.common.model.typeenum.KeyType; +import com.jd.platform.hotkey.client.log.JdLogger; +import com.jd.platform.hotkey.common.model.HotKeyModel; /** + * 收到来自于worker的新增key，或者etcd的新增和删除key事件 + * * @author wuweifeng wrote on 2020-02-24 * @version 1.0 */ -public class DefaultNewKeyListener extends AbsReceiveNewKey { - +public class DefaultNewKeyListener implements ReceiveNewKeyListener { @Override - void addKey(String key, KeyType keyType, long createTime) { + public void newKey(HotKeyModel hotKeyModel) { + long now = System.currentTimeMillis(); + //如果key到达时已经过去1秒了，记录一下。手工删除key时，没有CreateTime + if (hotKeyModel.getCreateTime() != 0 && Math.abs(now - hotKeyModel.getCreateTime()) > 1000) { + JdLogger.warn(getClass(), "the key comes too late : " + hotKeyModel.getKey() + " now " + + +now + " keyCreateAt " + hotKeyModel.getCreateTime()); + } + if (hotKeyModel.isRemove()) { + //如果是删除事件，就直接删除 + deleteKey(hotKeyModel.getKey()); + return; + } + //已经是热key了，又推过来同样的热key，做个日志记录，并刷新一下 + if (JdHotKeyStore.isHot(hotKeyModel.getKey())) { + JdLogger.warn(getClass(), "receive repeat hot key ：" + hotKeyModel.getKey() + " at " + now); + } + addKey(hotKeyModel.getKey()); + } + + private void addKey(String key) { ValueModel valueModel = ValueModel.defaultValue(key); if (valueModel == null) { //不符合任何规则 + deleteKey(key); return; } + //如果原来该key已经存在了，那么value就被重置，过期时间也会被重置。如果原来不存在，就新增的热key JdHotKeyStore.setValueDirectly(key, valueModel); } - @Override - void deleteKey(String key, KeyType keyType, long createTime) { + + private void deleteKey(String key) { CacheFactory.getNonNullCache(key).delete(key); } } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java index 65fa35a..13facca 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java @@ -134,6 +134,9 @@ public class JdHotKeyStore { getCache(key).set(key, value); } + /** + * 删除某key，会通知整个集群删除 + */ public static void remove(String key) { getCache(key).delete(key); HotKeyPusher.remove(key); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/ReceiveNewKeySubscribe.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/ReceiveNewKeySubscribe.java index 899a2f1..0483d69 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/ReceiveNewKeySubscribe.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/ReceiveNewKeySubscribe.java @@ -19,7 +19,7 @@ public class ReceiveNewKeySubscribe { if (hotKeyModel == null) { return; } - //收到新key推送，可能是新增，也可能是删除 + //收到新key推送 if (receiveNewKeyListener != null) { receiveNewKeyListener.newKey(hotKeyModel); } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/HotKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/HotKeyPusher.java index 2ebd0e2..50231eb 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/HotKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/HotKeyPusher.java @@ -5,6 +5,7 @@ import com.jd.platform.hotkey.client.core.rule.KeyRuleHolder; import com.jd.platform.hotkey.client.etcd.EtcdConfigFactory; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.typeenum.KeyType; +import com.jd.platform.hotkey.common.tool.Constant; import com.jd.platform.hotkey.common.tool.HotKeyPathTool; /** @@ -34,8 +35,13 @@ public class HotKeyPusher { if (remove) { - //如果是删除key，就直接发到etcd去，不用做聚合 + //如果是删除key，就直接发到etcd去，不用做聚合。但是有点问题现在，这个删除只能删手工添加的key，不能删worker探测出来的 + //因为各个client都在监听手工添加的那个path，没监听自动探测的path。所以如果手工的那个path下，没有该key，那么是删除不了的。 + //删不了，就达不到集群监听删除事件的效果，怎么办呢？可以通过新增的方式，新增一个热key，然后删除它 + EtcdConfigFactory.configCenter().putAndGrant(HotKeyPathTool.keyPath(hotKeyModel), Constant.DEFAULT_DELETE_VALUE, 1); EtcdConfigFactory.configCenter().delete(HotKeyPathTool.keyPath(hotKeyModel)); + //也删worker探测的目录 + EtcdConfigFactory.configCenter().delete(HotKeyPathTool.keyRecordPath(hotKeyModel)); } else { //如果key是规则内的要被探测的key，就积累等待传送 if (KeyRuleHolder.isKeyInRule(key)) { diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyRemover.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyRemover.java new file mode 100644 index 0000000..564c832 --- /dev/null +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyRemover.java @@ -0,0 +1,21 @@ +package com.jd.platform.hotkey.client.core.key; + +import com.jd.platform.hotkey.client.etcd.EtcdConfigFactory; +import com.jd.platform.hotkey.client.log.JdLogger; + +/** + * 删除某个key + * @author wuweifeng + * @version 1.0 + * @date 2020-07-16 + */ +public class KeyRemover { + + public static void remove(String key) { + try { + EtcdConfigFactory.configCenter().delete(key); + } catch (Exception e) { + JdLogger.error(KeyRemover.class, "remove key error"); + } + } +} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java index ee7d07b..d92564e 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java @@ -16,16 +16,14 @@ import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; +import com.jd.platform.hotkey.common.tool.Constant; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import io.grpc.StatusRuntimeException; import io.netty.util.internal.StringUtil; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; /** * etcd连接管理器 @@ -133,8 +131,9 @@ public class EtcdStarter { private void fetchWorkerInfo() { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 - scheduledExecutorService.scheduleAtFixedRate(() -> { JdLogger.info(getClass(), "trying to connect to etcd and fetch worker info"); - fetch(); + scheduledExecutorService.scheduleAtFixedRate(() -> { + JdLogger.info(getClass(), "trying to connect to etcd and fetch worker info"); + fetch(); }, 0, 30, TimeUnit.SECONDS); } @@ -183,10 +182,11 @@ public class EtcdStarter { } /** - * 异步开始监听热key变化信息 + * 异步开始监听热key变化信息，该目录里只有手工添加的key信息 */ private void startWatchHotKey() { - CompletableFuture.runAsync(() -> { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.submit(() -> { JdLogger.info(getClass(), "--- begin watch hotKey change ----"); IConfigCenter configCenter = EtcdConfigFactory.configCenter(); try { @@ -208,17 +208,19 @@ public class EtcdStarter { model.setKey(key); EventBusCenter.getInstance().post(new ReceiveNewKeyEvent(model)); } else { - JdLogger.info(getClass(), "etcd receive new key : " + key); HotKeyModel model = new HotKeyModel(); model.setRemove(false); - //value是uuid的，就是worker推到etcd推送过来的。value是时间戳的，就是手工创建的 - //该功能已摒弃，client不再监听worker发往etcd的key了，只监听etcd手工的和client发到etcd的 - if (keyValue.getValue().toStringUtf8().length() != 13) { - model.setCreateTime(System.currentTimeMillis()); - } else { - model.setCreateTime(Long.valueOf(keyValue.getValue().toStringUtf8())); + String value = keyValue.getValue().toStringUtf8(); + //新增热key + JdLogger.info(getClass(), "etcd receive new key : " + key + " --value:" + value); + //如果这是一个删除指令，就什么也不干 + if (Constant.DEFAULT_DELETE_VALUE.equals(value)) { + return; } + //手工创建的value是时间戳 + model.setCreateTime(Long.valueOf(keyValue.getValue().toStringUtf8())); + model.setKey(key); EventBusCenter.getInstance().post(new ReceiveNewKeyEvent(model)); } @@ -278,7 +280,8 @@ public class EtcdStarter { * 异步监听rule规则变化 */ private void startWatchRule() { - CompletableFuture.runAsync(() -> { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.submit(() -> { JdLogger.info(getClass(), "--- begin watch rule change ----"); try { IConfigCenter configCenter = EtcdConfigFactory.configCenter(); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 1e0765b..5c74754 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -25,6 +25,11 @@ public class Constant { public static String BAK_DELIMITER = "#\\*\\*#"; + /** + * 当客户端要删除某个key时，就往etcd里赋值这个value，设置1秒过期，就算删除了 + */ + public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; + //单次包最大2M public static int MAX_LENGTH = 2 * 1024 * 1024; -- Gitee From e09b59e2c5800a30e9b4c9d125dca7d1a596556e Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 16 Jul 2020 20:53:09 +0800 Subject: [PATCH 109/156] =?UTF-8?q?=E8=AE=BE=E7=BD=AEclient=E5=88=A0?= =?UTF-8?q?=E9=99=A4key=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/monitor/DataHandler.java | 8 ++++++++ .../service/impl/KeyServiceImpl.java | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 7c6775b..7abc437 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -60,6 +60,9 @@ public class DataHandler { TwoTuple twoTuple; try { twoTuple = handHotKey(queue.take()); + if (twoTuple == null) { + return; + } } catch (Exception e) { e.printStackTrace(); log.error("handHotKey error ," + e.getCause()); @@ -107,6 +110,11 @@ public class DataHandler { //组建成对象，供累计后批量插入、删除 TwoTuple timelyKeyRecordTwoTuple = new TwoTuple<>(); if (eventType.equals(Event.EventType.PUT)) { + //如果是客户端删除时发出的put指令 + if (com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE.equals(value)) { + log.info("client remove key event : " + appKey); + return null; + } //手工添加的是时间戳13位，worker传过来的是uuid String source = value.length() == 13 ? Constant.HAND : Constant.SYSTEM; timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).val(value).appName(appName).duration(ttl).uuid(appKey).createTime(date).build()); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 5b6bfa2..88a4263 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -13,8 +13,14 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; -import com.jd.platform.hotkey.dashboard.mapper.*; -import com.jd.platform.hotkey.dashboard.model.*; +import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; +import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; +import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; +import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.model.ChangeLog; +import com.jd.platform.hotkey.dashboard.model.KeyRecord; +import com.jd.platform.hotkey.dashboard.model.KeyTimely; +import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.service.KeyService; import com.jd.platform.hotkey.dashboard.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; @@ -178,8 +184,13 @@ public class KeyServiceImpl implements KeyService { //app + "_" + key String[] arr = keyTimely.getKey().split("/"); //删除client监听目录的key - String ectdKey = ConfigConstant.hotKeyPath + arr[0] + "/" + arr[1]; - configCenter.delete(ectdKey); + String etcdKey = ConfigConstant.hotKeyPath + arr[0] + "/" + arr[1]; + if (configCenter.get(etcdKey) == null) { + //如果手工目录也就是client监听的目录里没有该key，那么就往里面放一个，然后再删掉它，这样client才能监听到删除事件 + configCenter.putAndGrant(etcdKey, com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE, 1); + } + configCenter.delete(etcdKey); + //也删除Record目录下的该key，因为不确定要删的key到底在哪 String recordKey = ConfigConstant.hotKeyRecordPath + arr[0] + "/" + arr[1]; configCenter.delete(recordKey); -- Gitee From 98ad4f6f03cba28a8d0b6badd033539f53554cd0 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 16 Jul 2020 21:22:00 +0800 Subject: [PATCH 110/156] =?UTF-8?q?=E8=AE=BE=E7=BD=AEclient=E5=88=A0?= =?UTF-8?q?=E9=99=A4key=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/DataHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 7abc437..b5b1a1e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -61,7 +61,7 @@ public class DataHandler { try { twoTuple = handHotKey(queue.take()); if (twoTuple == null) { - return; + continue; } } catch (Exception e) { e.printStackTrace(); -- Gitee From 6dbc3ada4cee3ab9b22155244b357d2ee6828a4c Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 17 Jul 2020 10:06:17 +0800 Subject: [PATCH 111/156] =?UTF-8?q?=E8=AE=BE=E7=BD=AEclient=E5=88=A0?= =?UTF-8?q?=E9=99=A4key=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java index d92564e..c129fb1 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java @@ -215,7 +215,7 @@ public class EtcdStarter { JdLogger.info(getClass(), "etcd receive new key : " + key + " --value:" + value); //如果这是一个删除指令，就什么也不干 if (Constant.DEFAULT_DELETE_VALUE.equals(value)) { - return; + continue; } //手工创建的value是时间戳 -- Gitee From 99d74a25a5c43d754feb09f7a3cebdfb45e9ffa9 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 17 Jul 2020 13:58:00 +0800 Subject: [PATCH 112/156] =?UTF-8?q?=E6=A0=B7=E5=BC=8F=E4=BF=AE=E6=94=B9=20?= =?UTF-8?q?hotkey=E9=BB=98=E8=AE=A4APP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../static/admin/common/js/key-add.js | 31 +++++++++++++++++-- .../templates/admin/common/html/footer.html | 3 ++ .../templates/admin/common/html/leftMenu.html | 2 +- .../resources/templates/admin/key/add.html | 2 +- .../resources/templates/admin/key/edit.html | 2 +- .../main/resources/templates/admin/main.html | 2 +- .../resources/templates/admin/rule/list.html | 4 +-- 7 files changed, 37 insertions(+), 9 deletions(-) diff --git a/dashboard/src/main/resources/static/admin/common/js/key-add.js b/dashboard/src/main/resources/static/admin/common/js/key-add.js index 1ff6410..f9aadfe 100644 --- a/dashboard/src/main/resources/static/admin/common/js/key-add.js +++ b/dashboard/src/main/resources/static/admin/common/js/key-add.js @@ -4,9 +4,34 @@ $("#form-add").validate({ } }); -/** - * - */ +$.ajax({ + cache : true, + type : "POST", + url : "/user/info", + headers: { + "Authorization":getCookie("token") + }, + async : false, + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); + }, + success : function(data) { + console.log(data) + let role = data.role; + if(role === "ADMIN"){ + $("#apps").val(""); + } + let apps = data.appNames; + let appName = data.appName; + for (let i = 0; i < apps.length; i++) { + let app = apps[i]; + if(app === appName){ + $("#apps").val(apps[i]); + } + } + } +}); + function add() { var dataFormJson=$("#form-add").serialize(); $.ajax({ diff --git a/dashboard/src/main/resources/templates/admin/common/html/footer.html b/dashboard/src/main/resources/templates/admin/common/html/footer.html index a6cc3ff..662d4f3 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/footer.html +++ b/dashboard/src/main/resources/templates/admin/common/html/footer.html @@ -11,6 +11,9 @@ 2020 京东零售-平台业务中心-平台业务研发部-基础业务研发部 版权所有 | 使用支持：武伟峰 | 咚咚：85134158 +
接入文档：https://cf.jd.com/pages/viewpage.action?pageId=277756998 + +
diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 6b38ab3..29baffe 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -32,7 +32,7 @@
  • - + 秒级监控
    - +
    • diff --git a/dashboard/src/main/resources/templates/admin/common/html/title.html b/dashboard/src/main/resources/templates/admin/common/html/title.html index 08cf7c8..aa2fcaf 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/title.html +++ b/dashboard/src/main/resources/templates/admin/common/html/title.html @@ -36,8 +36,8 @@
    Widget
    @@ -34,33 +33,33 @@ @@ -156,7 +156,7 @@ var options = { formatter:function (val) { return changeDateFormat(val); } - },{ + }/*,{ title: '操作', formatter: function (value, row) { var id = row.id; @@ -165,7 +165,7 @@ var options = { actions.push('删除'); return actions.join(''); } - }] + }*/] }; $(function(){ var oTab=$.table.oTableInit(options); -- Gitee From 94d9997628cce9c7dab66a21f6d4689c8e4a3838 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 27 Aug 2020 12:03:16 +0800 Subject: [PATCH 143/156] =?UTF-8?q?=E5=9B=9E=E9=80=80view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/templates/admin/user/list.html | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dashboard/src/main/resources/templates/admin/user/list.html b/dashboard/src/main/resources/templates/admin/user/list.html index e446de4..54fa7b7 100644 --- a/dashboard/src/main/resources/templates/admin/user/list.html +++ b/dashboard/src/main/resources/templates/admin/user/list.html @@ -79,12 +79,12 @@ - - +
    @@ -108,7 +108,6 @@ - @@ -156,7 +155,7 @@ var options = { formatter:function (val) { return changeDateFormat(val); } - }/*,{ + },{ title: '操作', formatter: function (value, row) { var id = row.id; @@ -165,7 +164,7 @@ var options = { actions.push('删除'); return actions.join(''); } - }*/] + }] }; $(function(){ var oTab=$.table.oTableInit(options); -- Gitee From cbf8666502d92406628e470fb5dac28847e73269 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 27 Aug 2020 16:05:02 +0800 Subject: [PATCH 144/156] =?UTF-8?q?=E5=8F=98=E6=9B=B4=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E7=9A=84=E5=88=9B=E5=BB=BA=E6=97=B6=E9=97=B4=E5=B1=95=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/templates/admin/changeLog/list.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dashboard/src/main/resources/templates/admin/changeLog/list.html b/dashboard/src/main/resources/templates/admin/changeLog/list.html index a6fbf8d..d6ed97a 100644 --- a/dashboard/src/main/resources/templates/admin/changeLog/list.html +++ b/dashboard/src/main/resources/templates/admin/changeLog/list.html @@ -142,6 +142,11 @@ var options = { } } }, + { field: 'createTime', title: '创建时间', + formatter:function (val,row,index) { + return changeDateFormat(val); + } + }, { field: 'from', title: '初始值', -- Gitee From cf4614e82893b4e149db5cf911371462f60e5d94 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 14:32:55 +0800 Subject: [PATCH 145/156] merge --- .../platform/hotkey/client/ClientStarter.java | 9 +---- .../jd/platform/hotkey/client/Context.java | 2 +- common/pom.xml | 2 +- .../common/configcenter/ConfigConstant.java | 14 ++++++++ .../hotkey/common/model/HotKeyModel.java | 7 ++-- .../common/model/typeenum/MessageType.java | 3 +- .../hotkey/common/tool/flush/FlushUtil.java | 34 +++++++++++++++++++ 7 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java index 1102d8a..27474e7 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.client; -import com.jd.platform.hotkey.client.cache.LocalCache; import com.jd.platform.hotkey.client.callback.ReceiveNewKeySubscribe; import com.jd.platform.hotkey.client.core.eventbus.EventBusCenter; import com.jd.platform.hotkey.client.core.key.PushSchedulerStarter; @@ -40,9 +39,8 @@ public class ClientStarter { public static class Builder { private String appName; private String etcdServer; - private LocalCache localCache; private Long pushPeriod; - private int caffeineSize = 50000; + private int caffeineSize = 200000; public Builder() { } @@ -65,11 +63,6 @@ public class ClientStarter { return this; } - public Builder setLocalCache(LocalCache localCache) { - this.localCache = localCache; - return this; - } - public Builder setPushPeriod(Long pushPeriod) { this.pushPeriod = pushPeriod; return this; diff --git a/client/src/main/java/com/jd/platform/hotkey/client/Context.java b/client/src/main/java/com/jd/platform/hotkey/client/Context.java index f0455a2..fea4106 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/Context.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/Context.java @@ -12,5 +12,5 @@ public class Context { */ public static boolean NEED_RECONNECT = true; - public static int CAFFEINE_SIZE = 50000; + public static int CAFFEINE_SIZE; } diff --git a/common/pom.xml b/common/pom.xml index 59a7d76..a548184 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -18,7 +18,7 @@ 3.4.2 1.1.7.3 1.2.70 - 0.0.14 + 0.0.16 2.8.0 5.1.0 diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index e88389d..2a7eff0 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -13,6 +13,10 @@ public interface ConfigConstant { * 所有的workers，存这里 */ String workersPath = "/jd/workers/"; + /** + * dashboard的ip存这里 + */ + String dashboardPath = "/jd/dashboard/"; /** * 该app所有的workers地址的path。需要手工分配，默认每个app都用所有的worker */ @@ -54,4 +58,14 @@ public interface ConfigConstant { * 存放客户端hotKey访问次数和总访问次数的path */ String keyHitCountPath = "/jd/keyHitCount/"; + + /** + * 清理历史数据的配置的path + * time unit : day + */ + String clearCfgPath = "/jd/clearCfg/"; + /** + * 控制台启动的netty端口 + */ + int dashboardPort = 11112; } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java index 0451fce..11de0cc 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java @@ -23,11 +23,7 @@ public class HotKeyModel extends BaseModel { @Override public String toString() { - return "HotKeyModel{" + - "appName='" + appName + '\'' + - ", keyType=" + keyType + - ", remove=" + remove + - '}'; + return "appName:" + appName + "-key=" + getKey(); } public boolean isRemove() { @@ -53,4 +49,5 @@ public class HotKeyModel extends BaseModel { public void setKeyType(KeyType keyType) { this.keyType = keyType; } + } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java index cd3f983..5b07624 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java @@ -8,7 +8,8 @@ public enum MessageType { APP_NAME((byte) 1), REQUEST_NEW_KEY((byte) 2), RESPONSE_NEW_KEY((byte) 3), - REQUEST_HIT_COUNT((byte) 7), + REQUEST_HIT_COUNT((byte) 7), //命中率 + REQUEST_HOT_KEY((byte) 8), //热key，worker->dashboard PING((byte) 4), PONG((byte) 5), EMPTY((byte) 6); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java new file mode 100644 index 0000000..009eb47 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java @@ -0,0 +1,34 @@ +package com.jd.platform.hotkey.common.tool.flush; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author wuweifeng wrote on 2019-12-11 + * @version 1.0 + */ +public class FlushUtil { + private static Logger logger = LoggerFactory.getLogger("flushUtil"); + + /** + * 往channel里输出消息 + */ + public static void flush(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) { +// if (channelHandlerContext.channel().isWritable()) { +// channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { +// if (!future.isSuccess()) { +// logger.warn("flush error " + future.cause().getMessage()); +// } +// }); +// } else { + try { + //同步发送 + channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); + } catch (InterruptedException e) { + logger.error("flush error " + e.getMessage()); + } +// } + } +} -- Gitee From a86ff41209c55cf136dd91b7ab6050d044d39159 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 14:51:39 +0800 Subject: [PATCH 146/156] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/DashboardApplication.java | 6 - .../dashboard/cache/CaffeineBuilder.java | 30 ++ .../dashboard/common/domain/Constant.java | 6 + .../dashboard/common/domain/EventWrapper.java | 95 ------ .../dashboard/common/domain/IRecord.java | 27 ++ .../hotkey/dashboard/common/domain/Page.java | 18 +- .../common/domain/vo/ClearCfgVo.java | 47 +++ .../common/domain/vo/RuleLineChartVo.java | 35 +++ .../dashboard/common/monitor/DataHandler.java | 120 ++++---- .../dashboard/controller/ClearController.java | 62 ++++ .../dashboard/controller/KeyController.java | 10 +- .../{common/monitor => etcd}/EtcdMonitor.java | 134 ++++++--- .../hotkey/dashboard/etcd/EtcdRegister.java | 63 ++++ .../dashboard/mapper/KeyRecordMapper.java | 4 +- .../dashboard/mapper/KeyTimelyMapper.java | 31 -- .../dashboard/mapper/StatisticsMapper.java | 4 +- .../dashboard/mapper/SummaryMapper.java | 4 +- .../hotkey/dashboard/model/KeyRecord.java | 8 +- .../hotkey/dashboard/model/KeyTimely.java | 25 +- .../dashboard/netty/HotKeyReceiver.java | 175 +++++++++++ .../hotkey/dashboard/netty/NodesServer.java | 78 +++++ .../dashboard/netty/NodesServerHandler.java | 66 +++++ .../dashboard/service/ClearService.java | 23 ++ .../hotkey/dashboard/service/KeyService.java | 7 +- .../hotkey/dashboard/service/UserService.java | 5 + .../service/impl/ClearServiceImpl.java | 85 ++++++ .../service/impl/KeyServiceImpl.java | 271 ++++-------------- .../service/impl/UserServiceImpl.java | 43 +++ .../hotkey/dashboard/util/PageUtil.java | 36 +++ .../hotkey/dashboard/util/RuleUtil.java | 4 +- dashboard/src/main/resources/application.yml | 6 + .../dashboard/mapper/KeyRecordMapper.xml | 6 + .../dashboard/mapper/KeyTimelyMapper.xml | 207 ------------- .../dashboard/mapper/StatisticsMapper.xml | 7 +- .../hotkey/dashboard/mapper/SummaryMapper.xml | 7 + .../admin/assets/js/bootstrap/js/base_list.js | 19 +- .../static/admin/common/js/clearcfg-edit.js | 25 ++ .../resources/templates/admin/clear/edit.html | 69 +++++ .../resources/templates/admin/clear/list.html | 157 ++++++++++ .../templates/admin/common/html/leftMenu.html | 7 +- .../resources/templates/admin/key/list.html | 38 +-- .../templates/admin/key/listmaxhot.html | 4 +- .../templates/admin/key/listtimely.html | 2 +- .../main/resources/templates/admin/main.html | 2 +- .../templates/admin/rule/listhitcount.html | 43 +-- .../resources/templates/admin/user/list.html | 1 + 46 files changed, 1378 insertions(+), 744 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{common/monitor => etcd}/EtcdMonitor.java (71%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml create mode 100644 dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js create mode 100644 dashboard/src/main/resources/templates/admin/clear/edit.html create mode 100644 dashboard/src/main/resources/templates/admin/clear/list.html diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 074e45d..3070702 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.dashboard; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; @@ -9,7 +8,6 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import javax.annotation.Resource; @EnableAsync @@ -19,8 +17,6 @@ public class DashboardApplication implements CommandLineRunner { private Logger logger = LoggerFactory.getLogger(getClass()); - @Resource - private KeyTimelyMapper timelyMapper; public static void main(String[] args) { try { @@ -33,8 +29,6 @@ public class DashboardApplication implements CommandLineRunner { @Override public void run(String... args) { - int row = timelyMapper.clear(); - logger.info("clear db timely hotKey, effect row : {}",row); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java new file mode 100644 index 0000000..63612e8 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java @@ -0,0 +1,30 @@ +package com.jd.platform.hotkey.dashboard.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import java.util.concurrent.TimeUnit; + +/** + * @author wuweifeng wrote on 2019-12-12 + * @version 1.0 + */ +public class CaffeineBuilder { + + public static Cache cache(int duration) { + return cache(128, 1000000, duration); + } + + + /** + * 构建所有来的要缓存的key getCache + */ + public static Cache cache(int minSize, int maxSize, int expireSeconds) { + return Caffeine.newBuilder() + .initialCapacity(minSize)//初始大小 + .maximumSize(maxSize)//最大数量 + .expireAfterWrite(expireSeconds, TimeUnit.SECONDS)//过期时间 + .build(); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index b11fa73..952aa2a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -35,6 +35,9 @@ public class Constant { public static final String WORKER_VIEW = "节点信息"; + public static final String CLEAR_VIEW = "数据清理"; + + public static final int MAX_DAY_RANGE = 3; public static final int VERSION = 1; @@ -55,6 +58,9 @@ public class Constant { public static final String INFO = "info"; + public static final String THIRTY_DAY = "30"; + + public static final List HEAD = new ArrayList<>(); static { HEAD.add("热点key"); HEAD.add("次数"); HEAD.add("所属APP");} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java deleted file mode 100644 index 04d4a83..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.domain; - -import com.ibm.etcd.api.Event; - -import java.io.Serializable; -import java.util.Date; - -/** - * 包装类，用于保存事件最准确的时间 - */ -public class EventWrapper implements Serializable { - - private String key; - - private String value; - - private Event.EventType eventType; - - private Date date; - - private Long ttl; - - private long version; - - private String uuid; - - @Override - public String toString() { - return "EventWrapper{" + - "key='" + key + '\'' + - ", value='" + value + '\'' + - ", eventType=" + eventType + - ", date=" + date + - ", ttl=" + ttl + - ", version=" + version + - ", uuid='" + uuid + '\'' + - '}'; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public long getVersion() { - return version; - } - - public void setVersion(long version) { - this.version = version; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public Event.EventType getEventType() { - return eventType; - } - - public void setEventType(Event.EventType eventType) { - this.eventType = eventType; - } - - public void setDate(Date date) { - this.date = date; - } - - public void setTtl(Long ttl) { - this.ttl = ttl; - } - - public Date getDate() { - return date; - } - - public Long getTtl() { - return ttl; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java new file mode 100644 index 0000000..3585f73 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java @@ -0,0 +1,27 @@ +package com.jd.platform.hotkey.dashboard.common.domain; + +import java.util.Date; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-09-02 + */ +public interface IRecord { + /** + * appName + "/" + key + */ + String appNameKey(); + + /** + * 手工添加的是时间戳13位，worker传过来的是uuid + */ + String value(); + + /** + * 0插入，1删除 + */ + int type(); + + Date createTime(); +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java index 5f64c75..d10c3a5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java @@ -7,9 +7,13 @@ import java.util.List; public class Page implements Serializable { private static final long serialVersionUID = 1L; - + /** + * 当前第几页 + */ private Integer page; - + /** + * 总共多少条 + */ private int total; private List rows; @@ -20,6 +24,7 @@ public class Page implements Serializable { this.rows = rows; } + public Integer getPage() { return page; } @@ -43,4 +48,13 @@ public class Page implements Serializable { public void setRows(List rows) { this.rows = rows; } + + @Override + public String toString() { + return "Page{" + + "page=" + page + + ", total=" + total + + ", rows=" + rows + + '}'; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java new file mode 100644 index 0000000..14b872b --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java @@ -0,0 +1,47 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearCfgVo + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:54 + */ +public class ClearCfgVo { + + private String app; + + private String ttl; + + private Long version; + + public ClearCfgVo(String app, String ttl, Long version) { + this.app = app; + this.ttl = ttl; + this.version = version; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public String getTtl() { + return ttl; + } + + public void setTtl(String ttl) { + this.ttl = ttl; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java new file mode 100644 index 0000000..31ba530 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java @@ -0,0 +1,35 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +/** + * @author liyunfeng31 + */ +public class RuleLineChartVo { + + private String rule; + + private Integer count; + + public RuleLineChartVo() { + } + + public RuleLineChartVo(String rule, Integer count) { + this.rule = rule; + this.count = count; + } + + public String getRule() { + return rule; + } + + public void setRule(String rule) { + this.rule = rule; + } + + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index b5b1a1e..bd3a5e0 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -1,20 +1,22 @@ package com.jd.platform.hotkey.dashboard.common.monitor; +import cn.hutool.core.collection.CollectionUtil; import com.alibaba.fastjson.JSON; -import com.ibm.etcd.api.Event; +import com.google.common.collect.Queues; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.IRecord; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; -import com.jd.platform.hotkey.dashboard.util.TwoTuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -22,10 +24,13 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; @Component public class DataHandler { @@ -35,21 +40,25 @@ public class DataHandler { @Resource private KeyRecordMapper keyRecordMapper; @Resource - private KeyTimelyMapper keyTimelyMapper; - @Resource private StatisticsMapper statisticsMapper; + @Resource + private SummaryMapper summaryMapper; + + @Resource + private IConfigCenter configCenter; + /** * 队列 */ - private BlockingQueue queue = new LinkedBlockingQueue<>(); + private BlockingQueue queue = new LinkedBlockingQueue<>(); /** * 入队 */ - public void offer(EventWrapper eventWrapper) { + public void offer(IRecord record) { try { - queue.put(eventWrapper); + queue.put(record); } catch (InterruptedException e) { e.printStackTrace(); } @@ -57,35 +66,25 @@ public class DataHandler { public void insertRecords() { while (true) { - TwoTuple twoTuple; try { - twoTuple = handHotKey(queue.take()); - if (twoTuple == null) { + List records = new ArrayList<>(); + Queues.drain(queue, records, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(records)) { continue; } - } catch (Exception e) { - e.printStackTrace(); - log.error("handHotKey error ," + e.getCause()); - continue; - } - KeyRecord keyRecord = twoTuple.getSecond(); - KeyTimely keyTimely = twoTuple.getFirst(); - - if (keyTimely.getUuid() == null) { - keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); - } else { - try { - keyTimelyMapper.saveOrUpdate(keyTimely); - } catch (Exception e) { - log.info("insert timely error",e); + List keyRecordList = new ArrayList<>(); + for (IRecord iRecord : records) { + KeyRecord keyRecord = handHotKey(iRecord); + if (keyRecord != null) { + keyRecordList.add(keyRecord); + } } - } - if (keyRecord != null) { - //插入记录 - keyRecordMapper.insertSelective(keyRecord); - } + keyRecordMapper.batchInsert(keyRecordList); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } @@ -94,22 +93,19 @@ public class DataHandler { /** * 处理热点key和记录 */ - private TwoTuple handHotKey(EventWrapper eventWrapper) { - Date date = eventWrapper.getDate(); - long ttl = eventWrapper.getTtl(); - Event.EventType eventType = eventWrapper.getEventType(); - String appKey = eventWrapper.getKey(); - String value = eventWrapper.getValue(); + private KeyRecord handHotKey(IRecord record) { + Date date = record.createTime(); + String appKey = record.appNameKey(); + String value = record.value(); //appName+"/"+"key" String[] arr = appKey.split("/"); String appName = arr[0]; String key = arr[1]; - String uuid = eventWrapper.getUuid(); - int type = eventType.getNumber(); + String uuid = UUID.randomUUID().toString(); + int type = record.type(); //组建成对象，供累计后批量插入、删除 - TwoTuple timelyKeyRecordTwoTuple = new TwoTuple<>(); - if (eventType.equals(Event.EventType.PUT)) { + if (type == 0) { //如果是客户端删除时发出的put指令 if (com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE.equals(value)) { log.info("client remove key event : " + appKey); @@ -117,17 +113,14 @@ public class DataHandler { } //手工添加的是时间戳13位，worker传过来的是uuid String source = value.length() == 13 ? Constant.HAND : Constant.SYSTEM; - timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).val(value).appName(appName).duration(ttl).uuid(appKey).createTime(date).build()); String rule = RuleUtil.rule(appKey); - KeyRecord keyRecord = new KeyRecord(key, rule, appName, ttl, source, type, uuid, date); + KeyRecord keyRecord = new KeyRecord(key, rule, appName, 1, source, type, uuid, date); keyRecord.setRule(rule); - timelyKeyRecordTwoTuple.setSecond(keyRecord); - return timelyKeyRecordTwoTuple; - } else if (eventType.equals(Event.EventType.DELETE)) { - timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).appName(appName).build()); - return timelyKeyRecordTwoTuple; + return keyRecord; + } else { + //是删除 + return null; } - return timelyKeyRecordTwoTuple; } @@ -210,4 +203,29 @@ public class DataHandler { } + + /** + * 每天根据app的配置清理过期数据 + */ + @Scheduled(cron = "0 0 1 * * ?") + public void clearExpireData() { + try { + LocalDateTime now = LocalDateTime.now(); + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + for (KeyValue kv : keyValues) { + String key = kv.getKey().toStringUtf8(); + String ttl = kv.getValue().toStringUtf8(); + String app = key.replace(ConfigConstant.clearCfgPath, ""); + Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); + summaryMapper.clearExpireData(app, expireDate); + keyRecordMapper.clearExpireData(app, expireDate); + statisticsMapper.clearExpireData(app, expireDate); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java new file mode 100644 index 0000000..0ce81f0 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java @@ -0,0 +1,62 @@ +package com.jd.platform.hotkey.dashboard.controller; + +import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; +import com.jd.platform.hotkey.dashboard.service.ClearService; +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + +/** + * @ProjectName: hotkey + * @ClassName: ClearController + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:48 + */ +@Controller +@RequestMapping("/clear") +public class ClearController { + + + private String prefix = "admin/clear"; + + + @Resource + private ClearService clearService; + + @GetMapping("/view") + public String view(ModelMap modelMap){ + modelMap.put("title", Constant.CLEAR_VIEW); + return prefix + "/list"; + } + + + @PostMapping("/list") + @ResponseBody + public Page list(PageReq page, String app){ + PageInfo info = clearService.pageClearCfg(page, app); + return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + } + + + @GetMapping("/edit/{app}") + public String edit(@PathVariable("app") String app, ModelMap modelMap){ + modelMap.put("clearCfg", clearService.selectClearCfg(app)); + return prefix + "/edit"; + } + + @PostMapping("/save") + @ResponseBody + public Result save(ClearCfgVo cfg){ + int b = clearService.saveClearCfg(cfg); + return b == 0 ? Result.fail():Result.success(); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index 0b287a8..f3f362d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -79,8 +79,7 @@ public class KeyController extends BaseController { @PostMapping("/listTimely") @ResponseBody public Page listTimely(PageReq page, SearchReq searchReq){ - PageInfo info = keyService.pageKeyTimely(page, searchReq); - return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + return keyService.pageKeyTimely(page, searchReq); } @@ -123,13 +122,6 @@ public class KeyController extends BaseController { } - @GetMapping("/edit/{id}") - public String edit(@PathVariable("id") Long id, ModelMap modelMap){ - modelMap.put("key", keyService.selectByPk(id)); - return prefix + "/edit"; - } - - @PostMapping("/edit") @ResponseBody public Result editSave(KeyTimely key) { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java similarity index 71% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 92fa472..234e447 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.common.monitor; +package com.jd.platform.hotkey.dashboard.etcd; import cn.hutool.core.util.StrUtil; import com.ibm.etcd.api.Event; @@ -6,27 +6,27 @@ import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.client.kv.KvClient; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.IRecord; +import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.Worker; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.service.WorkerService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import io.grpc.StatusRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -55,7 +55,7 @@ public class EtcdMonitor { private DataHandler dataHandler; - public static final ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); + public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(16); /** * 监听新来的热key，该key的产生是来自于手工在控制台添加 @@ -65,48 +65,36 @@ public class EtcdMonitor { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); - EventWrapper eventWrapper = build(event); + KeyValue kv = event.getKv(); + String v = kv.getValue().toStringUtf8(); + Event.EventType eventType = event.getType(); String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyPath, ""); - eventWrapper.setKey(appKey); - dataHandler.offer(eventWrapper); - } - }); - } + dataHandler.offer(new IRecord() { + @Override + public String appNameKey() { + return appKey; + } - /** - * 监听新来的热key，该key的产生是来自于worker集群推送过来的 - */ - public void watchHotKeyRecord() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyRecordPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - EventWrapper eventWrapper = build(event); + @Override + public String value() { + return v; + } - String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyRecordPath, ""); - eventWrapper.setKey(appKey); + @Override + public int type() { + return eventType.getNumber(); + } - dataHandler.offer(eventWrapper); + @Override + public Date createTime() { + return new Date(); + } + }); } }); } - private EventWrapper build(Event event) { - KeyValue kv = event.getKv(); - long ttl = configCenter.timeToLive(kv.getLease()); - String v = kv.getValue().toStringUtf8(); - Event.EventType eventType = event.getType(); - EventWrapper eventWrapper = new EventWrapper(); - eventWrapper.setValue(v); - eventWrapper.setDate(new Date()); - eventWrapper.setTtl(ttl); - eventWrapper.setVersion(kv.getVersion()); - eventWrapper.setEventType(eventType); - eventWrapper.setUuid(v); - return eventWrapper; - } - @PostConstruct public void startWatch() { @@ -116,26 +104,83 @@ public class EtcdMonitor { //规则拉取完毕后才能去开始入库 insertRecords(); - //开始监听热key产生 - watchHotKeyRecord(); + //开始入库 + dealHotkey(); + //监听手工创建的key watchHandOperationHotKey(); //监听rule变化 watchRule(); - watchWorkers(); +// watchWorkers(); //观察热key访问次数和总访问次数，并做统计 watchHitCount(); } + /** + * 每秒去清理一次caffeine + */ + @Scheduled(fixedRate = 1000) + public void cleanCaffeine() { + try { + HotKeyReceiver.cleanUpCaffeine(); + } catch (Exception e) { + e.printStackTrace(); + } + } + private void insertRecords() { threadPoolExecutor.submit(() -> { dataHandler.insertRecords(); }); } + + /** + * 开始消费各worker发来的热key + */ + private void dealHotkey() { + threadPoolExecutor.submit(() -> { + while (true) { + try { + //获取发来的这个热key，存入本地caffeine，设置过期时间 + HotKeyModel model = HotKeyReceiver.take(); + + //将该key放入实时热key本地缓存中 + HotKeyReceiver.put(model); + + dataHandler.offer(new IRecord() { + @Override + public String appNameKey() { + return model.getAppName() + "/" + model.getKey(); + } + + @Override + public String value() { + return UUID.randomUUID().toString(); + } + + @Override + public int type() { + return 0; + } + + @Override + public Date createTime() { + return new Date(); + } + }); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + }); + } + /** * 异步监听rule规则变化 */ @@ -217,7 +262,6 @@ public class EtcdMonitor { } - /** * 监听热key访问次数和总访问次数 */ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java new file mode 100644 index 0000000..d0a44c5 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java @@ -0,0 +1,63 @@ +package com.jd.platform.hotkey.dashboard.etcd; + +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.tool.IpUtils; +import com.jd.platform.hotkey.dashboard.netty.NodesServer; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * dashboard也注册到etcd，供各个worker连接 + * + * @author wuweifeng + * @version 1.0 + * @date 2020-08-28 + */ +@Component +public class EtcdRegister { + @Resource + private IConfigCenter configCenter; + @Resource + private NodesServer nodesServer; + + /** + * 每隔一会去check一下，自己还在不在etcd里 + */ + @PostConstruct + public void makeSureSelfOn() { + new Thread(() -> nodesServer.startNettyServer(ConfigConstant.dashboardPort)).start(); + + //开启上传worker信息 + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.scheduleAtFixedRate(() -> { + + try { + uploadSelfInfo(); + } catch (Exception e) { + //do nothing + } + + }, 3, 30, TimeUnit.SECONDS); + } + + private void uploadSelfInfo() { + configCenter.putAndGrant(buildKey(), buildValue(), 32); + } + + private String buildKey() { + String hostName = IpUtils.getHostName(); + return ConfigConstant.dashboardPath + hostName; + } + + private String buildValue() { + String ip = IpUtils.getIp(); + return ip + ":" + ConfigConstant.dashboardPort; + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java index 3ef46b3..e135ff6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java @@ -1,11 +1,11 @@ package com.jd.platform.hotkey.dashboard.mapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -25,4 +25,6 @@ public interface KeyRecordMapper { List maxHotKey(SearchReq req); List statisticsByRule(SearchReq req); + + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java deleted file mode 100644 index 3423da3..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.hotkey.dashboard.mapper; - -import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; -import org.apache.ibatis.annotations.Mapper; - -import java.util.List; - -@Mapper -public interface KeyTimelyMapper { - - int clear(); - - int deleteByKeyAndApp(String key,String appName); - - int insertSelective(KeyTimely key); - - KeyTimely selectByPrimaryKey(Long id); - - KeyTimely selectByKey(String key); - - int updateByKey(KeyTimely key); - - List listKeyTimely(SearchReq param); - - int batchInsert(List list); - - void batchDeleted(List deleteList); - - void saveOrUpdate(KeyTimely keyTimely); -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java index dabdfc5..daf848e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java @@ -4,6 +4,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -42,7 +43,6 @@ public interface StatisticsMapper { /** * 清理 - * @param type type */ - void clear(int type); + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java index 4685e76..0305c4c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java @@ -2,10 +2,10 @@ package com.jd.platform.hotkey.dashboard.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; -import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -22,4 +22,6 @@ public interface SummaryMapper { int saveOrUpdate(Summary records); List listRuleHitCount(SearchReq req); + + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java index 97f0b4b..e8fdf0f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java @@ -16,7 +16,7 @@ public class KeyRecord implements Serializable { /** * 缓存时间 */ - private Long duration; + private int duration; /** * 来源： SYSTEM 系统探测；USERNAME创建人 @@ -43,7 +43,7 @@ public class KeyRecord implements Serializable { public KeyRecord() { } - public KeyRecord(String key,String val, String appName, Long duration, + public KeyRecord(String key,String val, String appName, int duration, String source, Integer type,String uuid, Date createTime) { this.key = key; this.val = val; @@ -103,11 +103,11 @@ public class KeyRecord implements Serializable { this.val = val; } - public Long getDuration() { + public int getDuration() { return duration; } - public void setDuration(Long duration) { + public void setDuration(int duration) { this.duration = duration; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java index 6efd7c5..67b9d0a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java @@ -22,7 +22,6 @@ public class KeyTimely implements Serializable { private Date createTime; - private String uuid; /** * 该rule的描述 */ @@ -44,12 +43,25 @@ public class KeyTimely implements Serializable { public KeyTimely(String key, String val, String appName, Long duration, String uuid, Date createTime) { this.key = key; this.val = val; - this.uuid = uuid; this.appName = appName; this.duration = duration; this.createTime = createTime; } + @Override + public String toString() { + return "KeyTimely{" + + "id=" + id + + ", key='" + key + '\'' + + ", appName='" + appName + '\'' + + ", val='" + val + '\'' + + ", duration=" + duration + + ", createTime=" + createTime + + ", ruleDesc='" + ruleDesc + '\'' + + ", updater='" + updater + '\'' + + '}'; + } + public String getRuleDesc() { return ruleDesc; } @@ -106,14 +118,6 @@ public class KeyTimely implements Serializable { this.createTime = createTime; } - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - public String getUpdater() { return updater; } @@ -186,7 +190,6 @@ public class KeyTimely implements Serializable { keyTimely.setVal(val); keyTimely.setDuration(duration); keyTimely.setCreateTime(createTime); - keyTimely.setUuid(uuid); keyTimely.setRuleDesc(ruleDesc); keyTimely.setUpdater(updater); return keyTimely; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java new file mode 100644 index 0000000..88057c9 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -0,0 +1,175 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import cn.hutool.core.util.StrUtil; +import com.github.benmanes.caffeine.cache.Cache; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.rule.KeyRule; +import com.jd.platform.hotkey.dashboard.cache.CaffeineBuilder; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.model.KeyTimely; +import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; + +/** + * 此处存储所有发来的热key，统一处理入库 + * + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class HotKeyReceiver { + + /** + * 发来的热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + /** + * 存储实时hotkey，供界面查询实时热key + */ + private static Map> aliveKeyStore = new ConcurrentHashMap<>(); + + private static Logger logger = LoggerFactory.getLogger("HotKeyReceiver"); + + /** + * netty收到的先存这里 + */ + public static void push(HotKeyModel model) { + hotKeyStoreQueue.offer(model); + } + + public static HotKeyModel take() { + try { + return hotKeyStoreQueue.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + return null; + } + } + + /** + * 将热key存入本地缓存，设置过期时间 + */ + public static void put(HotKeyModel hotKeyModel) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + logger.error("rule is null, hotkeyModel " + hotKeyModel.getAppName() + "-" + hotKeyModel.getKey()); + return; + } + Cache cache = aliveKeyStore.computeIfAbsent(keyRule.getDuration(), s -> CaffeineBuilder.cache(keyRule.getDuration())); + cache.put(appNameKey, hotKeyModel); + } + + /** + * 展示当前所有的实时热key + */ + public static List list(SearchReq searchReq) { + List timelyList = new ArrayList<>(); + + long now = System.currentTimeMillis(); + + for (Integer duration : aliveKeyStore.keySet()) { + Cache cache = aliveKeyStore.get(duration); + ConcurrentMap concurrentHashMap = cache.asMap(); + for (Map.Entry entry : concurrentHashMap.entrySet()) { + KeyTimely keyTimely = parse((HotKeyModel) entry.getValue(), now); + if (keyTimely == null) { + continue; + } + timelyList.add(keyTimely); + } + } + + if (searchReq != null) { + if (StrUtil.isNotEmpty(searchReq.getApp())) { + timelyList = timelyList.parallelStream().filter(keyTimely -> searchReq.getApp().equals(keyTimely.getAppName())).collect(Collectors.toList()); + } + if (StrUtil.isNotEmpty(searchReq.getKey())) { + timelyList = timelyList.parallelStream().filter(keyTimely -> keyTimely.getKey().startsWith(searchReq.getKey())).collect(Collectors.toList()); + } + + } + + timelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); + + return timelyList; + } + + /** + * 将hotkeyModel变成前端需要的对象 + */ + private static KeyTimely parse(HotKeyModel hotKeyModel, long now) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return null; + } + long remainTime = keyRule.getDuration() * 1000 - (now - hotKeyModel.getCreateTime()); + return KeyTimely.aKeyTimely() + .key(hotKeyModel.getKey()) + .val(UUID.randomUUID().toString()) + .appName(hotKeyModel.getAppName()) + .duration(remainTime / 1000) + .ruleDesc(RuleUtil.ruleDesc(appNameKey)) + .createTime(new Date(hotKeyModel.getCreateTime())).build(); + } + +// public static void main(String[] args) { +// List keyTimelyList = new ArrayList<>(); +// for (int i = 0; i < 24; i++) { +// KeyTimely keyTimely = new KeyTimely(); +// keyTimely.setKey(i + ""); +// keyTimely.setCreateTime(new Date()); +// try { +// Thread.sleep(10); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// keyTimelyList.add(keyTimely); +// } +// keyTimelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); +// +// Page page =PageUtil.pagination(keyTimelyList, 10, 3); +// System.out.println(page.getTotal()); +// System.out.println(page.getPage()); +// System.out.println(page.getRows()); +// } + + + + + + /** + * 删除实时热key + */ + public static boolean delete(String appNameKey) { + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return false; + } + Cache cache = aliveKeyStore.get(keyRule.getDuration()); + if (cache == null) { + return false; + } + cache.invalidate(appNameKey); + return true; + } + + /** + * 定时清理caffeine + */ + public static void cleanUpCaffeine() { + for (Integer duration : aliveKeyStore.keySet()) { + Cache cache = aliveKeyStore.get(duration); + cache.cleanUp(); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java new file mode 100644 index 0000000..a3dc7ee --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java @@ -0,0 +1,78 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import com.jd.platform.hotkey.common.tool.Constant; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.DelimiterBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.concurrent.TimeUnit; + +/** + * 该server用于给各个worker实例连接用。 + * + * @author wuweifeng wrote on 2019-11-05. + */ +@Component +public class NodesServer { + private Logger logger = LoggerFactory.getLogger(getClass()); + + public void startNettyServer(int port) { + //boss单线程 + EventLoopGroup bossGroup = new NioEventLoopGroup(1); + EventLoopGroup workerGroup = new NioEventLoopGroup(4); + try { + ServerBootstrap bootstrap = new ServerBootstrap(); + bootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .handler(new LoggingHandler(LogLevel.INFO)) + .option(ChannelOption.SO_BACKLOG, 1024) + //保持长连接 + .childOption(ChannelOption.SO_KEEPALIVE, true) + //出来网络io事件，如记录日志、对消息编解码等 + .childHandler(new ChildChannelHandler()); + //绑定端口，同步等待成功 + ChannelFuture future = bootstrap.bind(port).sync(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bossGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + workerGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + })); + //等待服务器监听端口关闭 + future.channel().closeFuture().sync(); + } catch (Exception e) { + e.printStackTrace(); + logger.error("dashboard netty server start failure"); + } finally { + //优雅退出，释放线程池资源 + bossGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + } + } + + /** + * handler类 + */ + private class ChildChannelHandler extends ChannelInitializer { + + @Override + protected void initChannel(Channel ch) { + NodesServerHandler serverHandler = new NodesServerHandler(); + + ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); + ch.pipeline() + .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) + .addLast(new StringDecoder()) + .addLast(serverHandler); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java new file mode 100644 index 0000000..9650c2f --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java @@ -0,0 +1,66 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.common.tool.flush.FlushUtil; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import java.util.List; + +import static com.jd.platform.hotkey.common.tool.Constant.PONG; + +/** + * 这里处理所有netty事件。 + * + * @author wuweifeng wrote on 2019-11-05. + */ +public class NodesServerHandler extends SimpleChannelInboundHandler { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, String message) { + if (StringUtils.isEmpty(message)) { + return; + } + try { + HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + if (MessageType.PING == msg.getMessageType()) { + String hotMsg = FastJsonUtils.convertObjectToJSON(new HotKeyMsg(MessageType.PONG, PONG)); + FlushUtil.flush(ctx, MsgBuilder.buildByteBuf(hotMsg)); + } else if (MessageType.REQUEST_HOT_KEY == msg.getMessageType()) { + List list = FastJsonUtils.toList(msg.getBody(), HotKeyModel.class); + for (HotKeyModel hotKeyModel : list) { + HotKeyReceiver.push(hotKeyModel); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("some thing is error , " + cause.getMessage()); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + ctx.close(); + super.channelInactive(ctx); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java new file mode 100644 index 0000000..5266ef3 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java @@ -0,0 +1,23 @@ +package com.jd.platform.hotkey.dashboard.service; + +import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearService + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:51 + */ +public interface ClearService { + + + PageInfo pageClearCfg(PageReq page, String app); + + ClearCfgVo selectClearCfg(String app); + + int saveClearCfg(ClearCfgVo cfg); + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java index 3dfcf18..558e814 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.service; import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; @@ -29,11 +30,7 @@ public interface KeyService { int delKeyByUser(KeyTimely keyTimely); - KeyTimely selectByKey(String key); - - KeyTimely selectByPk(Long key); - - PageInfo pageKeyTimely(PageReq page, SearchReq param); + Page pageKeyTimely(PageReq page, SearchReq param); PageInfo pageMaxHot(PageReq page, SearchReq param); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java index 67db3d3..71f57ea 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java @@ -5,6 +5,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.User; +import javax.servlet.http.Cookie; import java.util.List; /** @@ -22,6 +23,10 @@ public interface UserService { int insertUser(User user); + int insertUserByErp(User user); + + Cookie loginErpUser(User user); + int deleteByPrimaryKey(int id); User selectByPrimaryKey(int id); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java new file mode 100644 index 0000000..8227a95 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java @@ -0,0 +1,85 @@ +package com.jd.platform.hotkey.dashboard.service.impl; + +import com.github.pagehelper.PageInfo; +import com.github.pagehelper.util.StringUtil; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; +import com.jd.platform.hotkey.dashboard.service.ClearService; +import com.jd.platform.hotkey.dashboard.service.UserService; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; + +/** + * @ProjectName: hotkey + * @ClassName: ClearServiceImpl + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:57 + */ +@Service +public class ClearServiceImpl implements ClearService { + + + @Resource + private IConfigCenter configCenter; + + @Resource + private UserService userService; + + + @Override + public PageInfo pageClearCfg(PageReq page, String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + if(CollectionUtils.isEmpty(keyValues)){ + List apps = userService.listApp(); + for (String ap : apps) { + configCenter.put(ConfigConstant.clearCfgPath + ap, Constant.THIRTY_DAY); + } + keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + } + List cfgVos = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + String key = kv.getKey().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + configCenter.put(key, Constant.THIRTY_DAY); + continue; + } + long version = kv.getModRevision(); + String k = key.replace(ConfigConstant.clearCfgPath,""); + if(StringUtils.isEmpty(app)){ + cfgVos.add(new ClearCfgVo(k, v, version)); + }else{ + if(k.equals(app)){ + cfgVos.add(new ClearCfgVo(k, v, version)); + } + } + } + return new PageInfo<>(cfgVos); + } + + @Override + public ClearCfgVo selectClearCfg(String app) { + KeyValue kv = configCenter.getKv(ConfigConstant.clearCfgPath + app); + if(kv == null || kv.getValue() == null){ + configCenter.put(ConfigConstant.clearCfgPath + app, Constant.THIRTY_DAY); + return new ClearCfgVo(app, Constant.THIRTY_DAY, 0L); + } + String v = kv.getValue().toStringUtf8(); + return new ClearCfgVo(app,v,kv.getModRevision()); + } + + @Override + public int saveClearCfg(ClearCfgVo cfg) { + configCenter.put(ConfigConstant.clearCfgPath + cfg.getApp(), cfg.getTtl()); + return 1; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 88a4263..09dbd32 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -8,24 +8,29 @@ import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.service.KeyService; import com.jd.platform.hotkey.dashboard.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.PageUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -50,16 +55,17 @@ public class KeyServiceImpl implements KeyService { @Resource private KeyRecordMapper recordMapper; @Resource - private KeyTimelyMapper keyTimelyMapper; - @Resource private StatisticsMapper statisticsMapper; @Resource private RuleService ruleService; @Resource private ChangeLogMapper logMapper; + private Logger logger = LoggerFactory.getLogger(getClass()); + /** * 折线图 + * * @param req req * @return vo */ @@ -68,52 +74,48 @@ public class KeyServiceImpl implements KeyService { int type = req.getType(); String appReq = req.getApp(); // admin 全查 - if(StrUtil.isNotEmpty(appReq)){ + if (StrUtil.isNotEmpty(appReq)) { app = appReq; } req.setApp(null); LocalDateTime now = LocalDateTime.now(); req.setEndTime(req.getEndTime() == null ? DateUtil.ldtToDate(now) : req.getEndTime()); List rules = ruleService.listRules(null); - if(type == 4){ - LocalDateTime st = req.getStartTime() == null? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); + if (type == 4) { + LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); req.setStartTime(DateUtil.ldtToDate(st)); LocalDateTime et = DateUtil.dateToLdt(req.getEndTime()); - boolean longTime = Duration.between(st,et).toHours()>2; + boolean longTime = Duration.between(st, et).toHours() > 2; req.setType(longTime ? 6 : 5); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(st,et,list,!longTime,rules,app); + return CommonUtil.processData(st, et, list, !longTime, rules, app); } - if(type == 5){ + if (type == 5) { LocalDateTime startTime = now.minusMinutes(31); req.setStartTime(DateUtil.ldtToDate(startTime)); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime,now,list,true,rules,app); - }else if(type == 6){ + return CommonUtil.processData(startTime, now, list, true, rules, app); + } else if (type == 6) { LocalDateTime startTime2 = now.minusHours(25); req.setStartTime(DateUtil.ldtToDate(startTime2)); List list2 = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime2,now,list2,false,rules,app); - }else{ + return CommonUtil.processData(startTime2, now, list2, false, rules, app); + } else { LocalDateTime startTime3 = now.minusDays(7).minusHours(1); req.setStartTime(DateUtil.ldtToDate(startTime3)); req.setType(6); List list3 = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime3,now,list3,false,rules,app); + return CommonUtil.processData(startTime3, now, list3, false, rules, app); } } @Override - public PageInfo pageKeyTimely(PageReq page, SearchReq param) { - PageHelper.startPage(page.getPageNum(), page.getPageSize()); - List listKey = keyTimelyMapper.listKeyTimely(param); - for (KeyTimely timely : listKey) { - timely.setKey(CommonUtil.keyName(timely.getKey())); - timely.setRuleDesc(RuleUtil.ruleDesc(timely.getAppName() + "/" + timely.getKey())); - } - return new PageInfo<>(listKey); + public Page pageKeyTimely(PageReq page, SearchReq param) { + List keyTimelies = HotKeyReceiver.list(param); + return PageUtil.pagination(keyTimelies, page.getPageSize(), page.getPageNum()-1); + } @Override @@ -143,12 +145,12 @@ public class KeyServiceImpl implements KeyService { Map keyDateMap = keyDateMap(statistics, hours); // 获取时间x轴 List list = new ArrayList<>(); - for (int i = hours; i >0 ; i--) { - LocalDateTime time = LocalDateTime.now().minusHours(i-1); + for (int i = hours; i > 0; i--) { + LocalDateTime time = LocalDateTime.now().minusHours(i - 1); int hour = time.getHour(); - list.add(hour+"时"); + list.add(hour + "时"); } - return new HotKeyLineChartVo(list,keyDateMap); + return new HotKeyLineChartVo(list, keyDateMap); } @@ -168,6 +170,13 @@ public class KeyServiceImpl implements KeyService { public int insertKeyByUser(KeyTimely key) { configCenter.putAndGrant(ConfigConstant.hotKeyPath + key.getAppName() + "/" + key.getKey(), System.currentTimeMillis() + "", key.getDuration()); + + //写入本地缓存，实时热key信息 + HotKeyModel hotKeyModel = new HotKeyModel(); + hotKeyModel.setCreateTime(System.currentTimeMillis()); + hotKeyModel.setAppName(key.getAppName()); + hotKeyModel.setKey(key.getKey()); + HotKeyReceiver.put(hotKeyModel); return logMapper.insertSelective(new ChangeLog(key.getAppName(), Constant.HOTKEY_CHANGE, "", key.getKey(), key.getUpdater(), SystemClock.now() + "")); } @@ -185,56 +194,46 @@ public class KeyServiceImpl implements KeyService { String[] arr = keyTimely.getKey().split("/"); //删除client监听目录的key String etcdKey = ConfigConstant.hotKeyPath + arr[0] + "/" + arr[1]; + + //删除caffeine里的实时key + HotKeyReceiver.delete(arr[0] + "/" + arr[1]); + if (configCenter.get(etcdKey) == null) { //如果手工目录也就是client监听的目录里没有该key，那么就往里面放一个，然后再删掉它，这样client才能监听到删除事件 configCenter.putAndGrant(etcdKey, com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE, 1); } configCenter.delete(etcdKey); - //也删除Record目录下的该key，因为不确定要删的key到底在哪 - String recordKey = ConfigConstant.hotKeyRecordPath + arr[0] + "/" + arr[1]; - configCenter.delete(recordKey); - - KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, + KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); - return logMapper.insertSelective(new ChangeLog(arr[0], Constant.HOTKEY_CHANGE, arr[1],"", keyTimely.getUpdater(), SystemClock.now() + "")); + return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(), "", keyTimely.getUpdater(), SystemClock.now() + "")); } - @Override - public KeyTimely selectByKey(String key) { - return keyTimelyMapper.selectByKey(key); - } - - @Override - public KeyTimely selectByPk(Long id) { - return keyTimelyMapper.selectByPrimaryKey(id); - } - - - - private Map keyDateMap(List statistics, int hours){ + private Map keyDateMap(List statistics, int hours) { Map map = new HashMap<>(10); Map> listMap = statistics.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); for (Map.Entry> m : listMap.entrySet()) { int start = DateUtil.preHoursInt(5); - map.put(m.getKey(),new int[hours]); + map.put(m.getKey(), new int[hours]); int[] data = map.get(m.getKey()); int tmp = 0; for (int i = 0; i < hours; i++) { Statistics st; try { st = m.getValue().get(tmp); - if(String.valueOf(start).endsWith("24")){ start = start + 77; } - if(start != st.getHours()){ + if (String.valueOf(start).endsWith("24")) { + start = start + 77; + } + if (start != st.getHours()) { data[i] = 0; - }else{ - tmp ++; + } else { + tmp++; data[i] = st.getCount(); } start++; - }catch (Exception e){ + } catch (Exception e) { data[i] = 0; } } @@ -243,9 +242,8 @@ public class KeyServiceImpl implements KeyService { } - private void checkParam(SearchReq req) { - if(req.getStartTime() == null || req.getEndTime() == null){ + if (req.getStartTime() == null || req.getEndTime() == null) { req.setStartTime(DateUtil.preTime(5)); req.setEndTime(new Date()); } @@ -255,173 +253,6 @@ public class KeyServiceImpl implements KeyService { }*/ } - - private static List statisticsList(){ - Random rd = new Random(); - List records = new ArrayList<>(); - // statisticsMapper.clear(1); - for (int i = 0; i < 35; i++) { - if(i== 2||i== 3||i== 4||i== 25||i== 12||i== 13||i== 14||i== 30||i==32||i==33||i== 34){ - continue; - } - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusMinutes(35-i); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - int minus = DateUtil.nowMinus(now); - - Statistics s1 = new Statistics(); - s1.setKeyName("k21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(800)); - s1.setBizType(1); - s1.setMinutes(minus); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("k22"); - s11.setApp("key22APP"); - s11.setBizType(1); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setMinutes(minus); - s11.setCount(rd.nextInt(600)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - } - - if(1==1){ - return records; - } - // statisticsMapper.batchInsert(records); - List list = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 12 || i ==13){ - }else{ - Statistics st = new Statistics(); - st.setApp("k21"); - st.setKeyName("k21"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); - // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); - int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); - // System.out.println(time); - st.setMinutes(time); - // list.add(st); - } - } - List list2 = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ - - }else{ - Statistics st2 = new Statistics(); - st2.setApp("k22"); - st2.setKeyName("k22"); - st2.setCount(rd.nextInt(100)); - st2.setBizType(1); - st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); - list2.add(st2); - } - } - // list.addAll(list2); - return list; - - } - - private List statisticsList1(int type){ - Random rd = new Random(); - List records = new ArrayList<>(); - statisticsMapper.clear(6); - - if(type == 3){ - int temp = 1; - for (int j = 0; j < 9; j++) { - for (int i = 0; i < 24; i++) { - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusHours(9 * 24 - temp); - - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - Statistics s1 = new Statistics(); - s1.setKeyName("k21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(200)); - s1.setBizType(6); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("k22"); - s11.setApp("key22APP"); - s11.setBizType(6); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setCount(rd.nextInt(500)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - - Statistics s33 = new Statistics(); - s33.setKeyName("k33"); - s33.setApp("key33APP"); - s33.setBizType(6); - s33.setCreateTime(nowTime); - s33.setDays(day); - s33.setCount(rd.nextInt(300)); - s33.setHours(hour); - s33.setUuid(3 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s33); - temp++; - } - } - - - }else{ - for (int i = 0; i < 30; i++) { - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusHours(30-i); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - Statistics s1 = new Statistics(); - s1.setKeyName("key21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(800)); - s1.setBizType(2); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("key22"); - s11.setApp("key22APP"); - s11.setBizType(2); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setCount(rd.nextInt(600)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - } - } - statisticsMapper.batchInsert(records); - return records; - } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index d81f51e..c03ea2e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -34,6 +34,8 @@ import java.util.Set; @Service public class UserServiceImpl implements UserService { + @Value("${erp.defaultPwd:123}") + private String defaultPwd; @Resource private UserMapper userMapper; @Resource @@ -66,6 +68,46 @@ public class UserServiceImpl implements UserService { return userMapper.insertSelective(user); } + @Override + public int insertUserByErp(User user) { + User userParam = new User(); + userParam.setUserName(user.getUserName()); + List users = userMapper.selectHkUserList(userParam); + if(users.size() == 0){ + user.setCreateTime(new Date()); + user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); + int ret = userMapper.insertSelective(user); + System.out.println(user.getId()); + return ret; + } + return 0; + } + + @Override + public Cookie loginErpUser(User user){ + User userParam = new User(); + userParam.setUserName(user.getUserName()); + List users = userMapper.selectHkUserList(userParam); + if(users.size() == 0){ + user.setCreateTime(new Date()); + user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); + userMapper.insertSelective(user); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), "", user.getAppName(), user.getNickName()); + Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); + cookie.setMaxAge(3600*24*7); + //cookie.setDomain("localhost"); + cookie.setPath("/"); + return cookie; + }else{ + user =users.get(0); + String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), "", user.getAppName(), user.getNickName()); + Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); + cookie.setMaxAge(3600*24*7); + //cookie.setDomain("localhost"); + cookie.setPath("/"); + return cookie; + } + } @Override public int deleteByPrimaryKey(int id) { @@ -108,4 +150,5 @@ public class UserServiceImpl implements UserService { return new PageInfo<>(users); } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java new file mode 100644 index 0000000..cd4a987 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java @@ -0,0 +1,36 @@ +package com.jd.platform.hotkey.dashboard.util; + +import com.google.common.collect.Lists; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-09-01 + */ +public class PageUtil { + /** + * 通用分页工具类 + * + * @param data + * @param pageSize + * @param pageNum + * @param + * @return + */ + public static Page pagination(final List data, final int pageSize, final int pageNum) { + if (CollectionUtils.isEmpty(data)) { + return new Page<>(1, 0, new ArrayList<>()); + } + List> lists = Lists.partition(data, pageSize); + int localPageNum = pageNum; + if (localPageNum >= lists.size()) { + localPageNum = lists.size() - 1; + } + return new Page<>(localPageNum, data.size(), lists.get(localPageNum)); + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java index 40d82ff..d9352dc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java @@ -33,7 +33,7 @@ public class RuleUtil { } /** - * 根据APP的key，获取该key对应的rule + * 根据APP的key，获取该key对应的rule.如 cartpc-pu__ */ public static String rule(String key) { try { @@ -62,7 +62,7 @@ public class RuleUtil { return ""; } - private static KeyRule findByKey(String appNameKey) { + public static KeyRule findByKey(String appNameKey) { synchronized (RULE_MAP) { if (StrUtil.isEmpty(appNameKey)) { return null; diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 2e1b78e..fd37556 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -31,6 +31,12 @@ spring : password: ${MYSQL_PASS:JRTEST} url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 driver-class-name: com.mysql.cj.jdbc.Driver + hikari: + max-lifetime: 120000 + idle-timeout: 60000 + connection-timeout: 30000 + maximum-pool-size: 32 + minimum-idle: 10 pagehelper: helperDialect: mysql reasonable: true diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml index f869987..94037b7 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml @@ -199,4 +199,10 @@ + + delete from hk_key_record + where app_name = #{app} + + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml deleted file mode 100644 index 04704cf..0000000 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml +++ /dev/null @@ -1,207 +0,0 @@ - - - - - - - - - - - - - - id, key_name, val, uuid, app_name, duration, create_time - - - - - delete from hk_key_timely - where key_name = #{key} and app_name = #{appName} - - - - - delete from hk_key_timely - where (app_name,key_name) in - - (#{item.appName},#{item.key}) - - - - - delete from hk_key_timely where 1 = 1 - - - - - INSERT ignore INTO - hk_key_timely( - key_name, - val, - app_name, - uuid, - duration, - create_time) - VALUES - - (#{k.key}, - #{k.val}, - #{k.appName}, - #{k.uuid}, - #{k.duration}, - #{k.createTime}) - - - - - insert into hk_key_timely - - - id, - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{id,jdbcType=BIGINT}, - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - - - - update hk_key_timely - - - key_name = #{key,jdbcType=VARCHAR}, - - - val = #{val,jdbcType=VARCHAR}, - - - uuid = #{uuid,jdbcType=VARCHAR}, - - - app_name = #{appName,jdbcType=VARCHAR}, - - - duration = #{duration,jdbcType=BIGINT}, - - - create_time = #{createTime,jdbcType=TIMESTAMP}, - - - where key_name = #{key} - - - - - - - - insert into hk_key_timely - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - on duplicate key update - create_time = now() - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml index 8829d12..ac7f464 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml @@ -74,8 +74,9 @@ - + where app = #{app} + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml index 60f41a2..bd04545 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml @@ -51,4 +51,11 @@ + + delete from hk_summary + where app = #{app} + + + + \ No newline at end of file diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 59b7aac..01763f2 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -40,20 +40,13 @@ ajaxOptions:{ headers: {"Authorization": getCookie("token")} }, - exportOptions:{ - ignoreColumn: [0,1], //忽略某一列的索引 - fileName: '报表导出', //文件名称设置 - worksheetName: 'sheet1', //表格工作区名称 - tableName: '报表导出', - excelstyles: ['background-color', 'color', 'font-size', 'font-weight'] - }, onLoadSuccess: function(){ //加载成功时执行 var datas = $('#dataTable').bootstrapTable('getData'); console.info("加载成功"); }, onLoadError: function(status){ //加载失败时执行 let token = getCookie("token"); - if(status === 1000 && ( token == "undefined" || token =="")){ + if(status === 500 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); @@ -70,7 +63,15 @@ if(app!==undefined && app != null && app!==""){ search.app = app; } - return search; + let time1 = $("#startTime").val(); + if(time1!==undefined && time1 != null && time1!==""){ + search.startTime = time1; + } + let time2 = $("#endTime").val(); + if(time2!==undefined && time2 != null && time2!==""){ + search.endTime = time2; + } + return search; }, search:function(my){//查询条件 diff --git a/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js b/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js new file mode 100644 index 0000000..0d8de56 --- /dev/null +++ b/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js @@ -0,0 +1,25 @@ +$("#form-edit").validate({ + submitHandler:function(form){ + edit(); + } +}); + +function edit() { + let dataFormJson=$("#form-edit").serialize(); + $.ajax({ + cache : true, + type : "POST", + url : "/clear/save", + data : dataFormJson, + headers: { + "Authorization":getCookie("token") + }, + async : false, + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); + }, + success : function(data) { + $.operate.saveSuccess(data); + } + }); +} diff --git a/dashboard/src/main/resources/templates/admin/clear/edit.html b/dashboard/src/main/resources/templates/admin/clear/edit.html new file mode 100644 index 0000000..4ced25c --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/clear/edit.html @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    + +
    +
    + + +
    +
    + +
    +
    +
    +
    +
    +
    + + +
    + +
    + + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/clear/list.html b/dashboard/src/main/resources/templates/admin/clear/list.html new file mode 100644 index 0000000..7087f79 --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/clear/list.html @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + +
    +
    + +
    + +
    +
    + +
    +
    +
    搜索条件
    +
    + +
    +
    +
    +
    + + +
    + + +
    +
    +
    + + +
    +
    +
    表单
    +
    + + + +
    +
    + + + +
    +
    + +
    +
    +
    +
    +
    +
    + +
    + +
    +
    +
    + + + +
    + +
    + + + + + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 29baffe..668341d 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -52,7 +52,7 @@ - + 热点记录 变更记录 + + + 数据清理配置 + diff --git a/dashboard/src/main/resources/templates/admin/key/list.html b/dashboard/src/main/resources/templates/admin/key/list.html index 38c596f..6f755c8 100644 --- a/dashboard/src/main/resources/templates/admin/key/list.html +++ b/dashboard/src/main/resources/templates/admin/key/list.html @@ -163,7 +163,7 @@ var options = { { field: 'key', title: 'key'}, { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, - { field: 'duration', title: '缓存时间'}, + { field: 'duration', title: '剩余缓存时间'}, { field: 'source', title: '来源'}, { field: 'type', title: '操作类型', formatter:function (val) { @@ -174,8 +174,27 @@ var options = { { field: 'createTime', title: '创建时间' }] }; -$(function(){ + + $('.datetimepicker').datetimepicker({ + language: 'zh-CN', + CustomFormat: 'yyyy-mm-dd HH:ii:ss', + weekStart: 1, + todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 + autoclose: 1, //选完时间自动关闭 + todayHighlight: 1, //当天时间高亮 + startView: 2, //从月视图开始，选天 + minView: 0, //提供选择分钟的视图 + forceParse: 0, + minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 + }); + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); + + +$(function(){ $.ajax({ cache : true, type : "POST", @@ -206,24 +225,9 @@ $(function(){ } }); - var oTab=$.table.oTableInit(options); oTab.Init(); - $('.datetimepicker').datetimepicker({ - language: 'zh-CN', - CustomFormat: 'yyyy-mm-dd HH:ii:ss', - weekStart: 1, - todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 - autoclose: 1, //选完时间自动关闭 - todayHighlight: 1, //当天时间高亮 - startView: 2, //从月视图开始，选天 - minView: 0, //提供选择分钟的视图 - forceParse: 0, - minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 -}); - - }) function exportHot() { diff --git a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html index 366b156..5d7fc22 100644 --- a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html +++ b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html @@ -35,14 +35,14 @@
    - + - + diff --git a/dashboard/src/main/resources/templates/admin/key/listtimely.html b/dashboard/src/main/resources/templates/admin/key/listtimely.html index 37e85be..545caa7 100644 --- a/dashboard/src/main/resources/templates/admin/key/listtimely.html +++ b/dashboard/src/main/resources/templates/admin/key/listtimely.html @@ -162,7 +162,7 @@ var options = { { field: 'key', title: 'key'}, { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, - { field: 'duration', title: '缓存时间'}, + { field: 'duration', title: '剩余缓存时间'}, { field: 'createTime', title: '创建时间', formatter:function (val,row,index) { return changeDateFormat(val); diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index 75e714a..ee7b7fe 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -179,7 +179,7 @@ }, error: function(result){ //加载失败时执行 let token = getCookie("token"); - if(result.status == 1000 && ( token == "undefined" || token =="")){ + if(result.status == 500 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); diff --git a/dashboard/src/main/resources/templates/admin/rule/listhitcount.html b/dashboard/src/main/resources/templates/admin/rule/listhitcount.html index 7e501fa..f1be6e3 100644 --- a/dashboard/src/main/resources/templates/admin/rule/listhitcount.html +++ b/dashboard/src/main/resources/templates/admin/rule/listhitcount.html @@ -46,7 +46,7 @@
    - + @@ -55,7 +55,7 @@
    - + @@ -134,6 +134,27 @@ } }] }; + + + + $('.datetimepicker').datetimepicker({ + language: 'zh-CN', + CustomFormat: 'yyyy-mm-dd HH:ii:ss', + weekStart: 1, + todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 + autoclose: 1, //选完时间自动关闭 + todayHighlight: 1, //当天时间高亮 + startView: 2, //从月视图开始，选天 + minView: 0, //提供选择分钟的视图 + forceParse: 0, + minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 + }); + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); + + $(function(){ $.ajax({ cache : true, @@ -164,26 +185,12 @@ } } }); - var oTab=$.table.oTableInit(options); + + let oTab=$.table.oTableInit(options); oTab.Init(); }) - - $('.datetimepicker').datetimepicker({ - language: 'zh-CN', - CustomFormat: 'yyyy-mm-dd HH:ii:ss', - weekStart: 1, - todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 - autoclose: 1, //选完时间自动关闭 - todayHighlight: 1, //当天时间高亮 - startView: 2, //从月视图开始，选天 - minView: 0, //提供选择分钟的视图 - forceParse: 0, - minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 - }); - - diff --git a/dashboard/src/main/resources/templates/admin/user/list.html b/dashboard/src/main/resources/templates/admin/user/list.html index 54fa7b7..3c2df58 100644 --- a/dashboard/src/main/resources/templates/admin/user/list.html +++ b/dashboard/src/main/resources/templates/admin/user/list.html @@ -108,6 +108,7 @@ + -- Gitee From ef976e228138a64f729d8ba61ef06dc58683a1fe Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 14:58:39 +0800 Subject: [PATCH 147/156] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../keydispatcher/DispatcherConfig.java | 4 + .../worker/keydispatcher/KeyProducer.java | 5 +- .../netty/dashboard/DashboardHolder.java | 26 ++++++ .../worker/netty/dashboard/NettyClient.java | 86 +++++++++++++++++++ .../netty/dashboard/NettyClientHandler.java | 58 +++++++++++++ .../worker/netty/filter/HeartBeatFilter.java | 2 +- .../worker/netty/filter/HotKeyFilter.java | 14 +-- .../hotkey/worker/netty/flush/FlushUtil.java | 34 -------- .../netty/pusher/BatchToEtcdScheduler.java | 45 ---------- .../{EtcdPusher.java => DashboardPusher.java} | 28 ++---- .../worker/netty/pusher/KeyCollector.java | 53 ------------ .../worker/netty/pusher/KeyUploader.java | 46 ++++++++++ .../netty/pusher/store/HotkeyTempStore.java | 27 ++++++ .../worker/netty/server/NodesServer.java | 13 ++- .../hotkey/worker/starters/EtcdStarter.java | 25 ++++++ .../worker/starters/NodesServerStarter.java | 4 - .../platform/hotkey/worker/tool/CpuNum.java | 16 ++-- 17 files changed, 297 insertions(+), 189 deletions(-) create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java rename worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/{EtcdPusher.java => DashboardPusher.java} (31%) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java index defcf67..7e3094a 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java @@ -41,6 +41,10 @@ public class DispatcherConfig { //将实际值赋给static变量 if (threadCount != 0) { nowCount = threadCount; + } else { + if (nowCount >= 8) { + nowCount = nowCount / 2; + } } List consumerList = new ArrayList<>(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java index fc86ecc..2f3320b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.keydispatcher; -import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.tool.InitConstant; import org.springframework.stereotype.Component; @@ -17,12 +16,12 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @Component public class KeyProducer { - public void push(HotKeyModel model) { + public void push(HotKeyModel model, long now) { if (model == null || model.getKey() == null) { return; } //5秒前的过时消息就不处理了 - if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { + if (now - model.getCreateTime() > InitConstant.timeOut) { expireTotalCount.increment(); return; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java new file mode 100644 index 0000000..6e0d57e --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java @@ -0,0 +1,26 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import io.netty.channel.Channel; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class DashboardHolder { + /** + * 是否连上了dashboard + */ + public static boolean hasConnected = false; + /** + * channel + */ + public static Channel channel = null; + + public static void flushToDashboard(String message) { + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HOT_KEY, message))); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java new file mode 100644 index 0000000..5209c35 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java @@ -0,0 +1,86 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.tool.Constant; +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.DelimiterBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.timeout.IdleStateHandler; + +/** + * netty连接器 + * + * @author wuweifeng wrote on 2019-11-05. + */ +public class NettyClient { + private static final NettyClient nettyClient = new NettyClient(); + + private Bootstrap bootstrap; + + + public static NettyClient getInstance() { + return nettyClient; + } + + private NettyClient() { + if (bootstrap == null) { + bootstrap = initBootstrap(); + } + } + + private Bootstrap initBootstrap() { + //少线程 + EventLoopGroup group = new NioEventLoopGroup(2); + + Bootstrap bootstrap = new Bootstrap(); + NettyClientHandler nettyClientHandler = new NettyClientHandler(); + bootstrap.group(group).channel(NioSocketChannel.class) + .option(ChannelOption.SO_KEEPALIVE, true) + .option(ChannelOption.TCP_NODELAY, true) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); + ch.pipeline() + .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) + .addLast(new StringDecoder()) + //10秒没消息时，就发心跳包过去 + .addLast(new IdleStateHandler(0, 0, 30)) + .addLast(nettyClientHandler); + } + }); + return bootstrap; + } + + public synchronized void connect(String address) { + if (DashboardHolder.hasConnected) { + return; + } + String[] ss = address.split(":"); + try { + ChannelFuture channelFuture = bootstrap.connect(ss[0], Integer.parseInt(ss[1])).sync(); + DashboardHolder.channel = channelFuture.channel(); + DashboardHolder.hasConnected = true; + } catch (Exception e) { + DashboardHolder.hasConnected = false; + DashboardHolder.channel = null; + e.printStackTrace(); + } + + //这一步就阻塞了 +// channelFuture.channel().closeFuture().sync(); + //当server断开后才会走下面的 +// System.out.println("server is down"); + } + + public synchronized void disConnect() { + DashboardHolder.channel = null; + DashboardHolder.hasConnected = false; + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java new file mode 100644 index 0000000..5fdbe3a --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java @@ -0,0 +1,58 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.Constant; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author wuweifeng wrote on 2019-11-05. + */ +@ChannelHandler.Sharable +public class NettyClientHandler extends SimpleChannelInboundHandler { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent idleStateEvent = (IdleStateEvent) evt; + + if (idleStateEvent.state() == IdleState.ALL_IDLE) { + //向服务端发送消息 + ctx.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.PING, Constant.PING))); + } + } + + super.userEventTriggered(ctx, evt); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + logger.info("channelActive:" + ctx.name()); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + NettyClient.getInstance().disConnect(); + } + + @Override + protected void channelRead0(ChannelHandlerContext channelHandlerContext, String message) { + HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + if (MessageType.PONG == msg.getMessageType()) { + logger.info("heart beat"); + } + + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java index a013d53..7ef2cb8 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java @@ -4,7 +4,7 @@ import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; +import com.jd.platform.hotkey.common.tool.flush.FlushUtil; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 080a9d9..0a1521a 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -54,27 +54,19 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { } private void publishMsg(String message, ChannelHandlerContext ctx) { - //这个是给测试用的，实际走的是下面那个 - if (message.startsWith("{")) { - HotKeyModel model = FastJsonUtils.toBean(message, HotKeyModel.class); - if (WhiteListHolder.contains(model.getKey())) { - return; - } - keyProducer.push(model); - return; - } //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, HotKeyModel.class); + long now = SystemClock.now(); for (HotKeyModel model : models) { //白名单key不处理 if (WhiteListHolder.contains(model.getKey())) { continue; } - long timeOut = SystemClock.now() - model.getCreateTime(); + long timeOut = now - model.getCreateTime(); if (timeOut > 1000) { logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); } - keyProducer.push(model); + keyProducer.push(model, now); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java deleted file mode 100644 index 04f1354..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.flush; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public class FlushUtil { - private static Logger logger = LoggerFactory.getLogger("flushUtil"); - - /** - * 往channel里输出消息 - */ - public static void flush(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) { -// if (channelHandlerContext.channel().isWritable()) { -// channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { -// if (!future.isSuccess()) { -// logger.warn("flush error " + future.cause().getMessage()); -// } -// }); -// } else { - try { - //同步发送 - channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); - } catch (InterruptedException e) { - logger.error("flush error " + e.getMessage()); - } -// } - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java deleted file mode 100644 index fe958d5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.Set; -import java.util.UUID; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-26 - */ -@Component -@Deprecated -public class BatchToEtcdScheduler { - @Resource - private KeyCollector keyCollector; - @Resource - private IConfigCenter iConfigCenter; - - /** - * 每隔0.5秒上传一下已探测出的热key发往etcd供入库 - */ -// @Scheduled(fixedRate = 500) - public void uploadClientCount() { - Set set = keyCollector.lockAndGetResult(); - if (set.size() == 0) { - return; - } - - //worker将热key推送到该地址，供dashboard监听入库做记录 - String hotKeyRecordPath = ConfigConstant.hotKeyRecordPath + UUID.randomUUID().toString(); - try { - //推送到etcd，供dashboard监听入库 - iConfigCenter.putAndGrant(hotKeyRecordPath, FastJsonUtils.convertObjectToJSON(set), 10); - } catch (Exception e) { - e.printStackTrace(); - } - - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java similarity index 31% rename from worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java rename to worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java index 627d3eb..aa420cd 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java @@ -1,38 +1,24 @@ package com.jd.platform.hotkey.worker.netty.pusher; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.tool.HotKeyPathTool; -import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; +import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; import org.springframework.stereotype.Component; -import javax.annotation.Resource; -import java.util.UUID; - - /** - * @author wuweifeng wrote on 2020-02-24 + * 将热key推送到dashboard供入库 + * @author wuweifeng * @version 1.0 + * @date 2020-08-31 */ @Component -public class EtcdPusher implements IPusher { - @Resource - private IConfigCenter iConfigCenter; - - +public class DashboardPusher implements IPusher { @Override public void push(HotKeyModel model) { - //推送到etcd，供dashboard监听入库 - iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), - KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); + HotkeyTempStore.push(model); } @Override - @Deprecated public void remove(HotKeyModel model) { - //推送etcd删除 - iConfigCenter.delete(HotKeyPathTool.keyPath(model)); - } - + } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java deleted file mode 100644 index 1dad842..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import cn.hutool.core.collection.ConcurrentHashSet; -import cn.hutool.core.util.StrUtil; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import org.springframework.stereotype.Component; - -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - -import static com.jd.platform.hotkey.common.tool.KeyRecordKeyTool.key; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-26 - */ -@Component -@Deprecated -public class KeyCollector { - private Set keySet0 = new ConcurrentHashSet<>(); - private Set keySet1 = new ConcurrentHashSet<>(); - private AtomicLong atomicLong = new AtomicLong(0); - - public Set lockAndGetResult() { - //自增后，对应的map就会停止被写入，等待被读取 - atomicLong.addAndGet(1); - Set set; - if (atomicLong.get() % 2 == 0) { - set = new HashSet<>(keySet1); - keySet1.clear(); - } else { - set = new HashSet<>(keySet0); - keySet0.clear(); - } - return set; - } - - public void collect(HotKeyModel hotKeyModel) { - String key = hotKeyModel.getKey(); - if (StrUtil.isEmpty(key)) { - return; - } - if (atomicLong.get() % 2 == 0) { - keySet0.add(key(hotKeyModel)); - } else { - keySet1.add(key(hotKeyModel)); - } - - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java new file mode 100644 index 0000000..8ee6e11 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java @@ -0,0 +1,46 @@ +package com.jd.platform.hotkey.worker.netty.pusher; + +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.collect.Queues; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.worker.netty.dashboard.DashboardHolder; +import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; +import com.jd.platform.hotkey.worker.tool.AsyncPool; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * 定时批量往dashboard发送热key，供入库 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +@Component +public class KeyUploader { + + @PostConstruct + public void uploadToDashboard() { + AsyncPool.asyncDo(() -> { + while (true) { + try { + //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 + List tempModels = new ArrayList<>(); + Queues.drain(HotkeyTempStore.getQueue(), tempModels, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + + //将热key推到dashboard + DashboardHolder.flushToDashboard(FastJsonUtils.convertObjectToJSON(tempModels)); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java new file mode 100644 index 0000000..c93d18d --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java @@ -0,0 +1,27 @@ +package com.jd.platform.hotkey.worker.netty.pusher.store; + +import com.jd.platform.hotkey.common.model.HotKeyModel; + +import java.util.concurrent.LinkedBlockingQueue; + +/** + * 已热待上报入库的热key集中营 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class HotkeyTempStore { + /** + * 热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + + public static void push(HotKeyModel model) { + hotKeyStoreQueue.offer(model); + } + + public static LinkedBlockingQueue getQueue() { + return hotKeyStoreQueue; + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java index 6ea9b4f..0762742 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.netty.server; -import com.jd.platform.hotkey.common.coder.Codec; import com.jd.platform.hotkey.common.tool.Constant; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; @@ -17,6 +16,7 @@ import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import java.util.List; +import java.util.concurrent.TimeUnit; /** * 该server用于给各个微服务实例连接用。 @@ -26,7 +26,6 @@ import java.util.List; public class NodesServer { private IClientChangeListener clientChangeListener; private List messageFilters; - private Codec codec; public void startNettyServer(int port) throws Exception { //boss单线程 @@ -44,9 +43,14 @@ public class NodesServer { .childHandler(new ChildChannelHandler()); //绑定端口，同步等待成功 ChannelFuture future = bootstrap.bind(port).sync(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bossGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + workerGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + })); //等待服务器监听端口关闭 future.channel().closeFuture().sync(); } catch (Exception e) { + e.printStackTrace(); //do nothing System.out.println("netty stop"); } finally { @@ -70,8 +74,6 @@ public class NodesServer { ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); ch.pipeline() .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) -// .addLast(codec.newEncoder()) -// .addLast(codec.newDecoder()) .addLast(new StringDecoder()) .addLast(serverHandler); } @@ -85,7 +87,4 @@ public class NodesServer { this.messageFilters = messageFilters; } - public void setCodec(Codec codec) { - this.codec = codec; - } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 9fe62a4..2c60d6e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.worker.starters; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.StrUtil; import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; @@ -13,6 +14,7 @@ import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.worker.cache.CaffeineCacheHolder; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.model.TotalCount; +import com.jd.platform.hotkey.worker.netty.dashboard.NettyClient; import com.jd.platform.hotkey.worker.netty.filter.HotKeyFilter; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; @@ -217,6 +219,29 @@ public class EtcdStarter { } } + /** + * 每隔30秒去获取一下dashboard的地址 + */ + @Scheduled(fixedRate = 30000) + public void fetchDashboardIp() { + try { + //获取DashboardIp + List keyValues = configCenter.getPrefix(ConfigConstant.dashboardPath); + + //是空，给个警告 + if (CollectionUtil.isEmpty(keyValues)) { + logger.warn("very important warn !!! Dashboard ip is null!!!"); + return; + } + + String dashboardIp = keyValues.get(0).getValue().toStringUtf8(); + NettyClient.getInstance().connect(dashboardIp); + + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 校验一下receive的key数量，如果一段时间没变，考虑网络问题，就将worker注册自己到etcd的心跳给断掉30秒，让各client重连一下自己 */ diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java index 0b9c522..5759f5e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.starters; -import com.jd.platform.hotkey.common.coder.Codec; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import com.jd.platform.hotkey.worker.netty.server.NodesServer; @@ -29,8 +28,6 @@ public class NodesServerStarter { private IClientChangeListener iClientChangeListener; @Resource private List messageFilters; - @Resource - private Codec codec; @PostConstruct public void start() { @@ -40,7 +37,6 @@ public class NodesServerStarter { NodesServer nodesServer = new NodesServer(); nodesServer.setClientChangeListener(iClientChangeListener); nodesServer.setMessageFilters(messageFilters); - nodesServer.setCodec(codec); try { nodesServer.startNettyServer(port); } catch (Exception e) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java index acaa70e..55bd842 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java @@ -13,17 +13,13 @@ public class CpuNum { public static int workerCount() { //取cpu核数，新版jdk在docker里取的就是真实分配的，老版jdk取的是宿主机的，可能特别大，如32核 int count = Runtime.getRuntime().availableProcessors(); -// if (isNewerVersion()) { -// if (count >= 4) { -// count = count / 2; -// } -// } else { -// if (count >= 8) { -// count = 4; -// } -// } - if (count >= 4) { + if (isNewerVersion()) { + return count; + } else { count = count / 2; + if (count == 0) { + count = 1; + } } return count; } -- Gitee From 3702e705a1c4e89d506b84b159dd5bd73e9f55f0 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 4 Sep 2020 15:50:12 +0800 Subject: [PATCH 148/156] =?UTF-8?q?=E5=88=A0disruptor=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/client/netty/NettyClient.java | 7 -- common/pom.xml | 8 --- .../platform/hotkey/common/coder/Codec.java | 30 --------- .../hotkey/common/coder/MessageDecoder.java | 53 --------------- .../hotkey/common/coder/MessageEncoder.java | 47 -------------- .../hotkey/common/coder/NettyCodec.java | 36 ---------- .../common/configcenter/ConfigConstant.java | 6 ++ .../hotkey/worker/config/CodecConfig.java | 20 ------ .../hotkey/worker/config/DisruptorConfig.java | 33 ---------- .../worker/disruptor/AbsEventConsumer.java | 54 --------------- .../worker/disruptor/AbsWorkConsumer.java | 54 --------------- .../hotkey/worker/disruptor/BaseEvent.java | 26 -------- .../worker/disruptor/DisruptorBuilder.java | 65 ------------------- .../worker/disruptor/MessageProducer.java | 8 --- .../worker/disruptor/ProducerFactory.java | 50 -------------- .../worker/disruptor/hotkey/HotKeyEvent.java | 18 ----- .../disruptor/hotkey/HotKeyEventConsumer.java | 41 ------------ .../disruptor/hotkey/HotKeyEventProducer.java | 32 --------- .../hotkey/worker/model/ClientInfo.java | 62 ------------------ .../hotkey/worker/mq/IMqMessageReceiver.java | 9 --- .../worker/netty/filter/HotKeyFilter.java | 8 +-- .../worker/netty/filter/KeyCounterFilter.java | 8 +-- .../hotkey/worker/tool/InitConstant.java | 4 -- 23 files changed, 8 insertions(+), 671 deletions(-) delete mode 100755 common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java delete mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java delete mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java delete mode 100755 common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java index d2a36a4..d8ba892 100755 --- a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java @@ -2,8 +2,6 @@ package com.jd.platform.hotkey.client.netty; import com.jd.platform.hotkey.client.core.worker.WorkerInfoHolder; import com.jd.platform.hotkey.client.log.JdLogger; -import com.jd.platform.hotkey.common.coder.Codec; -import com.jd.platform.hotkey.common.coder.NettyCodec; import com.jd.platform.hotkey.common.tool.Constant; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; @@ -28,9 +26,6 @@ public class NettyClient { private Bootstrap bootstrap; - private Codec codec = new NettyCodec(); - - public static NettyClient getInstance() { return nettyClient; } @@ -56,8 +51,6 @@ public class NettyClient { ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); ch.pipeline() .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) -// .addLast(codec.newEncoder()) -// .addLast(codec.newDecoder()) .addLast(new StringDecoder()) //10秒没消息时，就发心跳包过去 .addLast(new IdleStateHandler(0, 0, 30)) diff --git a/common/pom.xml b/common/pom.xml index a548184..e11fe7a 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -15,7 +15,6 @@ 1.8 4.1.42.Final - 3.4.2 1.1.7.3 1.2.70 0.0.16 @@ -40,13 +39,6 @@ netty-all ${netty4.version} - - - com.lmax - disruptor - ${disruptor.version} - - com.alibaba fastjson diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java deleted file mode 100755 index c61805b..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jd.platform.hotkey.common.coder; - -import io.netty.channel.ChannelHandler; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public interface Codec { - - ChannelHandler newEncoder(); - - ChannelHandler newDecoder(); -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java deleted file mode 100644 index 24cb0ed..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jd.platform.hotkey.common.coder; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.charset.Charset; -import java.util.List; - -/** - * @author wuweifeng wrote on 2020-01-06 - * @version 1.0 - */ -public class MessageDecoder extends ByteToMessageDecoder { - private Logger logger = LoggerFactory.getLogger(getClass()); - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List out) { - if (byteBuf.readableBytes() < 10 || byteBuf.readableBytes() > Constant.MAX_LENGTH) { - logger.warn("数据包不正确，当前包大小为：" + byteBuf.readableBytes()); - return; - } - - byteBuf.markReaderIndex(); - - HotKeyMsg message = new HotKeyMsg(); - - int magicNumber = byteBuf.readInt(); - if (Constant.MAGIC_NUMBER != magicNumber) { - logger.warn("MAGIC_NUMBER不正确:" + magicNumber); - return; - } - - message.setMagicNumber(byteBuf.readInt()); // 读取魔数 - - MessageType messageType = MessageType.get(byteBuf.readByte()); - if (messageType == null) { - logger.error("messageType is null , byteBuf readByte = " + byteBuf.readByte()); - return; - } - message.setMessageType(messageType); // 读取当前的消息类型 - - int bodyLength = byteBuf.readInt(); // 读取消息体长度和数据 - CharSequence body = byteBuf.readCharSequence(bodyLength, Charset.defaultCharset()); - message.setBody(body.toString()); - out.add(message); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java deleted file mode 100644 index acf2162..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.jd.platform.hotkey.common.coder; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToByteEncoder; - -import java.nio.charset.Charset; - -/** - * @author wuweifeng wrote on 2020-01-06 - * @version 1.0 - */ -public class MessageEncoder extends MessageToByteEncoder { - - @Override - protected void encode(ChannelHandlerContext ctx, HotKeyMsg message, ByteBuf out) { - MessageType messageType = message.getMessageType(); - //非法类型 - if (MessageType.get(messageType.getType()) == null) { - return; - } - // 这里会判断消息类型是不是EMPTY类型，如果是EMPTY类型，则表示当前消息不需要写入到管道中 - if (messageType == MessageType.EMPTY) { - return; - } - //4个byte - out.writeInt(Constant.MAGIC_NUMBER); // 写入当前的魔数 - //1个byte - out.writeByte(message.getMessageType().getType()); // 写入当前消息的类型 - - if (null == message.getBody()) { - out.writeInt(0); // 如果消息体为空，则写入0，表示消息体长度为0 - } else { - System.out.println(message.getBody().length()); - out.writeInt(message.getBody().length()); - out.writeCharSequence(message.getBody(), Charset.defaultCharset()); - } - out.writeBytes(Constant.DELIMITER.getBytes()); - } - - public static void main(String[] args) { - System.out.println(Constant.MAGIC_NUMBER + ""); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java deleted file mode 100755 index 1aeac45..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jd.platform.hotkey.common.coder; - -import io.netty.channel.ChannelHandler; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public class NettyCodec implements Codec { - - @Override - public ChannelHandler newEncoder() { - return new MessageEncoder(); - } - - @Override - public ChannelHandler newDecoder() { - return new MessageDecoder(); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 2a7eff0..3d22e8c 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -64,6 +64,12 @@ public interface ConfigConstant { * time unit : day */ String clearCfgPath = "/jd/clearCfg/"; + + /** + * app配置 + */ + String appCfgPath = "/jd/appCfg/"; + /** * 控制台启动的netty端口 */ diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java deleted file mode 100644 index 0d62c90..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.common.coder.Codec; -import com.jd.platform.hotkey.common.coder.NettyCodec; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * netty的编解码器 - * @author wuweifeng wrote on 2020-02-21 - * @version 1.0 - */ -@Configuration -public class CodecConfig { - - @Bean - public Codec codec() { - return new NettyCodec(); - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java deleted file mode 100644 index 0a92f8f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.Resource; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -@Configuration -public class DisruptorConfig { - @Resource - private IKeyListener iKeyListener; - - @Value("${open.timeout}") - private boolean openTimeOut; - @Value("${thread.count}") - private int threadCount; - -// @Bean -// public MessageProducer messageProducer() { -// InitConstant.openTimeOut = openTimeOut; -// //将实际值赋给static变量 -// InitConstant.threadCount = threadCount; -// -// return ProducerFactory.createHotKeyProducer(iKeyListener); -// } - - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java deleted file mode 100755 index 47839ae..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java +++ /dev/null @@ -1,54 +0,0 @@ -//package com.jd.platform.hotkey.worker.disruptor; -// -//import cn.hutool.core.date.SystemClock; -//import com.jd.platform.hotkey.common.model.BaseModel; -//import com.jd.platform.hotkey.common.tool.Constant; -//import com.jd.platform.hotkey.worker.tool.InitConstant; -//import com.lmax.disruptor.EventHandler; -// -//import java.util.concurrent.atomic.LongAdder; -//import java.util.zip.CRC32; -// -///**各个消费者重复消费 -// * @author wuweifeng wrote on 2019-08-21. -// */ -//public abstract class AbsEventConsumer implements EventHandler { -// -// private int hashIndex; -// -// public static final LongAdder totalDealCount = new LongAdder(); -// //过期的 -// public static final LongAdder expireTotalCount = new LongAdder(); -// -// public AbsEventConsumer(int hashIndex) { -// this.hashIndex = hashIndex; -// } -// -// @Override -// public void onEvent(T t, long l, boolean b) { -// //每个消费者，只处理特定的key。保证相同的key，一定被同一个线程处理 -// BaseModel model = t.getModel(); -// if (model == null || model.getKey() == null) { -// return; -// } -// if (Math.abs(hash(model.getKey())) % Constant.Default_Threads == hashIndex) { -// //5秒前的过时消息就不处理了 -// if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { -// expireTotalCount.increment(); -// return; -// } -// onNewEvent(t); -// -// //处理完毕，将数量加1 -// totalDealCount.increment(); -// } -// } -// -// protected abstract void onNewEvent(T t); -// -// private long hash(String key) { -// CRC32 crc = new CRC32(); -// crc.update(key.getBytes()); -// return crc.getValue(); -// } -//} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java deleted file mode 100755 index bca2e70..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import cn.hutool.core.date.SystemClock; -import com.jd.platform.hotkey.common.model.BaseModel; -import com.jd.platform.hotkey.worker.tool.InitConstant; -import com.lmax.disruptor.WorkHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.atomic.LongAdder; - -/** - * 各个消费者不重复消费 - * - * @author wuweifeng wrote on 2019-08-21. - */ -public abstract class AbsWorkConsumer implements WorkHandler { - - private int hashIndex; - - public static final LongAdder totalDealCount = new LongAdder(); - //过期的 - public static final LongAdder expireTotalCount = new LongAdder(); - - private Logger logger = LoggerFactory.getLogger(getClass()); - - public AbsWorkConsumer(int hashIndex) { - this.hashIndex = hashIndex; - } - - @Override - public void onEvent(T t) { - BaseModel model = t.getModel(); - if (model == null || model.getKey() == null) { - return; - } - //5秒前的过时消息就不处理了 - if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { - expireTotalCount.increment(); - logger.warn("this key timeout : " + model.getKey() + " createAt : " + model.getCreateTime() + - " event at " + SystemClock.now()); - if (InitConstant.openTimeOut) { - return; - } - } - onNewEvent(t); - - //处理完毕，将数量加1 - totalDealCount.increment(); - } - - protected abstract void onNewEvent(T t); - -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java deleted file mode 100644 index e8483fe..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.jd.platform.hotkey.common.model.BaseModel; - -/** - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class BaseEvent { - private T model; - - public BaseEvent(T model) { - this.model = model; - } - - public BaseEvent() { - } - - public T getModel() { - return model; - } - - public void setModel(T model) { - this.model = model; - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java deleted file mode 100644 index 64dc4c0..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.lmax.disruptor.BlockingWaitStrategy; -import com.lmax.disruptor.EventFactory; -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.WorkHandler; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.dsl.ProducerType; - -import java.util.concurrent.Executors; - -/** - * 每个worker重复消费生产者消息 - * - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class DisruptorBuilder { - private int bufferSize; - private EventHandler[] eventHandlers; - private WorkHandler[] workHandlers; - private EventFactory eventFactory; - - public Disruptor build() { - Disruptor disruptor = new Disruptor<>(eventFactory, bufferSize, Executors.defaultThreadFactory(), - ProducerType.SINGLE, new BlockingWaitStrategy()); - - if (eventHandlers != null) { - disruptor.handleEventsWith(eventHandlers); - } - if (workHandlers != null) { - disruptor.handleEventsWithWorkerPool(workHandlers); - } - - disruptor.start(); - return disruptor; - } - - /** - * 每个worker会重复消费 - */ - public DisruptorBuilder setEventHandlers(EventHandler... eventHandlers) { - this.eventHandlers = eventHandlers; - return this; - } - - /** - * 每个worker不会重复消费 - */ - public DisruptorBuilder setWorkerHandlers(WorkHandler... workHandlers) { - this.workHandlers = workHandlers; - return this; - } - - public DisruptorBuilder setEventFactory(EventFactory eventFactory) { - this.eventFactory = eventFactory; - return this; - } - - public DisruptorBuilder setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - return this; - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java deleted file mode 100755 index 63d1421..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -/** - * @author wuweifeng wrote on 2018/4/20. - */ -public interface MessageProducer { - void publish(T t); -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java deleted file mode 100755 index 8854627..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEvent; -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventConsumer; -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventProducer; -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import com.jd.platform.hotkey.worker.tool.CpuNum; -import com.jd.platform.hotkey.worker.tool.InitConstant; -import com.lmax.disruptor.dsl.Disruptor; - -/** - * Disruptor创建producer的地方 - * - * @author wuweifeng wrote on 2019-11-5. - */ -public class ProducerFactory { - - /** - * 创建热key的producer - * - * @return HotKeyEventProducer - */ - public static MessageProducer createHotKeyProducer(IKeyListener iKeyListener) { - int threadCount = CpuNum.workerCount(); - //如果手工指定了线程数，就用手工指定的 - if (InitConstant.threadCount != 0) { - threadCount = InitConstant.threadCount; - } else { - if (threadCount >= 8) { - threadCount = threadCount / 2; - } - } - - HotKeyEventConsumer[] array = new HotKeyEventConsumer[threadCount]; - for (int i = 0; i < threadCount; i++) { - array[i] = new HotKeyEventConsumer(i); - array[i].setKeyListener(iKeyListener); - } - DisruptorBuilder builder = new DisruptorBuilder<>(); - Disruptor disruptor = builder - .setBufferSize(InitConstant.bufferSize * 1024 * 1024) - .setEventFactory(HotKeyEvent::new) -// .setEventHandlers(array) //重复消费的 - .setWorkerHandlers(array) //不重复消费的 - .build(); - - return new HotKeyEventProducer(disruptor); - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java deleted file mode 100755 index 8ec41c3..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.disruptor.BaseEvent; - -/** - * @author wuweifeng wrote on 2019-08-21. - */ -public class HotKeyEvent extends BaseEvent { - - public HotKeyEvent(HotKeyModel hotKeyModel) { - super(hotKeyModel); - } - - public HotKeyEvent() { - - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java deleted file mode 100755 index ca248f5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.disruptor.AbsWorkConsumer; -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import com.jd.platform.hotkey.worker.keylistener.KeyEventOriginal; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * 热key的消费者 - * @author wuweifeng wrote on 2019-12-12. - */ -public class HotKeyEventConsumer extends AbsWorkConsumer { - private Logger logger = LoggerFactory.getLogger(getClass()); - - private IKeyListener iKeyListener; - - public HotKeyEventConsumer(int hashIndex) { - super(hashIndex); - } - - @Override - protected void onNewEvent(HotKeyEvent hotKeyEvent) { - HotKeyModel model = hotKeyEvent.getModel(); - if (iKeyListener == null) { - logger.warn("new key is coming, but no consumer deal this key!"); - return; - } - - if (model.isRemove()) { - iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); - } else { - iKeyListener.newKey(model, KeyEventOriginal.CLIENT); - } - } - - public void setKeyListener(IKeyListener iKeyListener) { - this.iKeyListener = iKeyListener; - } -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java deleted file mode 100755 index 42dae5f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.worker.disruptor.MessageProducer; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; - -/** - * 所有节点发来的消息，都进入这里，然后publish出去，供消费者消费 - * - * @author wuweifeng wrote on 2019/11/6. - */ -public class HotKeyEventProducer implements MessageProducer { - - private Disruptor disruptor; - - public HotKeyEventProducer(Disruptor disruptor) { - this.disruptor = disruptor; - } - - @Override - public void publish(HotKeyEvent hotKeyEvent) { - RingBuffer ringBuffer = disruptor.getRingBuffer(); - long sequence = ringBuffer.next(); - try { - HotKeyEvent event = ringBuffer.get(sequence); - event.setModel(hotKeyEvent.getModel()); - } finally { - ringBuffer.publish(sequence); - } - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java b/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java deleted file mode 100644 index 081652b..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.jd.platform.hotkey.worker.model; - -import io.netty.channel.ChannelHandlerContext; - -/** - * @author wuweifeng wrote on 2019-12-05 - * @version 1.0 - */ -public class ClientInfo { - /** - * 应用名 - */ - private String appName; - /** - * 连接的id - */ - private String channelId; - - private ChannelHandlerContext channelHandlerContext; - - public ClientInfo() { - } - - public ClientInfo(String appName, String channelId, ChannelHandlerContext channelHandlerContext) { - this.appName = appName; - this.channelId = channelId; - this.channelHandlerContext = channelHandlerContext; - } - - @Override - public String toString() { - return "ClientInfo{" + - "appName='" + appName + '\'' + - ", channelId='" + channelId + '\'' + - ", channelHandlerContext=" + channelHandlerContext + - '}'; - } - - public ChannelHandlerContext getChannelHandlerContext() { - return channelHandlerContext; - } - - public void setChannelHandlerContext(ChannelHandlerContext channelHandlerContext) { - this.channelHandlerContext = channelHandlerContext; - } - - public String getAppName() { - return appName; - } - - public void setAppName(String appName) { - this.appName = appName; - } - - public String getChannelId() { - return channelId; - } - - public void setChannelId(String channelId) { - this.channelId = channelId; - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java b/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java deleted file mode 100644 index 155a476..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.jd.platform.hotkey.worker.mq; - -/** - * @author wuweifeng wrote on 2019-12-12 - * @version 1.0 - */ -public interface IMqMessageReceiver { - void receive(String msg); -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 0a1521a..2640fd7 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -7,7 +7,6 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.keydispatcher.KeyProducer; -import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -27,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; */ @Component @Order(3) -public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { +public class HotKeyFilter implements INettyMsgFilter { @Resource private KeyProducer keyProducer; @@ -48,11 +47,6 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { return true; } - @Override - public void receive(String msg) { - publishMsg(msg, null); - } - private void publishMsg(String message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, HotKeyModel.class); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index d8eae48..b268156 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -8,7 +8,6 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.counter.KeyCountItem; -import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; import com.jd.platform.hotkey.worker.tool.InitConstant; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -28,7 +27,7 @@ import static com.jd.platform.hotkey.worker.counter.CounterConfig.COUNTER_QUEUE; */ @Component @Order(4) -public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { +public class KeyCounterFilter implements INettyMsgFilter { private Logger logger = LoggerFactory.getLogger(getClass()); /** @@ -52,11 +51,6 @@ public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { return true; } - @Override - public void receive(String msg) { - publishMsg("", msg, null); - } - private void publishMsg(String appName, String message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 List models = FastJsonUtils.toList(message, KeyCountModel.class); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java index f9702f2..f603c1c 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java @@ -13,10 +13,6 @@ public class InitConstant { //单位是百万 public static int bufferSize = 2; - public static boolean openTimeOut; - - public static int threadCount; - public static final LongAdder expireTotalCount = new LongAdder(); public static final LongAdder totalDealCount = new LongAdder(); -- Gitee From 3bc9c0069324bec644b39dc0b240dd9905c42bc0 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Sat, 5 Sep 2020 14:30:19 +0800 Subject: [PATCH 149/156] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d7a858..f83b531 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ 1 探测性能：8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒平稳处理30万以上，实际压测达到37万，CPU平稳支撑，框架无异常。 -2 推送性能：在高并发写入的同时，对外推送目前性能约平稳推送每秒10-12万次，譬如有1千台server，一台worker上每秒产生了100个热key，那么这1秒会平稳推送100 * 1000 = 10万次，10万次推送会明确在1s内全部送达。如果是写入少，推送多，以纯推送来计数的话，该框架每秒可稳定对外推送40-60万次平稳，80次极限可撑几秒。 +2 推送性能：在高并发写入的同时，对外推送目前性能约平稳推送每秒10-12万次，譬如有1千台server，一台worker上每秒产生了100个热key，那么这1秒会平稳推送100 * 1000 = 10万次，10万次推送会明确在1s内全部送达。如果是写入少，推送多，以纯推送来计数的话，该框架每秒可稳定对外推送40-60万次平稳，80万次极限可撑几秒。 在真实业务场景中，可用1：1000的比例，即1台worker支撑1000台业务服务端的key探测任务，即可带来极大的数据存储资源节省（如对redis集群的扩充）。测试详情可去我[CSDN博客](https://blog.csdn.net/tianyaleixiaowu)查看。 -- Gitee From de65a2f5adf55b77f10ca0791e03cf62c6478e1f Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Fri, 11 Sep 2020 11:55:55 +0800 Subject: [PATCH 150/156] update README.md. --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index f83b531..b67b028 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,33 @@ https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw ![输入图片说明](https://images.gitee.com/uploads/images/2020/0807/170047_e37c882e_303698.jpeg "1596790826590.jpg") +#### 常见问题 + **1 worker挂了怎么办** + +client根据worker的数量对key进行hash后分发，同一个key一定会被发往同一个worker。譬如4台，挂了一台，key就自动hash到另外3台。那么这个过程中，就会丢失最多一个探测周期内的所有发来的key，譬如2秒10次算热，那么就可能全部被rehash，丢失这2秒的数据。 + +它的影响是什么呢？我要不要去存下来所有发来的key呢？很多人都会问的问题。 + +首先挂机，那是极其罕见的事件，即便挂了，对于特别热的key，完全不影响，hash丢几秒，不影响它继续瞬间变热。对于不热的key，它挂不挂，它也热不了。对于那些将热未热的，可能会这次让它热不起来，但没有什么影响，业务服务完全可以吃下这个热key。而加上一堆别的组件如存储、worker间通信传输key等，它的复杂度，性能都会影响很大。 + +所以它挂了对系统没有任何影响 + +**2 为什么全部要worker汇总计算，而不是客户端自己计算** + +首先，客户端是会本地累加的，在固定的上报周期内，如500ms内，本地就是在累加，每500ms批量上报给worker一次。如果上报频率很高，如10ms一次，那么大概率本地同一个key是没有累加。 + +有人会说，把这个间隔拉长，譬如本地计算3秒后，本地判定热key，再上报给其他机器。那么这种场景首先对于京东是不可行的，哪怕1秒都不行。譬如一个用户刷子，它在非常频繁地刷接口，一秒刷了500次，而正常用户一秒最多点5次，它已经是非常严重的刷子了。但我们本地还是判断不出来它是不是刷子。为什么？机器多。 + +随便一个app小组都有数千台机器，一秒500次请求，一个机器连1次都平均不到，大部分是0次，本地如何判断它是刷子呢？总不能访问我一次就算它刷吧。 + +然后抢购场景，有些秒杀商品，1-2秒就没了，流量就停了，你本地计算了3秒，才去上报，那活动已经结束了，你的热key再推送已经没价值了。我们就要在活动即将开始之前的可能在10ms内，就要该商品被推送到所有client的jvm里去，根本等不了1秒。 + +**3 为什么是etcd，不是zookeeper之类的** + +etcd里面具备一个过期删除的功能，你可以设置一个key几秒过期，etcd会自动删除它，删除时还会给所有监听的client回调，这个功能在框架里是在用的，别的配置中心没有这个功能。 + +etcd的性能和稳定性、低负载等各项指标非常优异，完全满足我们的需求。而zk在很多暴涨流量前和高负载下，并不是那么稳定，性能也差的远。 + #### 安装教程 1. 安装etcd -- Gitee From 8b1d33d0a4b1e3bc005793efc2c7cf1fea1a7d54 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 10:43:59 +0800 Subject: [PATCH 151/156] =?UTF-8?q?=E6=89=B9=E9=87=8F=E6=94=B91=E4=B8=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sample/src/main/java/com/jd/platform/sample/Starter.java | 8 ++++++++ sample/src/main/resources/application.yml | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index 15a4233..6960e59 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -5,6 +5,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; /** * @author wuweifeng wrote on 2020-01-14 @@ -24,4 +27,9 @@ public class Starter { starter.startPipeline(); } + public static void main(String[] args) { + Map> totalSkuSet = new HashMap<>(); + System.out.println(totalSkuSet.get("a")); + } + } diff --git a/sample/src/main/resources/application.yml b/sample/src/main/resources/application.yml index 879608b..8c025c8 100644 --- a/sample/src/main/resources/application.yml +++ b/sample/src/main/resources/application.yml @@ -1,9 +1,10 @@ #etcd的地址，如有多个用逗号分隔 etcd: - server: ${etcdServer:https://127.0.0.1:2379} + server: ${etcdServer:http://open-etcd.jd.com:2000} +# server: ${etcdServer:https://127.0.0.1:2379} spring: application: - name: sample + name: persona-api ###############################---redis---############################## redis: host: ${REDIS_HOST:127.0.0.1} -- Gitee From 781f2a354b6681ccdc5feb37ebf657c48ab65c9c Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 25 Sep 2020 15:33:34 +0800 Subject: [PATCH 152/156] =?UTF-8?q?JS=20=E7=89=88=E6=9C=AC=E9=99=8D?= =?UTF-8?q?=E4=BD=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/db.sql | 5 +++++ .../admin/assets/js/bootstrap/js/base_list.js | 16 ++++++++-------- .../static/admin/common/js/rule-edit.js | 2 +- .../static/admin/common/js/worker-edit.js | 2 +- .../templates/admin/key/listmaxhot.html | 4 ++-- .../src/main/resources/templates/admin/main.html | 16 ++++++++-------- .../resources/templates/admin/rule/list.html | 8 ++++---- .../resources/templates/admin/user/list.html | 2 +- 8 files changed, 30 insertions(+), 25 deletions(-) diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 98fbd82..fbbdc51 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -1,3 +1,8 @@ +-- !!! 注意设置sql model 否则可能sql报错 ！！！ +-- 查询你的sql_model参数：select @@global.sql_mode; 发现ONLY_FULL_GROUP_BY 则会导致报错 +-- 解决方式：set @@global.sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' +-- 详情查阅：https://www.cnblogs.com/hjhsblogs/p/11079356.html + DROP TABLE IF EXISTS `hk_change_log`; CREATE TABLE `hk_change_log` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 01763f2..de217e1 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -46,7 +46,7 @@ }, onLoadError: function(status){ //加载失败时执行 let token = getCookie("token"); - if(status === 500 && ( token == "undefined" || token =="")){ + if(status === 1000 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); @@ -303,7 +303,7 @@ title: title, content: url, success:function(dom){ - let $iframeDom=$(dom[0]).find("iframe").eq(0).contents(); + var $iframeDom=$(dom[0]).find("iframe").eq(0).contents(); $iframeDom.find("#test").html("我是从父级传来的值哟……") } }) @@ -495,19 +495,19 @@ modal_status = { function getCookie(cname){ - let token = window.localStorage.getItem('token'); + var token = window.localStorage.getItem('token'); if(token != null && token !==""){ - let time = window.localStorage.getItem('time'); + var time = window.localStorage.getItem('time'); if(Date.now()-time>7*24*360000){ localStorage.removeItem(time); return ""; } return token; } - let name = cname + "="; - let ca = document.cookie.split(';'); - for(let i=0; i"); } - let apps = data.appNames; - let appName = data.appName; - for (let i = 0; i < apps.length; i++) { + var apps = data.appNames; + var appName = data.appName; + for (var i = 0; i < apps.length; i++) { let app = apps[i]; if(app === appName){ $("#apps").append(""); diff --git a/dashboard/src/main/resources/templates/admin/user/list.html b/dashboard/src/main/resources/templates/admin/user/list.html index 3c2df58..55e31a2 100644 --- a/dashboard/src/main/resources/templates/admin/user/list.html +++ b/dashboard/src/main/resources/templates/admin/user/list.html @@ -176,7 +176,7 @@ $(function(){ + -